diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000000000000000000000000000000000000..d142467d14c8ae7495138ddb497722014306897d --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,75 @@ +--- +summary: "Repository script entry points and compatibility notes" +read_when: + - Looking for an existing script before adding a new one + - Running repository checks, tests, docs, Docker, release, or GitHub helper scripts + - Updating package scripts or CI workflow script references +title: "Scripts Directory" +--- + +# Scripts Directory + +The `scripts/` directory contains repository tooling used by local development, +CI, docs publishing, releases, Docker proof, and maintainer operations. Prefer +the package-script entry points in `package.json` when one exists, then read the +underlying script before running it directly. + +## Compatibility + +Many scripts are stable paths referenced by `package.json`, GitHub Actions, +docs, and maintainer runbooks. Do not move, rename, or regroup scripts only to +improve taxonomy. A directory migration needs an explicit maintainer-approved +compatibility plan for package scripts, workflows, docs snippets, and any raw +script paths users may have copied. + +This index is a discovery aid for the current flat layout. It does not define a +new directory taxonomy. + +## Common Entry Points + +| Area | Prefer | Notes | +| --------------- | ------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------- | +| Build | `pnpm build` | Runs `scripts/build-all.mts`; use specific build scripts only when debugging a build stage. | +| Changed checks | `pnpm changed:lanes --json`, `pnpm check:changed` | Lane classification lives in `scripts/changed-lanes.mjs`; changed-file checks live in `scripts/check-changed.mjs`. | +| Docs | `pnpm docs:list`, `pnpm docs:check-mdx`, `pnpm docs:check-links` | Backed by `scripts/docs-list.js`, `scripts/check-docs-mdx.mjs`, and `scripts/docs-link-audit.mjs`. | +| Formatting docs | `pnpm format:docs:check` | Uses `scripts/format-docs.mts`; use write mode only when intentionally formatting docs. | +| Lint | `pnpm lint`, `pnpm lint:core`, `pnpm lint:all` | Wrapper scripts keep oxlint behavior aligned with repo config. | +| Targeted tests | `pnpm test ` or `node scripts/run-vitest.mjs ` | Avoid bare `vitest`; it can start watch mode. | +| Changed tests | `pnpm test:changed` | Uses the repo's changed-test resolver instead of a broad Vitest run. | +| Docker proof | `pnpm test:docker:all`, `pnpm test:docker:rerun`, `pnpm test:docker:timings` | Use the planner/rerun helpers before launching broad Docker work. | +| Live proof | `pnpm test:live` | Live checks require the matching environment and credentials. | +| Release checks | `pnpm release:check`, `pnpm release:beta`, `pnpm release:candidate` | Release scripts are maintainer workflows; read release docs before use. | +| GitHub reads | `scripts/gh-read` | Uses a GitHub App read token when configured, leaving normal `gh` login for writes. | +| Commits | `git add ` then `git commit -m ""` | Stage only the intended files for each commit. | +| Remote proof | `node scripts/crabbox-wrapper.mjs ...` | Use when the current host lacks the required isolation, OS, device, or capacity; sync each run and reuse the lease. | + +## Script Families + +- `check-*.mts` / `check-*.ts` / retained `check-*.mjs`: guardrails for architecture, docs, package + contents, boundaries, workflows, and generated artifacts. +- `run-*.mjs` / `run-*.mts`: stable wrappers and typed implementations for Node, Vitest, + oxlint, tsgo, and environment setup. +- `test-*.mts` / retained `test-*.mjs` / `test-*.sh` / `test-*.ts`: test planners, Docker lanes, live + checks, and focused validation helpers. +- `docs-*` and `check-docs-*`: docs listing, link auditing, MDX checks, + spellcheck, sync, and i18n glossary checks. +- `release-*`, `openclaw-npm-*`, and `plugin-*-release-*`: release preparation, + package verification, and publishing helpers. +- `docker-*`, `test-docker-*`, and `test-live-*-docker.sh`: Docker E2E planning, + rerun, timing, and live/package lane helpers. +- `gh-read*`, `label-*`, `sync-labels.ts`, and PR helpers: GitHub read, + labeling, and maintainer workflow support. +- `generate-*`, `write-*`, `copy-*`, and `sync-*`: generated docs, metadata, + package surfaces, and build artifact support. +- `lib/`: shared helpers imported by script entry points. + +## Maintenance Rules + +- Read `scripts/AGENTS.md` before changing scripts. +- Keep package scripts, generators, generated-artifact checks, docs references, + and workflow references aligned when touching a script path. +- Prefer existing wrappers instead of introducing a raw tool invocation. +- Add or update focused tests under `test/scripts/` when changing script + behavior. + +See also [Scripts](https://docs.openclaw.ai/help/scripts) for public-facing script guidance. diff --git a/scripts/analyze-plugin-sdk-usage.ts b/scripts/analyze-plugin-sdk-usage.ts new file mode 100644 index 0000000000000000000000000000000000000000..c8951ea6837c02078aab85dcc8f8b3e23bfe861b --- /dev/null +++ b/scripts/analyze-plugin-sdk-usage.ts @@ -0,0 +1,10 @@ +#!/usr/bin/env node +// Analyze Plugin Sdk Usage script supports OpenClaw repository automation. +import { main } from "./ts-topology.ts"; + +const forwardedArgs = process.argv.slice(2); +const normalizedArgs = forwardedArgs[0] === "--" ? forwardedArgs.slice(1) : forwardedArgs; +const exitCode = await main(["--scope=plugin-sdk", ...normalizedArgs]); +if (exitCode !== 0) { + process.exit(exitCode); +} diff --git a/scripts/android-app-i18n.ts b/scripts/android-app-i18n.ts new file mode 100644 index 0000000000000000000000000000000000000000..5fbae58259fc2ed1e25965173a44f97519018f07 --- /dev/null +++ b/scripts/android-app-i18n.ts @@ -0,0 +1,1594 @@ +import { createHash } from "node:crypto"; +import { mkdir, readdir, readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { expectDefined } from "../packages/normalization-core/src/expect.js"; +import { isDirectRunUrl } from "./lib/direct-run.mjs"; +import { NATIVE_I18N_LOCALES } from "./native-i18n-locales.ts"; + +const HERE = path.dirname(fileURLToPath(import.meta.url)); +const ROOT = path.resolve(HERE, ".."); +const ANDROID_MAIN_ROOT = path.join(ROOT, "apps", "android", "app", "src", "main"); +const RESOURCE_ROOT = path.join(ANDROID_MAIN_ROOT, "res"); +const SOURCE_ROOT = path.join(ANDROID_MAIN_ROOT, "java"); +const ANDROID_PLAY_SOURCE_ROOT = path.join(ROOT, "apps", "android", "app", "src", "play", "java"); +const ANDROID_THIRD_PARTY_ROOT = path.join(ROOT, "apps", "android", "app", "src", "thirdParty"); +const ANDROID_THIRD_PARTY_RESOURCE_ROOT = path.join(ANDROID_THIRD_PARTY_ROOT, "res"); +const ANDROID_THIRD_PARTY_SOURCE_ROOT = path.join(ANDROID_THIRD_PARTY_ROOT, "java"); +const ANDROID_WEAR_MAIN_ROOT = path.join(ROOT, "apps", "android", "wear", "src", "main"); +const WEAR_RESOURCE_ROOT = path.join(ANDROID_WEAR_MAIN_ROOT, "res"); +const WEAR_SOURCE_ROOT = path.join(ANDROID_WEAR_MAIN_ROOT, "java"); +const ANDROID_SOURCE_ROOTS = [ + SOURCE_ROOT, + ANDROID_PLAY_SOURCE_ROOT, + ANDROID_THIRD_PARTY_SOURCE_ROOT, + WEAR_SOURCE_ROOT, +] as const; +const INVENTORY_PATH = path.join(ROOT, "apps", ".i18n", "native-source.json"); +const ARTIFACT_ROOT = path.join(ROOT, "apps", ".i18n", "native"); +const TOOL_DISPLAY_PATH = path.join( + ROOT, + "apps", + "shared", + "OpenClawKit", + "Sources", + "OpenClawKit", + "Resources", + "tool-display.json", +); +const GENERATED_KOTLIN_PATH = path.join( + SOURCE_ROOT, + "ai", + "openclaw", + "app", + "i18n", + "NativeStringResources.kt", +); +const MANAGED_PREFIX = "native_"; +const ANDROID_QUALIFIERS: Record = { + id: "in", + "zh-CN": "zh-rCN", + "zh-TW": "zh-rTW", + "pt-BR": "pt-rBR", + "ja-JP": "ja", +}; +const localeDirectory = (locale: string) => `values-${ANDROID_QUALIFIERS[locale] ?? locale}`; +const LOCALES = ["values", ...NATIVE_I18N_LOCALES.map(localeDirectory)] as const; +const STRING_RE = /]*)>([\s\S]*?)<\/string>/gu; +const ARRAY_RE = /]*>([\s\S]*?)<\/string-array>/gu; +const ARRAY_ITEM_RE = /([\s\S]*?)<\/item>/gu; +const ANDROID_REFERENCE_COMMENT_RE = + /("""[\s\S]*?"""|"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*')||\/\*[\s\S]*?\*\/|\/\/[^\r\n]*/gu; +const XML_COMMENT_RE = //gu; +const FORMAT_RE = /%\d+\$[a-z]/giu; +const INVALID_APOSTROPHE_RE = /(?:'|(?"; +const GENERATED_KOTLIN_HEADER = "// Generated by scripts/android-app-i18n.ts. Do not edit."; +const THIRD_PARTY_STRINGS_FILE = "accessibility_strings.xml"; +const THIRD_PARTY_STRINGS_REPO_PATH = + "apps/android/app/src/thirdParty/res/values/accessibility_strings.xml"; +const THIRD_PARTY_GENERATED_RESOURCE_RE = + /^apps\/android\/app\/src\/thirdParty\/res\/values-[^/]+\/accessibility_strings\.xml$/; +const WEAR_STRINGS_REPO_PATH = "apps/android/wear/src/main/res/values/strings.xml"; +const WEAR_GENERATED_RESOURCE_RE = + /^apps\/android\/wear\/src\/main\/res\/values-[^/]+\/strings\.xml$/; + +type NativeInventoryEntry = { + id: string; + source: string; + sites: Array<{ kind: string; path: string }>; + surface: "android" | "apple"; +}; + +type NativeTranslations = Record; + +type ResourceString = { + attrs: string; + key: string; + rawValue: string; + value: string; +}; + +const GENERATED_TRANSLATION_LINT_IGNORES = [ + "Typos", + "TypographyDashes", + "TypographyEllipsis", +] as const; + +type TranslationContradiction = { + locale: string; + selected: string; + source: string; + translations: string[]; +}; + +export type GeneratedCatalog = { + contradictions: TranslationContradiction[]; + kotlin: string; + resources: Map; + sources: Set; +}; + +export type AndroidUiLiteralFinding = { + line: number; + path: string; + source: string; +}; + +function compareText(left: string, right: string): number { + return left < right ? -1 : left > right ? 1 : 0; +} + +function decodeXml(value: string): string { + return value + .replaceAll("<", "<") + .replaceAll(">", ">") + .replaceAll(""", '"') + .replaceAll("&", "&") + .replaceAll("\\n", "\n") + .replaceAll("\\'", "'") + .replaceAll('\\"', '"') + .replaceAll("\\\\", "\\"); +} + +export function decodeAndroidResourceValue(rawValue: string): string { + const trimmed = rawValue.trim(); + const unquoted = + trimmed.startsWith('"') && trimmed.endsWith('"') ? trimmed.slice(1, -1) : trimmed; + return decodeXml(unquoted); +} + +type KotlinInterpolation = { + end: number; + start: number; + value: string; +}; + +function readKotlinInterpolations(source: string): KotlinInterpolation[] | null { + const interpolations: KotlinInterpolation[] = []; + for (let index = 0; index < source.length; index += 1) { + if (source[index] !== "$") { + continue; + } + const next = source[index + 1]; + if (next !== "{" && !/[A-Za-z_]/u.test(next ?? "")) { + continue; + } + const start = index; + if (next !== "{") { + index += 2; + while (/[A-Za-z0-9_]/u.test(source[index] ?? "")) { + index += 1; + } + interpolations.push({ start, end: index, value: source.slice(start, index) }); + index -= 1; + continue; + } + + let depth = 1; + let quote: '"' | "'" | null = null; + let escaped = false; + for (index += 2; index < source.length; index += 1) { + const character = source[index]; + if (escaped) { + escaped = false; + } else if (quote !== null && character === "\\") { + escaped = true; + } else if (character === quote) { + quote = null; + } else if (quote === null && (character === '"' || character === "'")) { + quote = character; + } else if (quote === null && character === "{") { + depth += 1; + } else if (quote === null && character === "}") { + depth -= 1; + if (depth === 0) { + const end = index + 1; + interpolations.push({ start, end, value: source.slice(start, end) }); + break; + } + } + } + if (depth !== 0) { + return null; + } + } + return interpolations; +} + +export function renderAndroidResourceValue(source: string, translated: string): string { + let rendered = translated; + const sourceInterpolations = readKotlinInterpolations(source); + const translatedInterpolations = readKotlinInterpolations(translated); + if (sourceInterpolations === null || translatedInterpolations === null) { + throw new Error( + `Android translation has unbalanced interpolation placeholders: ${JSON.stringify(source)} -> ${JSON.stringify(translated)}`, + ); + } + const sourceTokens = sourceInterpolations.map((interpolation) => interpolation.value); + const translatedTokens = translatedInterpolations.map((interpolation) => interpolation.value); + const tokenCounts = (tokens: readonly string[]) => { + const counts = new Map(); + for (const token of tokens) { + counts.set(token, (counts.get(token) ?? 0) + 1); + } + return [...counts].toSorted(([left], [right]) => compareText(left, right)); + }; + if (JSON.stringify(tokenCounts(sourceTokens)) !== JSON.stringify(tokenCounts(translatedTokens))) { + throw new Error( + `Android translation changed interpolation placeholders: ${JSON.stringify(source)} -> ${JSON.stringify(translated)}`, + ); + } + if (sourceTokens.length > 0) { + const sourceIndices = new Map(); + for (const [index, token] of sourceTokens.entries()) { + const indices = sourceIndices.get(token) ?? []; + indices.push(index + 1); + sourceIndices.set(token, indices); + } + const translatedOccurrences = new Map(); + let cursor = 0; + rendered = ""; + for (const interpolation of translatedInterpolations) { + rendered += translated.slice(cursor, interpolation.start).replaceAll("%", "%%"); + const token = interpolation.value; + const occurrence = translatedOccurrences.get(token) ?? 0; + translatedOccurrences.set(token, occurrence + 1); + const index = sourceIndices.get(token)?.[occurrence]; + rendered += index ? `%${index}$s` : token; + cursor = interpolation.end; + } + rendered += translated.slice(cursor).replaceAll("%", "%%"); + } + return escapeAndroidResourceValue(rendered); +} + +export function escapeAndroidResourceValue(value: string): string { + return value + .replaceAll("\\", "\\\\") + .replaceAll("\n", "\\n") + .replaceAll("'", "\\'") + .replaceAll('"', '\\"') + .replaceAll("&", "&") + .replaceAll("<", "<") + .replaceAll(">", ">"); +} + +function escapeKotlin(value: string): string { + return value + .replaceAll("\\", "\\\\") + .replaceAll('"', '\\"') + .replaceAll("$", "\\$") + .replaceAll("\n", "\\n"); +} + +function resourceKey(source: string): string { + return `${MANAGED_PREFIX}${createHash("sha256").update(source).digest("hex").slice(0, 16)}`; +} + +function parseStrings(source: string): ResourceString[] { + return [...source.matchAll(STRING_RE)].map((match) => ({ + key: match[1] ?? "", + attrs: match[2] ?? "", + rawValue: match[3] ?? "", + value: decodeXml(match[3] ?? ""), + })); +} + +function withGeneratedTranslationLintIgnores(attrs: string): string { + const existing = attrs.match(/\btools:ignore\s*=\s*"([^"]*)"/u); + const ignores = [ + ...(existing?.[1] + ?.split(",") + .map((value) => value.trim()) + .filter(Boolean) ?? []), + ...GENERATED_TRANSLATION_LINT_IGNORES, + ]; + const rendered = `tools:ignore="${[...new Set(ignores)].join(",")}"`; + return existing ? attrs.replace(existing[0], rendered) : `${attrs} ${rendered}`; +} + +function parseArrays(source: string): Map { + return new Map( + [...source.matchAll(ARRAY_RE)].map((match) => [ + match[1] ?? "", + [...(match[2] ?? "").matchAll(ARRAY_ITEM_RE)].map((item) => + decodeXml((item[1] ?? "").trim()), + ), + ]), + ); +} + +async function readStrings( + locale: string, + resourceRoot = RESOURCE_ROOT, + fileName = "strings.xml", +): Promise> { + const source = await readFile(path.join(resourceRoot, locale, fileName), "utf8"); + return new Map(parseStrings(source).map((entry) => [entry.key, entry])); +} + +async function readAndroidSource( + root = SOURCE_ROOT, +): Promise> { + const entries = await readdir(root, { withFileTypes: true }); + const sources: Array<{ path: string; source: string }> = []; + for (const entry of entries) { + const fullPath = path.join(root, entry.name); + if (entry.isDirectory()) { + sources.push(...(await readAndroidSource(fullPath))); + continue; + } + if (entry.isFile() && entry.name.endsWith(".kt")) { + sources.push({ + path: path.relative(ROOT, fullPath).split(path.sep).join("/"), + source: await readFile(fullPath, "utf8"), + }); + } + } + return sources; +} + +async function readAllAndroidSource(): Promise> { + return (await Promise.all(ANDROID_SOURCE_ROOTS.map((root) => readAndroidSource(root)))).flat(); +} + +type AndroidResourceReferenceSource = { + path: string; + source: string; +}; + +async function readAndroidResourceReferences( + root = ANDROID_MAIN_ROOT, +): Promise { + const entries = await readdir(root, { withFileTypes: true }); + const sources: AndroidResourceReferenceSource[] = []; + for (const entry of entries) { + const fullPath = path.join(root, entry.name); + if (entry.isDirectory()) { + // This walk is confined to app/src/main, so Gradle build output is never eligible. + sources.push(...(await readAndroidResourceReferences(fullPath))); + continue; + } + if (entry.isFile() && /\.(?:kt|kts|xml)$/u.test(entry.name)) { + sources.push({ + path: path.relative(ROOT, fullPath).split(path.sep).join("/"), + source: await readFile(fullPath, "utf8"), + }); + } + } + return sources; +} + +export function findUnusedAndroidResourceKeys( + keys: Iterable, + referenceSources: readonly AndroidResourceReferenceSource[], +): string[] { + const references = new Set(); + for (const reference of referenceSources) { + const isXml = reference.path.endsWith(".xml"); + const source = reference.source.replace( + isXml ? XML_COMMENT_RE : ANDROID_REFERENCE_COMMENT_RE, + (match) => match.replace(/[^\r\n]/gu, " "), + ); + const pattern = isXml ? /@string\/([A-Za-z0-9_]+)\b/gu : /\bR\.string\.([A-Za-z0-9_]+)\b/gu; + for (const match of source.matchAll(pattern)) { + if (match[1]) { + references.add(match[1]); + } + } + } + return [...keys].filter((key) => !references.has(key)); +} + +function lineNumber(source: string, offset: number): number { + return source.slice(0, offset).split("\n").length; +} + +function decodeKotlinLiteral(value: string): string { + return value.replace( + /\\(?:u([0-9a-fA-F]{4})|[n"$\\])/gu, + (escape, codeUnit: string | undefined) => { + if (codeUnit) { + return String.fromCharCode(Number.parseInt(codeUnit, 16)); + } + return escape === "\\n" ? "\n" : escape.slice(1); + }, + ); +} + +function collectExplicitRuntimeSources( + sourceFiles: readonly { path: string; source: string }[], +): Set { + const sources = new Set(); + const callPattern = /\bnativeString(?:Resource)?\(\s*"((?:\\.|[^"\\])+)"/gu; + for (const file of sourceFiles) { + if (file.path.endsWith("/i18n/NativeStringResources.kt")) { + continue; + } + for (const match of file.source.matchAll(callPattern)) { + if (match[1]) { + sources.add(decodeKotlinLiteral(match[1])); + } + } + } + return sources; +} + +function collectToolDisplaySources(value: unknown, sources = new Set()): Set { + if (Array.isArray(value)) { + for (const item of value) { + collectToolDisplaySources(item, sources); + } + return sources; + } + if (value === null || typeof value !== "object") { + return sources; + } + for (const [key, item] of Object.entries(value)) { + if ((key === "title" || key === "label") && typeof item === "string") { + sources.add(item); + } else { + collectToolDisplaySources(item, sources); + } + } + return sources; +} + +async function readToolDisplaySources(): Promise> { + return collectToolDisplaySources(JSON.parse(await readFile(TOOL_DISPLAY_PATH, "utf8"))); +} + +const DIRECT_UI_LITERAL_PATTERNS = [ + /\bText\s*\(\s*(?:text\s*=\s*)?"((?:\\.|[^"\\])+)"/gu, + /\b(?:ClawPrimaryButton|ClawSecondaryButton|ClawDangerButton|ClawLinkButton)\s*\([^)]*?\btext\s*=\s*"((?:\\.|[^"\\])+)"/gsu, + /\b(?:title|subtitle|body|text|label|statusText|confirmLabel|dismissLabel|contentDescription|placeholder|onClickLabel)\s*=\s*"((?:\\.|[^"\\])+)"/gu, + /\b(?:title|subtitle|body|text|label|statusText|confirmLabel|dismissLabel|contentDescription|placeholder|onClickLabel)\s*=\s*[^,\n]*\?:\s*"((?:\\.|[^"\\])+)"/gu, + /\b(?:_[A-Za-z0-9_]*(?:ErrorText|StatusText)|errorText|statusText)(?:\.value)?\s*=\s*"((?:\\.|[^"\\])+)"/gu, + /\bSettingsMetric\s*\(\s*"((?:\\.|[^"\\])+)"/gu, + /\bToast\.makeText\s*\([^,]+,\s*"((?:\\.|[^"\\])+)"/gu, + /\bset(?:Title|Message|PositiveButton|NegativeButton|NeutralButton)\s*\(\s*"((?:\\.|[^"\\])+)"/gu, + /\bnativeString(?:Resource)?\([^)]*\)\s*\+\s*"((?:\\.|[^"\\])+)"/gsu, +] as const; +const DIRECT_UI_RAW_LITERAL_PATTERNS = [ + /\bText\s*\(\s*(?:text\s*=\s*)?"""([\s\S]*?)"""/gu, + /\b(?:ClawPrimaryButton|ClawSecondaryButton|ClawDangerButton|ClawLinkButton)\s*\([^)]*?\btext\s*=\s*"""([\s\S]*?)"""/gsu, + /\b(?:title|subtitle|body|text|label|statusText|confirmLabel|dismissLabel|contentDescription|placeholder|onClickLabel)\s*=\s*"""([\s\S]*?)"""/gu, + /\b(?:title|subtitle|body|text|label|statusText|confirmLabel|dismissLabel|contentDescription|placeholder|onClickLabel)\s*=\s*[^,\n]*\?:\s*"""([\s\S]*?)"""/gu, + /\b(?:_[A-Za-z0-9_]*(?:ErrorText|StatusText)|errorText|statusText)(?:\.value)?\s*=\s*"""([\s\S]*?)"""/gu, + /\bSettingsMetric\s*\(\s*"""([\s\S]*?)"""/gu, + /\bToast\.makeText\s*\([^,]+,\s*"""([\s\S]*?)"""/gu, + /\bset(?:Title|Message|PositiveButton|NegativeButton|NeutralButton)\s*\(\s*"""([\s\S]*?)"""/gu, + /\bnativeString(?:Resource)?\([^)]*\)\s*\+\s*"""([\s\S]*?)"""/gsu, +] as const; +const UI_STRING_HELPER_NAME_RE = + /(?:description|detail|error|label|message|nextRun|notice|report|status|subtitle|summary|text|title)$/iu; +const UI_STRING_FUNCTION_RE = /\bfun\s+(?:<[^>{}]*>\s*)?([A-Za-z_][A-Za-z0-9_]*)\s*/gu; +const UI_STRING_PROPERTY_RE = + /\bval\s+(?:[A-Za-z_][A-Za-z0-9_]*\.)?([A-Za-z_][A-Za-z0-9_]*)\s*:\s*String\s*(?:\n\s*)?get\(\)\s*=\s*/gu; +const UI_BRANCH_LITERAL_RE = /(?:->|return|\?:)\s*"((?:\\.|[^"\\])+)"/gu; +const UI_ELSE_BRANCH_LITERAL_RE = /\belse\s*\{?\s*"((?:\\.|[^"\\])+)"/gu; +const UI_BRANCH_RAW_LITERAL_RE = /(?:->|return|\?:)\s*"""([\s\S]*?)"""/gu; +const UI_ELSE_BRANCH_RAW_LITERAL_RE = /\belse\s*\{?\s*"""([\s\S]*?)"""/gu; +const UI_MODEL_STRING_NAME_RE = + /^(?:contentDescription|errorText|helperText|onClickLabel|statusText)$/u; +const UI_MODEL_CLASS_FIELDS = new Map>([ + ["CommandItem", new Set(["subtitle", "title"])], + ["HomeAttentionRow", new Set(["subtitle", "title"])], + ["OverviewMetricCardSpec", new Set(["subtitle", "title", "value"])], + ["SettingsToggleRow", new Set(["subtitle", "title"])], +]); +const KOTLIN_STRING_LITERAL_RE = /"""([\s\S]*?)"""|"((?:\\.|[^"\\])+?)"/gu; + +// These literals are either technical data or immutable source tokens localized at a typed render edge. +const ALLOWED_UI_LITERALS = new Map>([ + [ + "*", + new Set([ + "0 = exact", + "Cron expression, e.g. 0 9 * * *", + "D", + "Google Chat", + "ID", + "ISO time, e.g. 2026-07-09T09:30:00Z", + "LOG", + "O", + "OC", + "OK", + "OPENCLAW", + "OpenClaw", + "U", + "e.g. America/New_York", + "current-step-alpha", + "gateway-progress", + "iMessage", + "main, isolated, current, or session:", + "n/a", + "openclaw gateway", + "openclaw qr", + "PTT_BUSY: previous push-to-talk turn is still finishing", + "WhatsApp", + ]), + ], + ["apps/android/app/src/main/java/ai/openclaw/app/chat/ChatController.kt", new Set(["Off"])], + [ + "apps/android/app/src/main/java/ai/openclaw/app/ui/SkillWorkshopSettingsScreen.kt", + new Set(["all", "applied", "held", "pending", "rejected"]), + ], + [ + "apps/android/app/src/main/java/ai/openclaw/app/ui/GatewayDiagnostics.kt", + new Set(["$versionName-dev"]), + ], + [ + "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatScreen.kt", + // Plan checklist chrome: numeric done-counter and checkmark glyph. + new Set(["$completedCount/${steps.size}", "✓"]), + ], + [ + "apps/android/app/src/main/java/ai/openclaw/app/ui/SettingsScreens.kt", + // Discovered-gateway subtitles are host:port endpoints, not translatable copy. + new Set(["${endpoint.host}:${endpoint.port}"]), + ], + [ + "apps/android/app/src/main/java/ai/openclaw/app/ui/SidebarShell.kt", + // Compose animation labels are tooling identifiers, not rendered copy. + new Set(["sidebar-content-translation"]), + ], + [ + "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatCommandControls.kt", + new Set(["/$name", "help"]), + ], + [ + "apps/android/app/src/main/java/ai/openclaw/app/ui/chat/ChatMessageActions.kt", + new Set([">", "> $line"]), + ], + [ + "apps/android/wear/src/main/java/ai/openclaw/wear/WearScreens.kt", + // Compose animation labels are tooling identifiers, not rendered copy. + new Set(["voice-swipe-hint", "voice-swipe-hint-offset"]), + ], +]); + +function isAllowedUiLiteral(repoPath: string, source: string): boolean { + return ( + source.trim().length === 0 || + ALLOWED_UI_LITERALS.get("*")?.has(source) === true || + ALLOWED_UI_LITERALS.get(repoPath)?.has(source) === true + ); +} + +function shouldScanUiLiterals(repoPath: string): boolean { + if (repoPath.endsWith("/i18n/NativeStringResources.kt")) { + return false; + } + if ( + repoPath.endsWith("/AndroidScreenshotFixture.kt") || + repoPath.endsWith("/WearScreenshotMode.kt") + ) { + return false; + } + if (repoPath.endsWith("/ui/design/ClawComponents.kt")) { + return false; + } + if (repoPath.endsWith("/ui/design/OpenClawMascot.kt")) { + return false; + } + return ( + repoPath.includes("/ui/") || + repoPath.endsWith("/accessibility/AccessibilityDevActivity.kt") || + repoPath.includes("/wear/src/main/java/ai/openclaw/wear/") || + repoPath.endsWith("/MainActivity.kt") || + repoPath.endsWith("/NodeRuntime.kt") || + repoPath.endsWith("/PermissionRequester.kt") || + repoPath.endsWith("/NodeForegroundService.kt") || + repoPath.endsWith("/chat/ChatController.kt") || + repoPath.endsWith("/voice/MicCaptureManager.kt") || + repoPath.endsWith("/voice/TalkModeManager.kt") || + repoPath.endsWith("/node/SystemHandler.kt") + ); +} + +function findClosingDelimiter( + source: string, + openingOffset: number, + opening: string, + closing: string, +): number | null { + let depth = 0; + let quoted = false; + let escaped = false; + for (let index = openingOffset; index < source.length; index += 1) { + const character = source[index]; + if (escaped) { + escaped = false; + continue; + } + if (quoted && character === "\\") { + escaped = true; + continue; + } + if (character === '"') { + quoted = !quoted; + continue; + } + if (quoted) { + continue; + } + if (character === opening) { + depth += 1; + } else if (character === closing) { + depth -= 1; + if (depth === 0) { + return index; + } + } + } + return null; +} + +function expressionEnd(source: string, expressionStart: number): number { + const cursor = source.slice(expressionStart).search(/\S/u); + if (cursor < 0) { + return source.length; + } + const start = expressionStart + cursor; + let quoted = false; + let escaped = false; + let lineStart = start; + const depths = { "(": 0, "[": 0, "{": 0 }; + const closingToOpening = { ")": "(", "]": "[", "}": "{" } as const; + for (let index = start; index < source.length; index += 1) { + const character = source[index]; + if (escaped) { + escaped = false; + continue; + } + if (quoted && character === "\\") { + escaped = true; + continue; + } + if (character === '"') { + quoted = !quoted; + continue; + } + if (quoted) { + continue; + } + if (character === "(" || character === "[" || character === "{") { + depths[character] += 1; + continue; + } + if (character === ")" || character === "]" || character === "}") { + depths[closingToOpening[character]] -= 1; + continue; + } + if (character !== "\n" || !Object.values(depths).every((depth) => depth === 0)) { + continue; + } + const line = source.slice(lineStart, index).trimEnd(); + const nextLine = + source + .slice(index + 1) + .match(/^[^\S\n]*([^\n]*)/u)?.[1] + ?.trimStart() ?? ""; + const continues = + line.length === 0 || + /(?:\?:|[+*/%&|=.,([{])$/u.test(line) || + /^(?:else\b|\?:|[+*/%&|.])/u.test(nextLine); + if (!continues) { + return index; + } + lineStart = index + 1; + } + return source.length; +} + +function splitTopLevelSegments( + source: string, + start: number, + end: number, + options: { trackTypeArguments?: boolean } = {}, +): Array<{ end: number; start: number; value: string }> { + const segments: Array<{ end: number; start: number; value: string }> = []; + let segmentStart = start; + let quoted = false; + let escaped = false; + let typeArgumentDepth = 0; + let inDefaultValue = false; + const depths = { "(": 0, "[": 0, "{": 0 }; + const closingToOpening = { ")": "(", "]": "[", "}": "{" } as const; + for (let index = start; index < end; index += 1) { + const character = source[index]; + if (escaped) { + escaped = false; + continue; + } + if (quoted && character === "\\") { + escaped = true; + continue; + } + if (character === '"') { + quoted = !quoted; + continue; + } + if (quoted) { + continue; + } + if (options.trackTypeArguments && !inDefaultValue && character === "<") { + typeArgumentDepth += 1; + continue; + } + if (options.trackTypeArguments && character === ">" && typeArgumentDepth > 0) { + typeArgumentDepth -= 1; + continue; + } + if (character === "(" || character === "[" || character === "{") { + depths[character] += 1; + continue; + } + if (character === ")" || character === "]" || character === "}") { + depths[closingToOpening[character]] -= 1; + continue; + } + if ( + options.trackTypeArguments && + character === "=" && + typeArgumentDepth === 0 && + Object.values(depths).every((depth) => depth === 0) + ) { + inDefaultValue = true; + continue; + } + if ( + character === "," && + typeArgumentDepth === 0 && + Object.values(depths).every((depth) => depth === 0) + ) { + segments.push({ end: index, start: segmentStart, value: source.slice(segmentStart, index) }); + segmentStart = index + 1; + inDefaultValue = false; + } + } + segments.push({ end, start: segmentStart, value: source.slice(segmentStart, end) }); + return segments; +} + +function isLocalizedLiteral(expression: string, literalOffset: number): boolean { + return /\bnativeString(?:Resource)?\(\s*$/u.test(expression.slice(0, literalOffset)); +} + +function isComparisonLiteral(expression: string, literalOffset: number): boolean { + return /(?:==|!=)\s*$/u.test(expression.slice(0, literalOffset)); +} + +function collectHelperLiteralFindings(source: string, repoPath: string): AndroidUiLiteralFinding[] { + const findings: AndroidUiLiteralFinding[] = []; + const collectRange = (name: string, start: number, end: number) => { + if (!UI_STRING_HELPER_NAME_RE.test(name)) { + return; + } + const body = source.slice(start, end); + const direct = body.match(/^\s*"((?:\\.|[^"\\])+)"/u); + const directRaw = body.match(/^\s*"""([\s\S]*?)"""/u); + const matches = [ + ...(direct ? [direct] : []), + ...(directRaw ? [directRaw] : []), + ...body.matchAll(UI_BRANCH_LITERAL_RE), + ...body.matchAll(UI_ELSE_BRANCH_LITERAL_RE), + ...body.matchAll(UI_BRANCH_RAW_LITERAL_RE), + ...body.matchAll(UI_ELSE_BRANCH_RAW_LITERAL_RE), + ]; + const seenOffsets = new Set(); + const addLiteral = (literal: string, literalOffset: number) => { + if (seenOffsets.has(literalOffset)) { + return; + } + seenOffsets.add(literalOffset); + findings.push({ + line: lineNumber(source, start + literalOffset), + path: repoPath, + source: decodeKotlinLiteral(literal), + }); + }; + for (const match of matches) { + const literal = match[1]; + if (!literal) { + continue; + } + const delimiter = match[0]?.includes('"""') ? '"""' : '"'; + const literalOffset = body.indexOf(`${delimiter}${literal}${delimiter}`, match.index ?? 0); + addLiteral(literal, Math.max(0, literalOffset)); + } + for (const match of body.matchAll(/\bif\s*/gu)) { + let cursor = (match.index ?? 0) + match[0].length; + if (body[cursor] !== "(") { + continue; + } + const closingParen = findClosingDelimiter(body, cursor, "(", ")"); + if (closingParen === null) { + continue; + } + cursor = closingParen + 1; + while (/\s/u.test(body[cursor] ?? "")) { + cursor += 1; + } + if (body[cursor] === "{") { + cursor += 1; + while (/\s/u.test(body[cursor] ?? "")) { + cursor += 1; + } + } + const expression = body.slice(cursor); + const literal = + expression.match(/^"""([\s\S]*?)"""/u)?.[1] ?? + expression.match(/^"((?:\\.|[^"\\])+)"/u)?.[1]; + if (literal) { + addLiteral(literal, cursor); + } + } + }; + for (const match of source.matchAll(UI_STRING_FUNCTION_RE)) { + const name = match[1]; + if (!name) { + continue; + } + let openingParen = (match.index ?? 0) + match[0].length; + while (/\s/u.test(source[openingParen] ?? "")) { + openingParen += 1; + } + if (source[openingParen] !== "(") { + continue; + } + const closingParen = findClosingDelimiter(source, openingParen, "(", ")"); + if (closingParen === null) { + continue; + } + const returnType = source.slice(closingParen + 1).match(/^\s*:\s*String\s*(=|\{)/u); + const bodyKind = returnType?.[1]; + if (!bodyKind || returnType?.index === undefined) { + continue; + } + const bodyStart = closingParen + 1 + returnType.index + returnType[0].length; + if (bodyKind === "{") { + const openingBrace = bodyStart - 1; + const closingBrace = findClosingDelimiter(source, openingBrace, "{", "}"); + if (closingBrace !== null) { + collectRange(name, bodyStart, closingBrace); + } + } else { + collectRange(name, bodyStart, expressionEnd(source, bodyStart)); + } + } + for (const match of source.matchAll(UI_STRING_PROPERTY_RE)) { + const name = match[1]; + if (!name) { + continue; + } + const bodyStart = (match.index ?? 0) + match[0].length; + collectRange(name, bodyStart, expressionEnd(source, bodyStart)); + } + return findings; +} + +function collectTypedModelLiteralFindings( + source: string, + repoPath: string, +): AndroidUiLiteralFinding[] { + const findings: AndroidUiLiteralFinding[] = []; + const classPattern = /\b(?:data\s+class|class)\s+([A-Za-z_][A-Za-z0-9_]*)\s*/gu; + for (const declaration of source.matchAll(classPattern)) { + const className = declaration[1]; + if (!className) { + continue; + } + let openingParen = (declaration.index ?? 0) + declaration[0].length; + while (/\s/u.test(source[openingParen] ?? "")) { + openingParen += 1; + } + if (source[openingParen] === "<") { + const closingTypeArguments = findClosingDelimiter(source, openingParen, "<", ">"); + if (closingTypeArguments === null) { + continue; + } + openingParen = closingTypeArguments + 1; + while (/\s/u.test(source[openingParen] ?? "")) { + openingParen += 1; + } + } + if (source[openingParen] !== "(") { + continue; + } + const closingParen = findClosingDelimiter(source, openingParen, "(", ")"); + if (closingParen === null) { + continue; + } + const parameters = splitTopLevelSegments(source, openingParen + 1, closingParen, { + trackTypeArguments: true, + }); + const userFacingParameters = new Map(); + const classFields = UI_MODEL_CLASS_FIELDS.get(className); + parameters.forEach((parameter, index) => { + const field = parameter.value.match( + /\bval\s+([A-Za-z_][A-Za-z0-9_]*)\s*:\s*String\??(?=\s*(?:=|$))/u, + )?.[1]; + if (field && (UI_MODEL_STRING_NAME_RE.test(field) || classFields?.has(field))) { + userFacingParameters.set(field, index); + } + }); + if (userFacingParameters.size === 0) { + continue; + } + const callPattern = new RegExp(`\\b${className}\\b`, "gu"); + for (const call of source.matchAll(callPattern)) { + let callOpening = (call.index ?? 0) + call[0].length; + while (/\s/u.test(source[callOpening] ?? "")) { + callOpening += 1; + } + if (source[callOpening] === "<") { + const closingTypeArguments = findClosingDelimiter(source, callOpening, "<", ">"); + if (closingTypeArguments === null) { + continue; + } + callOpening = closingTypeArguments + 1; + while (/\s/u.test(source[callOpening] ?? "")) { + callOpening += 1; + } + } + if (source[callOpening] !== "(") { + continue; + } + if (callOpening === openingParen) { + continue; + } + const callClosing = findClosingDelimiter(source, callOpening, "(", ")"); + if (callClosing === null) { + continue; + } + const argumentsList = splitTopLevelSegments(source, callOpening + 1, callClosing); + const namedArguments = new Map(); + let positionalArgumentCount = argumentsList.length; + argumentsList.forEach((argument, index) => { + const name = argument.value.match(/^\s*([A-Za-z_][A-Za-z0-9_]*)\s*=(?!=)/u)?.[1]; + if (!name) { + return; + } + namedArguments.set(name, argument); + positionalArgumentCount = Math.min(positionalArgumentCount, index); + }); + for (const [field, parameterIndex] of userFacingParameters) { + const argument = + namedArguments.get(field) ?? + (parameterIndex < positionalArgumentCount ? argumentsList[parameterIndex] : undefined); + if (!argument) { + continue; + } + for (const literal of argument.value.matchAll(KOTLIN_STRING_LITERAL_RE)) { + const literalValue = literal[1] ?? literal[2]; + if ( + !literalValue || + isLocalizedLiteral(argument.value, literal.index ?? 0) || + isComparisonLiteral(argument.value, literal.index ?? 0) + ) { + continue; + } + findings.push({ + line: lineNumber(source, argument.start + (literal.index ?? 0)), + path: repoPath, + source: literal[1] === undefined ? decodeKotlinLiteral(literalValue) : literalValue, + }); + } + } + } + } + return findings; +} + +export function findUnlocalizedAndroidUiLiterals( + source: string, + repoPath: string, +): AndroidUiLiteralFinding[] { + if (!shouldScanUiLiterals(repoPath)) { + return []; + } + const findings = new Map(); + for (const [patterns, delimiter] of [ + [DIRECT_UI_LITERAL_PATTERNS, '"'], + [DIRECT_UI_RAW_LITERAL_PATTERNS, '"""'], + ] as const) { + for (const pattern of patterns) { + pattern.lastIndex = 0; + for (const match of source.matchAll(pattern)) { + const literal = match[1]; + if (!literal) { + continue; + } + const findingSource = delimiter === '"' ? decodeKotlinLiteral(literal) : literal; + if ( + isAllowedUiLiteral(repoPath, findingSource) || + literal.startsWith("http") || + literal.startsWith("content://") + ) { + continue; + } + const matchText = match[0] ?? ""; + const offset = + (match.index ?? 0) + matchText.lastIndexOf(`${delimiter}${literal}${delimiter}`); + const finding = { + line: lineNumber(source, Math.max(0, offset)), + path: repoPath, + source: findingSource, + }; + findings.set(`${finding.line}\u0000${finding.source}`, finding); + } + } + } + for (const finding of [ + ...collectHelperLiteralFindings(source, repoPath), + ...collectTypedModelLiteralFindings(source, repoPath), + ]) { + if (!isAllowedUiLiteral(repoPath, finding.source)) { + findings.set(`${finding.line}\u0000${finding.source}`, finding); + } + } + return [...findings.values()].toSorted( + (left, right) => left.line - right.line || compareText(left.source, right.source), + ); +} + +function findInvalidResourceSyntax(strings: ReadonlyMap): string[] { + return [...strings] + .filter(([, entry]) => { + const trimmed = entry.rawValue.trim(); + const isQuoted = trimmed.startsWith('"') && trimmed.endsWith('"'); + return !isQuoted && INVALID_APOSTROPHE_RE.test(trimmed); + }) + .map(([key]) => key); +} + +export function selectDeterministicTranslation(source: string, values: readonly string[]): string { + const translated = values.filter((value) => value !== source); + const candidates = translated.length > 0 ? translated : values; + const counts = new Map(); + for (const value of candidates) { + counts.set(value, (counts.get(value) ?? 0) + 1); + } + return ( + [...counts] + .toSorted( + ([left, leftCount], [right, rightCount]) => + rightCount - leftCount || compareText(left, right), + ) + .at(0)?.[0] ?? "" + ); +} + +export function selectGeneratedTranslation( + source: string, + artifactTranslations: readonly string[], + existing?: { source: string; translation: string }, +): string { + // Tool-display sources can remain live after their matching UI inventory entries are retired. + // Preserve the checked-in locale value only while its English source is unchanged. + const candidates = + artifactTranslations.length > 0 + ? artifactTranslations + : existing?.source === source && existing.translation !== source + ? [existing.translation] + : []; + return selectDeterministicTranslation(source, candidates); +} + +async function readInventory(): Promise { + const parsed = JSON.parse(await readFile(INVENTORY_PATH, "utf8")) as { + entries?: NativeInventoryEntry[]; + }; + return (parsed.entries ?? []).filter((entry) => entry.surface === "android"); +} + +async function readArtifacts(): Promise> { + return new Map( + await Promise.all( + NATIVE_I18N_LOCALES.map(async (locale) => { + const parsed = JSON.parse( + await readFile(path.join(ARTIFACT_ROOT, `${locale}.json`), "utf8"), + ) as { translations?: NativeTranslations }; + return [locale, parsed.translations ?? {}] as const; + }), + ), + ); +} + +function translationsBySource( + inventory: readonly NativeInventoryEntry[], + translationsById: Readonly, +): Map { + const translations = new Map(); + for (const entry of inventory) { + const translated = translationsById[entry.id]; + if (translated !== undefined) { + translations.set(entry.source, [translated]); + } + } + return translations; +} + +function localizeManualStrings( + base: ReadonlyMap, + inventoryBySource: ReadonlyMap, + translations: Readonly, + surface: string, +): ResourceString[] { + return [...base.values()].map((entry) => { + const source = decodeAndroidResourceValue(entry.rawValue); + const translatable = !/\btranslatable\s*=\s*"false"/u.test(entry.attrs); + const inventoryEntry = inventoryBySource.get(source); + if (translatable && !inventoryEntry) { + throw new Error( + `${surface} string is missing from native inventory: ${JSON.stringify(source)}`, + ); + } + const value = (translatable && inventoryEntry && translations[inventoryEntry.id]) || source; + return { + attrs: translatable ? withGeneratedTranslationLintIgnores(entry.attrs) : entry.attrs, + key: entry.key, + rawValue: `"${escapeAndroidResourceValue(value)}"`, + value, + }; + }); +} + +function renderStringsXml( + manual: readonly ResourceString[], + generated: ReadonlyMap, +): string { + const usesToolsNamespace = + generated.size > 0 || manual.some((entry) => entry.attrs.includes("tools:")); + const lines = [ + usesToolsNamespace + ? '' + : "", + ]; + for (const entry of manual) { + lines.push(` ${entry.rawValue}`); + } + if (generated.size > 0) { + lines.push(GENERATED_HEADER); + for (const [key, entry] of [...generated].toSorted(([left], [right]) => + compareText(left, right), + )) { + const formatted = + (readKotlinInterpolations(entry.source)?.length ?? 0) > 0 ? "" : ' formatted="false"'; + // Translation-memory text intentionally preserves technical tokens and source punctuation, + // so Android's English dictionary and typography suggestions do not apply to managed keys. + lines.push( + ` "${renderAndroidResourceValue(entry.source, entry.value)}"`, + ); + } + } + lines.push("", ""); + return lines.join("\n"); +} + +function renderAssistantXml(items: readonly string[]): string { + return [ + "", + ' ', + ...items.map((item) => ` "${renderAndroidResourceValue(item, item)}"`), + " ", + "", + "", + ].join("\n"); +} + +function renderKotlin(sourceToKey: ReadonlyMap): string { + const entries = [...sourceToKey].toSorted(([left], [right]) => compareText(left, right)); + return [ + GENERATED_KOTLIN_HEADER, + "package ai.openclaw.app.i18n", + "", + "import ai.openclaw.app.R", + "", + "internal val nativeStringResourceIds: Map =", + " mapOf(", + ...entries.map(([source, key]) => ` "${escapeKotlin(source)}" to R.string.${key},`), + " )", + "", + ].join("\n"); +} + +export async function buildAndroidAppI18nCatalog(): Promise { + const inventory = await readInventory(); + const [ + artifacts, + localeStrings, + thirdPartyBaseStrings, + wearBaseStrings, + sourceFiles, + toolDisplaySources, + ] = await Promise.all([ + readArtifacts(), + Promise.all(LOCALES.map((locale) => readStrings(locale))), + readStrings("values", ANDROID_THIRD_PARTY_RESOURCE_ROOT, THIRD_PARTY_STRINGS_FILE), + readStrings("values", WEAR_RESOURCE_ROOT), + readAllAndroidSource(), + readToolDisplaySources(), + ]); + const baseStrings = expectDefined(localeStrings[0], "English Android string resources"); + const translatedStrings = localeStrings.slice(1); + const wearInventoryBySource = new Map( + inventory + .filter((entry) => entry.sites.some((site) => site.path === WEAR_STRINGS_REPO_PATH)) + .map((entry) => [entry.source, entry]), + ); + const thirdPartyInventoryBySource = new Map( + inventory + .filter((entry) => entry.sites.some((site) => site.path === THIRD_PARTY_STRINGS_REPO_PATH)) + .map((entry) => [entry.source, entry]), + ); + const manualBase = [...baseStrings.values()].filter( + (entry) => !entry.key.startsWith(MANAGED_PREFIX), + ); + const manualSourceToKey = new Map(manualBase.map((entry) => [entry.value, entry.key])); + const entriesBySource = new Map(); + for (const entry of inventory) { + if ( + entry.surface !== "android" || + entry.sites.every((site) => site.kind === "resource-string") + ) { + continue; + } + const group = entriesBySource.get(entry.source) ?? []; + group.push(entry); + entriesBySource.set(entry.source, group); + } + for (const source of collectExplicitRuntimeSources(sourceFiles)) { + if (!entriesBySource.has(source)) { + entriesBySource.set(source, []); + } + } + for (const source of toolDisplaySources) { + if (!entriesBySource.has(source)) { + entriesBySource.set(source, []); + } + } + const sourceToKey = new Map(); + for (const source of entriesBySource.keys()) { + sourceToKey.set(source, manualSourceToKey.get(source) ?? resourceKey(source)); + } + const resources = new Map(); + const contradictions: TranslationContradiction[] = []; + for (const [localeIndex, locale] of NATIVE_I18N_LOCALES.entries()) { + const manualTranslations = translatedStrings[localeIndex] ?? new Map(); + const localeTranslations = artifacts.get(locale) ?? {}; + const artifactTranslationsBySource = translationsBySource(inventory, localeTranslations); + const generated = new Map(); + for (const source of entriesBySource.keys()) { + const key = sourceToKey.get(source); + if (!key || !key.startsWith(MANAGED_PREFIX)) { + continue; + } + const translations = artifactTranslationsBySource.get(source) ?? []; + const existingSource = decodeAndroidResourceValue(baseStrings.get(key)?.rawValue ?? ""); + const existingTranslation = decodeAndroidResourceValue( + manualTranslations.get(key)?.rawValue ?? "", + ); + const selected = selectGeneratedTranslation( + source, + translations, + existingSource && existingTranslation + ? { source: existingSource, translation: existingTranslation } + : undefined, + ); + if (selected === source && translations.some((translation) => translation !== source)) { + throw new Error( + `Android translation selection kept the source despite a translated candidate: ${locale} ${JSON.stringify(source)}`, + ); + } + const unique = [...new Set(translations)].toSorted(compareText); + if (unique.length > 1) { + contradictions.push({ locale, selected, source, translations: unique }); + } + generated.set(key, { + source, + value: selected || source, + }); + } + const manual = [...manualTranslations.values()].filter( + (entry) => !entry.key.startsWith(MANAGED_PREFIX), + ); + resources.set( + path.join(RESOURCE_ROOT, localeDirectory(locale), "strings.xml"), + renderStringsXml(manual, generated), + ); + const wearManual = localizeManualStrings( + wearBaseStrings, + wearInventoryBySource, + localeTranslations, + "Wear", + ); + resources.set( + path.join(WEAR_RESOURCE_ROOT, localeDirectory(locale), "strings.xml"), + renderStringsXml(wearManual, new Map()), + ); + const thirdPartyManual = localizeManualStrings( + thirdPartyBaseStrings, + thirdPartyInventoryBySource, + localeTranslations, + "Android third-party", + ); + resources.set( + path.join( + ANDROID_THIRD_PARTY_RESOURCE_ROOT, + localeDirectory(locale), + THIRD_PARTY_STRINGS_FILE, + ), + renderStringsXml(thirdPartyManual, new Map()), + ); + } + const generatedBase = new Map(); + for (const [source, key] of sourceToKey) { + if (!key.startsWith(MANAGED_PREFIX)) { + continue; + } + generatedBase.set(key, { + source, + value: source, + }); + } + resources.set( + path.join(RESOURCE_ROOT, "values", "strings.xml"), + renderStringsXml(manualBase, generatedBase), + ); + + const assistantSource = await readFile( + path.join(RESOURCE_ROOT, "values", "assistant.xml"), + "utf8", + ); + const assistantItems = parseArrays(assistantSource).get("ask_openclaw_query_patterns") ?? []; + for (const [locale, localeTranslations] of artifacts) { + const translatedBySource = translationsBySource(inventory, localeTranslations); + const translatedItems = assistantItems.map( + (source) => + selectDeterministicTranslation(source, translatedBySource.get(source) ?? []) || source, + ); + resources.set( + path.join(RESOURCE_ROOT, localeDirectory(locale), "assistant.xml"), + renderAssistantXml(translatedItems), + ); + } + + return { + contradictions, + kotlin: renderKotlin(sourceToKey), + resources, + sources: new Set(sourceToKey.keys()), + }; +} + +function formatProblems(problems: Array): string { + return [ + "Android i18n resources are out of sync.", + ...problems.map(([label, keys]) => `${label}=${keys.join(",") || "none"}`), + ].join("\n"); +} + +/** + * Managed app rows (native_*) and localized Wear catalogs are owned end-to-end + * by the post-merge locale refresh workflow, so a source PR may legitimately + * carry stale generated output. Every other delta is real drift. + */ +function onlyManagedRowsPending(current: string, expected: string): boolean { + const currentRows = new Map(parseStrings(current).map((entry) => [entry.key, entry])); + const expectedRows = new Map(parseStrings(expected).map((entry) => [entry.key, entry])); + for (const [key, entry] of currentRows) { + const expectedEntry = expectedRows.get(key); + if (!expectedEntry) { + if (!key.startsWith(MANAGED_PREFIX)) { + return false; + } + continue; + } + if (expectedEntry.attrs !== entry.attrs || expectedEntry.rawValue !== entry.rawValue) { + return false; + } + } + for (const key of expectedRows.keys()) { + if (!currentRows.has(key) && !key.startsWith(MANAGED_PREFIX)) { + return false; + } + } + return true; +} + +export async function syncAndroidAppI18n( + options: { check?: boolean; tolerateManagedPending?: boolean } = {}, +) { + const catalog = await buildAndroidAppI18nCatalog(); + const drift: string[] = []; + let sawUnmanagedDrift = false; + for (const [filePath, expected] of catalog.resources) { + const current = await readFile(filePath, "utf8").catch(() => ""); + if (current === expected) { + continue; + } + const relativeFilePath = path.relative(ROOT, filePath).split(path.sep).join("/"); + if ( + options.check && + options.tolerateManagedPending && + (WEAR_GENERATED_RESOURCE_RE.test(relativeFilePath) || + THIRD_PARTY_GENERATED_RESOURCE_RE.test(relativeFilePath) || + (filePath.endsWith("strings.xml") && onlyManagedRowsPending(current, expected))) + ) { + continue; + } + sawUnmanagedDrift = true; + drift.push(relativeFilePath); + if (!options.check) { + await mkdir(path.dirname(filePath), { recursive: true }); + await writeFile(filePath, expected); + } + } + const currentKotlin = await readFile(GENERATED_KOTLIN_PATH, "utf8").catch(() => ""); + if (currentKotlin !== catalog.kotlin) { + // The Kotlin map derives 1:1 from the same managed catalog: tolerate it + // exactly when every resource delta above was managed-pending. + if (!(options.check && options.tolerateManagedPending && !sawUnmanagedDrift)) { + drift.push(path.relative(ROOT, GENERATED_KOTLIN_PATH).split(path.sep).join("/")); + if (!options.check) { + await writeFile(GENERATED_KOTLIN_PATH, catalog.kotlin); + } + } + } + if (options.check && drift.length > 0) { + throw new Error(`Android generated localization drift:\n${drift.join("\n")}`); + } + if (catalog.contradictions.length > 0) { + const limit = 20; + const visible = catalog.contradictions.slice(0, limit); + const remaining = catalog.contradictions.length - visible.length; + process.stderr.write( + [ + `android-app-i18n: contradictions=${catalog.contradictions.length}`, + ...visible.map( + (finding) => + `${finding.locale}: ${JSON.stringify(finding.source)} -> ${JSON.stringify(finding.selected)} (${finding.translations.map((translation) => JSON.stringify(translation)).join(", ")})`, + ), + ...(remaining > 0 ? [`android-app-i18n: ${remaining} more contradictions omitted`] : []), + "", + ].join("\n"), + ); + } + return catalog; +} + +export async function verifyAndroidAppI18n() { + const [ + sourceFiles, + base, + referenceSource, + thirdPartyBase, + thirdPartyReferenceSource, + wearBase, + wearReferenceSource, + ] = await Promise.all([ + readAllAndroidSource(), + readStrings("values"), + readAndroidResourceReferences(), + readStrings("values", ANDROID_THIRD_PARTY_RESOURCE_ROOT, THIRD_PARTY_STRINGS_FILE), + readAndroidResourceReferences(ANDROID_THIRD_PARTY_ROOT), + readStrings("values", WEAR_RESOURCE_ROOT), + readAndroidResourceReferences(ANDROID_WEAR_MAIN_ROOT), + ]); + const baseKeys = new Set(base.keys()); + const thirdPartyBaseKeys = new Set(thirdPartyBase.keys()); + const wearBaseKeys = new Set(wearBase.keys()); + const problems: Array = [ + ["App English syntax", findInvalidResourceSyntax(base)], + ["Third-party English syntax", findInvalidResourceSyntax(thirdPartyBase)], + ["Wear English syntax", findInvalidResourceSyntax(wearBase)], + ]; + const manualBaseKeys = [...baseKeys].filter((key) => !key.startsWith(MANAGED_PREFIX)); + problems.push([ + "App English unused", + findUnusedAndroidResourceKeys(manualBaseKeys, referenceSource), + ]); + problems.push([ + "Third-party English unused", + findUnusedAndroidResourceKeys(thirdPartyBaseKeys, thirdPartyReferenceSource), + ]); + problems.push([ + "Wear English unused", + findUnusedAndroidResourceKeys(wearBaseKeys, wearReferenceSource), + ]); + const uiFindings = sourceFiles.flatMap((file) => + findUnlocalizedAndroidUiLiterals(file.source, file.path), + ); + problems.push([ + "Unlocalized UI literals", + uiFindings.map((finding) => `${finding.path}:${finding.line}:${finding.source}`), + ]); + if (problems.some(([, keys]) => keys.length)) { + throw new Error(formatProblems(problems)); + } + process.stdout.write( + `android-app-i18n: appSourceKeys=${baseKeys.size} thirdPartySourceKeys=${thirdPartyBaseKeys.size} wearSourceKeys=${wearBaseKeys.size}\n`, + ); +} + +export async function checkAndroidAppI18n(options: { tolerateManagedPending?: boolean } = {}) { + await verifyAndroidAppI18n(); + await syncAndroidAppI18n({ check: true, ...options }); + const localeStrings = await Promise.all(LOCALES.map((locale) => readStrings(locale))); + const wearLocaleStrings = options.tolerateManagedPending + ? [] + : await Promise.all(LOCALES.map((locale) => readStrings(locale, WEAR_RESOURCE_ROOT))); + const thirdPartyLocaleStrings = options.tolerateManagedPending + ? [] + : await Promise.all( + LOCALES.map((locale) => + readStrings(locale, ANDROID_THIRD_PARTY_RESOURCE_ROOT, THIRD_PARTY_STRINGS_FILE), + ), + ); + const base = expectDefined(localeStrings[0], "English Android string resources"); + const localeProblems = ( + surface: string, + surfaceBase: ReadonlyMap, + translations: ReadonlyArray>, + ): Array => { + const surfaceBaseKeys = new Set(surfaceBase.keys()); + return translations.flatMap((strings, index) => { + const locale = NATIVE_I18N_LOCALES[index]; + const keys = new Set(strings.keys()); + const placeholderMismatches = [...surfaceBase].flatMap(([key, sourceEntry]) => { + const translatedEntry = strings.get(key); + if (!translatedEntry) { + return []; + } + const expected = [...sourceEntry.rawValue.matchAll(FORMAT_RE)] + .map((match) => match[0]) + .toSorted(); + const actual = [...translatedEntry.rawValue.matchAll(FORMAT_RE)] + .map((match) => match[0]) + .toSorted(); + return expected.join("\u0000") === actual.join("\u0000") ? [] : [key]; + }); + return [ + [`${surface} ${locale} missing`, [...surfaceBaseKeys].filter((key) => !keys.has(key))], + [`${surface} ${locale} extra`, [...keys].filter((key) => !surfaceBaseKeys.has(key))], + [`${surface} ${locale} placeholders`, placeholderMismatches], + [`${surface} ${locale} syntax`, findInvalidResourceSyntax(strings)], + ] as const; + }); + }; + const problems = localeProblems("App", base, localeStrings.slice(1)); + if (thirdPartyLocaleStrings.length > 0) { + const thirdPartyBase = expectDefined( + thirdPartyLocaleStrings[0], + "English Android third-party string resources", + ); + problems.push( + ...localeProblems("Third-party", thirdPartyBase, thirdPartyLocaleStrings.slice(1)), + ); + } + if (wearLocaleStrings.length > 0) { + const wearBase = expectDefined(wearLocaleStrings[0], "English Wear string resources"); + problems.push(...localeProblems("Wear", wearBase, wearLocaleStrings.slice(1))); + } + if (problems.some(([, keys]) => keys.length)) { + throw new Error(formatProblems(problems)); + } + process.stdout.write( + `android-app-i18n: appKeys=${base.size} locales=${NATIVE_I18N_LOCALES.join(",")}\n`, + ); +} + +if (isDirectRunUrl(process.argv[1], import.meta.url)) { + const [command] = process.argv.slice(2); + if (command === "sync") { + await syncAndroidAppI18n(); + } else if (command === "check") { + await checkAndroidAppI18n(); + } else { + throw new Error("usage: node --import tsx scripts/android-app-i18n.ts "); + } +} diff --git a/scripts/android-native-ci.mjs b/scripts/android-native-ci.mjs new file mode 100644 index 0000000000000000000000000000000000000000..cfb0c34c09369963d3f60cafbd1a1efd6053b804 --- /dev/null +++ b/scripts/android-native-ci.mjs @@ -0,0 +1,58 @@ +#!/usr/bin/env node + +import { readFileSync, realpathSync } from "node:fs"; +import { validateReleaseChildRunProvenance } from "./full-release-validation-policy.mjs"; +import { isDirectRunUrl } from "./lib/direct-run.mjs"; +import { isRecord } from "./lib/record-shared.mjs"; +import { runReleaseToolingGh } from "./release-tooling-identity.mjs"; + +export function verifyAndroidNativeCi(approval, { runGh = runReleaseToolingGh } = {}) { + // Stable v2026.8.2's tag-owned publisher consumes v2 approvals from full FRV. + // The npm-only policy requires v3 so its separate native proof survives dispatch. + if (approval.version === 2 && approval.nativeCi === undefined) { + return; + } + const nativeCi = approval.nativeCi; + if ( + approval.version !== 3 || + !isRecord(nativeCi) || + Object.keys(nativeCi).length !== 3 || + typeof nativeCi.runId !== "string" || + !/^[1-9][0-9]*$/u.test(nativeCi.runId) || + nativeCi.runAttempt !== 1 || + typeof nativeCi.workflowRef !== "string" || + nativeCi.workflowRef.length === 0 + ) { + throw new Error("Android approval requires an exact native CI qualification tuple."); + } + const run = JSON.parse( + runGh([ + "api", + `repos/${approval.repository}/actions/runs/${nativeCi.runId}`, + "--method", + "GET", + ]), + ); + const proof = validateReleaseChildRunProvenance(run, { + key: "nativeAndroid", + plannedRunAttempt: nativeCi.runAttempt, + repository: approval.repository, + runId: nativeCi.runId, + workflow: "ci.yml", + displayTitle: `CI release-native-android-${approval.parentRunId}-${approval.parentRunAttempt}-${approval.targetSha}`, + workflowRef: nativeCi.workflowRef, + workflowSha: approval.parentWorkflowSha, + }); + if ( + proof.effectiveRunAttempt !== nativeCi.runAttempt || + run.status !== "completed" || + run.conclusion !== "success" + ) { + throw new Error("Android approval requires the exact completed successful native CI attempt."); + } +} + +// Isolated release harnesses can invoke the verifier through a scripts symlink. +if (process.argv[1] && isDirectRunUrl(realpathSync(process.argv[1]), import.meta.url)) { + verifyAndroidNativeCi(JSON.parse(readFileSync(process.argv[2], "utf8"))); +} diff --git a/scripts/android-pin-version.ts b/scripts/android-pin-version.ts new file mode 100644 index 0000000000000000000000000000000000000000..8034a10d47d66696a07b02449b0d1e0a01dc34bb --- /dev/null +++ b/scripts/android-pin-version.ts @@ -0,0 +1,25 @@ +// Android pinning is retired; the shared mobile cutter owns release metadata. +import { isDirectRunUrl } from "./lib/direct-run.mjs"; + +const RETIRED_MESSAGE = + "Android version pinning is retired. Run the shared mobile cutter in scripts/mobile-release-version.ts."; + +function printUsage(): void { + process.stdout.write(`${RETIRED_MESSAGE}\n`); +} + +export async function main(argv: string[]): Promise { + if (argv.includes("-h") || argv.includes("--help")) { + printUsage(); + return 0; + } + process.stderr.write(`${RETIRED_MESSAGE}\n`); + return 1; +} + +if (isDirectRunUrl(process.argv[1], import.meta.url)) { + const exitCode = await main(process.argv.slice(2)); + if (exitCode !== 0) { + process.exit(exitCode); + } +} diff --git a/scripts/android-screenshots.sh b/scripts/android-screenshots.sh new file mode 100644 index 0000000000000000000000000000000000000000..b7d683a1b0ccb8dd35b8cdd8ed8ee457e5387437 --- /dev/null +++ b/scripts/android-screenshots.sh @@ -0,0 +1,770 @@ +#!/usr/bin/env bash +# Bash 5.3+ can deadlock writing heredoc pipes on macOS before the reader starts. +if [[ ${OSTYPE:-} == darwin* && $BASH != /bin/bash ]] && ((BASH_VERSINFO[0] > 5 || (BASH_VERSINFO[0] == 5 && BASH_VERSINFO[1] >= 3))); then + exec /bin/bash "$0" "$@" +fi +set -euo pipefail + +usage() { + cat <<'EOF' +Usage: + scripts/android-screenshots.sh [--form-factor all|phone|wear] [--device ] [--avd ] [--locale en-US] [--skip-build] [--skip-install] [--keep-emulator] [--dry-run] + +Builds and installs the phone and Wear OS debug apps on matching emulators, +launches production screens with deterministic local fixture state, and writes +Google Play screenshots under: + apps/android/fastlane/metadata/android//images/phoneScreenshots/ + apps/android/fastlane/metadata/android//images/wearScreenshots/ + +Capture evidence is saved under: + .artifacts/android-screenshots/latest/ + +By default, the script captures both form factors using retained Pixel 2 and +Wear OS Large Round AVDs. Use --form-factor with --avd or --device to capture +one form factor on an explicitly selected emulator. +EOF +} + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +ANDROID_DIR="${ROOT_DIR}/apps/android" +DEFAULT_SCREENSHOT_AVD="OpenClaw_Screenshots_API36" +DEFAULT_SCREENSHOT_DEVICE_PROFILE="pixel_2" +DEFAULT_WEAR_SCREENSHOT_AVD="OpenClaw_Wear_Screenshots_API34" +DEFAULT_WEAR_SCREENSHOT_DEVICE_PROFILE="wearos_large_round" +case "$(uname -m)" in + arm64|aarch64) DEFAULT_SCREENSHOT_ABI="arm64-v8a" ;; + *) DEFAULT_SCREENSHOT_ABI="x86_64" ;; +esac +DEFAULT_SCREENSHOT_SYSTEM_IMAGE="system-images;android-36;google_apis;${DEFAULT_SCREENSHOT_ABI}" +DEFAULT_WEAR_SCREENSHOT_SYSTEM_IMAGE="system-images;android-34;android-wear;${DEFAULT_SCREENSHOT_ABI}" +FORM_FACTOR="${ANDROID_SCREENSHOT_FORM_FACTOR:-all}" +LOCALE="en-US" +DEVICE="${ANDROID_SCREENSHOT_DEVICE:-}" +AVD="${ANDROID_SCREENSHOT_AVD:-$DEFAULT_SCREENSHOT_AVD}" +SCREENSHOT_DEVICE_PROFILE="${ANDROID_SCREENSHOT_DEVICE_PROFILE:-$DEFAULT_SCREENSHOT_DEVICE_PROFILE}" +SCREENSHOT_SYSTEM_IMAGE="${ANDROID_SCREENSHOT_SYSTEM_IMAGE:-$DEFAULT_SCREENSHOT_SYSTEM_IMAGE}" +KEEP_EMULATOR="${ANDROID_SCREENSHOT_KEEP_EMULATOR:-0}" +SKIP_BUILD=0 +SKIP_INSTALL=0 +DRY_RUN=0 +SCENES=(home chat settings gateway voice-wake) +OUTPUT_TYPE="phoneScreenshots" +GRADLE_ASSEMBLE_TASK=":app:assemblePlayDebug" +APP_PACKAGE="ai.openclaw.app.debug" +ACTIVITY_COMPONENT="$APP_PACKAGE/ai.openclaw.app.MainActivity" +EMULATOR_PID="" +EMULATOR_LOG="" +STARTED_EMULATOR=0 +ARTIFACT_ROOT="${ROOT_DIR}/.artifacts/android-screenshots/latest" +ARTIFACT_DIR="${ARTIFACT_ROOT}/phone" +SCREENSHOT_SIZE="${ANDROID_SCREENSHOT_SIZE:-1440x2560}" +DISPLAY_OVERRIDDEN=0 +ORIGINAL_WM_SIZE="" +ORIGINAL_WM_DENSITY="" +SCREENSHOT_DENSITY="" +TIMEZONE_OVERRIDDEN=0 +ORIGINAL_AUTO_TIME_ZONE="" +ORIGINAL_TIME_ZONE="" +DEVICE_EXPLICIT=0 +AVD_EXPLICIT=0 + +while [[ $# -gt 0 ]]; do + case "$1" in + --) + shift + ;; + --form-factor) + FORM_FACTOR="${2:-}" + shift 2 + ;; + --device) + DEVICE="${2:-}" + DEVICE_EXPLICIT=1 + shift 2 + ;; + --avd) + AVD="${2:-}" + AVD_EXPLICIT=1 + shift 2 + ;; + --locale) + LOCALE="${2:-}" + shift 2 + ;; + --skip-build) + SKIP_BUILD=1 + shift + ;; + --skip-install) + SKIP_INSTALL=1 + shift + ;; + --keep-emulator) + KEEP_EMULATOR=1 + shift + ;; + --dry-run) + DRY_RUN=1 + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unknown argument: $1" >&2 + usage + exit 1 + ;; + esac +done + +case "$FORM_FACTOR" in + all) + if [[ "$DEVICE_EXPLICIT" == "1" || "$AVD_EXPLICIT" == "1" ]]; then + echo "--device and --avd require --form-factor phone or --form-factor wear." >&2 + exit 1 + fi + if [[ "$KEEP_EMULATOR" == "1" ]]; then + echo "--keep-emulator requires --form-factor phone or --form-factor wear." >&2 + exit 1 + fi + child_args=(--locale "$LOCALE") + [[ "$SKIP_BUILD" == "1" ]] && child_args+=(--skip-build) + [[ "$SKIP_INSTALL" == "1" ]] && child_args+=(--skip-install) + [[ "$DRY_RUN" == "1" ]] && child_args+=(--dry-run) + bash "$0" --form-factor phone "${child_args[@]}" + bash "$0" --form-factor wear "${child_args[@]}" + exit 0 + ;; + phone) + ;; + wear) + if [[ "$DEVICE_EXPLICIT" != "1" ]]; then + DEVICE="${ANDROID_WEAR_SCREENSHOT_DEVICE:-}" + fi + if [[ "$AVD_EXPLICIT" != "1" ]]; then + AVD="${ANDROID_WEAR_SCREENSHOT_AVD:-$DEFAULT_WEAR_SCREENSHOT_AVD}" + fi + SCREENSHOT_DEVICE_PROFILE="${ANDROID_WEAR_SCREENSHOT_DEVICE_PROFILE:-$DEFAULT_WEAR_SCREENSHOT_DEVICE_PROFILE}" + SCREENSHOT_SYSTEM_IMAGE="${ANDROID_WEAR_SCREENSHOT_SYSTEM_IMAGE:-$DEFAULT_WEAR_SCREENSHOT_SYSTEM_IMAGE}" + SCREENSHOT_SIZE="${ANDROID_WEAR_SCREENSHOT_SIZE:-454x454}" + SCENES=(chat voice controls) + OUTPUT_TYPE="wearScreenshots" + GRADLE_ASSEMBLE_TASK=":wear:assembleDebug" + ACTIVITY_COMPONENT="$APP_PACKAGE/ai.openclaw.wear.MainActivity" + ARTIFACT_DIR="${ARTIFACT_ROOT}/wear" + ;; + *) + echo "Invalid Android screenshot form factor: ${FORM_FACTOR}" >&2 + echo "Use all, phone, or wear." >&2 + exit 1 + ;; +esac + +validate_locale() { + local locale="$1" + if [[ "$locale" =~ ^[A-Za-z0-9][A-Za-z0-9_-]*$ ]]; then + return + fi + echo "Invalid Android screenshot locale: ${locale}" >&2 + echo "Use a locale tag like en-US or pt-BR; path separators and dot segments are not allowed." >&2 + exit 1 +} + +validate_locale "$LOCALE" + +if [[ ! "$SCREENSHOT_SIZE" =~ ^[0-9]+x[0-9]+$ ]]; then + echo "Invalid Android screenshot size: ${SCREENSHOT_SIZE}" >&2 + echo "Use a pixel size like 1080x1920." >&2 + exit 1 +fi +SCREENSHOT_WIDTH=$((10#${SCREENSHOT_SIZE%x*})) +SCREENSHOT_HEIGHT=$((10#${SCREENSHOT_SIZE#*x})) +SCREENSHOT_MIN_DIMENSION=$((SCREENSHOT_WIDTH < SCREENSHOT_HEIGHT ? SCREENSHOT_WIDTH : SCREENSHOT_HEIGHT)) +SCREENSHOT_MAX_DIMENSION=$((SCREENSHOT_WIDTH > SCREENSHOT_HEIGHT ? SCREENSHOT_WIDTH : SCREENSHOT_HEIGHT)) +if (( SCREENSHOT_MIN_DIMENSION < 320 || SCREENSHOT_MAX_DIMENSION > 3840 || SCREENSHOT_MAX_DIMENSION > SCREENSHOT_MIN_DIMENSION * 2 )); then + echo "Android screenshot size ${SCREENSHOT_SIZE} does not meet Google Play dimension and aspect-ratio limits." >&2 + exit 1 +fi + +cleanup_started_emulator() { + local stopped=0 + + if [[ "$STARTED_EMULATOR" != "1" || "$KEEP_EMULATOR" == "1" ]]; then + return + fi + if [[ -n "${ADB_BIN:-}" && -n "${ADB_SERIAL:-}" ]]; then + if "$ADB_BIN" -s "$ADB_SERIAL" emu kill >/dev/null 2>&1; then + stopped=1 + local deadline=$((SECONDS + 30)) + while (( SECONDS < deadline )); do + if ! "$ADB_BIN" devices | awk 'NR > 1 && $2 == "device" { print $1 }' | grep -Fxq "$ADB_SERIAL"; then + break + fi + sleep 1 + done + fi + fi + if [[ "$stopped" != "1" && -n "$EMULATOR_PID" ]]; then + kill "$EMULATOR_PID" >/dev/null 2>&1 || true + fi +} + +restore_device_display() { + if [[ "$DISPLAY_OVERRIDDEN" != "1" || -z "${ADB_BIN:-}" || -z "${ADB_SERIAL:-}" ]]; then + return + fi + if [[ -n "$ORIGINAL_WM_SIZE" ]]; then + "$ADB_BIN" -s "$ADB_SERIAL" shell wm size "$ORIGINAL_WM_SIZE" >/dev/null 2>&1 || true + else + "$ADB_BIN" -s "$ADB_SERIAL" shell wm size reset >/dev/null 2>&1 || true + fi + if [[ -n "$ORIGINAL_WM_DENSITY" ]]; then + "$ADB_BIN" -s "$ADB_SERIAL" shell wm density "$ORIGINAL_WM_DENSITY" >/dev/null 2>&1 || true + else + "$ADB_BIN" -s "$ADB_SERIAL" shell wm density reset >/dev/null 2>&1 || true + fi +} + +restore_device_timezone() { + if [[ "$TIMEZONE_OVERRIDDEN" != "1" || -z "${ADB_BIN:-}" || -z "${ADB_SERIAL:-}" ]]; then + return + fi + if [[ -n "$ORIGINAL_TIME_ZONE" ]]; then + "$ADB_BIN" -s "$ADB_SERIAL" shell cmd alarm set-timezone "$ORIGINAL_TIME_ZONE" >/dev/null 2>&1 || true + fi + if [[ "$ORIGINAL_AUTO_TIME_ZONE" == "true" || "$ORIGINAL_AUTO_TIME_ZONE" == "false" ]]; then + "$ADB_BIN" -s "$ADB_SERIAL" shell cmd time_zone_detector set_auto_detection_enabled "$ORIGINAL_AUTO_TIME_ZONE" >/dev/null 2>&1 || true + fi +} + +cleanup_emulator_log() { + if [[ -n "$EMULATOR_LOG" && -f "$EMULATOR_LOG" ]]; then + rm -f "$EMULATOR_LOG" + fi +} + +cleanup() { + restore_device_display + restore_device_timezone + cleanup_started_emulator + cleanup_emulator_log +} + +trap cleanup EXIT + +adb_bin() { + if [[ -n "${ADB:-}" ]]; then + printf '%s\n' "$ADB" + return + fi + if command -v adb >/dev/null 2>&1; then + command -v adb + return + fi + for sdk_root in "${ANDROID_HOME:-}" "${ANDROID_SDK_ROOT:-}" "$HOME/Library/Android/sdk"; do + if [[ -n "$sdk_root" && -x "$sdk_root/platform-tools/adb" ]]; then + printf '%s\n' "$sdk_root/platform-tools/adb" + return + fi + done + echo "adb not found. Install Android platform-tools or set ADB." >&2 + return 127 +} + +emulator_bin() { + if [[ -n "${ANDROID_EMULATOR:-}" ]]; then + printf '%s\n' "$ANDROID_EMULATOR" + return + fi + if command -v emulator >/dev/null 2>&1; then + command -v emulator + return + fi + for sdk_root in "${ANDROID_HOME:-}" "${ANDROID_SDK_ROOT:-}" "$HOME/Library/Android/sdk"; do + if [[ -n "$sdk_root" && -x "$sdk_root/emulator/emulator" ]]; then + printf '%s\n' "$sdk_root/emulator/emulator" + return + fi + done + echo "Android emulator binary not found. Install the Android emulator or set ANDROID_EMULATOR." >&2 + return 127 +} + +avdmanager_bin() { + if [[ -n "${AVDMANAGER:-}" ]]; then + printf '%s\n' "$AVDMANAGER" + return + fi + for sdk_root in "${ANDROID_HOME:-}" "${ANDROID_SDK_ROOT:-}" "$HOME/Library/Android/sdk"; do + for relative_path in cmdline-tools/latest/bin/avdmanager cmdline-tools/bin/avdmanager tools/bin/avdmanager; do + if [[ -n "$sdk_root" && -x "$sdk_root/$relative_path" ]]; then + printf '%s\n' "$sdk_root/$relative_path" + return + fi + done + done + if command -v avdmanager >/dev/null 2>&1; then + command -v avdmanager + return + fi + echo "avdmanager not found. Install Android SDK command-line tools or set AVDMANAGER." >&2 + return 127 +} + +ensure_screenshot_avd() { + local avd="$1" + local emulator + local avdmanager + + emulator="$(emulator_bin)" + if "$emulator" -list-avds | grep -Fxq "$avd"; then + return + fi + + avdmanager="$(avdmanager_bin)" + echo "Creating no-cutout screenshot AVD '${avd}' from device profile '${SCREENSHOT_DEVICE_PROFILE}'." >&2 + if ! printf 'no\n' | "$avdmanager" create avd \ + --force \ + --name "$avd" \ + --package "$SCREENSHOT_SYSTEM_IMAGE" \ + --device "$SCREENSHOT_DEVICE_PROFILE"; then + echo "Could not create Android screenshot AVD '${avd}'." >&2 + echo "Install SDK package '${SCREENSHOT_SYSTEM_IMAGE}' or set ANDROID_SCREENSHOT_SYSTEM_IMAGE." >&2 + return 1 + fi +} + +connected_devices() { + local adb="$1" + "$adb" devices | awk 'NR > 1 && $2 == "device" { print $1 }' +} + +device_count() { + local devices="$1" + printf '%s\n' "$devices" | sed '/^$/d' | wc -l | tr -d ' ' +} + +running_avd_name() { + local adb="$1" + local serial="$2" + "$adb" -s "$serial" emu avd name 2>/dev/null | tr -d '\r' | sed -n '1p' +} + +wait_for_single_device() { + local adb="$1" + local timeout_seconds="${ANDROID_SCREENSHOT_EMULATOR_TIMEOUT_SECONDS:-180}" + local deadline=$((SECONDS + timeout_seconds)) + local devices + local count + + while (( SECONDS < deadline )); do + devices="$(connected_devices "$adb")" + count="$(device_count "$devices")" + if [[ "$count" == "1" ]]; then + printf '%s\n' "$devices" + return + fi + sleep 2 + done + + echo "Timed out waiting for exactly one Android emulator device." >&2 + "$adb" devices -l >&2 || true + return 1 +} + +wait_for_boot_completed() { + local adb="$1" + local serial="$2" + local timeout_seconds="${ANDROID_SCREENSHOT_EMULATOR_TIMEOUT_SECONDS:-180}" + local deadline=$((SECONDS + timeout_seconds)) + local boot_completed + + "$adb" -s "$serial" wait-for-device + while (( SECONDS < deadline )); do + boot_completed="$("$adb" -s "$serial" shell getprop sys.boot_completed 2>/dev/null | tr -d '\r' || true)" + if [[ "$boot_completed" == "1" ]]; then + "$adb" -s "$serial" shell input keyevent 82 >/dev/null 2>&1 || true + return + fi + sleep 2 + done + + echo "Timed out waiting for Android emulator boot completion on ${serial}." >&2 + return 1 +} + +wait_for_explicit_device() { + local adb="$1" + local serial="$2" + local timeout_seconds="${ANDROID_SCREENSHOT_DEVICE_TIMEOUT_SECONDS:-30}" + local deadline=$((SECONDS + timeout_seconds)) + local state + + while (( SECONDS < deadline )); do + state="$("$adb" devices | awk -v serial="$serial" '$1 == serial { print $2 }')" + if [[ "$state" == "device" ]]; then + return + fi + sleep 2 + done + + if [[ -n "$state" ]]; then + echo "Android device '${serial}' did not become usable within ${timeout_seconds}s; current adb state is '${state}'." >&2 + else + echo "Android device '${serial}' was not found within ${timeout_seconds}s." >&2 + fi + "$adb" devices -l >&2 || true + return 1 +} + +stabilize_device_for_screenshots() { + local adb="$1" + local serial="$2" + ORIGINAL_AUTO_TIME_ZONE="$("$adb" -s "$serial" shell cmd time_zone_detector is_auto_detection_enabled 2>/dev/null | tr -d '\r')" + ORIGINAL_TIME_ZONE="$("$adb" -s "$serial" shell getprop persist.sys.timezone 2>/dev/null | tr -d '\r')" + if [[ "$ORIGINAL_AUTO_TIME_ZONE" != "true" && "$ORIGINAL_AUTO_TIME_ZONE" != "false" ]]; then + echo "Could not determine emulator automatic timezone setting." >&2 + return 1 + fi + if [[ -z "$ORIGINAL_TIME_ZONE" ]]; then + echo "Could not determine emulator timezone." >&2 + return 1 + fi + # Arm cleanup before the first mutation; restoring an unchanged snapshot is safe. + TIMEZONE_OVERRIDDEN=1 + # Seeded chat timestamps use the device timezone, so disable detection before + # pinning UTC to avoid inheriting the runner's locale. + "$adb" -s "$serial" shell cmd time_zone_detector set_auto_detection_enabled false >/dev/null + "$adb" -s "$serial" shell cmd alarm set-timezone UTC >/dev/null + "$adb" -s "$serial" shell settings put global window_animation_scale 0 >/dev/null 2>&1 || true + "$adb" -s "$serial" shell settings put global transition_animation_scale 0 >/dev/null 2>&1 || true + "$adb" -s "$serial" shell settings put global animator_duration_scale 0 >/dev/null 2>&1 || true + "$adb" -s "$serial" shell settings put global stay_on_while_plugged_in 3 >/dev/null 2>&1 || true + "$adb" -s "$serial" shell svc power stayon true >/dev/null 2>&1 || true + "$adb" -s "$serial" shell input keyevent 224 >/dev/null 2>&1 || true + "$adb" -s "$serial" shell wm dismiss-keyguard >/dev/null 2>&1 || true + "$adb" -s "$serial" shell settings put system font_scale 1.0 >/dev/null 2>&1 || true +} + +require_emulator_device() { + local adb="$1" + local serial="$2" + local qemu + + qemu="$("$adb" -s "$serial" shell getprop ro.kernel.qemu 2>/dev/null | tr -d '\r')" + if [[ "$qemu" == "1" ]]; then + return + fi + echo "Android screenshot capture requires an emulator; '${serial}' is not an emulator." >&2 + echo "Pass --avd or --device ." >&2 + return 1 +} + +configure_screenshot_display() { + local adb="$1" + local serial="$2" + local current_size + local current_density + local effective_size + local effective_density + local effective_width + + current_size="$("$adb" -s "$serial" shell wm size 2>/dev/null | tr -d '\r')" + current_density="$("$adb" -s "$serial" shell wm density 2>/dev/null | tr -d '\r')" + ORIGINAL_WM_SIZE="$(printf '%s\n' "$current_size" | sed -n 's/^Override size: //p')" + ORIGINAL_WM_DENSITY="$(printf '%s\n' "$current_density" | sed -n 's/^Override density: //p')" + effective_size="${ORIGINAL_WM_SIZE:-$(printf '%s\n' "$current_size" | sed -n 's/^Physical size: //p')}" + effective_density="${ORIGINAL_WM_DENSITY:-$(printf '%s\n' "$current_density" | sed -n 's/^Physical density: //p')}" + if [[ ! "$effective_size" =~ ^[0-9]+x[0-9]+$ || ! "$effective_density" =~ ^[0-9]+$ ]]; then + echo "Could not determine emulator display size and density." >&2 + return 1 + fi + + effective_width=$((10#${effective_size%x*})) + SCREENSHOT_DENSITY=$(((10#$effective_density * SCREENSHOT_WIDTH + effective_width / 2) / effective_width)) + DISPLAY_OVERRIDDEN=1 + "$adb" -s "$serial" shell wm size "$SCREENSHOT_SIZE" >/dev/null + "$adb" -s "$serial" shell wm density "$SCREENSHOT_DENSITY" >/dev/null + echo "Android screenshot density: ${SCREENSHOT_DENSITY} dpi" +} + +boot_emulator() { + local adb="$1" + local avd="$2" + local emulator + local emulator_args + local extra_args + local serial + + ensure_screenshot_avd "$avd" + emulator="$(emulator_bin)" + EMULATOR_LOG="$(mktemp "${TMPDIR:-/tmp}/openclaw-android-screenshot-emulator.XXXXXX.log")" + echo "No connected Android device found. Booting AVD '${avd}'." >&2 + emulator_args=(-avd "$avd" -no-window -no-audio -no-boot-anim) + if [[ -n "${ANDROID_SCREENSHOT_EMULATOR_ARGS:-}" ]]; then + read -r -a extra_args <<<"$ANDROID_SCREENSHOT_EMULATOR_ARGS" + emulator_args+=(${extra_args[@]+"${extra_args[@]}"}) + fi + "$emulator" "${emulator_args[@]}" >"$EMULATOR_LOG" 2>&1 & + EMULATOR_PID="$!" + STARTED_EMULATOR=1 + + serial="$(wait_for_single_device "$adb")" + wait_for_boot_completed "$adb" "$serial" + ADB_SERIAL="$serial" +} + +resolve_device() { + local adb="$1" + local devices + local count + local connected_avd + + if [[ -n "$DEVICE" ]]; then + wait_for_explicit_device "$adb" "$DEVICE" + ADB_SERIAL="$DEVICE" + return + fi + devices="$(connected_devices "$adb")" + count="$(device_count "$devices")" + if [[ "$count" == "1" ]]; then + if connected_avd="$(running_avd_name "$adb" "$devices")"; then + if [[ "$connected_avd" != "$AVD" ]]; then + echo "Connected emulator '${connected_avd:-unknown}' is not the screenshot AVD '${AVD}'." >&2 + echo "Stop it so the script can boot '${AVD}', or pass --device '${devices}' to override the no-cutout profile." >&2 + return 1 + fi + fi + ADB_SERIAL="$devices" + return + fi + if [[ "$count" == "0" ]]; then + boot_emulator "$adb" "$AVD" + return + else + echo "Multiple Android devices are connected. Pass --device ." >&2 + fi + return 1 +} + +latest_debug_apk() { + local output_dir + local pattern + + if [[ "$FORM_FACTOR" == "wear" ]]; then + output_dir="${ANDROID_DIR}/wear/build/outputs/apk/debug" + pattern='*-debug.apk' + else + output_dir="${ANDROID_DIR}/app/build/outputs/apk/play/debug" + pattern='*-play-debug.apk' + fi + if [[ ! -d "$output_dir" ]]; then + return 0 + fi + find "$output_dir" -maxdepth 1 -name "$pattern" -print 2>/dev/null | sort | tail -n 1 +} + +scene_ready_text() { + if [[ "$FORM_FACTOR" == "wear" ]]; then + case "$1" in + chat) printf '%s\n' "Release planning" ;; + voice) printf '%s\n' "Dictate" ;; + controls) printf '%s\n' "Gateway connected" ;; + *) + echo "Unknown Wear OS screenshot scene: $1" >&2 + return 1 + ;; + esac + return + fi + case "$1" in + home) printf '%s\n' "Overview" ;; + # The screenshot fixture seeds chat history and restores at the live edge, + # so wait for the latest reply instead of empty-chat copy. + chat) printf '%s\n' "The Android release is close." ;; + settings) printf '%s\n' "OpenClaw mobile" ;; + voice-wake) printf '%s\n' "Wake listener" ;; + # Connected fixtures can push Add Gateway below the composed viewport, so + # wait for the gateway detail's always-visible subtitle instead. + gateway) printf '%s\n' "Connection between this phone and OpenClaw." ;; + *) + echo "Unknown Android screenshot scene: $1" >&2 + return 1 + ;; + esac +} + +wait_for_scene_ready() { + local adb="$1" + local serial="$2" + local scene="$3" + local dump_path="$4" + local marker + local timeout_seconds="${ANDROID_SCREENSHOT_SCENE_TIMEOUT_SECONDS:-45}" + local deadline=$((SECONDS + timeout_seconds)) + + marker="$(scene_ready_text "$scene")" + while (( SECONDS < deadline )); do + if "$adb" -s "$serial" exec-out uiautomator dump /dev/tty >"$dump_path" 2>/dev/null; then + # A newly created Wear AVD may cover the resumed app with its charging + # overlay after stay-awake is enabled. Dismiss it or first-run capture stalls. + if [[ "$FORM_FACTOR" == "wear" ]] && grep -Fq 'com.google.android.wearable.sysui:id/charging_container' "$dump_path"; then + "$adb" -s "$serial" shell input keyevent 4 >/dev/null 2>&1 || true + sleep 0.5 + continue + fi + if grep -Fq "$marker" "$dump_path"; then + return + fi + fi + sleep 1 + done + + echo "Timed out waiting for scene '${scene}' to expose '${marker}' in the Android UI tree." >&2 + return 1 +} + +sips_bin() { + if [[ -n "${SIPS:-}" ]]; then + printf '%s\n' "$SIPS" + return + fi + if command -v sips >/dev/null 2>&1; then + command -v sips + return + fi + echo "sips not found. Android release screenshots require macOS or an explicit SIPS executable." >&2 + return 127 +} + +normalize_capture_for_play() { + local input_path="$1" + local output_path="$2" + local sips + local description + + sips="$(sips_bin)" + "$sips" -s format jpeg -s formatOptions best "$input_path" --out "$output_path" >/dev/null + rm -f "$input_path" + + description="$(file "$output_path")" + if [[ "$description" != *"${SCREENSHOT_SIZE/x/x}"* || "$description" != *"JPEG image data"* ]]; then + echo "Invalid Google Play screenshot output: ${description}" >&2 + return 1 + fi +} + +write_artifact_manifest() { + local serial="$1" + local avd_name + local git_sha + local checksum_command + + avd_name="$(running_avd_name "$ADB_BIN" "$serial")" + git_sha="$(git -C "$ROOT_DIR" rev-parse HEAD)" + if command -v shasum >/dev/null 2>&1; then + checksum_command=(shasum -a 256) + else + checksum_command=(sha256sum) + fi + + { + printf 'git_sha=%s\n' "$git_sha" + printf 'form_factor=%s\n' "$FORM_FACTOR" + printf 'output_type=%s\n' "$OUTPUT_TYPE" + printf 'device=%s\n' "$serial" + printf 'avd=%s\n' "${avd_name:-unknown}" + printf 'locale=%s\n' "$LOCALE" + printf 'size=%s\n' "$SCREENSHOT_SIZE" + printf 'density=%s\n' "$SCREENSHOT_DENSITY" + printf 'format=high-quality JPEG\n' + printf 'scenes=%s\n' "${SCENES[*]}" + printf 'screenshots:\n' + "${checksum_command[@]}" "$ARTIFACT_DIR"/screenshots/*.jpg + } >"$ARTIFACT_DIR/manifest.txt" +} + +OUTPUT_DIR="${ANDROID_DIR}/fastlane/metadata/android/${LOCALE}/images/${OUTPUT_TYPE}" +ADB_SERIAL="" +ADB_DISPLAY="${DEVICE:-}" + +echo "Android screenshot output: ${OUTPUT_DIR}" +echo "Android screenshot artifacts: ${ARTIFACT_DIR}" +echo "Android screenshot form factor: ${FORM_FACTOR}" +echo "Android screenshot size: ${SCREENSHOT_SIZE}" +echo "Scenes: ${SCENES[*]}" +echo "ADB device: ${ADB_DISPLAY}" +echo "Screenshot AVD: ${AVD}" +echo "Screenshot device profile: ${SCREENSHOT_DEVICE_PROFILE}" +echo "Screenshot system image: ${SCREENSHOT_SYSTEM_IMAGE}" + +if [[ "$DRY_RUN" == "1" ]]; then + echo "Dry run complete. No build, install, or capture commands were executed." + exit 0 +fi + +ADB_BIN="$(adb_bin)" +resolve_device "$ADB_BIN" +require_emulator_device "$ADB_BIN" "$ADB_SERIAL" +stabilize_device_for_screenshots "$ADB_BIN" "$ADB_SERIAL" +configure_screenshot_display "$ADB_BIN" "$ADB_SERIAL" +mkdir -p "$OUTPUT_DIR" +rm -f "$OUTPUT_DIR"/*.png "$OUTPUT_DIR"/*.jpg "$OUTPUT_DIR"/*.jpeg +rm -rf "$ARTIFACT_DIR" +mkdir -p "$ARTIFACT_DIR/screenshots" "$ARTIFACT_DIR/ui-dumps" "$ARTIFACT_DIR/activity-start" + +if [[ "$SKIP_INSTALL" != "1" ]]; then + if [[ "$SKIP_BUILD" != "1" ]]; then + ( + cd "$ANDROID_DIR" + ./gradlew "$GRADLE_ASSEMBLE_TASK" + ) + fi + APK_PATH="$(latest_debug_apk)" + if [[ -z "$APK_PATH" ]]; then + echo "No existing ${FORM_FACTOR} debug APK found. Run without --skip-build first." >&2 + exit 1 + fi + "$ADB_BIN" -s "$ADB_SERIAL" install -r "$APK_PATH" >/dev/null +elif [[ "$SKIP_BUILD" != "1" ]]; then + ( + cd "$ANDROID_DIR" + ./gradlew "$GRADLE_ASSEMBLE_TASK" + ) +fi + +"$ADB_BIN" -s "$ADB_SERIAL" shell pm clear "$APP_PACKAGE" >/dev/null +"$ADB_BIN" -s "$ADB_SERIAL" shell pm grant "$APP_PACKAGE" android.permission.RECORD_AUDIO >/dev/null +"$ADB_BIN" -s "$ADB_SERIAL" logcat -c >/dev/null 2>&1 || true + +for scene in "${SCENES[@]}"; do + output_path="${OUTPUT_DIR}/openclaw-${scene}.jpg" + raw_path="${OUTPUT_DIR}/openclaw-${scene}.raw.png" + artifact_path="${ARTIFACT_DIR}/screenshots/openclaw-${scene}.jpg" + ui_dump_path="${ARTIFACT_DIR}/ui-dumps/openclaw-${scene}.xml" + activity_start_path="${ARTIFACT_DIR}/activity-start/openclaw-${scene}.txt" + "$ADB_BIN" -s "$ADB_SERIAL" shell am force-stop "$APP_PACKAGE" >/dev/null + "$ADB_BIN" -s "$ADB_SERIAL" shell input keyevent 224 >/dev/null 2>&1 || true + "$ADB_BIN" -s "$ADB_SERIAL" shell wm dismiss-keyguard >/dev/null 2>&1 || true + "$ADB_BIN" -s "$ADB_SERIAL" shell am start -W \ + -n "$ACTIVITY_COMPONENT" \ + --ez openclaw.screenshotMode true \ + --es openclaw.screenshotScene "$scene" >"$activity_start_path" + wait_for_scene_ready "$ADB_BIN" "$ADB_SERIAL" "$scene" "$ui_dump_path" + sleep "${ANDROID_SCREENSHOT_SETTLE_SECONDS:-0.5}" + "$ADB_BIN" -s "$ADB_SERIAL" exec-out screencap -p >"$raw_path" + normalize_capture_for_play "$raw_path" "$output_path" + cp "$output_path" "$artifact_path" + echo "Captured ${output_path}" +done + +"$ADB_BIN" -s "$ADB_SERIAL" logcat -d >"$ARTIFACT_DIR/logcat.txt" 2>&1 || true +if [[ -n "$EMULATOR_LOG" && -f "$EMULATOR_LOG" ]]; then + cp "$EMULATOR_LOG" "$ARTIFACT_DIR/emulator.log" +fi +write_artifact_manifest "$ADB_SERIAL" + +echo "Android screenshots written to ${OUTPUT_DIR}" +echo "Android screenshot artifacts written to ${ARTIFACT_DIR}" diff --git a/scripts/android-sips-linux.sh b/scripts/android-sips-linux.sh new file mode 100644 index 0000000000000000000000000000000000000000..926b11d3fa6561c50138af9224daa6cbb498ebd4 --- /dev/null +++ b/scripts/android-sips-linux.sh @@ -0,0 +1,77 @@ +#!/bin/bash +set -euo pipefail + +fail() { + printf 'android-sips-linux: %s\n' "$1" >&2 + exit "${2:-1}" +} + +if [[ $# -ne 9 || + "$1" != "-s" || "$2" != "format" || "$3" != "jpeg" || + "$4" != "-s" || "$5" != "formatOptions" || "$6" != "best" || + "$8" != "--out" ]]; then + fail "unsupported arguments" 2 +fi + +input_path="$7" +output_path="$9" +convert_bin="/usr/bin/convert" +identify_bin="/usr/bin/identify" + +[[ "$input_path" == /* && "$output_path" == /* ]] || + fail "input and output paths must be absolute" 2 +[[ "$input_path" != "$output_path" ]] || fail "input and output paths must differ" 2 +[[ -f "$input_path" && ! -L "$input_path" ]] || fail "input is not a regular file" +[[ ! -e "$output_path" && ! -L "$output_path" ]] || fail "output already exists" +[[ -d "${output_path%/*}" ]] || fail "output directory does not exist" + +[[ -x "$convert_bin" && -x "$identify_bin" ]] || + fail "fixed ImageMagick executables are unavailable" 2 + +if ! input_dimensions="$("$identify_bin" -ping -format '%w %h' "$input_path" 2>/dev/null)"; then + fail "input is not a readable image" +fi +read -r input_width input_height extra <<<"$input_dimensions" +[[ "$input_width" =~ ^[1-9][0-9]*$ && "$input_height" =~ ^[1-9][0-9]*$ && -z "${extra:-}" ]] || + fail "input dimensions are invalid" + +umask 077 +temporary_dir="$(mktemp -d "${output_path}.tmp.XXXXXX")" +temporary_output="$temporary_dir/output.jpg" +cleanup() { + rm -rf "$temporary_dir" +} +trap cleanup EXIT + +if ! "$convert_bin" "$input_path" \ + -colorspace sRGB \ + -background white \ + -alpha remove \ + -alpha off \ + -type TrueColor \ + -quality 95 \ + "jpeg:$temporary_output" >/dev/null 2>&1; then + fail "ImageMagick conversion failed" +fi + +if ! output_description="$( + "$identify_bin" +ping -format '%m|%w|%h|%[colorspace]|%[type]|%[channels]|%Q' \ + "$temporary_output" 2>/dev/null +)"; then + fail "converted output is not a readable image" +fi +IFS='|' read -r output_format output_width output_height output_colorspace \ + output_type output_channels output_quality <<<"$output_description" +output_colorspace="$(printf '%s' "$output_colorspace" | tr '[:upper:]' '[:lower:]')" +output_channels="$(printf '%s' "$output_channels" | tr '[:upper:]' '[:lower:]')" + +[[ "$output_format" == "JPEG" ]] || fail "converted output is not JPEG" +[[ "$output_width" == "$input_width" && "$output_height" == "$input_height" ]] || + fail "converted output dimensions changed" +[[ "$output_colorspace" == "srgb" ]] || fail "converted output is not sRGB" +[[ "$output_type" == "TrueColor" ]] || fail "converted output is not true color" +[[ "$output_channels" != *a* ]] || fail "converted output retained an alpha channel" +[[ "$output_quality" =~ ^[0-9]+$ && "$output_quality" -ge 90 ]] || + fail "converted output quality is too low" + +mv "$temporary_output" "$output_path" diff --git a/scripts/apple-app-i18n.ts b/scripts/apple-app-i18n.ts new file mode 100644 index 0000000000000000000000000000000000000000..128a00b8e03097c3281314ea279f379dd4b52b7c --- /dev/null +++ b/scripts/apple-app-i18n.ts @@ -0,0 +1,984 @@ +import { mkdir, readdir, readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { NATIVE_I18N_LOCALES } from "./native-i18n-locales.ts"; + +const HERE = path.dirname(fileURLToPath(import.meta.url)); +const ROOT = path.resolve(HERE, ".."); +const REQUIRED_LOCALES = ["en", ...NATIVE_I18N_LOCALES]; +const FORMAT_RE = /%(?:%|(?:\d+\$)?(?:lld|ld|[@a-z]))/giu; +const INFLECTED_COUNT_INTERPOLATION_RE = /\\\([A-Za-z_][A-Za-z0-9_]*\)/gu; +const INFLECTED_COUNT_INTERPOLATION_EXACT_RE = /^\\\([A-Za-z_][A-Za-z0-9_]*\)$/u; +const INFLECTED_COUNT_SEGMENT_RE = + /\^\[[^\]]*\\\([A-Za-z_][A-Za-z0-9_]*\)[^\]]*\]\(inflect: true\)/gu; +const INFLECTED_COUNT_MARKER = "](inflect: true)"; +const IOS_CATALOG_PATH = "apps/ios/Resources/Localizable.xcstrings"; +const MACOS_CATALOG_PATH = "apps/macos/Sources/OpenClaw/Resources/Localizable.xcstrings"; +const MACOS_INFO_PLIST_PATH = "apps/macos/Sources/OpenClaw/Resources/Info.plist"; +const NATIVE_SOURCE_PATH = "apps/.i18n/native-source.json"; +const NATIVE_TRANSLATIONS_DIR = "apps/.i18n/native"; +const SHARED_CHAT_UI_SOURCE_PREFIX = "apps/shared/OpenClawKit/Sources/OpenClawChatUI/"; +const SHARED_GATEWAY_DISCOVERY_STATUS_SOURCE = + "apps/shared/OpenClawKit/Sources/OpenClawKit/GatewayDiscoveryStatusText.swift"; +const IOS_SOURCE_PREFIXES = [ + "apps/ios/", + SHARED_CHAT_UI_SOURCE_PREFIX, + "apps/shared/OpenClawKit/Sources/OpenClawKit/", +] as const; +const IOS_CATALOG_EXCLUSIONS = new Set([ + // Product names and preview-only single-character fixtures are intentionally verbatim. + "OpenClaw", + "z", +]); +const MACOS_SOURCE_PREFIXES = [ + "apps/macos/Sources/OpenClaw/", + SHARED_CHAT_UI_SOURCE_PREFIX, + SHARED_GATEWAY_DISCOVERY_STATUS_SOURCE, +] as const; +const MACOS_CATALOG_EXCLUSIONS = new Set([ + // Product names are intentionally verbatim. + "OpenClaw", +]); +const IOS_INFO_PLIST_TARGETS = [ + { + outputRoot: "apps/ios/Sources", + sourcePath: "apps/ios/Sources/Info.plist", + }, + { + outputRoot: "apps/ios/WatchApp", + sourcePath: "apps/ios/WatchApp/Info.plist", + }, + { + outputRoot: "apps/ios/ShareExtension", + sourcePath: "apps/ios/ShareExtension/Info.plist", + }, + { + outputRoot: "apps/ios/ActivityWidget", + sourcePath: "apps/ios/ActivityWidget/Info.plist", + }, +] as const; +const INFO_PLIST_LOCALIZABLE_KEYS = new Set(["NSScreenCaptureDescription"]); +const AMBIGUOUS_RUNTIME_INTERPOLATIONS = [ + { + label: "interpolated localized resource", + pattern: + /(?:\bString\s*\(\s*localized:|\bAttributedString\s*\(\s*localized:|\bLocalizedString(?:Key|Resource)\s*\(|(?:\b[A-Za-z_]\w*)?\.localized(?:Format)?\s*\()\s*"((?:\\.|[^"\\])*)"/gu, + allowsInflection: true, + }, + { + label: "interpolated multiline localized resource", + pattern: + /\b(?:String\s*\(\s*localized:|AttributedString\s*\(\s*localized:|LocalizedString(?:Key|Resource)\s*\()\s*"""([\s\S]*?)"""/gu, + allowsInflection: true, + }, + { + label: "interpolated SwiftUI text literal", + pattern: + /\b(?:Text|Label|Button|TextField|SecureField|Picker|Section|LabeledContent|Toggle|Menu|ShareLink|Link|TextEditor|ProgressView|Gauge|DisclosureGroup|ControlGroup|DatePicker|Stepper)\s*\(\s*"((?:\\.|[^"\\])*)"/gu, + allowsInflection: false, + }, + { + label: "interpolated multiline SwiftUI text literal", + pattern: + /\b(?:Text|Label|Button|TextField|SecureField|Picker|Section|LabeledContent|Toggle|Menu|ShareLink|Link|TextEditor|ProgressView|Gauge|DisclosureGroup|ControlGroup|DatePicker|Stepper)\s*\(\s*"""([\s\S]*?)"""/gu, + allowsInflection: false, + }, + { + label: "interpolated multiline SwiftUI modifier literal", + pattern: + /\.(?:accessibilityLabel|accessibilityHint|alert|confirmationDialog|help|navigationTitle)\s*\(\s*"""([\s\S]*?)"""/gu, + allowsInflection: false, + }, + { + label: "interpolated SwiftUI modifier literal", + pattern: + /\.(?:accessibilityLabel|accessibilityHint|alert|confirmationDialog|help|navigationTitle)\s*\(\s*"((?:\\.|[^"\\])*)"/gu, + allowsInflection: false, + }, + { + label: "interpolated accessibility model literal", + pattern: /\baccessibilityLabel\s*:\s*"((?:\\.|[^"\\])*)"/gu, + allowsInflection: false, + }, +] as const; + +export function findAmbiguousRuntimeInterpolations(source: string): string[] { + const violations: string[] = []; + for (const { label, pattern, allowsInflection } of AMBIGUOUS_RUNTIME_INTERPOLATIONS) { + pattern.lastIndex = 0; + for (const match of source.matchAll(pattern)) { + const literal = match[1] ?? ""; + if (!literal.includes("\\(") || (allowsInflection && isInflectedCountSource(literal))) { + continue; + } + violations.push(label); + break; + } + } + return violations; +} +const APPLE_LOCALE_DIRECTORIES: Record = { + "ja-JP": "ja", + "zh-CN": "zh-Hans", + "zh-TW": "zh-Hant", +}; +const LOCALIZED_WRAPPER_CONTRACTS: Record = { + "apps/macos/Sources/OpenClaw/DeviceSettingsPanels.swift": [ + 'String(localized: "Quick Chat shortcut")', + 'String(localized: "Microphone Test")', + 'Button("Done")', + ], + "apps/macos/Sources/OpenClaw/DeviceMicrophonePanel.swift": [ + 'String(localized: "Stopped")', + 'String(localized: "Timeout: no trigger heard")', + "Text(verbatim: meterError)", + ], + "apps/ios/Sources/Design/OpenClawProComponents.swift": [ + "enum OpenClawTextValue: ExpressibleByStringLiteral", + "struct OpenClawNoticeBanner: View {\n let icon: String\n let title: OpenClawTextValue\n let message: OpenClawTextValue", + "struct OpenClawAdaptiveHeaderRow: View {\n let title: OpenClawTextValue\n let subtitle: OpenClawTextValue?", + "struct OpenClawStatusBadge: View {\n @Environment(\\.colorScheme) private var colorScheme\n let label: OpenClawTextValue", + "struct ProStatusRow: View {\n let icon: String\n let title: OpenClawTextValue\n let detail: OpenClawTextValue", + ], + "apps/ios/Sources/Design/SettingsProTabSupport.swift": [ + "struct SettingsDetailRow: View {\n let label: LocalizedStringKey\n let value: OpenClawTextValue", + "init(_ label: LocalizedStringKey, value: OpenClawTextValue)", + "self.value.text", + "struct SettingsApprovalItem: Identifiable {\n let id: String\n let icon: String\n let title: OpenClawTextValue\n let detail: OpenClawTextValue\n let priority: OpenClawTextValue", + "self.item.title.text", + "self.item.detail.text", + "self.item.priority.text", + ], + "apps/ios/Sources/Design/SettingsProTabActions.swift": [ + "func detailStatusCard(\n icon: String,\n title: OpenClawTextValue,\n detail: OpenClawTextValue,\n value: OpenClawTextValue", + "func diagnosticCheckRow(\n icon: String,\n title: OpenClawTextValue,\n detail: OpenClawTextValue,\n value: OpenClawTextValue", + "pendingApproval.commandPreview.map(OpenClawTextValue.verbatim)", + 'format: String(localized: "Agent: %@")', + ], + "apps/ios/Sources/Design/SettingsProTabSections.swift": [ + "func settingsListRow(\n icon: String,\n iconColor: Color,\n title: LocalizedStringKey", + "func aboutLinkRow(\n title: LocalizedStringKey", + "func gatewaySecureField(\n _ placeholder: LocalizedStringKey", + "func settingsToggle(\n _ title: LocalizedStringKey", + ".accessibilityLabel(Text(placeholder))", + ".accessibilityLabel(Text(title))", + ], + "apps/ios/Sources/Gateway/GatewayConnectionController+Capabilities.swift": [ + 'String(localized: "Secure connection is required for this host.")', + 'String(localized: "Use only on a trusted private network.")', + ], + "apps/ios/Sources/Gateway/ExecApprovalPromptDialog.swift": [ + "private struct ExecApprovalPromptMetadataRow: View {\n let label: LocalizedStringKey", + 'localized: "about ^[\\(minutes) minute](inflect: true)"', + 'localized: "about ^[\\(hours) hour](inflect: true)"', + ], + "apps/ios/Sources/Voice/TalkGatewayPermissionState.swift": [ + 'String(format: String(localized: "Missing %@"), scope)', + 'String(localized: "Requesting approval")', + ], + "apps/ios/Sources/Voice/TalkModeManager.swift": [ + 'format: String(localized: "Speech error: %@")', + 'format: String(localized: "Speak failed: %@")', + 'self.setStatus(String(localized: "Listening (PTT)"), phase: .listening)', + ], + "apps/ios/Sources/Voice/VoiceWakeManager.swift": [ + 'format: String(localized: "Recognizer error: %@")', + 'self.statusText = String(localized: "Triggered")', + ], + "apps/ios/Sources/Design/CommandCenterSupport.swift": [ + "Text(verbatim: self.item.title)", + "Text(verbatim: self.item.trailing)", + "Text(verbatim: self.item.detail)", + "struct CommandEmptyStateRow: View {\n let icon: String\n let title: OpenClawTextValue\n let detail: OpenClawTextValue", + "private func actionButton(\n _ title: OpenClawTextValue", + "self.actionButton(.verbatim(category)", + ], + "apps/ios/Sources/Gateway/GatewayQuickSetupSheet.swift": [ + "fullRowToggle(_ title: LocalizedStringKey", + "private struct GatewayQuickSetupStatusRow: View {\n let title: LocalizedStringKey\n let value: String", + "Text(LocalizedStringKey(availability.actionTitle))", + "Text(verbatim: self.message)", + ], + "apps/ios/Sources/Gateway/GatewayProblemView.swift": [ + "title: .verbatim(self.problem.localizedTitle)", + "message: .verbatim(self.problem.localizedMessage)", + "Text(verbatim: primaryActionTitle)", + ], + "apps/ios/Sources/LiveActivity/LiveActivityManager.swift": [ + 'String(localized: "Connecting...")', + "status: .disconnected", + ], + "apps/ios/WatchApp/Sources/WatchInboxView.swift": [ + "enum WatchTextValue {\n case localized(LocalizedStringResource)", + "private struct WatchFaceHeader: View {\n let section: WatchTextValue\n let title: WatchTextValue\n let subtitle: WatchTextValue", + "private struct WatchHeroCard: View {\n let label: WatchTextValue\n let title: WatchTextValue\n let subtitle: WatchTextValue", + "private struct WatchStackCard: View {\n let label: WatchTextValue\n let title: WatchTextValue\n let subtitle: WatchTextValue", + "private struct WatchDecisionButton: View {\n let title: LocalizedStringKey", + 'format: String(localized: "Expires in %@")', + 'String(localized: "Pending review")', + 'String(localized: "Review command below")', + "title: .verbatim(record.approval.commandPreview", + ], + "apps/ios/WatchApp/Sources/WatchDirectNode.swift": [ + 'private(set) var statusText = String(\n localized: "Use iPhone Settings to enable direct connection.")', + 'format: String(localized: "Direct connection failed: %@")', + ], + "apps/ios/WatchApp/Sources/WatchInboxStore.swift": [ + "WatchAppCommandStatus(command: command, code: .sending)", + "WatchAppCommandStatus(command: command, code: .sent)", + "WatchAppCommandStatus(command: command, code: .queued)", + ], +}; +const RAW_LOCALIZATION_BYPASSES: Record = { + "apps/ios/Sources/Design/SettingsProTabSections.swift": [ + "func settingsListRow(\n icon: String,\n iconColor: Color,\n title: String", + "func aboutLinkRow(title: String", + "func gatewayActionButton(\n title: String", + "func gatewaySecureField(_ placeholder: String", + "func settingsToggle(\n _ title: String", + ".accessibilityLabel(title)", + ], + "apps/ios/Sources/Gateway/GatewayConnectionController+Capabilities.swift": [ + 'helperText: "Secure connection is required for this host."', + 'helperText: "Use only on a trusted private network."', + ], + "apps/ios/Sources/Gateway/ExecApprovalPromptDialog.swift": [ + "private struct ExecApprovalPromptMetadataRow: View {\n let label: String", + 'return "under a minute"', + 'return "about 1 minute"', + ], + "apps/ios/Sources/Voice/TalkModeManager.swift": [ + 'self.statusText = "', + 'streamingOwner.terminalStatus = "', + ], + "apps/ios/Sources/Voice/VoiceWakeManager.swift": ['self.statusText = "'], + "apps/ios/Sources/Design/SettingsProTabSupport.swift": [ + "struct SettingsDetailRow: View {\n let label: LocalizedStringKey\n let value: String", + "init(_ label: LocalizedStringKey, value: String)", + "Text(self.value)", + "Text(self.item.title)", + "Text(self.item.detail)", + "Text(self.item.priority)", + ], + "apps/ios/Sources/Design/CommandCenterSupport.swift": [ + "Text(self.item.title)", + "Text(self.item.trailing)", + "Text(self.item.detail)", + "struct CommandEmptyStateRow: View {\n let icon: String\n let title: String", + "private func actionButton(\n _ title: String", + ], + "apps/ios/Sources/Design/SettingsProTabActions.swift": [ + "func detailStatusCard(\n icon: String,\n title: String", + "func diagnosticCheckRow(\n icon: String,\n title: String", + ], + "apps/ios/WatchApp/Sources/WatchInboxView.swift": [ + 'parts.append("Expires in \\(expiresText)")', + 'return "Expires in <1m"', + 'return "Expires in \\(deltaSeconds / 60)m"', + 'return "Pending review"', + 'return "Review command below"', + 'return "Stop speaking"', + 'return "Cancel voice turn"', + 'return "Start voice turn"', + ], + "apps/ios/WatchApp/Sources/WatchDirectNode.swift": [ + 'private(set) var statusText = "', + 'self.statusText = "', + ], + "apps/ios/WatchApp/Sources/WatchInboxStore.swift": [ + 'self.appSnapshotStatusText = "', + 'self.appCommandStatusText = "', + 'self.execApprovals[index].statusText = "', + 'self.replyStatusText = "', + ], +}; + +type StringUnit = { + state?: string; + value?: string; +}; + +type CatalogLocalization = Record & { + stringUnit?: StringUnit; +}; + +type CatalogEntry = Record & { + localizations?: Record; +}; + +type Catalog = { + sourceLanguage?: string; + strings?: Record; + version?: string; +}; + +type NativeSourceEntry = { + id: string; + source: string; + sites: Array<{ kind: string; path: string }>; + surface: string; +}; + +type NativeSourceArtifact = { + entries: NativeSourceEntry[]; + version: number; +}; + +type NativeTranslationArtifact = { + locale: string; + translations: Record; + version: number; +}; + +export type AppleTranslationContradiction = { + locale: string; + source: string; + translations: string[]; +}; + +export type AppleCatalogBuild = { + catalog: Catalog; + contradictions: AppleTranslationContradiction[]; +}; + +function formatTokens(value: string): string[] { + return [...value.matchAll(FORMAT_RE)].map((match) => match[0]).toSorted(); +} + +function stringsLiteral(value: string): string { + return JSON.stringify(value); +} + +function compareCodeUnits(left: string, right: string): number { + return left < right ? -1 : left > right ? 1 : 0; +} + +export function serializeAppleCatalog(catalog: Catalog): string { + const topLevelEntries = Object.entries(catalog); + const lines = ["{"]; + + for (const [topLevelIndex, [key, value]] of topLevelEntries.entries()) { + const comma = topLevelIndex === topLevelEntries.length - 1 ? "" : ","; + if (key !== "strings" || !value || typeof value !== "object" || Array.isArray(value)) { + lines.push(` ${JSON.stringify(key)}: ${JSON.stringify(value)}${comma}`); + continue; + } + + const entries = Object.entries(value); + lines.push(` ${JSON.stringify(key)}: {`); + for (const [entryIndex, [entryKey, entry]] of entries.entries()) { + const entryComma = entryIndex === entries.length - 1 ? "" : ","; + lines.push(` ${JSON.stringify(entryKey)}: ${JSON.stringify(entry)}${entryComma}`); + } + lines.push(` }${comma}`); + } + + lines.push("}", ""); + return lines.join("\n"); +} + +function decodeXml(value: string): string { + return value + .replaceAll(""", '"') + .replaceAll("'", "'") + .replaceAll("<", "<") + .replaceAll(">", ">") + .replaceAll("&", "&"); +} + +function parseInfoPlistStrings(source: string): Array<{ key: string; source: string }> { + return [...source.matchAll(/([^<]+)<\/key>\s*([\s\S]*?)<\/string>/gu)] + .map((match) => ({ + key: decodeXml(match[1] ?? ""), + source: decodeXml(match[2] ?? ""), + })) + .filter( + (entry) => + entry.key.endsWith("UsageDescription") || INFO_PLIST_LOCALIZABLE_KEYS.has(entry.key), + ); +} + +type InfoPlistTranslation = { + source?: string; + value: string; +}; + +function parseStringsFile(source: string): Map { + const values = new Map(); + for (const match of source.matchAll( + /(?:^\/\* OpenClaw source: ("(?:\\.|[^"\\])*") \*\/\n)?^\s*("(?:\\.|[^"\\])*")\s*=\s*("(?:\\.|[^"\\])*");/gmu, + )) { + values.set(JSON.parse(match[2] ?? '""') as string, { + source: match[1] ? (JSON.parse(match[1]) as string) : undefined, + value: JSON.parse(match[3] ?? '""') as string, + }); + } + return values; +} + +export function selectInfoPlistTranslation( + source: string, + candidates: readonly string[], + existing?: InfoPlistTranslation, +): string { + const translatedCandidates = candidates.filter( + (candidate) => candidate.trim() && candidate.trim() !== source.trim(), + ); + if (translatedCandidates.length > 0) { + return chooseTranslation(source, translatedCandidates); + } + return existing?.source === source && existing.value.trim() ? existing.value : source; +} + +export function infoPlistTranslationCandidates( + artifact: NativeTranslationArtifact | undefined, + sourceId: string, + _source: string, +): string[] { + const translated = artifact?.translations[sourceId]; + return typeof translated === "string" ? [translated] : []; +} + +function infoPlistSourceIds(nativeSource: NativeSourceArtifact): Map { + return new Map( + nativeSource.entries.flatMap((entry) => + entry.sites + .filter((site) => site.kind === "plist-string") + .map((site) => [[site.path, entry.source].join("\u0000"), entry.id] as const), + ), + ); +} + +function renderInfoPlistStrings( + sourcePath: string, + sourceEntries: ReadonlyArray<{ key: string; source: string }>, + sourceIds: ReadonlyMap, + artifact: NativeTranslationArtifact | undefined, + existing: ReadonlyMap = new Map(), +): string { + const lines = sourceEntries.map(({ key, source }) => { + const sourceId = sourceIds.get([sourcePath, source].join("\u0000")); + if (!sourceId) { + throw new Error(`missing native InfoPlist source id for ${sourcePath}:${key}`); + } + const candidates = infoPlistTranslationCandidates(artifact, sourceId, source); + const value = selectInfoPlistTranslation(source, candidates, existing.get(key)); + return [ + `/* OpenClaw source: ${stringsLiteral(source)} */`, + `${stringsLiteral(key)} = ${stringsLiteral(value)};`, + ].join("\n"); + }); + return `${lines.join("\n")}\n`; +} + +async function readOptionalFile(filePath: string): Promise { + try { + return await readFile(filePath, "utf8"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + return null; + } + throw error; + } +} + +function isAppleCatalogKind(kind: string): boolean { + return kind === "conditional-branch" || kind.startsWith("ui-"); +} + +function isAppleCatalogEntry( + entry: NativeSourceEntry, + sourcePrefixes: readonly string[], + exclusions: ReadonlySet, +): boolean { + return ( + entry.surface === "apple" && + entry.sites.some( + (site) => + sourcePrefixes.some((prefix) => site.path.startsWith(prefix)) && + isAppleCatalogKind(site.kind), + ) && + (!entry.source.includes("\\(") || isInflectedCountSource(entry.source)) && + !exclusions.has(entry.source) + ); +} + +function isInflectedCountSource(value: string): boolean { + if (!value.includes(INFLECTED_COUNT_MARKER)) { + return false; + } + const interpolations = value.match(/\\\([^)]*\)/gu) ?? []; + const segments = [...value.matchAll(INFLECTED_COUNT_SEGMENT_RE)]; + return ( + segments.length > 0 && + interpolations.length === segments.length && + interpolations.every((interpolation) => + INFLECTED_COUNT_INTERPOLATION_EXACT_RE.test(interpolation), + ) && + segments.every( + ([segment]) => (segment.match(INFLECTED_COUNT_INTERPOLATION_RE) ?? []).length === 1, + ) + ); +} + +function appleCatalogValue(value: string): string { + if (!isInflectedCountSource(value)) { + return value; + } + return value.replace(INFLECTED_COUNT_SEGMENT_RE, (segment) => + segment.replace(INFLECTED_COUNT_INTERPOLATION_RE, "%lld"), + ); +} + +function chooseTranslation(source: string, translations: readonly string[]): string { + // Apple catalogs key by source, so duplicate native contexts must converge. + // Preserve shipped values first; otherwise choose deterministically and report every conflict. + const counts = new Map(); + for (const translation of translations) { + counts.set(translation, (counts.get(translation) ?? 0) + 1); + } + return ( + [...counts].toSorted(([leftValue, leftCount], [rightValue, rightCount]) => { + const sourcePenalty = Number(leftValue === source) - Number(rightValue === source); + return sourcePenalty || rightCount - leftCount || compareCodeUnits(leftValue, rightValue); + })[0]?.[0] ?? source + ); +} + +function buildAppleCatalog( + existingCatalog: Catalog, + nativeSource: NativeSourceArtifact, + translations: readonly NativeTranslationArtifact[], + includesEntry: (entry: NativeSourceEntry) => boolean, +): AppleCatalogBuild { + const catalogEntries = nativeSource.entries + .filter(includesEntry) + .map((entry) => [entry, appleCatalogValue(entry.source)] as const); + const sources = [...new Set(catalogEntries.map(([, source]) => source))].toSorted( + compareCodeUnits, + ); + const catalogIds = new Set(catalogEntries.map(([entry]) => entry.id)); + const existingStrings = existingCatalog.strings ?? {}; + const nativeEntryById = new Map(nativeSource.entries.map((entry) => [entry.id, entry])); + const translationsByLocale = new Map( + translations.map((artifact) => { + const bySource = new Map(); + for (const [id, translated] of Object.entries(artifact.translations)) { + const entry = nativeEntryById.get(id); + if (!entry || !catalogIds.has(id)) { + continue; + } + const source = appleCatalogValue(entry.source); + const values = bySource.get(source) ?? []; + values.push(appleCatalogValue(translated)); + bySource.set(source, values); + } + return [artifact.locale, bySource] as const; + }), + ); + const contradictions: AppleTranslationContradiction[] = []; + const strings: Record = {}; + + for (const source of sources) { + const existing = existingStrings[source]; + const localizations: NonNullable = {}; + for (const locale of REQUIRED_LOCALES) { + const candidates = translationsByLocale.get(locale)?.get(source) ?? []; + const distinct = [...new Set(candidates)].toSorted(compareCodeUnits); + if (distinct.length > 1) { + contradictions.push({ locale, source, translations: distinct }); + } + const existingUnit = existing?.localizations?.[locale]?.stringUnit; + if (existingUnit?.value && existingUnit.state === "translated") { + localizations[locale] = { stringUnit: { ...existingUnit } }; + continue; + } + if (locale === "en") { + localizations.en = { + stringUnit: { + state: "translated", + value: source, + }, + }; + continue; + } + const value = chooseTranslation(source, candidates); + localizations[locale] = { + stringUnit: { + state: value === source ? "new" : "translated", + value, + }, + }; + } + strings[source] = { localizations }; + } + + return { + catalog: { + sourceLanguage: "en", + strings, + version: "1.0", + }, + contradictions: contradictions.toSorted( + (left, right) => + compareCodeUnits(left.source, right.source) || compareCodeUnits(left.locale, right.locale), + ), + }; +} + +export function buildIosCatalog( + existingCatalog: Catalog, + nativeSource: NativeSourceArtifact, + translations: readonly NativeTranslationArtifact[], +): AppleCatalogBuild { + return buildAppleCatalog(existingCatalog, nativeSource, translations, (entry) => + isAppleCatalogEntry(entry, IOS_SOURCE_PREFIXES, IOS_CATALOG_EXCLUSIONS), + ); +} + +export function buildMacosCatalog( + existingCatalog: Catalog, + nativeSource: NativeSourceArtifact, + translations: readonly NativeTranslationArtifact[], +): AppleCatalogBuild { + return buildAppleCatalog(existingCatalog, nativeSource, translations, (entry) => + isAppleCatalogEntry(entry, MACOS_SOURCE_PREFIXES, MACOS_CATALOG_EXCLUSIONS), + ); +} + +async function listSwiftFiles(directory: string): Promise { + const entries = await readdir(directory, { withFileTypes: true }); + const nested = await Promise.all( + entries.map(async (entry) => { + if ( + entry.isDirectory() && + /^(?:\.build|build|DerivedData|Previews?|Tests?|UITests?)$/u.test(entry.name) + ) { + return []; + } + const entryPath = path.join(directory, entry.name); + if (entry.isDirectory()) { + return listSwiftFiles(entryPath); + } + return entry.isFile() && entry.name.endsWith(".swift") ? [entryPath] : []; + }), + ); + return nested.flat(); +} + +async function validateRuntimeInterpolationPaths(): Promise { + const roots = [...new Set([...IOS_SOURCE_PREFIXES, ...MACOS_SOURCE_PREFIXES])]; + const files = new Set( + ( + await Promise.all( + roots.map((prefix) => { + const sourcePath = path.join(ROOT, prefix); + return prefix.endsWith(".swift") + ? Promise.resolve([sourcePath]) + : listSwiftFiles(sourcePath); + }), + ) + ).flat(), + ); + const violations: string[] = []; + for (const file of [...files].toSorted()) { + const source = await readFile(file, "utf8"); + for (const label of findAmbiguousRuntimeInterpolations(source)) { + violations.push(`${path.relative(ROOT, file)}: ${label}`); + } + } + if (violations.length) { + throw new Error( + `Apple i18n runtime interpolation bypasses generated catalog coverage:\n${violations.join("\n")}`, + ); + } +} + +async function readNativeTranslations(): Promise { + const expectedFiles = NATIVE_I18N_LOCALES.map((locale) => `${locale}.json`).toSorted(); + const actualFiles = ( + await readdir(path.join(ROOT, NATIVE_TRANSLATIONS_DIR), { + withFileTypes: true, + }) + ) + .filter((entry) => entry.isFile() && entry.name.endsWith(".json")) + .map((entry) => entry.name) + .toSorted(); + if (JSON.stringify(actualFiles) !== JSON.stringify(expectedFiles)) { + throw new Error( + `Apple/native locale parity drift: expected ${JSON.stringify(expectedFiles)}, got ${JSON.stringify(actualFiles)}`, + ); + } + return Promise.all( + NATIVE_I18N_LOCALES.map(async (locale) => { + const artifact = JSON.parse( + await readFile(path.join(ROOT, NATIVE_TRANSLATIONS_DIR, `${locale}.json`), "utf8"), + ) as NativeTranslationArtifact; + if (artifact.locale !== locale) { + throw new Error( + `native Apple translation artifact ${locale} declares locale ${artifact.locale}`, + ); + } + return artifact; + }), + ); +} + +async function readAppleCatalogBuild( + catalogPath: string, + buildCatalog: typeof buildIosCatalog, +): Promise { + const existingCatalog = JSON.parse( + await readFile(path.join(ROOT, catalogPath), "utf8"), + ) as Catalog; + const nativeSource = JSON.parse( + await readFile(path.join(ROOT, NATIVE_SOURCE_PATH), "utf8"), + ) as NativeSourceArtifact; + const translations = await readNativeTranslations(); + return buildCatalog(existingCatalog, nativeSource, translations); +} + +function validateCatalog(pathName: string, catalog: Catalog): number { + if (catalog.sourceLanguage !== "en" || catalog.version !== "1.0" || !catalog.strings) { + throw new Error(`invalid Apple string catalog: ${pathName}`); + } + let checked = 0; + for (const [key, entry] of Object.entries(catalog.strings)) { + const sourceTokens = formatTokens(key); + for (const locale of REQUIRED_LOCALES) { + const unit = entry.localizations?.[locale]?.stringUnit; + const value = unit?.value?.trim(); + if (!value || (locale === "en" && unit?.state !== "translated")) { + throw new Error( + `Apple catalog ${pathName} is missing ${locale} for ${JSON.stringify(key)}`, + ); + } + if (locale !== "en" && unit?.state !== "translated" && unit?.state !== "new") { + throw new Error( + `Apple catalog ${pathName} has untranslated ${locale} copy for ${JSON.stringify(key)}`, + ); + } + if (formatTokens(value).join("\u0000") !== sourceTokens.join("\u0000")) { + throw new Error( + `Apple catalog ${pathName} has placeholder drift in ${locale} for ${JSON.stringify(key)}`, + ); + } + } + checked += 1; + } + return checked; +} + +async function syncIosInfoPlist(write: boolean): Promise { + const translations = await readNativeTranslations(); + const nativeSource = JSON.parse( + await readFile(path.join(ROOT, NATIVE_SOURCE_PATH), "utf8"), + ) as NativeSourceArtifact; + const sourceIds = infoPlistSourceIds(nativeSource); + let checked = 0; + for (const target of IOS_INFO_PLIST_TARGETS) { + const sourceEntries = parseInfoPlistStrings( + await readFile(path.join(ROOT, target.sourcePath), "utf8"), + ); + for (const locale of NATIVE_I18N_LOCALES) { + const localeDir = APPLE_LOCALE_DIRECTORIES[locale] ?? locale; + const outputPath = path.join( + ROOT, + target.outputRoot, + `${localeDir}.lproj`, + "InfoPlist.strings", + ); + const existingSource = await readOptionalFile(outputPath); + const existing = parseStringsFile(existingSource ?? ""); + const artifact = translations.find((candidate) => candidate.locale === locale); + const expected = renderInfoPlistStrings( + target.sourcePath, + sourceEntries, + sourceIds, + artifact, + existing, + ); + if (existingSource !== expected) { + if (!write) { + throw new Error( + `Apple InfoPlist localization ${path.relative(ROOT, outputPath)} is stale; run apple-app-i18n.ts sync-ios --write`, + ); + } + await mkdir(path.dirname(outputPath), { recursive: true }); + await writeFile(outputPath, expected, "utf8"); + } + checked += 1; + } + } + return checked; +} + +export async function syncIosCatalog(write: boolean): Promise { + const build = await readAppleCatalogBuild(IOS_CATALOG_PATH, buildIosCatalog); + const catalogPath = path.join(ROOT, IOS_CATALOG_PATH); + const expected = serializeAppleCatalog(build.catalog); + const actual = await readFile(catalogPath, "utf8"); + if (actual !== expected) { + if (!write) { + throw new Error( + `Apple catalog ${IOS_CATALOG_PATH} is stale; run apple-app-i18n.ts sync-ios --write`, + ); + } + await writeFile(catalogPath, expected, "utf8"); + } + return build; +} + +export async function syncMacosCatalog(write: boolean): Promise { + const build = await readAppleCatalogBuild(MACOS_CATALOG_PATH, buildMacosCatalog); + const catalogPath = path.join(ROOT, MACOS_CATALOG_PATH); + const expected = serializeAppleCatalog(build.catalog); + const actual = await readFile(catalogPath, "utf8"); + if (actual !== expected) { + if (!write) { + assertMacosCatalogCurrent(actual, build); + return build; + } + await writeFile(catalogPath, expected, "utf8"); + } + return build; +} + +export function assertMacosCatalogCurrent(actual: string, build: AppleCatalogBuild): void { + if (actual !== serializeAppleCatalog(build.catalog)) { + throw new Error( + `Apple catalog ${MACOS_CATALOG_PATH} is stale; run native-app-i18n.ts sync --write`, + ); + } +} + +/** + * Regenerates every Apple derived artifact (app catalogs and InfoPlist strings). + * Shared by this CLI and native-app-i18n's sync so the inventory can never be + * rewritten without its derived catalogs. + */ +export async function syncAppleAppI18n(): Promise<{ + build: AppleCatalogBuild; + infoPlistFiles: number; + macosBuild: AppleCatalogBuild; +}> { + const [build, macosBuild] = await Promise.all([syncIosCatalog(true), syncMacosCatalog(true)]); + const infoPlistFiles = await syncIosInfoPlist(true); + return { build, infoPlistFiles, macosBuild }; +} + +export async function verifyAppleAppI18n() { + await validateRuntimeInterpolationPaths(); + for (const [sourcePath, contracts] of Object.entries(LOCALIZED_WRAPPER_CONTRACTS)) { + const source = await readFile(path.join(ROOT, sourcePath), "utf8"); + const missing = contracts.filter((contract) => !source.includes(contract)); + if (missing.length) { + throw new Error( + `Apple i18n wrapper ${sourcePath} bypasses localized string lookup: ${missing.join(", ")}`, + ); + } + } + for (const [sourcePath, bypasses] of Object.entries(RAW_LOCALIZATION_BYPASSES)) { + const source = await readFile(path.join(ROOT, sourcePath), "utf8"); + const present = bypasses.filter((bypass) => source.includes(bypass)); + if (present.length) { + throw new Error( + `Apple i18n runtime path ${sourcePath} contains raw app-owned strings: ${present.join(", ")}`, + ); + } + } + + const macosBuild = await readAppleCatalogBuild(MACOS_CATALOG_PATH, buildMacosCatalog); + const macosKeys = validateCatalog(MACOS_CATALOG_PATH, macosBuild.catalog); + + process.stdout.write(`apple-app-i18n: sourceMacosKeys=${macosKeys}\n`); +} + +export async function checkAppleAppI18n() { + await verifyAppleAppI18n(); + const [iosBuild, macosBuild] = await Promise.all([ + syncIosCatalog(false), + syncMacosCatalog(false), + ]); + const iosKeys = validateCatalog(IOS_CATALOG_PATH, iosBuild.catalog); + const macosKeys = validateCatalog(MACOS_CATALOG_PATH, macosBuild.catalog); + const infoPlistFiles = await syncIosInfoPlist(false); + + process.stdout.write( + [ + `apple-app-i18n: iosKeys=${iosKeys}`, + `macosKeys=${macosKeys}`, + `infoPlistFiles=${infoPlistFiles}`, + `translationContradictions=${iosBuild.contradictions.length}`, + `macosTranslationContradictions=${macosBuild.contradictions.length}`, + `locales=${NATIVE_I18N_LOCALES.join(",")}`, + "\n", + ].join(" "), + ); +} + +export async function compileMacosLocalizations(outputDir: string) { + // Source PRs intentionally leave generated Apple catalogs for the serialized + // post-merge refresh. Package from the derived catalog so source changes + // cannot ship stale localization coverage before that refresh lands. + await verifyAppleAppI18n(); + const catalog = (await readAppleCatalogBuild(MACOS_CATALOG_PATH, buildMacosCatalog)).catalog; + if (!catalog.strings) { + throw new Error(`invalid Apple string catalog: ${MACOS_CATALOG_PATH}`); + } + const [nativeSource, translations, infoPlistSource] = await Promise.all([ + readFile(path.join(ROOT, NATIVE_SOURCE_PATH), "utf8").then( + (source) => JSON.parse(source) as NativeSourceArtifact, + ), + readNativeTranslations(), + readFile(path.join(ROOT, MACOS_INFO_PLIST_PATH), "utf8"), + ]); + const sourceIds = infoPlistSourceIds(nativeSource); + const infoPlistEntries = parseInfoPlistStrings(infoPlistSource); + + for (const locale of REQUIRED_LOCALES) { + const localeDir = APPLE_LOCALE_DIRECTORIES[locale] ?? locale; + const lprojDir = path.join(outputDir, `${localeDir}.lproj`); + const lines = Object.entries(catalog.strings) + .toSorted(([left], [right]) => compareCodeUnits(left, right)) + .map(([key, entry]) => { + const value = entry.localizations?.[locale]?.stringUnit?.value; + if (!value) { + throw new Error( + `Apple catalog ${MACOS_CATALOG_PATH} is missing ${locale} for ${JSON.stringify(key)}`, + ); + } + return `${stringsLiteral(key)} = ${stringsLiteral(value)};`; + }); + await mkdir(lprojDir, { recursive: true }); + await writeFile(path.join(lprojDir, "Localizable.strings"), `${lines.join("\n")}\n`, "utf8"); + if (locale !== "en") { + const artifact = translations.find((candidate) => candidate.locale === locale); + const infoPlistStrings = renderInfoPlistStrings( + MACOS_INFO_PLIST_PATH, + infoPlistEntries, + sourceIds, + artifact, + ); + await writeFile(path.join(lprojDir, "InfoPlist.strings"), infoPlistStrings, "utf8"); + } + } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) { + const [command, flag, value] = process.argv.slice(2); + if (command === "check") { + await checkAppleAppI18n(); + } else if (command === "sync-ios" && flag === "--write") { + const { build, infoPlistFiles, macosBuild } = await syncAppleAppI18n(); + process.stdout.write( + `apple-app-i18n: synced Apple catalogs and ${infoPlistFiles} InfoPlist files; contradictions=${build.contradictions.length + macosBuild.contradictions.length}\n`, + ); + } else if (command === "compile-macos" && flag === "--output" && value) { + await compileMacosLocalizations(path.resolve(value)); + } else { + throw new Error( + "usage: node --import tsx scripts/apple-app-i18n.ts check|sync-ios --write|compile-macos --output ", + ); + } +} diff --git a/scripts/audit-seams.mts b/scripts/audit-seams.mts new file mode 100644 index 0000000000000000000000000000000000000000..2e19048699f09c867d35ae7d60d11ee382bed335 --- /dev/null +++ b/scripts/audit-seams.mts @@ -0,0 +1,1049 @@ +#!/usr/bin/env node +// Audits repo ownership seams, optional plugin leaks, and nearby test coverage signals. + +import { promises as fs } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import ts from "typescript"; +import { + BUNDLED_PLUGIN_PATH_PREFIX, + BUNDLED_PLUGIN_ROOT_DIR, +} from "./lib/bundled-plugin-paths.mjs"; +import { visitModuleSpecifiers } from "./lib/guard-inventory-utils.mjs"; +import { optionalBundledClusterSet } from "./lib/optional-bundled-clusters.mjs"; +import { escapeRegExp } from "./lib/regexp.mjs"; +import { resolveRepoRoot } from "./lib/repo-root.mjs"; +import { toLine } from "./lib/ts-guard-utils.mts"; + +type ImportEntry = { + family: string; + file: string; + kind: string; + line: number; + resolvedPath: string; + specifier: string; +}; +type OptionalClusterImportEntry = Omit & { cluster: string }; +type ModuleSpecifierVisit = { + kind: string; + specifierNode: ts.Node; + specifier: string; +}; +const MATCH_QUALITY_RANK = { + "exact-stem": 0, + "path-nearby": 1, + "direct-import": 2, + "dir-token": 3, +} as const satisfies Record; +type MatchQuality = keyof typeof MATCH_QUALITY_RANK; +type RelatedTestMatch = { file: string; matchQuality: MatchQuality }; +type PackageJson = { + dependencies?: Record; + optionalDependencies?: Record; + devDependencies?: Record; + name?: string; + private?: boolean; + openclaw?: { install?: { npmSpec?: string } }; +}; + +const repoRoot = resolveRepoRoot(import.meta.url); +const srcRoot = path.join(repoRoot, "src"); +const extensionsRoot = path.join(repoRoot, BUNDLED_PLUGIN_ROOT_DIR); +const testRoot = path.join(repoRoot, "test"); +const workspacePackagePaths = ["ui/package.json"]; +const MAX_SCAN_BYTES = 2 * 1024 * 1024; +const compareStrings = (left: string, right: string) => left.localeCompare(right); +const HELP_TEXT = `Usage: node --import tsx scripts/audit-seams.mts [--help] + +Audit repo seam inventory and emit JSON to stdout. + +Sections: + duplicatedSeamFamilies Plugin SDK seam families imported from multiple production files + overlapFiles Production files that touch multiple seam families + optionalClusterStaticLeaks Optional extension/plugin clusters referenced from the static graph + missingPackages Workspace packages whose deps are not mirrored at the root + seamTestInventory High-signal seam candidates with nearby-test gap signals, + including cron orchestration seams for agent handoff, + outbound/media delivery, heartbeat/followup handoff, + and scheduler state crossings, plus subagent seams + for spawn/session handoff, announce delivery, + lifecycle registry, cleanup, and parent streaming + +Notes: + - Output is JSON only. + - For clean redirected JSON through package scripts, prefer: + pnpm --silent audit:seams > seam-inventory.json +`; + +async function collectWorkspacePackagePaths() { + const entries = await fs.readdir(extensionsRoot, { withFileTypes: true }); + for (const entry of entries) { + if (entry.isDirectory()) { + workspacePackagePaths.push(path.join("extensions", entry.name, "package.json")); + } + } +} + +function normalizePath(filePath: string) { + return path.relative(repoRoot, filePath).split(path.sep).join("/"); +} + +async function readScannableText(filePath: string, maxBytes = MAX_SCAN_BYTES) { + const stat = await fs.stat(filePath); + if (stat.size <= maxBytes) { + return fs.readFile(filePath, "utf8"); + } + const handle = await fs.open(filePath, "r"); + try { + const buffer = Buffer.alloc(maxBytes); + const { bytesRead } = await handle.read(buffer, 0, maxBytes, 0); + return buffer.subarray(0, bytesRead).toString("utf8"); + } finally { + await handle.close(); + } +} + +function redactNpmSpec(npmSpec: unknown) { + if (typeof npmSpec !== "string") { + return npmSpec ?? null; + } + return npmSpec + .replace(/(https?:\/\/)([^/\s:@]+):([^/\s@]+)@/gi, "$1***:***@") + .replace(/(https?:\/\/)([^/\s:@]+)@/gi, "$1***@"); +} + +function isCodeFile(fileName: string) { + return /\.(ts|tsx|mts|cts|js|jsx|mjs|cjs)$/.test(fileName); +} + +function isTestLikePath(relativePath: string) { + return ( + /(^|\/)(__tests__|fixtures|test-utils|test-fixtures)\//.test(relativePath) || + /(?:^|\/)[^/]*(?:[.-](?:test|spec))(?:[.-][^/]+)?\.(ts|tsx|mts|cts|js|jsx|mjs|cjs)$/.test( + relativePath, + ) + ); +} + +function isProductionLikeFile(relativePath: string) { + return !isTestLikePath(relativePath); +} + +async function walkCodeFiles(rootDir: string) { + const out: string[] = []; + async function walk(dir: string): Promise { + const entries = await fs.readdir(dir, { withFileTypes: true }); + for (const entry of entries) { + if (entry.name === "dist" || entry.name === "node_modules") { + continue; + } + const fullPath = path.join(dir, entry.name); + if (entry.isDirectory()) { + await walk(fullPath); + continue; + } + if (!entry.isFile() || !isCodeFile(entry.name)) { + continue; + } + const relativePath = normalizePath(fullPath); + if (!isProductionLikeFile(relativePath)) { + continue; + } + out.push(fullPath); + } + } + await walk(rootDir); + return out.toSorted((left, right) => normalizePath(left).localeCompare(normalizePath(right))); +} + +async function walkAllCodeFiles(rootDir: string, options: { includeTests?: boolean } = {}) { + const out: string[] = []; + const includeTests = options.includeTests === true; + + async function walk(dir: string): Promise { + let entries; + try { + entries = await fs.readdir(dir, { withFileTypes: true }); + } catch { + return; + } + for (const entry of entries) { + if (entry.name === "dist" || entry.name === "node_modules") { + continue; + } + const fullPath = path.join(dir, entry.name); + if (entry.isDirectory()) { + await walk(fullPath); + continue; + } + if (!entry.isFile() || !isCodeFile(entry.name)) { + continue; + } + const relativePath = normalizePath(fullPath); + if (!includeTests && !isProductionLikeFile(relativePath)) { + continue; + } + out.push(fullPath); + } + } + + await walk(rootDir); + return out.toSorted((left, right) => normalizePath(left).localeCompare(normalizePath(right))); +} + +function resolveRelativeSpecifier(specifier: string, importerFile: string) { + if (!specifier.startsWith(".")) { + return null; + } + return normalizePath(path.resolve(path.dirname(importerFile), specifier)); +} + +function normalizePluginSdkFamily(resolvedPath: string) { + const relative = resolvedPath.replace(/^src\/plugin-sdk\//, ""); + return relative.replace(/\.(m|c)?[jt]sx?$/, ""); +} + +function resolveOptionalClusterFromPath(resolvedPath: string) { + if (resolvedPath.startsWith(BUNDLED_PLUGIN_PATH_PREFIX)) { + const [, cluster] = resolvedPath.split("/"); + return cluster && optionalBundledClusterSet.has(cluster) ? cluster : null; + } + if (resolvedPath.startsWith("src/plugin-sdk/")) { + const [cluster] = normalizePluginSdkFamily(resolvedPath).split("/"); + return cluster && optionalBundledClusterSet.has(cluster) ? cluster : null; + } + return null; +} + +function compareImports(left: ImportEntry, right: ImportEntry) { + return ( + left.family.localeCompare(right.family) || + left.file.localeCompare(right.file) || + left.line - right.line || + left.kind.localeCompare(right.kind) || + left.specifier.localeCompare(right.specifier) + ); +} + +function collectPluginSdkImports(filePath: string, sourceFile: ts.SourceFile): ImportEntry[] { + const entries: ImportEntry[] = []; + + function push(kind: string, specifierNode: ts.Node, specifier: string) { + const resolvedPath = resolveRelativeSpecifier(specifier, filePath); + if (!resolvedPath?.startsWith("src/plugin-sdk/")) { + return; + } + entries.push({ + family: normalizePluginSdkFamily(resolvedPath), + file: normalizePath(filePath), + kind, + line: toLine(sourceFile, specifierNode), + resolvedPath, + specifier, + }); + } + + const visit = ({ kind, specifierNode, specifier }: ModuleSpecifierVisit) => + push(kind, specifierNode, specifier); + visitModuleSpecifiers(ts, sourceFile, visit); + return entries; +} + +async function collectCorePluginSdkImports() { + const files = await walkCodeFiles(srcRoot); + const inventory: ImportEntry[] = []; + for (const filePath of files) { + if (normalizePath(filePath).startsWith("src/plugin-sdk/")) { + continue; + } + const source = await fs.readFile(filePath, "utf8"); + const sourceFile = ts.createSourceFile(filePath, source, ts.ScriptTarget.Latest, true); + inventory.push(...collectPluginSdkImports(filePath, sourceFile)); + } + return inventory.toSorted(compareImports); +} + +function collectOptionalClusterStaticImports( + filePath: string, + sourceFile: ts.SourceFile, +): OptionalClusterImportEntry[] { + const entries: OptionalClusterImportEntry[] = []; + + function push(kind: string, specifierNode: ts.Node, specifier: string) { + if (!specifier.startsWith(".")) { + return; + } + const resolvedPath = resolveRelativeSpecifier(specifier, filePath); + if (!resolvedPath) { + return; + } + const cluster = resolveOptionalClusterFromPath(resolvedPath); + if (!cluster) { + return; + } + entries.push({ + cluster, + file: normalizePath(filePath), + kind, + line: toLine(sourceFile, specifierNode), + resolvedPath, + specifier, + }); + } + + const visit = ({ kind, specifierNode, specifier }: ModuleSpecifierVisit) => { + if (kind !== "dynamic-import") { + push(kind, specifierNode, specifier); + } + }; + visitModuleSpecifiers(ts, sourceFile, visit); + return entries; +} + +async function collectOptionalClusterStaticLeaks() { + const files = await walkCodeFiles(srcRoot); + const inventory: OptionalClusterImportEntry[] = []; + for (const filePath of files) { + const relativePath = normalizePath(filePath); + if (relativePath.startsWith("src/plugin-sdk/")) { + continue; + } + const source = await fs.readFile(filePath, "utf8"); + const sourceFile = ts.createSourceFile(filePath, source, ts.ScriptTarget.Latest, true); + inventory.push(...collectOptionalClusterStaticImports(filePath, sourceFile)); + } + return inventory.toSorted((left, right) => { + return ( + left.cluster.localeCompare(right.cluster) || + left.file.localeCompare(right.file) || + left.line - right.line || + left.kind.localeCompare(right.kind) || + left.specifier.localeCompare(right.specifier) + ); + }); +} + +function buildDuplicatedSeamFamilies(inventory: ImportEntry[]) { + const grouped = new Map(); + for (const entry of inventory) { + const bucket = grouped.get(entry.family) ?? []; + bucket.push(entry); + grouped.set(entry.family, bucket); + } + + const duplicated = Object.fromEntries( + [...grouped.entries()] + .map(([family, entries]) => { + const files = [...new Set(entries.map((entry) => entry.file))].toSorted(compareStrings); + return [ + family, + { + count: files.length, + importCount: entries.length, + files, + imports: entries, + }, + ] as const; + }) + .filter(([, value]) => value.files.length > 1) + .toSorted((left, right) => { + return ( + right[1].count - left[1].count || + right[1].importCount - left[1].importCount || + left[0].localeCompare(right[0]) + ); + }), + ); + + return duplicated; +} + +function buildOverlapFiles(inventory: ImportEntry[]) { + const byFile = new Map(); + for (const entry of inventory) { + const bucket = byFile.get(entry.file) ?? []; + bucket.push(entry); + byFile.set(entry.file, bucket); + } + + return [...byFile.entries()] + .map(([file, entries]) => { + const families = [...new Set(entries.map((entry) => entry.family))].toSorted(compareStrings); + return { + file, + families, + imports: entries, + }; + }) + .filter((entry) => entry.families.length > 1) + .toSorted((left, right) => { + return ( + right.families.length - left.families.length || + right.imports.length - left.imports.length || + left.file.localeCompare(right.file) + ); + }); +} + +function buildOptionalClusterStaticLeaks(inventory: OptionalClusterImportEntry[]) { + const grouped = new Map(); + for (const entry of inventory) { + const bucket = grouped.get(entry.cluster) ?? []; + bucket.push(entry); + grouped.set(entry.cluster, bucket); + } + + return Object.fromEntries( + [...grouped.entries()] + .map( + ([cluster, entries]) => + [ + cluster, + { + count: entries.length, + files: [...new Set(entries.map((entry) => entry.file))].toSorted(compareStrings), + imports: entries, + }, + ] as const, + ) + .toSorted((left, right) => { + return right[1].count - left[1].count || left[0].localeCompare(right[0]); + }), + ); +} + +function packageClusterMeta(relativePackagePath: string) { + if (relativePackagePath === "ui/package.json") { + return { + cluster: "ui", + packageName: "openclaw-control-ui", + packagePath: relativePackagePath, + reachability: "workspace-ui", + }; + } + const cluster = path.basename(path.dirname(relativePackagePath)); + return { + cluster, + packageName: null, + packagePath: relativePackagePath, + reachability: relativePackagePath.startsWith(BUNDLED_PLUGIN_PATH_PREFIX) + ? "extension-workspace" + : "workspace", + }; +} + +function classifyMissingPackageCluster(params: { + cluster: string; + pluginSdkEntries: string[]; + hasStaticLeak: boolean; +}) { + if (params.hasStaticLeak) { + return { + decision: "required", + reason: + "Cluster already appears in the static graph in this audit run, so treating it as optional would be misleading.", + }; + } + if (optionalBundledClusterSet.has(params.cluster)) { + if (params.cluster === "ui") { + return { + decision: "optional", + reason: + "Private UI workspace. Repo-wide CLI/plugin CI should not require UI-only packages.", + }; + } + if (params.pluginSdkEntries.length > 0) { + return { + decision: "optional", + reason: + "Public plugin-sdk entry exists, but repo-wide default check/build should isolate this optional cluster from the static graph.", + }; + } + return { + decision: "optional", + reason: + "Workspace package is intentionally not mirrored into the root dependency set by default CI policy.", + }; + } + return { + decision: "required", + reason: + "Cluster is statically visible to repo-wide check/build and has not been classified optional.", + }; +} + +async function buildMissingPackages(params: { staticLeakClusters?: Set } = {}) { + const rootPackage: PackageJson = JSON.parse( + await fs.readFile(path.join(repoRoot, "package.json"), "utf8"), + ); + const rootDeps = new Set([ + ...Object.keys(rootPackage.dependencies ?? {}), + ...Object.keys(rootPackage.optionalDependencies ?? {}), + ...Object.keys(rootPackage.devDependencies ?? {}), + ]); + + const pluginSdkEntrySources = await walkCodeFiles(path.join(repoRoot, "src", "plugin-sdk")); + const pluginSdkReachability = new Map>(); + for (const filePath of pluginSdkEntrySources) { + const source = await fs.readFile(filePath, "utf8"); + const matches = [...source.matchAll(/from\s+"(\.\.\/\.\.\/extensions\/([^/]+)\/[^"]+)"/g)]; + for (const match of matches) { + const cluster = match[2]; + if (!cluster) { + continue; + } + const bucket = pluginSdkReachability.get(cluster) ?? new Set(); + bucket.add(normalizePath(filePath)); + pluginSdkReachability.set(cluster, bucket); + } + } + + const output = []; + for (const relativePackagePath of workspacePackagePaths.toSorted(compareStrings)) { + const packagePath = path.join(repoRoot, relativePackagePath); + let pkg: PackageJson; + try { + pkg = JSON.parse(await fs.readFile(packagePath, "utf8")); + } catch { + continue; + } + const missing = Object.keys(pkg.dependencies ?? {}) + .filter((dep) => dep !== "openclaw" && !rootDeps.has(dep)) + .toSorted(compareStrings); + if (missing.length === 0) { + continue; + } + const meta = packageClusterMeta(relativePackagePath); + const pluginSdkEntries = [...(pluginSdkReachability.get(meta.cluster) ?? new Set())].toSorted( + compareStrings, + ); + const classification = classifyMissingPackageCluster({ + cluster: meta.cluster, + pluginSdkEntries, + hasStaticLeak: params.staticLeakClusters?.has(meta.cluster) === true, + }); + output.push({ + cluster: meta.cluster, + decision: classification.decision, + decisionReason: classification.reason, + packageName: pkg.name ?? meta.packageName, + packagePath: relativePackagePath, + npmSpec: redactNpmSpec(pkg.openclaw?.install?.npmSpec), + private: pkg.private === true, + pluginSdkReachability: + pluginSdkEntries.length > 0 ? { staticEntryPoints: pluginSdkEntries } : undefined, + missing, + }); + } + + return output.toSorted((left, right) => { + return right.missing.length - left.missing.length || left.cluster.localeCompare(right.cluster); + }); +} + +function stemFromRelativePath(relativePath: string) { + return relativePath.replace(/\.(m|c)?[jt]sx?$/, ""); +} + +function splitNameTokens(name: string) { + return name + .split(/[^a-zA-Z0-9]+/) + .map((token) => token.trim().toLowerCase()) + .filter(Boolean); +} + +function hasImportSource(source: string, specifier: string, matchSuffix = false) { + const escaped = escapeRegExp(specifier); + const importSource = matchSuffix ? `(?:[^"']*/)?${escaped}` : escaped; + return new RegExp( + `from\\s+["']${importSource}["']|import\\s*\\(\\s*["']${importSource}["']\\s*\\)`, + ).test(source); +} + +function hasAnyImportSource(source: string, specifiers: string[]) { + return specifiers.some((specifier) => hasImportSource(source, specifier)); +} + +function isCronProductionPath(relativePath: string) { + return relativePath.startsWith("src/cron/") && isProductionLikeFile(relativePath); +} + +function isSubagentProductionPath(relativePath: string) { + return ( + (relativePath.startsWith("src/agents/") || relativePath.startsWith("src/cron/")) && + isProductionLikeFile(relativePath) && + (/subagent|sessions-spawn|acp-spawn/.test(relativePath) || + relativePath === "src/agents/tools/sessions-spawn-tool.ts" || + relativePath === "src/agents/tools/subagents-tool.ts") + ); +} + +function describeCronSeamKinds(relativePath: string, source: string) { + if (!isCronProductionPath(relativePath)) { + return []; + } + + const seamKinds = []; + const importsAgentRunner = hasAnyImportSource(source, [ + "../../agents/cli-runner.js", + "../../agents/embedded-agent.js", + "../../agents/model-fallback.js", + "../../agents/subagent-registry.js", + "../../infra/agent-events.js", + ]); + const importsOutboundDelivery = hasAnyImportSource(source, [ + "../infra/outbound/deliver.js", + "../../infra/outbound/deliver.js", + "../infra/outbound/session-context.js", + "../../infra/outbound/session-context.js", + "../infra/outbound/identity.js", + "../../infra/outbound/identity.js", + "../cli/outbound-send-deps.js", + "../../cli/outbound-send-deps.js", + ]); + const importsHeartbeat = hasAnyImportSource(source, [ + "../auto-reply/heartbeat.js", + "../../auto-reply/heartbeat.js", + "../infra/heartbeat-wake.js", + "../../infra/heartbeat-wake.js", + ]); + const importsFollowup = hasAnyImportSource(source, [ + "./subagent-followup.js", + "../../agents/subagent-registry.js", + "../../agents/tools/agent-step.js", + "../../gateway/call.js", + ]); + const importsSchedulerModules = + relativePath.startsWith("src/cron/service/") && + hasAnyImportSource(source, [ + "./jobs.js", + "./store.js", + "./timer.js", + "./state.js", + "../schedule.js", + "../store.js", + ]); + + if ( + importsAgentRunner && + /\brunCliAgent\b|\brunEmbeddedAgent\b|\brunWithModelFallback\b|\bregisterAgentRunContext\b/.test( + source, + ) + ) { + seamKinds.push("cron-agent-handoff"); + } + + if ( + importsOutboundDelivery && + /\bdeliverOutboundPayloads\b|\bbuildOutboundSessionContext\b|\bresolveAgentOutboundIdentity\b/.test( + source, + ) + ) { + seamKinds.push("cron-outbound-delivery"); + } + + if ( + importsHeartbeat && + /\bstripHeartbeatToken\b|\bHeartbeat\b|\bheartbeat\b|\bnext-heartbeat\b/.test(source) + ) { + seamKinds.push("cron-heartbeat-handoff"); + } + + if ( + importsSchedulerModules && + /\bensureLoaded\b|\bpersist\b|\barmTimer\b|\brunMissedJobs\b|\bcomputeJobNextRunAtMs\b|\brecomputeNextRuns\b|\bnextWakeAtMs\b/.test( + source, + ) + ) { + seamKinds.push("cron-scheduler-state"); + } + + if ( + importsOutboundDelivery && + /\bmediaUrl\b|\bmediaUrls\b|\bfilename\b|\baudioAsVoice\b|\bdeliveryPayloads\b|\bdeliveryPayloadHasStructuredContent\b/.test( + source, + ) + ) { + seamKinds.push("cron-media-delivery"); + } + + if ( + importsFollowup && + /\bwaitForDescendantSubagentSummary\b|\breadDescendantSubagentFallbackReply\b|\bexpectsSubagentFollowup\b|\bcallGateway\b|\blistDescendantRunsForRequester\b/.test( + source, + ) + ) { + seamKinds.push("cron-followup-handoff"); + } + + return seamKinds; +} + +function describeSubagentSeamKinds(relativePath: string, source: string) { + if (!isSubagentProductionPath(relativePath)) { + return []; + } + + const seamKinds = []; + const isAnnounceDispatchPath = + relativePath === "src/agents/subagents/announce/subagent-announce.ts" || + relativePath === "src/agents/subagents/announce/subagent-announce-dispatch.ts"; + const importsSpawnRuntime = hasAnyImportSource(source, [ + "./subagent-spawn.js", + "./acp-spawn.js", + "./subagents/spawn/subagent-spawn.js", + "../subagents/spawn/subagent-spawn.js", + "./subagents/spawn/acp-spawn.js", + "../subagents/spawn/acp-spawn.js", + "./subagent-registry.js", + "../subagent-registry.js", + "../../subagent-registry.js", + "../acp/control-plane/manager.js", + "../../../acp/control-plane/manager.js", + ]); + const importsLifecycleRegistry = + hasImportSource(source, "subagent-registry.js", true) || + hasAnyImportSource(source, [ + "./subagent-registry-completion.js", + "./subagent-registry-cleanup.js", + "./subagent-registry-state.js", + "./subagent-lifecycle-events.js", + "../context-engine/init.js", + "../context-engine/registry.js", + "../sessions/session-lifecycle-events.js", + "../../../context-engine/init.js", + "../../../context-engine/registry.js", + "../../../sessions/session-lifecycle-events.js", + ]); + const importsAnnounceDelivery = hasAnyImportSource(source, [ + "./subagent-announce.js", + "./subagent-announce-dispatch.js", + "../announce/subagent-announce.js", + "../announce/subagent-announce-dispatch.js", + "./subagent-announce-queue.js", + "../infra/outbound/bound-delivery-router.js", + "../utils/delivery-context.shared.js", + "../gateway/call.js", + "../../../infra/outbound/bound-delivery-router.js", + "../../../utils/delivery-context.shared.js", + "../../../gateway/call.js", + ]); + const importsCleanup = hasAnyImportSource(source, [ + "../gateway/call.js", + "./subagent-registry-cleanup.js", + "../acp/control-plane/spawn.js", + "../../../gateway/call.js", + "../../../acp/control-plane/spawn.js", + ]); + const importsParentStream = hasAnyImportSource(source, [ + "./acp-spawn-parent-stream.js", + "../infra/heartbeat-wake.js", + "../infra/system-events.js", + "../infra/agent-events.js", + "../../../infra/heartbeat-wake.js", + "../../../infra/system-events.js", + "../../../infra/agent-events.js", + ]); + + if ( + importsSpawnRuntime && + /\bspawnSubagentDirect\b|\bspawnAcpDirect\b|\bregisterSubagentRun\b|\bgetAcpSessionManager\b|\bspawnSubagent\b|\bspawnAcp\b/.test( + source, + ) + ) { + seamKinds.push("subagent-session-spawn"); + } + + if ( + importsLifecycleRegistry && + /\bemitSubagentEndedHookOnce\b|\bresolveDeferredCleanupDecision\b|\bpersistSubagentRunsToDisk\b|\brestoreSubagentRunsFromDisk\b|\bresolveContextEngine\b|\bemitSessionLifecycleEvent\b|\bcaptureSubagentCompletionReply\b/.test( + source, + ) + ) { + seamKinds.push("subagent-lifecycle-registry"); + } + + if ( + (importsAnnounceDelivery || isAnnounceDispatchPath) && + /\brunSubagentAnnounceFlow\b|\brunSubagentAnnounceDispatch\b|\benqueueAnnounce\b|\bcreateBoundDeliveryRouter\b|\bqueueEmbeddedAgentMessage\b|\bwaitForEmbeddedAgentRunEnd\b|\bqueue-fallback\b|\bdirect-primary\b/.test( + source, + ) + ) { + seamKinds.push("subagent-announce-delivery"); + } + + if ( + importsCleanup && + /\bsessions\.delete\b|\bdeleteTranscript\b|\bcleanupFailedAcpSpawn\b|\bcleanupProvisionalSession\b|\bcleanupFailedSpawnBeforeAgentStart\b|\bresolveDeferredCleanupDecision\b/.test( + source, + ) + ) { + seamKinds.push("subagent-session-cleanup"); + } + + if ( + importsParentStream && + /\bstartAcpSpawnParentStreamRelay\b|\brequestHeartbeatNow\b|\benqueueSystemEvent\b|\bonAgentEvent\b|\bstreamTo\b/.test( + source, + ) + ) { + seamKinds.push("subagent-parent-stream"); + } + + return seamKinds; +} + +export function describeSeamKinds(relativePath: string, source: string) { + const seamKinds = []; + const isReplyDeliveryPath = + /reply-delivery|reply-dispatcher|deliver-reply|reply\/.*delivery|monitor\/(?:replies|deliver|native-command)|outbound\/deliver|outbound\/message/.test( + relativePath, + ); + const isChannelMediaAdapterPath = + (relativePath.startsWith(BUNDLED_PLUGIN_PATH_PREFIX) && + /(outbound|outbound-adapter|reply-delivery|send|delivery|messenger|channel(?:\.runtime)?)\.ts$/.test( + relativePath, + )) || + /^src\/channels\/plugins\/outbound\/[^/]+\.ts$/.test(relativePath); + if ( + relativePath.startsWith("src/agents/tools/") && + source.includes("details") && + source.includes("media") && + /details\s*:\s*{[\s\S]*\bmedia\b\s*:/.test(source) + ) { + seamKinds.push("tool-result-media"); + } + if ( + isReplyDeliveryPath && + /\bmediaUrl\b|\bmediaUrls\b|resolveSendableOutboundReplyParts/.test(source) + ) { + seamKinds.push("reply-delivery-media"); + } + if ( + isChannelMediaAdapterPath && + (/sendMedia\b/.test(source) || /\bmediaUrl\b|\bmediaUrls\b|filename|audioAsVoice/.test(source)) + ) { + seamKinds.push("channel-media-adapter"); + } + if ( + isReplyDeliveryPath && + /blockStreamingEnabled|directlySentBlockKeys|resolveSendableOutboundReplyParts/.test(source) && + /\bmediaUrl\b|\bmediaUrls\b/.test(source) + ) { + seamKinds.push("streaming-media-handoff"); + } + seamKinds.push(...describeCronSeamKinds(relativePath, source)); + seamKinds.push(...describeSubagentSeamKinds(relativePath, source)); + return [...new Set(seamKinds)].toSorted(compareStrings); +} + +async function buildTestIndex(testFiles: string[]) { + return Promise.all( + testFiles.map(async (filePath) => { + const relativePath = normalizePath(filePath); + const stem = stemFromRelativePath(relativePath) + .replace(/\.test$/, "") + .replace(/\.spec$/, ""); + const baseName = path.basename(stem); + const source = await readScannableText(filePath); + return { + filePath, + relativePath, + stem, + baseName, + source, + }; + }), + ); +} + +type TestIndexEntry = Awaited>[number]; + +function hasExecutableImportReference(source: string, importPath: string) { + const escapedImportPath = escapeRegExp(importPath); + const suffix = String.raw`(?:\.[^"'\\\`]+)?`; + const patterns = [ + new RegExp(String.raw`\bfrom\s*["'\`]${escapedImportPath}${suffix}["'\`]`), + new RegExp(String.raw`\bimport\s*["'\`]${escapedImportPath}${suffix}["'\`]`), + new RegExp(String.raw`\brequire\s*\(\s*["'\`]${escapedImportPath}${suffix}["'\`]\s*\)`), + new RegExp(String.raw`\bimport\s*\(\s*["'\`]${escapedImportPath}${suffix}["'\`]\s*\)`), + ]; + return patterns.some((pattern) => pattern.test(source)); +} + +function hasModuleMockReference(source: string, importPath: string) { + const escapedImportPath = escapeRegExp(importPath); + const suffix = String.raw`(?:\.[^"'\\\`]+)?`; + const patterns = [ + new RegExp(String.raw`\bvi\.mock\s*\(\s*["'\`]${escapedImportPath}${suffix}["'\`]`), + new RegExp(String.raw`\bjest\.mock\s*\(\s*["'\`]${escapedImportPath}${suffix}["'\`]`), + ]; + return patterns.some((pattern) => pattern.test(source)); +} + +const matchQualityRank = (quality: MatchQuality) => MATCH_QUALITY_RANK[quality] ?? 4; + +function findRelatedTests(relativePath: string, testIndex: TestIndexEntry[]): RelatedTestMatch[] { + const stem = stemFromRelativePath(relativePath); + const baseName = path.basename(stem); + const dirName = path.dirname(relativePath); + const normalizedDir = dirName.split(path.sep).join("/"); + const baseTokens = new Set(splitNameTokens(baseName).filter((token) => token.length >= 7)); + + const matches: RelatedTestMatch[] = testIndex.flatMap((entry): RelatedTestMatch[] => { + if (entry.stem === stem) { + return [{ file: entry.relativePath, matchQuality: "exact-stem" }]; + } + if (entry.stem.startsWith(`${stem}.`)) { + return [{ file: entry.relativePath, matchQuality: "path-nearby" }]; + } + const entryDir = path.dirname(entry.relativePath).split(path.sep).join("/"); + const importPath = + path.posix.relative(entryDir, stem) === path.basename(stem) + ? `./${path.basename(stem)}` + : path.posix.relative(entryDir, stem).startsWith(".") + ? path.posix.relative(entryDir, stem) + : `./${path.posix.relative(entryDir, stem)}`; + if ( + hasExecutableImportReference(entry.source, importPath) && + !hasModuleMockReference(entry.source, importPath) + ) { + return [{ file: entry.relativePath, matchQuality: "direct-import" }]; + } + if (entryDir === normalizedDir && baseTokens.size > 0) { + const entryTokens = splitNameTokens(entry.baseName); + const sharedToken = entryTokens.find((token) => baseTokens.has(token)); + if (sharedToken) { + return [{ file: entry.relativePath, matchQuality: "dir-token" }]; + } + } + return []; + }); + + const byFile = new Map(); + for (const match of matches) { + const existing = byFile.get(match.file); + if ( + !existing || + matchQualityRank(match.matchQuality) < matchQualityRank(existing.matchQuality) + ) { + byFile.set(match.file, match); + } + } + + return [...byFile.values()].toSorted((left, right) => { + return ( + matchQualityRank(left.matchQuality) - matchQualityRank(right.matchQuality) || + left.file.localeCompare(right.file) + ); + }); +} + +export function determineSeamTestStatus( + seamKinds: string[], + relatedTestMatches: RelatedTestMatch[], +) { + if (relatedTestMatches.length === 0) { + return { + status: "gap", + reason: "No nearby test file references this seam candidate.", + }; + } + + const bestMatch = relatedTestMatches[0]?.matchQuality ?? "unknown"; + if ( + seamKinds.includes("reply-delivery-media") || + seamKinds.includes("streaming-media-handoff") || + seamKinds.includes("tool-result-media") || + seamKinds.includes("cron-agent-handoff") || + seamKinds.includes("cron-outbound-delivery") || + seamKinds.includes("cron-heartbeat-handoff") || + seamKinds.includes("cron-scheduler-state") || + seamKinds.includes("cron-media-delivery") || + seamKinds.includes("cron-followup-handoff") || + seamKinds.includes("subagent-session-spawn") || + seamKinds.includes("subagent-lifecycle-registry") || + seamKinds.includes("subagent-announce-delivery") || + seamKinds.includes("subagent-session-cleanup") || + seamKinds.includes("subagent-parent-stream") + ) { + return { + status: "partial", + reason: `Nearby tests exist (best match: ${bestMatch}), but this inventory does not prove cross-layer seam coverage end to end.`, + }; + } + return { + status: "heuristic-nearby", + reason: `Nearby tests exist (best match: ${bestMatch}), but this remains a filename/path heuristic rather than proof of seam assertions.`, + }; +} + +async function buildSeamTestInventory() { + const productionFiles = [ + ...(await walkCodeFiles(srcRoot)), + ...(await walkCodeFiles(extensionsRoot)), + ].toSorted((left, right) => normalizePath(left).localeCompare(normalizePath(right))); + const testFiles = [ + ...(await walkAllCodeFiles(srcRoot, { includeTests: true })), + ...(await walkAllCodeFiles(extensionsRoot, { includeTests: true })), + ...(await walkAllCodeFiles(testRoot, { includeTests: true })), + ] + .filter((filePath) => /\.(test|spec)\.(ts|tsx|mts|cts|js|jsx|mjs|cjs)$/.test(filePath)) + .toSorted((left, right) => normalizePath(left).localeCompare(normalizePath(right))); + const testIndex = await buildTestIndex(testFiles); + const inventory = []; + + for (const filePath of productionFiles) { + const relativePath = normalizePath(filePath); + const source = await readScannableText(filePath); + const seamKinds = describeSeamKinds(relativePath, source); + if (seamKinds.length === 0) { + continue; + } + const relatedTestMatches = findRelatedTests(relativePath, testIndex); + const status = determineSeamTestStatus(seamKinds, relatedTestMatches); + inventory.push({ + file: relativePath, + seamKinds, + relatedTests: relatedTestMatches.map((entry) => entry.file), + relatedTestMatches, + status: status.status, + reason: status.reason, + }); + } + + return inventory.toSorted((left, right) => { + return ( + left.status.localeCompare(right.status) || + left.file.localeCompare(right.file) || + left.seamKinds.join(",").localeCompare(right.seamKinds.join(",")) + ); + }); +} + +export async function main(argv: string[] = process.argv.slice(2)) { + const args = new Set(argv); + if (args.has("--help") || args.has("-h")) { + process.stdout.write(`${HELP_TEXT}\n`); + return; + } + + await collectWorkspacePackagePaths(); + const inventory = await collectCorePluginSdkImports(); + const optionalClusterStaticLeaks = await collectOptionalClusterStaticLeaks(); + const staticLeakClusters = new Set(optionalClusterStaticLeaks.map((entry) => entry.cluster)); + const result = { + duplicatedSeamFamilies: buildDuplicatedSeamFamilies(inventory), + overlapFiles: buildOverlapFiles(inventory), + optionalClusterStaticLeaks: buildOptionalClusterStaticLeaks(optionalClusterStaticLeaks), + missingPackages: await buildMissingPackages({ staticLeakClusters }), + seamTestInventory: await buildSeamTestInventory(), + }; + + process.stdout.write(`${JSON.stringify(result, null, 2)}\n`); +} + +const entryFilePath = process.argv[1] ? path.resolve(process.argv[1]) : null; +if (entryFilePath === fileURLToPath(import.meta.url)) { + await main(); +} diff --git a/scripts/authorized-beta-focused-policy.json b/scripts/authorized-beta-focused-policy.json new file mode 100644 index 0000000000000000000000000000000000000000..c65491a815574e589968561db6ae6a5f35e39594 --- /dev/null +++ b/scripts/authorized-beta-focused-policy.json @@ -0,0 +1,77 @@ +{ + "schema": "openclaw.authorized-beta-focused-policy.v1", + "mode": "authorized-beta-focused-v1", + "releaseTag": "v2026.8.1-beta.3", + "releaseVersion": "2026.8.1-beta.3", + "baseCandidateSha": "3203a6f7f8d79644fde2b4f091a694f4c1698538", + "candidateSha": "3fbe94065c2b94f4c08acb6742a69938bf408d94", + "reviewedHeadSha": "e347223a5858bda5907acda858404e5f7a4c3720", + "candidateTreeSha": "71fb83bcab002be3da4057e7cffacf77d4cdbd2d", + "baseTreeSha": "10d1f3ab0ff52383c025c5846427e0dd5d14598a", + "packageProjectionSha256": "d1d8f60b8a14aeb567a0d5215a6b0265c74a07ebea04b288e43c31cba473bbca", + "eligibilityPlanDigest": "sha256:e05226cfd77716b262882b3e2525037a506cd8b6af2affa0a876499074b1671b", + "historicalToolingSha": "eed27fdb88c0c60cb79c5b159a0284bd519271c2", + "historicalToolingRef": "refs/tags/release-publish/eed27fdb88c0-1787493831", + "historicalFrv": { + "runId": "32644377679", + "runAttempt": 1, + "workflowPath": ".github/workflows/full-release-validation.yml", + "workflowRef": "release-ci/eed27fdb88c0-1787493942617", + "targetSha": "3203a6f7f8d79644fde2b4f091a694f4c1698538", + "ciRunId": "32644407381", + "ciFailedJobId": "97206458686", + "ciAggregateJobId": "97216945772", + "pluginRunId": "32645134710", + "pluginFailedJobId": "97208293666", + "pluginAggregateJobId": "97210400113", + "releaseChecksRunId": "32645133620", + "releaseChecksVerifierJobId": "97211299203", + "performanceRunId": "32644407718", + "performanceFailedJobId": "97206336416" + }, + "focusedProof": { + "ciRunId": "32664685168", + "ciTargetLogJobId": "97256163016", + "ciSuccessJobId": "97256296219", + "pluginRunId": "32664686635", + "pluginTargetLogJobId": "97256164326", + "pluginSuccessJobId": "97256329353" + }, + "changedPaths": [ + { + "path": "extensions/slack/src/monitor.test-helpers.ts", + "status": "M", + "added": 12, + "deleted": 18 + }, + { + "path": "extensions/slack/src/monitor/provider.auth-test-token.test.ts", + "status": "M", + "added": 10, + "deleted": 569 + }, + { + "path": "extensions/slack/src/monitor/provider.event-identity-recovery.test.ts", + "status": "D", + "added": 0, + "deleted": 155 + }, + { + "path": "extensions/slack/src/monitor/provider.identity.test.ts", + "status": "A", + "added": 658, + "deleted": 0 + } + ], + "inventory": { + "npmCount": 93, + "npmNamesSha256": "778eb6aeb324c02942e17755557ff9d5055376fb4d39096a60022f0ae75d67f3", + "clawHubCount": 89, + "clawHubNamesSha256": "1b161a4fde3c003e23db306d4a86908086caac1cd33e75feeaf27474366c1c2b", + "trustedPublisherCount": 75, + "trustedPublisherNamesSha256": "4266416f20fe5c8b7a4ce732121a2206fc8cfa6af25deae0aec70d552ceb5df8", + "bootstrapCount": 14, + "bootstrapNamesSha256": "83c79c6099d12dc2ebb58e2fc5ca8e078776dfa21b50dee741bac6aed5526eef", + "missingTrustedPublisherCount": 0 + } +} diff --git a/scripts/bench-agent-concurrency-worker.ts b/scripts/bench-agent-concurrency-worker.ts new file mode 100644 index 0000000000000000000000000000000000000000..2ef4e42ca4413636d57c52f949b9a538b07e7add --- /dev/null +++ b/scripts/bench-agent-concurrency-worker.ts @@ -0,0 +1,836 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { performance } from "node:perf_hooks"; +import { pathToFileURL } from "node:url"; +import { toErrorObject } from "@openclaw/normalization-core/error-coercion"; +import type { SubagentRunRecord } from "../src/agents/subagents/registry/subagent-registry.types.js"; +import type { DB as OpenClawStateKyselyDatabase } from "../src/state/openclaw-state-db.generated.js"; +import { + WORKER_RESULT_SENTINEL, + type WorkerResult, + type WorkerScenario, +} from "./bench-agent-concurrency.js"; +import { classifyBoundedUnsignedDecimal } from "./lib/arg-utils.mts"; + +type WorkerOptions = { + scenario: WorkerScenario; + size: number; + runs: number; + warmup: number; +}; + +type Sample = { + durationMs: number; + invariant: Record; +}; + +const SCENARIOS = new Set([ + "spawnPipelineInMemory", + "spawnPipelineDurable", + "admission", + "recoverySweep", + "duplicateSuppression", +]); + +function parseInteger(raw: string | undefined, flag: string, min: number, max: number): number { + const result = classifyBoundedUnsignedDecimal(raw, min, max); + if (result.kind === "syntax") { + throw new Error(`${flag} must be an integer`); + } + if (result.kind !== "value") { + throw new Error(`${flag} must be between ${min} and ${max}`); + } + return result.value; +} + +function parseOptions(argv: string[]): WorkerOptions { + const values = new Map(); + for (let index = 0; index < argv.length; index += 2) { + const flag = argv[index]; + const value = argv[index + 1]; + if (!flag?.startsWith("--") || !value || value.startsWith("--")) { + throw new Error(`invalid worker argument near ${flag ?? "end"}`); + } + if (values.has(flag)) { + throw new Error(`${flag} was provided more than once`); + } + values.set(flag, value); + } + const scenario = values.get("--scenario") as WorkerScenario | undefined; + if (!scenario || !SCENARIOS.has(scenario)) { + throw new Error(`unknown worker scenario: ${scenario ?? "missing"}`); + } + return { + scenario, + size: parseInteger(values.get("--size"), "--size", 1, 4096), + runs: parseInteger(values.get("--runs"), "--runs", 1, 100), + warmup: parseInteger(values.get("--warmup"), "--warmup", 0, 20), + }; +} + +function processMaxRssBytes(): number { + return Math.max(0, Math.round(process.resourceUsage().maxRSS * 1024)); +} + +async function waitForCondition(check: () => boolean): Promise { + const deadline = Date.now() + 30_000; + while (Date.now() < deadline) { + if (check()) { + return true; + } + await new Promise((resolve) => { + setImmediate(resolve); + }); + } + return check(); +} + +async function drainSpawnSampleActiveWork( + waitForActiveWork?: ( + timeoutMs: number, + ) => Promise<{ drained: boolean; snapshot: { counts: { totalActive: number } } }>, +): Promise { + const wait = + waitForActiveWork ?? + (await import("../src/infra/gateway-active-work.js")).waitForGatewayActiveWork; + const result = await wait(30_000); + const active = result.snapshot.counts.totalActive; + if (!result.drained || active !== 0) { + throw new Error(`spawn sample left ${active} active gateway work items`); + } +} + +async function resetRuntime(persist: boolean): Promise { + const [subagents, tasks, stateDb, agentDb] = await Promise.all([ + import("../src/agents/subagents/registry/subagent-registry.test-helpers.js"), + import("../src/tasks/task-runtime.test-helpers.js"), + import("../src/state/openclaw-state-db.js"), + import("../src/state/openclaw-agent-db.js"), + ]); + subagents.resetSubagentRegistryForTests({ persist }); + subagents.testing.setDepsForTest(); + tasks.resetTaskRegistryControlRuntimeForTests(); + tasks.resetTaskRegistryDeliveryRuntimeForTests(); + tasks.resetDetachedTaskLifecycleRuntimeForTests(); + tasks.resetTaskRegistryForTests({ persist }); + tasks.resetTaskFlowRegistryForTests({ persist }); + stateDb.closeOpenClawStateDatabaseForTest(); + agentDb.closeOpenClawAgentDatabasesForTest(); +} + +function createTerminalWaitBarrier() { + const waiters = new Map void>(); + let outstanding = 0; + let releasesStarted = false; + const callGateway = (async (request: { method?: string; params?: unknown }) => { + if (request.method !== "agent.wait") { + return {} as T; + } + if (releasesStarted) { + throw new Error("agent.wait registered after the benchmark barrier released"); + } + const runId = + request.params && + typeof request.params === "object" && + !Array.isArray(request.params) && + typeof (request.params as { runId?: unknown }).runId === "string" + ? (request.params as { runId: string }).runId + : undefined; + if (!runId || waiters.has(runId)) { + throw new Error(`invalid or duplicate benchmark agent.wait run id: ${runId ?? "missing"}`); + } + outstanding += 1; + return await new Promise((resolve) => { + waiters.set(runId, (value) => { + outstanding -= 1; + resolve(value as T); + }); + }); + }) as typeof import("../src/gateway/call.js").callGateway; + return { + callGateway, + get outstanding() { + return outstanding; + }, + async waitUntilBlocked(expected: number) { + return await waitForCondition(() => outstanding === expected); + }, + release(runId: string) { + releasesStarted = true; + const resolve = waiters.get(runId); + if (!resolve) { + throw new Error(`missing blocked benchmark agent.wait for ${runId}`); + } + waiters.delete(runId); + const endedAt = Date.now(); + resolve({ + status: "ok", + startedAt: endedAt - 1, + endedAt, + stopReason: "stop", + }); + }, + releaseAll() { + releasesStarted = true; + for (const runId of waiters.keys()) { + this.release(runId); + } + }, + }; +} + +async function configureSpawnRuntime( + mode: "memory" | "durable", + callGateway: typeof import("../src/gateway/call.js").callGateway, +): Promise { + const [subagents, registry, taskStore, flowStore] = await Promise.all([ + import("../src/agents/subagents/registry/subagent-registry.test-helpers.js"), + import("../src/agents/subagents/registry/subagent-registry-memory.js"), + import("../src/tasks/task-registry.store.js"), + import("../src/tasks/task-flow-registry.store.test-support.js"), + ]); + const sharedDeps = { + callGateway, + getRuntimeConfig: () => ({}), + onAgentEvent: () => () => {}, + resolveAgentTimeoutMs: () => 1_000, + captureSubagentCompletionReply: async (childSessionKey: string) => { + const entry = [...registry.subagentRuns.values()].find( + (candidate) => candidate.childSessionKey === childSessionKey, + ); + if (entry) { + // Completion already owns the row at this awaited seam. Suppress only + // its unrelated session projection, not the terminal registry/task transition. + entry.execution.suppressSessionEffects = true; + } + return undefined; + }, + cleanupBrowserSessionsForLifecycleEnd: async () => {}, + runSubagentAnnounceFlow: async () => "retryable" as const, + maybeWakeRequesterAfterAllChildrenSettled: async () => false, + ensureContextEnginesInitialized: () => {}, + loadAgentRuntimePluginRegistryHandle: () => undefined, + resolveContextEngine: async () => + ({ + info: { id: "bench", name: "bench", version: "1" }, + ingest: async () => ({ ok: true }), + assemble: async () => ({ messages: [] }), + onSubagentEnded: async () => {}, + }) as unknown as import("../src/context-engine/types.js").ContextEngine, + }; + if (mode === "memory") { + subagents.testing.setDepsForTest({ + ...sharedDeps, + persistSubagentRunsToDisk: () => {}, + persistSubagentRunsToDiskOrThrow: () => {}, + }); + taskStore.configureTaskRegistryRuntime({ + store: { + loadSnapshot: () => ({ tasks: new Map(), deliveryStates: new Map() }), + upsertTaskWithDeliveryState: () => {}, + deleteTaskWithDeliveryState: () => {}, + upsertDeliveryState: () => {}, + close: () => {}, + }, + }); + const { createInMemoryTaskFlowRegistryStore } = + await import("../src/test-utils/task-registry-store.js"); + flowStore.configureTaskFlowRegistryRuntime({ + store: createInMemoryTaskFlowRegistryStore(), + }); + return; + } + subagents.testing.setDepsForTest(sharedDeps); +} + +type BenchmarkStateDatabase = Pick; + +async function readDurableRows() { + const [{ executeSqliteQuerySync, getNodeSqliteKysely }, stateDb] = await Promise.all([ + import("../src/infra/kysely-sync.js"), + import("../src/state/openclaw-state-db.js"), + ]); + const database = stateDb.openOpenClawStateDatabase(); + const db = getNodeSqliteKysely(database.db); + return { + path: database.path, + subagentRows: executeSqliteQuerySync( + database.db, + db + .selectFrom("subagent_runs") + .select((eb) => [ + "run_id", + eb + .fn("json_extract", ["payload_json", eb.val("$.execution.endedAt")]) + .as("ended_at"), + ]) + .orderBy("run_id"), + ).rows, + taskRows: executeSqliteQuerySync( + database.db, + db + .selectFrom("task_runs") + .select(["run_id", "status", "ended_at"]) + .where("runtime", "=", "subagent") + .orderBy("run_id"), + ).rows, + }; +} + +function listSqliteFiles(root: string): string[] { + return fs + .readdirSync(root, { recursive: true }) + .map(String) + .filter((entry) => entry.includes(".sqlite")) + .toSorted(); +} + +async function listBenchmarkTasks() { + const tasks = await import("../src/tasks/task-registry.js"); + return tasks.listTaskRecords().filter((task) => task.runtime === "subagent"); +} + +async function listBenchmarkTaskMemory() { + const state = await import("../src/tasks/task-registry-state.js"); + return [...state.tasks.values()].filter((task) => task.runtime === "subagent"); +} + +function assertExactRunIds(actual: Array, expected: string[], label: string): void { + const normalized = actual + .filter((value): value is string => typeof value === "string") + .toSorted(); + const wanted = expected.toSorted(); + if (JSON.stringify(normalized) !== JSON.stringify(wanted)) { + throw new Error( + `${label} mismatch: ${JSON.stringify({ actual: normalized, expected: wanted })}`, + ); + } +} + +async function runSpawnSample( + fanout: number, + serial: number, + mode: "memory" | "durable", + stateDir: string, +): Promise { + const [pipeline, registry] = await Promise.all([ + import("../src/agents/spawn-pipeline.js"), + import("../src/agents/subagents/registry/subagent-registry-memory.js"), + ]); + await resetRuntime(mode === "durable"); + const barrier = createTerminalWaitBarrier(); + await configureSpawnRuntime(mode, barrier.callGateway); + const taskRegistry = await import("../src/tasks/task-registry.js"); + let releases = 0; + const runIds: string[] = []; + const params = Array.from({ length: fanout }, (_, index) => { + const runId = `bench-${mode}-${serial}-${index}`; + runIds.push(runId); + let released = false; + return { + adapter: { + initialize: async () => ({ index }), + dispatchTurn: async () => ({ runId }), + cleanupOnFailure: async () => {}, + }, + admissionReservation: { + release: () => { + if (!released) { + released = true; + releases += 1; + } + }, + }, + buildRegistration: () => ({ + runId, + childSessionKey: `agent:bench:subagent:${mode}:${serial}:${index}`, + requesterSessionKey: "agent:bench:main", + requesterDisplayKey: "bench", + task: `benchmark child ${index}`, + cleanup: "keep" as const, + expectsCompletionMessage: false, + }), + progressSessionKey: "agent:bench:main", + }; + }); + let result: Sample | undefined; + let failure: unknown; + try { + const startedAt = performance.now(); + const pipelineResults = await Promise.all( + params.map((entry) => pipeline.runSpawnPipeline(entry)), + ); + const durationMs = performance.now() - startedAt; + const blocked = await barrier.waitUntilBlocked(fanout); + const successful = pipelineResults.filter((pipelineResult) => pipelineResult.ok); + const uniqueRuns = new Set(successful.map((pipelineResult) => pipelineResult.runId)).size; + const registeredRuns = registry.subagentRuns.size; + const tasksWhileBlocked = await listBenchmarkTasks(); + assertExactRunIds( + tasksWhileBlocked.map((task) => task.runId ?? null), + runIds, + `${mode} in-memory task rows`, + ); + if (!blocked || barrier.outstanding !== fanout) { + throw new Error(`spawn ${mode} did not block ${fanout} agent.wait calls`); + } + if ( + successful.length !== fanout || + uniqueRuns !== fanout || + registeredRuns !== fanout || + releases !== fanout + ) { + throw new Error( + `spawn ${mode} registration invariant failed: ${JSON.stringify({ fanout, uniqueRuns, registeredRuns, releases })}`, + ); + } + let durableSubagentRows = 0; + let durableTaskRows = 0; + let durableStateFile = listSqliteFiles(stateDir).length > 0; + if (mode === "durable") { + const durable = await readDurableRows(); + assertExactRunIds( + durable.subagentRows.map((row) => row.run_id), + runIds, + "durable subagent rows", + ); + assertExactRunIds( + durable.taskRows.map((row) => row.run_id), + runIds, + "durable task rows", + ); + if ( + durable.subagentRows.some((row) => row.ended_at !== null) || + durable.taskRows.some((row) => row.status !== "running" || row.ended_at !== null) + ) { + throw new Error("durable rows settled before the benchmark barrier released"); + } + durableSubagentRows = durable.subagentRows.length; + durableTaskRows = durable.taskRows.length; + durableStateFile = fs.existsSync(durable.path); + if (!durableStateFile) { + throw new Error(`durable spawn pipeline database is missing at ${durable.path}`); + } + } else if (durableStateFile) { + throw new Error("in-memory spawn pipeline created durable SQLite state"); + } + for (const runId of runIds) { + barrier.release(runId); + const runSettled = await waitForCondition(() => { + const run = registry.subagentRuns.get(runId); + const task = taskRegistry.findTaskByRunId(runId); + return ( + run?.execution.status === "terminal" && + typeof run.execution.endedAt === "number" && + typeof run.cleanupCompletedAt === "number" && + task?.status === "succeeded" + ); + }); + if (!runSettled) { + throw new Error(`spawn ${mode} did not settle released run ${runId}`); + } + } + const settledTasks = await listBenchmarkTasks(); + const settledRuns = [...registry.subagentRuns.values()].filter( + (entry) => + entry.execution.status === "terminal" && + typeof entry.execution.endedAt === "number" && + typeof entry.cleanupCompletedAt === "number", + ).length; + const succeededTasks = settledTasks.filter((task) => task.status === "succeeded").length; + if (settledRuns !== fanout || succeededTasks !== fanout || barrier.outstanding !== 0) { + throw new Error( + `spawn ${mode} settlement invariant failed: ${JSON.stringify({ fanout, settledRuns, succeededTasks, outstandingWaits: barrier.outstanding })}`, + ); + } + // Terminal rows can settle before detached cleanup and requester-wake roots. + // Drain before reset so leaked work stays visible and cannot reach the next sample. + await drainSpawnSampleActiveWork(); + result = { + durationMs, + invariant: { + ok: true, + registeredRuns, + reservationsReleased: releases, + blockedWaits: fanout, + settledRuns, + settledTasks: succeededTasks, + outstandingWaits: barrier.outstanding, + durableSubagentRows, + durableTaskRows, + durableStateFile, + postTeardownRegistryRows: -1, + postTeardownTaskRows: -1, + postTeardownDurableSubagentRows: -1, + postTeardownDurableTaskRows: -1, + postTeardownActiveRootWork: 0, + }, + }; + } catch (error) { + failure = error; + } finally { + try { + barrier.releaseAll(); + } catch (error) { + failure ??= error; + } + try { + await resetRuntime(mode === "durable"); + } catch (error) { + failure ??= error; + } + } + if (failure) { + throw toErrorObject(failure, "Agent concurrency benchmark failed"); + } + const postTeardownTasks = await listBenchmarkTaskMemory(); + const postTeardownRegistryRows = registry.subagentRuns.size; + const postTeardownTaskRows = postTeardownTasks.length; + let postTeardownDurableSubagentRows = 0; + let postTeardownDurableTaskRows = 0; + if (mode === "durable") { + const durable = await readDurableRows(); + postTeardownDurableSubagentRows = durable.subagentRows.length; + postTeardownDurableTaskRows = durable.taskRows.length; + await resetRuntime(false); + } + if ( + postTeardownRegistryRows !== 0 || + postTeardownTaskRows !== 0 || + postTeardownDurableSubagentRows !== 0 || + postTeardownDurableTaskRows !== 0 || + barrier.outstanding !== 0 + ) { + throw new Error( + `spawn ${mode} teardown invariant failed: ${JSON.stringify({ postTeardownRegistryRows, postTeardownTaskRows, postTeardownDurableSubagentRows, postTeardownDurableTaskRows, outstandingWaits: barrier.outstanding })}`, + ); + } + if (mode === "memory" && listSqliteFiles(stateDir).length > 0) { + throw new Error("in-memory spawn pipeline created SQLite state during teardown"); + } + if (!result) { + throw new Error(`spawn ${mode} did not produce a benchmark result`); + } + result.invariant.postTeardownRegistryRows = postTeardownRegistryRows; + result.invariant.postTeardownTaskRows = postTeardownTaskRows; + result.invariant.postTeardownDurableSubagentRows = postTeardownDurableSubagentRows; + result.invariant.postTeardownDurableTaskRows = postTeardownDurableTaskRows; + return result; +} + +async function runAdmissionSample(fanout: number, serial: number): Promise { + const { reserveChildAdmissionSlot } = await import("../src/agents/child-admission.js"); + const controllerSessionKey = `agent:bench:admission:${serial}`; + const reservations: Array<{ release: () => void }> = []; + const reserve = () => + reserveChildAdmissionSlot({ + controllerSessionKey, + resolveAdmission: (pendingChildren) => + pendingChildren < fanout + ? { ok: true as const } + : { ok: false as const, governingCap: "benchmark" }, + }); + const startedAt = performance.now(); + for (let index = 0; index < fanout; index += 1) { + const reservation = reserve(); + if (!reservation.ok) { + throw new Error(`admission rejected slot ${index + 1}/${fanout}`); + } + reservations.push(reservation); + } + const overflow = reserve(); + for (const reservation of reservations) { + reservation.release(); + reservation.release(); + } + const replacement = reserve(); + if (replacement.ok) { + replacement.release(); + } + const durationMs = performance.now() - startedAt; + const ok = !overflow.ok && replacement.ok; + if (!ok) { + throw new Error(`admission invariant failed at fanout ${fanout}`); + } + return { + durationMs, + invariant: { ok, admissionCap: reservations.length, overflowRejected: true, released: true }, + }; +} + +function sweepRow(child: number, generation: number, now: number): SubagentRunRecord { + const current = generation === 3; + return { + runId: `bench-sweep-${child}-${generation}`, + childSessionKey: `agent:bench:subagent:sweep-${child}`, + requesterSessionKey: "agent:bench:main", + requesterDisplayKey: "bench", + task: `sweep child ${child}`, + cleanup: current ? "keep" : "delete", + generation, + createdAt: now - generation, + archiveAtMs: current ? undefined : now - 1, + terminalOwner: current ? "interrupted-recovery" : undefined, + endedReason: current ? "subagent-error" : undefined, + execution: current + ? { + status: "terminal", + startedAt: now - 2_000, + endedAt: now - 1_000, + outcome: { status: "error", error: "interrupted recovery replay" }, + suppressSessionEffects: true, + } + : { + status: "terminal", + startedAt: now - 2_000, + endedAt: now - 1_000, + outcome: { status: "error", error: "retired recovery generation" }, + suppressSessionEffects: true, + }, + }; +} + +async function runSweepSample(childCount: number): Promise { + const [ + { getSubagentRunsForChildSession, subagentRuns: runs }, + { createSubagentRegistrySweeper }, + ] = await Promise.all([ + import("../src/agents/subagents/registry/subagent-registry-memory.js"), + import("../src/agents/subagents/registry/subagent-registry-sweeper.js"), + ]); + const now = Date.now(); + runs.clear(); + for (let child = 0; child < childCount; child += 1) { + for (const generation of [3, 2, 1]) { + const entry = sweepRow(child, generation, now); + runs.set(entry.runId, entry); + } + } + let sessionEffects = 0; + let recoveryProjections = 0; + let lostContextCompletions = 0; + const sweeper = createSubagentRegistrySweeper({ + runs, + resumedRuns: new Set(), + persist: () => {}, + clearPendingLifecycleError: () => {}, + clearPendingLifecycleTimeout: () => {}, + clearPendingSubagentRecoveryNotice: () => true, + sweepPendingLifecycle: () => {}, + completeSubagentRunWithRecovery: async () => { + lostContextCompletions += 1; + }, + getGatewayRecoveryRuntime: () => undefined, + abandonSubagentRestartRecoveryLaunch: () => true, + clearAcceptedSubagentRestartRecovery: () => true, + resumeSettledSubagentRestartRecovery: () => true, + replaceSubagentRunAfterSteer: () => true, + markSubagentRestartRecoveryLaunchAttempted: () => undefined, + markSubagentRestartRecoveryLaunchAccepted: () => undefined, + markSubagentRestartRecoveryLaunchConsumed: () => undefined, + reserveSubagentRestartRecoveryLaunch: () => undefined, + resetSubagentRestartRecoveryLaunchAttempt: () => true, + finalizeInterruptedSubagentRun: async ({ runId, expectedEntry }) => { + if (runs.get(runId) !== expectedEntry || expectedEntry?.generation !== 3) { + throw new Error(`unexpected recovery projection owner: ${runId}`); + } + recoveryProjections += 1; + return 1; + }, + resumeRequesterSettleWake: () => {}, + startSubagentAnnounceCleanupFlow: () => true, + completeCleanupBookkeeping: () => {}, + discardTerminalDelivery: () => {}, + shouldEmitEndedHookForRun: () => false, + emitSubagentEndedHookForRun: async () => {}, + callGateway: (async () => { + sessionEffects += 1; + return {} as T; + }) as typeof import("../src/gateway/call.js").callGateway, + cleanupCollectorLaunchResources: async () => true, + runContextEngineSubagentEnded: async () => { + sessionEffects += 1; + }, + notifyContextEngineSubagentEnded: async () => { + sessionEffects += 1; + }, + retireSupersededRun: async () => {}, + getRunsForChildSession: getSubagentRunsForChildSession, + getRunsForCollectorGroup: () => [], + warn: () => {}, + }); + try { + const startedAt = performance.now(); + await sweeper.sweepOnce(); + const durationMs = performance.now() - startedAt; + const retainedCurrent = [...runs.values()].filter((entry) => entry.generation === 3).length; + const removedRows = childCount * 3 - runs.size; + const ok = + removedRows === childCount * 2 && + retainedCurrent === childCount && + sessionEffects === 0 && + recoveryProjections === childCount && + lostContextCompletions === 0; + if (!ok) { + throw new Error( + `registry sweep invariant failed: ${JSON.stringify({ childCount, removedRows, retainedCurrent, sessionEffects, recoveryProjections, lostContextCompletions })}`, + ); + } + return { + durationMs, + invariant: { + ok, + seededRows: childCount * 3, + removedRows, + retainedCurrent, + sessionEffects, + recoveryProjections, + lostContextCompletions, + }, + }; + } finally { + sweeper.reset(); + runs.clear(); + } +} + +async function runDedupeSample(childCount: number): Promise { + const { dedupeLatestChildCompletionRows } = + await import("../src/agents/subagents/announce/subagent-announce-output.js"); + const rowsForOrder = (generations: number[]) => + Array.from({ length: childCount }, (_, child) => + generations.map((generation) => ({ + runId: `bench-dedupe-${child}-${generation}`, + childSessionKey: `agent:bench:subagent:dedupe-${child}`, + task: `dedupe child ${child}`, + generation, + createdAt: generation, + execution: { status: "terminal" as const, endedAt: generation }, + })), + ).flat(); + const newestFirstRows = rowsForOrder([3, 2, 1]); + const oldestFirstRows = rowsForOrder([1, 2, 3]); + const startedAt = performance.now(); + const newestFirst = dedupeLatestChildCompletionRows(newestFirstRows); + const oldestFirst = dedupeLatestChildCompletionRows(oldestFirstRows); + const durationMs = performance.now() - startedAt; + const newestFirstSelectedNewest = newestFirst.every((row) => row.generation === 3); + const oldestFirstSelectedNewest = oldestFirst.every((row) => row.generation === 3); + const ok = + newestFirst.length === childCount && + oldestFirst.length === childCount && + newestFirstSelectedNewest && + oldestFirstSelectedNewest; + if (!ok) { + throw new Error(`completion dedupe invariant failed for ${childCount} children`); + } + return { + durationMs, + invariant: { + ok, + inputRowsPerOrdering: newestFirstRows.length, + newestFirstSelectedRows: newestFirst.length, + oldestFirstSelectedRows: oldestFirst.length, + newestFirstSelectedNewest, + oldestFirstSelectedNewest, + }, + }; +} + +async function runScenario(options: WorkerOptions, stateDir: string): Promise { + const timingsMs: number[] = []; + let invariant: Record = {}; + const rssStartBytes = process.memoryUsage().rss; + for (let index = 0; index < options.warmup + options.runs; index += 1) { + let sample: Sample; + if (options.scenario === "spawnPipelineInMemory") { + sample = await runSpawnSample(options.size, index, "memory", stateDir); + } else if (options.scenario === "spawnPipelineDurable") { + sample = await runSpawnSample(options.size, index, "durable", stateDir); + } else if (options.scenario === "admission") { + sample = await runAdmissionSample(options.size, index); + } else if (options.scenario === "recoverySweep") { + sample = await runSweepSample(options.size); + } else { + sample = await runDedupeSample(options.size); + } + invariant = sample.invariant; + if (index >= options.warmup) { + timingsMs.push(sample.durationMs); + } + } + return { + scenario: options.scenario, + size: options.size, + timingsMs, + memory: { + rssStartBytes, + rssEndBytes: process.memoryUsage().rss, + processMaxRssBytes: processMaxRssBytes(), + }, + invariant, + }; +} + +async function main(): Promise { + const options = parseOptions(process.argv.slice(2)); + const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-agent-concurrency-worker-")); + const previousStateDir = process.env.OPENCLAW_STATE_DIR; + const previousNodeEnv = process.env.NODE_ENV; + let result: WorkerResult | undefined; + let failure: unknown; + process.env.OPENCLAW_STATE_DIR = stateDir; + process.env.NODE_ENV = "test"; + try { + const { pinRuntimePaths } = await import("../src/config/paths.js"); + pinRuntimePaths(); + result = await runScenario(options, stateDir); + } catch (error) { + failure = error; + } finally { + try { + await resetRuntime(false); + } catch (error) { + failure ??= error; + } finally { + if (previousStateDir === undefined) { + delete process.env.OPENCLAW_STATE_DIR; + } else { + process.env.OPENCLAW_STATE_DIR = previousStateDir; + } + if (previousNodeEnv === undefined) { + delete process.env.NODE_ENV; + } else { + process.env.NODE_ENV = previousNodeEnv; + } + try { + fs.rmSync(stateDir, { recursive: true, force: true }); + } catch (error) { + failure ??= error; + } + } + } + if (failure) { + throw toErrorObject(failure, "Agent concurrency benchmark failed"); + } + if (!result) { + throw new Error("benchmark worker completed without a result"); + } + process.stdout.write(`${WORKER_RESULT_SENTINEL}${JSON.stringify(result)}\n`); +} + +export const testing = { drainSpawnSampleActiveWork }; + +if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { + try { + await main(); + } catch (error) { + console.error(error instanceof Error ? error.stack : String(error)); + process.exitCode = 1; + } finally { + if (process.exitCode && process.exitCode !== 0) { + console.error(`[bench-agent-concurrency-worker] FAILED (exit ${process.exitCode})`); + } + } +} diff --git a/scripts/bench-agent-concurrency.ts b/scripts/bench-agent-concurrency.ts new file mode 100644 index 0000000000000000000000000000000000000000..4ba915dfeedc7f43e9e6471eea9a122f8e100680 --- /dev/null +++ b/scripts/bench-agent-concurrency.ts @@ -0,0 +1,394 @@ +import { pathToFileURL } from "node:url"; +import { isRecord } from "@openclaw/normalization-core/record-coerce"; +import { + emitBenchmarkReport, + parseBenchmarkInteger, + parseBenchmarkIntegerList, + parseBenchmarkOptions, + parseBenchmarkWorkerResult, + runBenchmarkEntrypoint, + runBenchmarkJobs, + runBenchmarkWorker, + summarizeBenchmarkTimings, + type BenchmarkTimingSummary, + type BenchmarkWorkerProcessResult, +} from "./lib/benchmark-harness.mts"; + +const DEFAULT_FANOUT = [1, 8, 32, 64]; +const DEFAULT_SWEEP_ROWS = [32, 128, 512]; +const WORKER_TIMEOUT_MS = 300_000; +export const WORKER_RESULT_SENTINEL = "[bench-agent-concurrency-result] "; + +export type WorkerScenario = + | "spawnPipelineInMemory" + | "spawnPipelineDurable" + | "admission" + | "recoverySweep" + | "duplicateSuppression"; + +export type WorkerResult = { + scenario: WorkerScenario; + size: number; + timingsMs: number[]; + memory: { + rssStartBytes: number; + rssEndBytes: number; + processMaxRssBytes: number; + }; + invariant: Record; +}; + +type Options = { + runs: number; + warmup: number; + fanout: number[]; + sweepRows: number[]; + output?: string; + json: boolean; + help: boolean; +}; + +const SCENARIO_SPECS: ReadonlyArray<{ + scenario: WorkerScenario; + sizes: "fanout" | "sweepRows"; +}> = [ + { scenario: "spawnPipelineInMemory", sizes: "fanout" }, + { scenario: "spawnPipelineDurable", sizes: "fanout" }, + { scenario: "admission", sizes: "fanout" }, + { scenario: "recoverySweep", sizes: "sweepRows" }, + { scenario: "duplicateSuppression", sizes: "sweepRows" }, +]; + +const REQUIRED_INVARIANT_FIELDS: Record = { + spawnPipelineInMemory: [ + "ok", + "registeredRuns", + "reservationsReleased", + "blockedWaits", + "settledRuns", + "settledTasks", + "outstandingWaits", + "durableSubagentRows", + "durableTaskRows", + "durableStateFile", + "postTeardownRegistryRows", + "postTeardownTaskRows", + "postTeardownDurableSubagentRows", + "postTeardownDurableTaskRows", + "postTeardownActiveRootWork", + ], + spawnPipelineDurable: [ + "ok", + "registeredRuns", + "reservationsReleased", + "blockedWaits", + "settledRuns", + "settledTasks", + "outstandingWaits", + "durableSubagentRows", + "durableTaskRows", + "durableStateFile", + "postTeardownRegistryRows", + "postTeardownTaskRows", + "postTeardownDurableSubagentRows", + "postTeardownDurableTaskRows", + "postTeardownActiveRootWork", + ], + admission: ["ok", "admissionCap", "overflowRejected", "released"], + recoverySweep: [ + "ok", + "seededRows", + "removedRows", + "retainedCurrent", + "sessionEffects", + "recoveryProjections", + "lostContextCompletions", + ], + duplicateSuppression: [ + "ok", + "inputRowsPerOrdering", + "newestFirstSelectedRows", + "oldestFirstSelectedRows", + "newestFirstSelectedNewest", + "oldestFirstSelectedNewest", + ], +}; + +type BenchmarkRuntime = { + runWorker?: typeof runWorker; + writeProgress?: (line: string) => void; + now?: () => number; +}; + +function usage(): string { + return `OpenClaw agent concurrency benchmark + +Usage: + node --import tsx scripts/bench-agent-concurrency.ts [options] + +Options: + --runs Measured samples per scenario (default: 5) + --warmup Warmup samples per scenario (default: 1) + --fanout Comma-separated spawn/admission sizes (default: 1,8,32,64) + --sweep-rows Comma-separated child counts, with 3 generations each (default: 32,128,512) + --output Write the JSON report to a file + --json Print only the JSON report + --help Show this text +`; +} + +function parseOptions(argv: string[]): Options { + return parseBenchmarkOptions( + argv, + { + runs: 5, + warmup: 1, + fanout: DEFAULT_FANOUT, + sweepRows: DEFAULT_SWEEP_ROWS, + json: false, + help: false, + }, + { + "--runs": (options, value) => { + options.runs = parseBenchmarkInteger(value, "--runs", 1, 100); + }, + "--warmup": (options, value) => { + options.warmup = parseBenchmarkInteger(value, "--warmup", 0, 20); + }, + "--fanout": (options, value) => { + options.fanout = parseBenchmarkIntegerList(value, "--fanout", 256); + }, + "--sweep-rows": (options, value) => { + options.sweepRows = parseBenchmarkIntegerList(value, "--sweep-rows", 4096); + }, + "--output": (options, value) => { + options.output = value; + }, + }, + ); +} + +function expectedWorkerKeys(options: Options): string[] { + return SCENARIO_SPECS.flatMap(({ scenario, sizes }) => + options[sizes].map((size) => `${scenario}:${size}`), + ); +} + +function aggregateWorkerResults( + options: Options, + workers: WorkerResult[], + parentMemory = { + rssStartBytes: process.memoryUsage().rss, + rssEndBytes: process.memoryUsage().rss, + }, +) { + const expected = expectedWorkerKeys(options); + const byKey = new Map(workers.map((worker) => [`${worker.scenario}:${worker.size}`, worker])); + if (byKey.size !== workers.length) { + throw new Error("worker results contain duplicate scenario/size pairs"); + } + const missing = expected.filter((key) => !byKey.has(key)); + const unexpected = [...byKey.keys()].filter((key) => !expected.includes(key)); + if (missing.length > 0 || unexpected.length > 0) { + throw new Error( + `worker result mismatch: missing=${missing.join(",") || "none"} unexpected=${unexpected.join(",") || "none"}`, + ); + } + + const scenarios = Object.fromEntries( + SCENARIO_SPECS.map(({ scenario, sizes }) => [ + scenario, + options[sizes].map((size) => { + const worker = byKey.get(`${scenario}:${size}`); + if (!worker) { + throw new Error(`missing worker result for ${scenario}:${size}`); + } + return { + size, + timingsMs: summarizeBenchmarkTimings(worker.timingsMs), + memory: worker.memory, + invariant: worker.invariant, + }; + }), + ]), + ) as Record< + WorkerScenario, + Array<{ + size: number; + timingsMs: BenchmarkTimingSummary; + memory: WorkerResult["memory"]; + invariant: WorkerResult["invariant"]; + }> + >; + + const checks = { + spawnPipelineInMemory: scenarios.spawnPipelineInMemory.every( + (entry) => entry.invariant.ok === true, + ), + spawnPipelineDurable: scenarios.spawnPipelineDurable.every( + (entry) => entry.invariant.ok === true, + ), + admissionCapOverflowRelease: scenarios.admission.every((entry) => entry.invariant.ok === true), + sweepRecoveryRowsWithoutSessionEffects: scenarios.recoverySweep.every( + (entry) => entry.invariant.ok === true, + ), + dedupeNewestPerChild: scenarios.duplicateSuppression.every( + (entry) => entry.invariant.ok === true, + ), + }; + const failures = Object.entries(checks) + .filter(([, ok]) => !ok) + .map(([name]) => name); + + return { + schemaVersion: 2, + generatedAt: new Date().toISOString(), + runtime: { node: process.version, platform: process.platform, arch: process.arch }, + options: { + runs: options.runs, + warmup: options.warmup, + fanout: options.fanout, + sweepRows: options.sweepRows, + }, + memory: { + ...parentMemory, + workerProcessMaxRssBytes: Math.max( + ...workers.map((worker) => worker.memory.processMaxRssBytes), + ), + }, + scenarios, + invariants: { ok: failures.length === 0, failures, ...checks }, + }; +} + +function assertFiniteNonNegative(value: unknown, field: string): asserts value is number { + if (typeof value !== "number" || !Number.isFinite(value) || value < 0) { + throw new Error(`worker result field ${field} must be a finite nonnegative number`); + } +} + +function validateWorkerResult( + value: unknown, + expected: { scenario: WorkerScenario; size: number; runs: number }, +): WorkerResult { + if (!isRecord(value)) { + throw new Error("worker result must be an object"); + } + if (value.scenario !== expected.scenario || value.size !== expected.size) { + throw new Error(`worker ${expected.scenario}:${expected.size} returned mismatched identity`); + } + if (!Array.isArray(value.timingsMs) || value.timingsMs.length !== expected.runs) { + throw new Error( + `worker ${expected.scenario}:${expected.size} returned ${Array.isArray(value.timingsMs) ? value.timingsMs.length : "invalid"} samples; expected ${expected.runs}`, + ); + } + value.timingsMs.forEach((timing, index) => + assertFiniteNonNegative(timing, `timingsMs[${index}]`), + ); + if (!isRecord(value.memory)) { + throw new Error("worker result memory must be an object"); + } + assertFiniteNonNegative(value.memory.rssStartBytes, "memory.rssStartBytes"); + assertFiniteNonNegative(value.memory.rssEndBytes, "memory.rssEndBytes"); + assertFiniteNonNegative(value.memory.processMaxRssBytes, "memory.processMaxRssBytes"); + if (!isRecord(value.invariant)) { + throw new Error("worker result invariant must be an object"); + } + for (const field of REQUIRED_INVARIANT_FIELDS[expected.scenario]) { + const invariantValue = value.invariant[field]; + if (typeof invariantValue !== "number" && typeof invariantValue !== "boolean") { + throw new Error(`worker result invariant.${field} is missing or invalid`); + } + } + if (value.invariant.ok !== true) { + throw new Error(`worker ${expected.scenario}:${expected.size} reported a failed invariant`); + } + return value as WorkerResult; +} + +function parseWorkerProcessResult( + result: BenchmarkWorkerProcessResult, + expected: { scenario: WorkerScenario; size: number; runs: number }, +): WorkerResult { + return parseBenchmarkWorkerResult({ + result, + label: `${expected.scenario}:${expected.size}`, + sentinel: WORKER_RESULT_SENTINEL, + timeoutMs: WORKER_TIMEOUT_MS, + validate: (value) => validateWorkerResult(value, expected), + }); +} + +function runWorker(options: Options, scenario: WorkerScenario, size: number): WorkerResult { + return runBenchmarkWorker({ + args: [ + "--import", + "tsx", + "scripts/bench-agent-concurrency-worker.ts", + "--scenario", + scenario, + "--size", + String(size), + "--runs", + String(options.runs), + "--warmup", + String(options.warmup), + ], + label: `${scenario}:${size}`, + sentinel: WORKER_RESULT_SENTINEL, + timeoutMs: WORKER_TIMEOUT_MS, + validate: (value) => validateWorkerResult(value, { scenario, size, runs: options.runs }), + }); +} + +function benchmark(options: Options, runtime: BenchmarkRuntime = {}) { + const rssStartBytes = process.memoryUsage().rss; + const jobs = SCENARIO_SPECS.flatMap(({ scenario, sizes }) => + options[sizes].map((size) => ({ scenario, size })), + ); + const run = runtime.runWorker ?? runWorker; + const workers = runBenchmarkJobs(jobs, { + prefix: "bench-agent-concurrency", + describe: ({ scenario, size }) => `scenario=${scenario} size=${size}`, + run: ({ scenario, size }) => run(options, scenario, size), + now: runtime.now, + writeProgress: runtime.writeProgress, + }); + return aggregateWorkerResults(options, workers, { + rssStartBytes, + rssEndBytes: process.memoryUsage().rss, + }); +} + +async function main(argv = process.argv.slice(2)): Promise { + const options = parseOptions(argv); + if (options.help) { + process.stdout.write(usage()); + return; + } + const report = benchmark(options); + emitBenchmarkReport(report, options, (result) => [ + ...Object.entries(result.scenarios).flatMap(([name, scenarios]) => + scenarios.map((scenario) => { + const tail = + scenario.timingsMs.p95 === undefined + ? "" + : ` p95=${scenario.timingsMs.p95.toFixed(3)}ms p99=${scenario.timingsMs.p99?.toFixed(3)}ms`; + return `${name} size=${scenario.size} p50=${scenario.timingsMs.p50.toFixed(3)}ms max=${scenario.timingsMs.max.toFixed(3)}ms${tail}`; + }), + ), + `max worker RSS ${(result.memory.workerProcessMaxRssBytes / 1024 / 1024).toFixed(1)} MiB`, + ]); +} + +export const testing = { + aggregateWorkerResults, + benchmark, + parseOptions, + parseWorkerProcessResult, + summarizeTimings: summarizeBenchmarkTimings, +}; + +if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { + await runBenchmarkEntrypoint("bench-agent-concurrency", main); +} diff --git a/scripts/bench-cli-startup.ts b/scripts/bench-cli-startup.ts new file mode 100644 index 0000000000000000000000000000000000000000..29f8ae5a4250e383585ba67b7dd676509b5aa7ab --- /dev/null +++ b/scripts/bench-cli-startup.ts @@ -0,0 +1,1747 @@ +// Bench Cli Startup script supports OpenClaw repository automation. +import { execFileSync, spawn } from "node:child_process"; +import { + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + realpathSync, + rmSync, + writeFileSync, +} from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { expectDefined } from "../packages/normalization-core/src/expect.js"; +import { + assertCompatibleCliStartupExecutionModes, + assertCompatibleCliStartupMemoryMetrics, + CLI_RUNTIME_MEMORY_METRIC, + type CliStartupExecutionMode, + cliStartupMemoryMetric, +} from "./lib/cli-startup-memory-contract.mts"; +import { + inspectManagedProcessGroup, + terminateManagedChild, + waitForManagedProcessGroupExit, +} from "./lib/managed-child-process.mts"; + +type CommandCase = { + id: string; + name: string; + args: string[]; + presets: readonly string[]; + stateScope?: "case" | "sample"; + expectedExitCodes?: readonly number[]; + expectedNonzeroOutputIncludes?: readonly string[]; + firstOutputBudgetMs?: number; + exitBudgetMs?: number; +}; + +type Sample = { + ms: number; + firstOutputMs: number | null; + maxRssMb: number | null; + memory?: SampleMemory; + exitCode: number | null; + signal: string | null; + startedAt?: string; + endedAt?: string; + timedOut?: boolean; + stdoutTail?: string; + stderrTail?: string; +}; + +type RssObservation = { + pid: number; + parentPid: number; + matchesArguments: boolean; + matchesInvocation: boolean; + maxRssBytes: number | null; +}; + +type SampleMemory = { + runtimePid: number | null; + processes: Array<{ + pid: number; + parentPid: number; + role: "runtime" | "launcher" | "auxiliary" | "unresolved"; + metricKind: "process-high-water-rss"; + maxRssBytes: number | null; + }>; + error?: string; +}; + +type CaseRuns = { + warmupSamples: Sample[]; + samples: Sample[]; +}; + +type SummaryStats = { + avg: number; + p50: number; + p95: number; + min: number; + max: number; +}; + +type CaseSummary = { + sampleCount: number; + durationMs: SummaryStats; + firstOutputMs: SummaryStats | null; + maxRssMb: SummaryStats | null; + exitSummary: string; +}; + +type SuiteResult = { + entry: string; + executionMode?: CliStartupExecutionMode; + memoryMetric?: string; + cases: Array<{ + id: string; + name: string; + args: string[]; + expectedExitCodes?: number[]; + expectedNonzeroOutputIncludes?: string[]; + contract: { + firstOutputBudgetMs: number | null; + exitBudgetMs: number | null; + } | null; + warmupSamples?: Sample[]; + samples: Sample[]; + summary: CaseSummary; + }>; +}; + +type BenchmarkReport = { + primary: SuiteResult; + secondary?: SuiteResult | null; +}; + +type CaseDelta = { + id: string; + name: string; + durationAvgDeltaMs: number; + durationAvgDeltaPct: number; + maxRssAvgDeltaMb: number | null; + maxRssAvgDeltaPct: number | null; +}; + +type BenchmarkComparison = { + baseline: string; + candidate: string; + deltas: CaseDelta[]; +}; + +type BenchmarkComparisonResult = { + baseline: SuiteResult; + candidate: SuiteResult; + comparison: BenchmarkComparison; +}; + +type CliOptions = { + cases: CommandCase[]; + compareBaseline?: string; + compareCandidate?: string; + entryPrimary: string; + entrySecondary?: string; + runs: number; + warmup: number; + timeoutMs: number; + runtimeRss: boolean; + json: boolean; + output?: string; + cpuProfDir?: string; + heapProfDir?: string; +}; + +const DEFAULT_RUNS = 5; +const DEFAULT_WARMUP = 1; +const DEFAULT_TIMEOUT_MS = 30_000; +const DEFAULT_TIMEOUT_KILL_GRACE_MS = 1_000; +const TIMEOUT_KILL_GRACE_MS = resolveTimeoutKillGraceMs(process.env); +const DEFAULT_ENTRY = "openclaw.mjs"; +const MAX_RSS_MARKER = "__OPENCLAW_MAX_RSS_KB__="; + +type SampleTransport = { + prefix: string[]; + binary: string; + env: Record; + cwd: string; +}; + +function sampleTransport(): SampleTransport | undefined { + const raw = process.env.OPENCLAW_BENCH_TRANSPORT_JSON; + if (raw === undefined) { + return undefined; + } + const value: unknown = JSON.parse(raw); + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error("Invalid benchmark transport"); + } + const prefix = "prefix" in value ? value.prefix : undefined; + const binary = "binary" in value ? value.binary : undefined; + const env = "env" in value ? value.env : undefined; + const cwd = "cwd" in value ? value.cwd : undefined; + if ( + !Array.isArray(prefix) || + prefix.length === 0 || + !prefix.every( + (part: unknown) => typeof part === "string" && part.length > 0 && !part.includes("\0"), + ) || + typeof prefix[0] !== "string" || + !path.isAbsolute(prefix[0]) || + typeof binary !== "string" || + !path.isAbsolute(binary) || + binary.includes("\0") || + !env || + typeof env !== "object" || + Array.isArray(env) + ) { + throw new Error("Invalid benchmark transport"); + } + const fixedEnv: Record = {}; + for (const [key, entry] of Object.entries(env)) { + if (!/^[A-Z_][A-Z0-9_]*$/u.test(key) || typeof entry !== "string" || entry.includes("\0")) { + throw new Error("Invalid benchmark transport environment"); + } + fixedEnv[key] = entry; + } + if (!fixedEnv.HOME || !path.isAbsolute(fixedEnv.HOME) || !fixedEnv.PATH) { + throw new Error("Benchmark transport requires fixed HOME and PATH"); + } + const directory = cwd === undefined ? fixedEnv.HOME : cwd; + if (typeof directory !== "string" || !path.isAbsolute(directory) || directory.includes("\0")) { + throw new Error("Invalid benchmark transport cwd"); + } + return { prefix, binary, env: fixedEnv, cwd: directory }; +} + +function transportedCommand( + transport: SampleTransport, + args: string[], + env: Record = {}, + timeoutMs?: number, +): { command: string; args: string[] } { + return { + command: expectDefined(transport.prefix[0], "benchmark transport command"), + args: [ + ...transport.prefix.slice(1), + "/usr/bin/env", + "-C", + transport.cwd, + "-i", + ...Object.entries({ ...transport.env, ...env }).map(([key, value]) => `${key}=${value}`), + ...(timeoutMs === undefined + ? [] + : ["/usr/bin/timeout", "--signal=TERM", "--kill-after=1s", `${timeoutMs / 1000}s`]), + transport.binary, + ...args, + ], + }; +} + +function sampleFilesystem( + transport: SampleTransport, + operation: "create" | "prepare" | "remove", + root?: string, + config?: Record | null, + hook?: string, +): string { + // Only the SUT opens these paths. Its replies are never read as paths on the runner. + const launch = transportedCommand(transport, [ + "--input-type=module", + "-e", + `import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +const {operation,root,config,hook} = JSON.parse(fs.readFileSync(0,"utf8")); +if (operation === "create") process.stdout.write(fs.mkdtempSync(path.join(os.tmpdir(),"openclaw-cli-bench-home-"))); +else if (operation === "prepare") { + fs.mkdirSync(path.join(root,".openclaw"),{recursive:true}); + if (config) fs.writeFileSync(path.join(root,".openclaw/openclaw.json"),JSON.stringify(config)+"\\n"); + fs.writeFileSync(path.join(root,"measure-rss.mjs"),hook); +} else if (operation === "remove") fs.rmSync(root,{recursive:true,force:true}); +else throw new Error("Invalid sample filesystem operation");`, + ]); + return execFileSync(launch.command, launch.args, { + input: JSON.stringify({ operation, root, config, hook }), + encoding: "utf8", + timeout: 10_000, + maxBuffer: 64 * 1024, + env: { PATH: process.env.PATH }, + }); +} + +function createSampleRoot(transport?: SampleTransport): string { + if (!transport) { + return mkdtempSync(path.join(os.tmpdir(), "openclaw-cli-bench-home-")); + } + const root = sampleFilesystem(transport, "create"); + if (!path.isAbsolute(root) || /[\0\r\n]/u.test(root)) { + throw new Error("Invalid SUT sample directory"); + } + return root; +} + +function removeSampleRoot(root: string, transport?: SampleTransport): void { + if (transport) { + sampleFilesystem(transport, "remove", root); + } else { + rmSync(root, { recursive: true, force: true }); + } +} + +function resolveTimeoutKillGraceMs(env: NodeJS.ProcessEnv): number { + const raw = env.VITEST ? env.OPENCLAW_TEST_CLI_STARTUP_TIMEOUT_KILL_GRACE_MS : undefined; + if (!raw || !/^\d+$/u.test(raw)) { + return DEFAULT_TIMEOUT_KILL_GRACE_MS; + } + const parsed = Number(raw); + return Number.isSafeInteger(parsed) ? parsed : DEFAULT_TIMEOUT_KILL_GRACE_MS; +} +const VALUE_FLAGS = new Set([ + "--case", + "--compare-baseline", + "--compare-candidate", + "--cpu-prof-dir", + "--entry", + "--entry-primary", + "--entry-secondary", + "--heap-prof-dir", + "--output", + "--preset", + "--runs", + "--timeout-ms", + "--warmup", +]); +const BOOLEAN_FLAGS = new Set(["--help", "--json", "--runtime-rss"]); + +const COMMAND_CASES: readonly CommandCase[] = [ + { + id: "version", + name: "--version", + args: ["--version"], + presets: ["startup", "response"], + firstOutputBudgetMs: 1_000, + exitBudgetMs: 2_000, + }, + { + id: "help", + name: "--help", + args: ["--help"], + presets: ["startup", "response"], + firstOutputBudgetMs: 1_000, + exitBudgetMs: 2_000, + }, + { + id: "onboardHelp", + name: "onboard --help", + args: ["onboard", "--help"], + presets: ["response"], + firstOutputBudgetMs: 2_500, + exitBudgetMs: 6_000, + }, + { + id: "setupHelp", + name: "setup --help", + args: ["setup", "--help"], + presets: ["response"], + firstOutputBudgetMs: 2_500, + exitBudgetMs: 6_000, + }, + { + id: "configureHelp", + name: "configure --help", + args: ["configure", "--help"], + presets: ["response"], + firstOutputBudgetMs: 2_500, + exitBudgetMs: 6_000, + }, + { + id: "channelsAddHelp", + name: "channels add --help", + args: ["channels", "add", "--help"], + presets: ["response"], + firstOutputBudgetMs: 2_500, + exitBudgetMs: 6_000, + }, + { + id: "channelsParent", + name: "channels", + args: ["channels"], + presets: ["response"], + firstOutputBudgetMs: 2_500, + exitBudgetMs: 6_000, + }, + { + id: "doctorHelp", + name: "doctor --help", + args: ["doctor", "--help"], + presets: ["response"], + firstOutputBudgetMs: 2_500, + exitBudgetMs: 6_000, + }, + { + id: "modelsHelp", + name: "models --help", + args: ["models", "--help"], + presets: ["response"], + firstOutputBudgetMs: 2_500, + exitBudgetMs: 6_000, + }, + { + id: "pluginsHelp", + name: "plugins --help", + args: ["plugins", "--help"], + presets: ["response"], + firstOutputBudgetMs: 2_500, + exitBudgetMs: 6_000, + }, + { + id: "pluginsParent", + name: "plugins", + args: ["plugins"], + presets: ["response"], + firstOutputBudgetMs: 2_500, + exitBudgetMs: 6_000, + }, + { + id: "pluginsListJson", + name: "plugins list --json", + args: ["plugins", "list", "--json"], + presets: ["response", "real"], + firstOutputBudgetMs: 2_500, + exitBudgetMs: 6_000, + }, + { + id: "gatewayHelp", + name: "gateway --help", + args: ["gateway", "--help"], + presets: ["response"], + firstOutputBudgetMs: 2_500, + exitBudgetMs: 6_000, + }, + { + id: "agentsHelp", + name: "agents --help", + args: ["agents", "--help"], + presets: ["response"], + firstOutputBudgetMs: 3_500, + exitBudgetMs: 8_000, + }, + { + id: "sessionsHelp", + name: "sessions --help", + args: ["sessions", "--help"], + presets: ["response"], + firstOutputBudgetMs: 2_500, + exitBudgetMs: 6_000, + }, + { + id: "tasksHelp", + name: "tasks --help", + args: ["tasks", "--help"], + presets: ["response"], + firstOutputBudgetMs: 2_500, + exitBudgetMs: 6_000, + }, + { + id: "messageHelp", + name: "message --help", + args: ["message", "--help"], + presets: ["response"], + firstOutputBudgetMs: 2_500, + exitBudgetMs: 6_000, + }, + { + id: "pairingHelp", + name: "pairing --help", + args: ["pairing", "--help"], + presets: ["response"], + firstOutputBudgetMs: 2_500, + exitBudgetMs: 6_000, + }, + { + id: "authHelp", + name: "auth --help", + args: ["auth", "--help"], + presets: ["response"], + firstOutputBudgetMs: 2_500, + exitBudgetMs: 6_000, + }, + { + id: "configHelp", + name: "config --help", + args: ["config", "--help"], + presets: ["response"], + firstOutputBudgetMs: 2_500, + exitBudgetMs: 6_000, + }, + { + id: "secretsHelp", + name: "secrets --help", + args: ["secrets", "--help"], + presets: ["response"], + firstOutputBudgetMs: 2_500, + exitBudgetMs: 6_000, + }, + { + id: "skillsHelp", + name: "skills --help", + args: ["skills", "--help"], + presets: ["response"], + firstOutputBudgetMs: 2_500, + exitBudgetMs: 6_000, + }, + { + id: "nodesHelp", + name: "nodes --help", + args: ["nodes", "--help"], + presets: ["response"], + firstOutputBudgetMs: 3_500, + exitBudgetMs: 8_000, + }, + { + id: "directoryHelp", + name: "directory --help", + args: ["directory", "--help"], + presets: ["response"], + firstOutputBudgetMs: 2_500, + exitBudgetMs: 6_000, + }, + { + id: "sandboxHelp", + name: "sandbox --help", + args: ["sandbox", "--help"], + presets: ["response"], + firstOutputBudgetMs: 2_500, + exitBudgetMs: 6_000, + }, + { + id: "devicesParent", + name: "devices", + args: ["devices"], + presets: ["response"], + firstOutputBudgetMs: 2_500, + exitBudgetMs: 6_000, + }, + { + id: "mcpParent", + name: "mcp", + args: ["mcp"], + presets: ["response"], + firstOutputBudgetMs: 2_500, + exitBudgetMs: 6_000, + }, + { + id: "browserHelp", + name: "browser --help", + args: ["browser", "--help"], + presets: ["response"], + firstOutputBudgetMs: 1_500, + exitBudgetMs: 3_000, + }, + { + id: "webhooksHelp", + name: "webhooks --help", + args: ["webhooks", "--help"], + presets: ["response"], + firstOutputBudgetMs: 2_500, + exitBudgetMs: 6_000, + }, + { + id: "health", + name: "health", + args: ["health"], + presets: ["startup", "real"], + expectedExitCodes: [0, 1], + expectedNonzeroOutputIncludes: ["Gateway target:"], + }, + { + id: "healthJson", + name: "health --json", + args: ["health", "--json"], + presets: ["startup"], + expectedExitCodes: [0, 1], + expectedNonzeroOutputIncludes: ['"ok"', '"gateway_transport_error"'], + }, + { + id: "statusJson", + name: "status --json", + args: ["status", "--json"], + presets: ["startup", "real"], + }, + { id: "status", name: "status", args: ["status"], presets: ["startup", "real"] }, + { id: "sessions", name: "sessions", args: ["sessions"], presets: ["real"] }, + { + id: "sessionsJson", + name: "sessions --json", + args: ["sessions", "--json"], + presets: ["real"], + }, + { + id: "tasksJson", + name: "tasks --json", + args: ["tasks", "--json"], + presets: ["real"], + }, + { + id: "tasksListJson", + name: "tasks list --json", + args: ["tasks", "list", "--json"], + presets: ["real"], + }, + { + id: "tasksAuditJson", + name: "tasks audit --json", + args: ["tasks", "audit", "--json"], + presets: ["real"], + }, + { + id: "agentsListJson", + name: "agents list --json", + args: ["agents", "list", "--json"], + presets: ["real"], + }, + { + id: "gatewayStatus", + name: "gateway status", + args: ["gateway", "status"], + presets: ["real"], + }, + { + id: "gatewayStatusJson", + name: "gateway status --json", + args: ["gateway", "status", "--json"], + presets: ["real"], + }, + { + id: "gatewayHealthJson", + name: "gateway health --json", + args: ["gateway", "health", "--json"], + presets: ["real"], + expectedExitCodes: [0, 1], + expectedNonzeroOutputIncludes: ['"ok"', '"gateway_transport_error"'], + }, + { + id: "gatewayHealthJsonWarmState", + name: "gateway health --json (warm state)", + args: ["gateway", "health", "--json"], + presets: [], + stateScope: "case", + }, + { + id: "gatewayHealthJsonFreshState", + name: "gateway health --json (fresh state)", + args: ["gateway", "health", "--json"], + presets: [], + stateScope: "sample", + }, + { + id: "configGetGatewayPort", + name: "config get gateway.port", + args: ["config", "get", "gateway.port"], + presets: ["real"], + expectedExitCodes: [0, 1], + expectedNonzeroOutputIncludes: ["Config path not found: gateway.port"], + }, +] as const; + +function parseFlagValue(flag: string): string | undefined { + const idx = process.argv.indexOf(flag); + if (idx === -1) { + return undefined; + } + const value = process.argv[idx + 1]; + if (!value || value.startsWith("-")) { + throw new Error(`${flag} requires a value`); + } + return value; +} + +function hasFlag(flag: string): boolean { + return process.argv.includes(flag); +} + +function parseRepeatableFlag(flag: string): string[] { + const values: string[] = []; + for (let i = 0; i < process.argv.length; i += 1) { + const value = process.argv[i + 1]; + if (process.argv[i] === flag && value && !value.startsWith("-")) { + values.push(value); + } + } + return values; +} + +function validateCliArgs(argv: readonly string[] = process.argv.slice(2)): void { + const seenSingleValueFlags = new Set(); + for (let index = 0; index < argv.length; index += 1) { + const arg = expectDefined(argv[index], `CLI benchmark argument at index ${index}`); + if (VALUE_FLAGS.has(arg)) { + if (arg !== "--case") { + if (seenSingleValueFlags.has(arg)) { + throw new Error(`${arg} was provided more than once`); + } + seenSingleValueFlags.add(arg); + } + const value = argv[index + 1]; + if (!value || value.startsWith("-")) { + throw new Error(`${arg} requires a value`); + } + index += 1; + continue; + } + if (BOOLEAN_FLAGS.has(arg)) { + continue; + } + throw new Error(`Unknown argument: ${arg}`); + } +} + +function parsePositiveInt(raw: string | undefined, fallback: number, label = "value"): number { + return parseIntegerOption(raw, fallback, label, 1); +} + +function parseNonNegativeInt(raw: string | undefined, fallback: number, label = "value"): number { + return parseIntegerOption(raw, fallback, label, 0); +} + +// This runner is checked out from trusted main beside frozen candidates, whose +// root dependencies need not include current workspace packages. +function parseIntegerOption( + raw: string | undefined, + fallback: number, + label: string, + min: number, +): number { + const value = raw?.trim(); + if (!value) { + return fallback; + } + if (!/^\d+$/u.test(value)) { + throw new Error(`${label} must be an integer >= ${min}; got ${JSON.stringify(raw)}`); + } + const parsed = Number(value); + if (!Number.isSafeInteger(parsed) || parsed < min) { + throw new Error(`${label} must be an integer >= ${min}; got ${JSON.stringify(raw)}`); + } + return parsed; +} + +function parseGatewayPortEnv(raw: string | undefined): number { + const value = raw?.trim(); + if (!value) { + return 32123; + } + const bracketHostMatch = /^\[[^\]]+\]:(\d+)$/u.exec(value); + if (bracketHostMatch) { + return parsePositiveInt(bracketHostMatch[1], 32123, "OPENCLAW_GATEWAY_PORT"); + } + if (value.startsWith("[") && value.endsWith("]")) { + return 32123; + } + const colonCount = value.split(":").length - 1; + if (colonCount > 1) { + return 32123; + } + const portRaw = colonCount === 1 ? value.split(":")[1] : value; + return parsePositiveInt(portRaw, 32123, "OPENCLAW_GATEWAY_PORT"); +} + +function parsePresets(raw: string | undefined): string[] { + if (!raw) { + return ["startup"]; + } + const values = raw + .split(",") + .map((value) => value.trim()) + .filter(Boolean); + if (values.includes("all")) { + return ["startup", "real", "response"]; + } + return values.length > 0 ? values : ["startup"]; +} + +function resolveCases(options: { presets: string[]; caseIds: string[] }): CommandCase[] { + const byId = new Map(COMMAND_CASES.map((commandCase) => [commandCase.id, commandCase])); + if (options.caseIds.length > 0) { + const seenIds = new Set(); + return options.caseIds.map((id) => { + if (seenIds.has(id)) { + throw new Error(`Duplicate --case "${id}"`); + } + seenIds.add(id); + const commandCase = byId.get(id); + if (!commandCase) { + throw new Error(`Unknown --case "${id}"`); + } + return commandCase; + }); + } + return COMMAND_CASES.filter((commandCase) => + commandCase.presets.some((preset) => options.presets.includes(preset)), + ); +} + +function median(values: number[]): number { + if (values.length === 0) { + return 0; + } + const sorted = [...values].toSorted((a, b) => a - b); + const mid = Math.floor(sorted.length / 2); + if (sorted.length % 2 === 0) { + return ( + (expectDefined(sorted[mid - 1], "lower middle CLI benchmark sample") + + expectDefined(sorted[mid], "upper middle CLI benchmark sample")) / + 2 + ); + } + return expectDefined(sorted[mid], "middle CLI benchmark sample"); +} + +function percentile(values: number[], p: number): number { + if (values.length === 0) { + return 0; + } + const sorted = [...values].toSorted((a, b) => a - b); + const index = Math.min(sorted.length - 1, Math.floor((p / 100) * sorted.length)); + return sorted[index] ?? 0; +} + +function summarizeNumbers(values: number[]): SummaryStats { + const total = values.reduce((sum, value) => sum + value, 0); + const avg = values.length > 0 ? total / values.length : 0; + const min = values.length > 0 ? Math.min(...values) : 0; + const max = values.length > 0 ? Math.max(...values) : 0; + return { + avg, + p50: median(values), + p95: percentile(values, 95), + min, + max, + }; +} + +function summarizeSamples(samples: Sample[]): CaseSummary { + const durations = summarizeNumbers(samples.map((sample) => sample.ms)); + const firstOutputValues = samples + .map((sample) => sample.firstOutputMs) + .filter((value): value is number => typeof value === "number" && Number.isFinite(value)); + const rssValues = samples + .map((sample) => sample.maxRssMb) + .filter((value): value is number => typeof value === "number" && Number.isFinite(value)); + return { + sampleCount: samples.length, + durationMs: durations, + firstOutputMs: firstOutputValues.length > 0 ? summarizeNumbers(firstOutputValues) : null, + maxRssMb: rssValues.length > 0 ? summarizeNumbers(rssValues) : null, + exitSummary: collectExitSummary(samples), + }; +} + +function formatMs(value: number): string { + return `${value.toFixed(1)}ms`; +} + +function formatMb(value: number): string { + return `${value.toFixed(1)}MiB`; +} + +function collectExitSummary(samples: Sample[]): string { + const buckets = new Map(); + for (const sample of samples) { + const key = + sample.signal != null + ? `signal:${sample.signal}` + : `code:${sample.exitCode == null ? "null" : String(sample.exitCode)}`; + buckets.set(key, (buckets.get(key) ?? 0) + 1); + } + return [...buckets.entries()].map(([key, count]) => `${key}x${count}`).join(", "); +} + +function buildConfigFixture(commandCase: CommandCase): Record | null { + const usesSharedToken = + commandCase.id === "gatewayHealthJsonWarmState" || + commandCase.id === "gatewayHealthJsonFreshState"; + if ( + commandCase.id !== "configGetGatewayPort" && + commandCase.id !== "gatewayHealthJson" && + !usesSharedToken && + commandCase.id !== "health" && + commandCase.id !== "healthJson" + ) { + return null; + } + const port = parseGatewayPortEnv(process.env.OPENCLAW_GATEWAY_PORT); + return { + gateway: { + auth: { mode: usesSharedToken ? "token" : "none" }, + bind: "loopback", + mode: "local", + port, + }, + }; +} + +function buildRssHook(tmpDir: string): string { + const rssHookPath = path.join(tmpDir, "measure-rss.mjs"); + writeFileSync( + rssHookPath, + [ + "process.on('exit', () => {", + " const usage = typeof process.resourceUsage === 'function' ? process.resourceUsage() : null;", + ` if (usage && typeof usage.maxRSS === 'number') console.error('${MAX_RSS_MARKER}' + String(usage.maxRSS));`, + "});", + "", + ].join("\n"), + "utf8", + ); + return rssHookPath; +} + +function parseMaxRssMb(stderr: string): number | null { + const matches = [...stderr.matchAll(new RegExp(`^${MAX_RSS_MARKER}(\\d+)\\s*$`, "gm"))]; + const lastMatch = matches.at(-1); + if (!lastMatch?.[1]) { + return null; + } + return Number(lastMatch[1]) / 1024; +} + +function buildRuntimeRssHook(tmpDir: string): string { + const rssHookPath = path.join(tmpDir, "measure-rss.mjs"); + writeFileSync( + rssHookPath, + `import { writeFileSync, realpathSync } from "node:fs"; +import { join } from "node:path"; +import { isMainThread } from "node:worker_threads"; +if (isMainThread) { + const { directory, entries, args } = JSON.parse(process.env.OPENCLAW_BENCH_MEMORY); + let entry; + try { + if (process.argv[1]) entry = realpathSync(process.argv[1]); + } catch (error) { + if (error.code !== "ENOENT") throw error; + } + const matchesArguments = JSON.stringify(process.argv.slice(2)) === JSON.stringify(args); + const record = { + pid: process.pid, + parentPid: process.ppid, + matchesArguments, + matchesInvocation: entries.includes(entry) && matchesArguments, + maxRssBytes: null, + }; + const file = join(directory, String(process.pid) + ".json"); + writeFileSync(file, JSON.stringify(record)); + process.on("exit", () => { + record.maxRssBytes = process.resourceUsage().maxRSS * 1024; + writeFileSync(file, JSON.stringify(record)); + }); +} +`, + "utf8", + ); + return rssHookPath; +} + +function readSampleMemory(directory: string, entryPid: number | undefined): SampleMemory { + const memory: SampleMemory = { runtimePid: null, processes: [] }; + try { + const observations: RssObservation[] = readdirSync(directory).map((file) => { + const value: unknown = JSON.parse(readFileSync(path.join(directory, file), "utf8")); + if ( + typeof value !== "object" || + value === null || + !("pid" in value) || + typeof value.pid !== "number" || + !Number.isSafeInteger(value.pid) || + value.pid <= 0 || + file !== `${value.pid}.json` || + !("parentPid" in value) || + typeof value.parentPid !== "number" || + !Number.isSafeInteger(value.parentPid) || + value.parentPid < 0 || + !("matchesArguments" in value) || + typeof value.matchesArguments !== "boolean" || + !("matchesInvocation" in value) || + typeof value.matchesInvocation !== "boolean" || + !("maxRssBytes" in value) || + !( + value.maxRssBytes === null || + (typeof value.maxRssBytes === "number" && + Number.isSafeInteger(value.maxRssBytes) && + value.maxRssBytes > 0) + ) + ) { + throw new Error("invalid process RSS observation"); + } + return { + pid: value.pid, + parentPid: value.parentPid, + matchesArguments: value.matchesArguments, + matchesInvocation: value.matchesInvocation, + maxRssBytes: value.maxRssBytes, + }; + }); + memory.processes = observations.map((record) => ({ + pid: record.pid, + parentPid: record.parentPid, + role: record.matchesInvocation ? "unresolved" : "auxiliary", + metricKind: "process-high-water-rss", + maxRssBytes: record.maxRssBytes, + })); + if (observations.some((record) => record.matchesArguments && !record.matchesInvocation)) { + throw new Error("unrecognized CLI entry with matching command arguments"); + } + const invocation = observations.filter((record) => record.matchesInvocation); + const entryObservation = invocation.find((record) => record.pid === entryPid); + if (!entryObservation) { + throw new Error("missing CLI entry process identity"); + } + let current: RssObservation = entryObservation; + const lineage = new Set(); + // Respawns preserve CLI argv and the preload. Follow the unique invocation + // chain, not exit order, RSS magnitude, or platform-specific ready flags. + while (true) { + if (lineage.has(current.pid)) { + throw new Error("cyclic CLI process identity"); + } + lineage.add(current.pid); + if (current.maxRssBytes === null) { + throw new Error(`missing process high-water RSS for CLI PID ${current.pid}`); + } + const parentPid = current.pid; + const children = invocation.filter((record) => record.parentPid === parentPid); + if (children.length > 1) { + throw new Error("ambiguous CLI runtime identity: multiple matching children"); + } + const child = children[0]; + if (!child) { + break; + } + current = child; + } + if (lineage.size !== invocation.length) { + throw new Error("disconnected CLI runtime identity"); + } + memory.runtimePid = current.pid; + for (const record of memory.processes) { + if (lineage.has(record.pid)) { + record.role = record.pid === current.pid ? "runtime" : "launcher"; + } + } + } catch (error) { + memory.error = error instanceof Error ? error.message : String(error); + } + return memory; +} + +function memoryInvocationEntries(entry: string): string[] { + const resolvedEntry = realpathSync(entry); + const entries = [resolvedEntry]; + // The wrapper's compile-cache handoff can enter dist directly. + if (path.basename(resolvedEntry) === "openclaw.mjs") { + for (const name of ["entry.js", "entry.mjs"]) { + try { + entries.push(realpathSync(path.join(path.dirname(resolvedEntry), "dist", name))); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") { + throw error; + } + } + } + } + return entries; +} + +function nodeImportSpecifierForPath(filePath: string): string { + return pathToFileURL(filePath).href; +} + +function buildCpuOrHeapFlags(options: { cpuProfDir?: string; heapProfDir?: string }): string[] { + const flags: string[] = []; + if (options.cpuProfDir) { + flags.push("--cpu-prof", "--cpu-prof-dir", options.cpuProfDir); + } + if (options.heapProfDir) { + flags.push("--heap-prof", "--heap-prof-dir", options.heapProfDir); + } + return flags; +} + +function appendLimited(current: string, chunk: Buffer | string, maxLength: number): string { + const next = current + String(chunk); + return next.length > maxLength ? next.slice(next.length - maxLength) : next; +} + +async function runSample(params: { + entry: string; + commandCase: CommandCase; + timeoutMs: number; + runtimeRss: boolean; + cpuProfDir?: string; + heapProfDir?: string; + rssHookPath: string; + runRoot?: string; +}): Promise { + const transport = sampleTransport(); + const runRoot = params.runRoot ?? createSampleRoot(transport); + const ownsRunRoot = params.runRoot == null; + const stateDir = path.join(runRoot, ".openclaw"); + const configPath = path.join(stateDir, "openclaw.json"); + const configFixture = buildConfigFixture(params.commandCase); + let rssHookPath = params.rssHookPath; + if (transport) { + sampleFilesystem( + transport, + "prepare", + runRoot, + configFixture, + readFileSync(rssHookPath, "utf8"), + ); + rssHookPath = path.join(runRoot, "measure-rss.mjs"); + } else if (configFixture) { + mkdirSync(stateDir, { recursive: true }); + writeFileSync(configPath, `${JSON.stringify(configFixture, null, 2)}\n`, "utf8"); + } + const nodeArgs = [ + ...(transport + ? [ + "--import", + `data:text/javascript,${encodeURIComponent(`process.chdir(${JSON.stringify(path.dirname(params.entry))});`)}`, + ] + : []), + "--import", + nodeImportSpecifierForPath(rssHookPath), + ...buildCpuOrHeapFlags({ + cpuProfDir: params.cpuProfDir, + heapProfDir: params.heapProfDir, + }), + params.entry, + ...params.commandCase.args, + ]; + const startedAt = new Date(); + const started = process.hrtime.bigint(); + let firstOutputMs: number | null = null; + let stdout = ""; + let stderr = ""; + let settled = false; + let timedOut = false; + let forceKillAt: number | null = null; + let forceKillTimer: ReturnType | null = null; + const maxOutputLength = 32 * 1024 * 1024; + const memoryDirectory = params.runtimeRss + ? mkdtempSync(path.join(path.dirname(params.rssHookPath), "sample-")) + : undefined; + + try { + return await new Promise((resolve) => { + const sampleEnv = { + HOME: runRoot, + USERPROFILE: runRoot, + OPENCLAW_HOME: runRoot, + OPENCLAW_STATE_DIR: stateDir, + OPENCLAW_CONFIG_PATH: configPath, + OPENCLAW_HIDE_BANNER: "1", + NO_COLOR: "1", + FORCE_COLOR: "0", + ...(memoryDirectory + ? { + OPENCLAW_BENCH_MEMORY: JSON.stringify({ + directory: memoryDirectory, + entries: memoryInvocationEntries(params.entry), + args: params.commandCase.args, + }), + } + : {}), + }; + const launch = transport + ? transportedCommand( + transport, + nodeArgs, + { + ...sampleEnv, + ...(process.env.OPENCLAW_GATEWAY_TOKEN + ? { OPENCLAW_GATEWAY_TOKEN: process.env.OPENCLAW_GATEWAY_TOKEN } + : {}), + ...(process.env.OPENCLAW_GATEWAY_PORT + ? { OPENCLAW_GATEWAY_PORT: process.env.OPENCLAW_GATEWAY_PORT } + : {}), + }, + params.timeoutMs, + ) + : { command: process.execPath, args: nodeArgs }; + const proc = spawn(launch.command, launch.args, { + cwd: process.cwd(), + detached: process.platform !== "win32", + env: transport ? { PATH: process.env.PATH } : { ...process.env, ...sampleEnv }, + stdio: ["ignore", "pipe", "pipe"], + }); + + const finish = (sample: Omit) => { + if (settled) { + return; + } + settled = true; + if (forceKillTimer) { + clearTimeout(forceKillTimer); + forceKillTimer = null; + } + const ms = Number(process.hrtime.bigint() - started) / 1e6; + const memory = memoryDirectory ? readSampleMemory(memoryDirectory, proc.pid) : undefined; + const runtimeRss = memory?.processes.find( + (record) => record.role === "runtime", + )?.maxRssBytes; + resolve({ + ms, + firstOutputMs, + maxRssMb: memory + ? runtimeRss == null + ? null + : runtimeRss / 1024 / 1024 + : parseMaxRssMb(stderr), + ...(memory ? { memory } : {}), + startedAt: startedAt.toISOString(), + endedAt: new Date().toISOString(), + ...(timedOut ? { timedOut } : {}), + ...sample, + }); + }; + + const markFirstOutput = () => { + if (firstOutputMs == null) { + firstOutputMs = Number(process.hrtime.bigint() - started) / 1e6; + } + }; + + const timeout = setTimeout(() => { + timedOut = true; + signalSampleProcess(proc, "SIGTERM"); + forceKillAt = Date.now() + TIMEOUT_KILL_GRACE_MS; + forceKillTimer = setTimeout(() => { + signalSampleProcess(proc, "SIGKILL"); + }, TIMEOUT_KILL_GRACE_MS).unref?.(); + }, params.timeoutMs); + timeout.unref?.(); + + proc.stdout?.on("data", (chunk) => { + markFirstOutput(); + stdout = appendLimited(stdout, chunk, maxOutputLength); + }); + proc.stderr?.on("data", (chunk) => { + markFirstOutput(); + stderr = appendLimited(stderr, chunk, maxOutputLength); + }); + proc.once("error", (error) => { + clearTimeout(timeout); + stderr = appendLimited( + stderr, + error instanceof Error ? error.message : String(error), + maxOutputLength, + ); + finish({ + exitCode: null, + signal: null, + stdoutTail: tailLines(stdout, 20), + stderrTail: tailLines(stderr, 20), + }); + }); + proc.once("close", (code, signal) => { + clearTimeout(timeout); + const complete = () => + finish({ + exitCode: code, + signal, + ...(code === 0 && signal == null + ? {} + : { + stdoutTail: tailLines(stdout, 20), + stderrTail: tailLines(stderr, 20), + }), + }); + if (timedOut && isSampleProcessGroupAlive(proc)) { + void finishAfterTimeoutCleanup({ + complete, + forceKillAt, + proc, + }); + return; + } + complete(); + }); + }); + } finally { + if (memoryDirectory) { + rmSync(memoryDirectory, { recursive: true, force: true }); + } + if (ownsRunRoot) { + removeSampleRoot(runRoot, transport); + } + } +} + +async function finishAfterTimeoutCleanup(params: { + complete: () => void; + forceKillAt: number | null; + proc: ReturnType; +}): Promise { + const graceRemainingMs = + params.forceKillAt === null + ? TIMEOUT_KILL_GRACE_MS + : Math.max(0, params.forceKillAt - Date.now()); + if (graceRemainingMs > 0) { + await waitForSampleProcessGroupExit(params.proc, graceRemainingMs); + } + if (isSampleProcessGroupAlive(params.proc)) { + signalSampleProcess(params.proc, "SIGKILL"); + } + await waitForSampleProcessGroupExit(params.proc, TIMEOUT_KILL_GRACE_MS); + params.complete(); +} + +function signalSampleProcess(proc: ReturnType, signal: NodeJS.Signals): void { + if (!proc.pid) { + return; + } + const handleSignalError = (error: unknown) => { + const code = (error as NodeJS.ErrnoException | undefined)?.code; + if (code !== "ESRCH" && code !== "EPERM") { + throw error; + } + }; + terminateManagedChild(proc, signal, { + onChildSignalError: handleSignalError, + onProcessGroupSignalError: handleSignalError, + processGroupFallback: "never", + useWindowsTaskkill: false, + }); +} + +function isSampleProcessGroupAlive(proc: ReturnType): boolean { + return inspectManagedProcessGroup(proc, { errorPolicy: "alive-on-eperm" }) === "live"; +} + +function waitForSampleProcessGroupExit( + proc: ReturnType, + timeoutMs: number, +): Promise { + return waitForManagedProcessGroupExit(proc, timeoutMs, { errorPolicy: "alive-on-eperm" }); +} + +async function runCase(params: { + entry: string; + commandCase: CommandCase; + runs: number; + warmup: number; + timeoutMs: number; + runtimeRss: boolean; + cpuProfDir?: string; + heapProfDir?: string; + rssHookPath: string; +}): Promise { + const warmupSamples: Sample[] = []; + const samples: Sample[] = []; + const totalRuns = params.warmup + params.runs; + const transport = sampleTransport(); + const caseRunRoot = + params.commandCase.stateScope === "case" ? createSampleRoot(transport) : undefined; + try { + for (let i = 0; i < totalRuns; i += 1) { + const sample = await runSample({ ...params, runRoot: caseRunRoot }); + if (i < params.warmup) { + warmupSamples.push(sample); + continue; + } + samples.push(sample); + } + return { warmupSamples, samples }; + } finally { + if (caseRunRoot) { + removeSampleRoot(caseRunRoot, transport); + } + } +} + +function tailLines(value: string, maxLines: number): string { + return value.split(/\r?\n/).filter(Boolean).slice(-maxLines).join("\n"); +} + +function printSuite(result: SuiteResult): void { + console.log(`Entry: ${result.entry}`); + console.log(`RSS metric: ${cliStartupMemoryMetric(result)}`); + for (const commandCase of result.cases) { + const { durationMs, firstOutputMs, maxRssMb, exitSummary } = commandCase.summary; + const rssSummary = + maxRssMb == null + ? "rss=n/a" + : `rss(avg=${formatMb(maxRssMb.avg)} p50=${formatMb(maxRssMb.p50)} p95=${formatMb(maxRssMb.p95)})`; + const firstOutputSummary = + firstOutputMs == null + ? "first-output=n/a" + : `first-output(avg=${formatMs(firstOutputMs.avg)} p50=${formatMs( + firstOutputMs.p50, + )} p95=${formatMs(firstOutputMs.p95)})`; + console.log( + `${commandCase.name.padEnd(24)} avg=${formatMs(durationMs.avg)} p50=${formatMs( + durationMs.p50, + )} p95=${formatMs(durationMs.p95)} min=${formatMs(durationMs.min)} max=${formatMs( + durationMs.max, + )} ${firstOutputSummary} ${rssSummary} exits=[${exitSummary}]`, + ); + } + console.log(""); +} + +function printDelta(primary: SuiteResult, secondary: SuiteResult): void { + const deltas = buildCaseDeltas(primary, secondary); + console.log("Delta (secondary - primary, avg)"); + for (const delta of deltas) { + const durationDelta = delta.durationAvgDeltaMs; + const durationPct = delta.durationAvgDeltaPct; + const durationSign = durationDelta > 0 ? "+" : ""; + let line = `${delta.name.padEnd(24)} ${durationSign}${formatMs(durationDelta)} (${durationSign}${durationPct.toFixed(1)}%)`; + if (delta.maxRssAvgDeltaMb != null && delta.maxRssAvgDeltaPct != null) { + const rssDelta = delta.maxRssAvgDeltaMb; + const rssPct = delta.maxRssAvgDeltaPct; + const rssSign = rssDelta > 0 ? "+" : ""; + line += ` rss ${rssSign}${formatMb(rssDelta)} (${rssSign}${rssPct.toFixed(1)}%)`; + } + console.log(line); + } +} + +function buildCaseDeltas(primary: SuiteResult, secondary: SuiteResult): CaseDelta[] { + assertCompatibleCliStartupExecutionModes(primary, secondary); + assertCompatibleCliStartupMemoryMetrics(primary, secondary); + const primaryById = new Map(primary.cases.map((commandCase) => [commandCase.id, commandCase])); + const deltas: CaseDelta[] = []; + for (const commandCase of secondary.cases) { + const baseline = primaryById.get(commandCase.id); + if (!baseline) { + continue; + } + const durationDelta = commandCase.summary.durationMs.avg - baseline.summary.durationMs.avg; + const durationPct = + baseline.summary.durationMs.avg > 0 + ? (durationDelta / baseline.summary.durationMs.avg) * 100 + : 0; + const rssDelta = + baseline.summary.maxRssMb && commandCase.summary.maxRssMb + ? commandCase.summary.maxRssMb.avg - baseline.summary.maxRssMb.avg + : null; + const rssPct = + rssDelta != null && baseline.summary.maxRssMb && baseline.summary.maxRssMb.avg > 0 + ? (rssDelta / baseline.summary.maxRssMb.avg) * 100 + : null; + deltas.push({ + id: commandCase.id, + name: commandCase.name, + durationAvgDeltaMs: durationDelta, + durationAvgDeltaPct: durationPct, + maxRssAvgDeltaMb: rssDelta, + maxRssAvgDeltaPct: rssPct, + }); + } + return deltas; +} + +export function collectFailedSamples(result: SuiteResult): string[] { + const failures: string[] = []; + for (const commandCase of result.cases) { + if (commandCase.samples.length === 0) { + failures.push(`${result.entry} ${commandCase.id}: no measured samples`); + } + for (const [sampleKind, samples] of [ + ["warmup", commandCase.warmupSamples ?? []], + ["sample", commandCase.samples], + ] as const) { + for (const [sampleIndex, sample] of samples.entries()) { + const label = `${result.entry} ${commandCase.id} ${sampleKind} ${sampleIndex + 1}`; + const expectedExitCodes = new Set(commandCase.expectedExitCodes ?? [0]); + if (sample.timedOut === true) { + failures.push(`${label}: timed out`); + } else if (sample.signal !== null) { + failures.push(`${label}: exited via signal ${sample.signal}`); + } else if (!expectedExitCodes.has(sample.exitCode ?? -1)) { + failures.push(`${label}: exited with code ${String(sample.exitCode)}`); + } else if (sample.maxRssMb === null) { + failures.push( + `${label}: did not report max RSS${sample.memory?.error ? ` (${sample.memory.error})` : ""}`, + ); + } else if (sample.exitCode !== 0) { + const output = `${sample.stdoutTail ?? ""}\n${sample.stderrTail ?? ""}`; + const missing = (commandCase.expectedNonzeroOutputIncludes ?? []).filter( + (snippet) => !output.includes(snippet), + ); + if (missing.length > 0) { + failures.push( + `${label}: exited with expected code ${String( + sample.exitCode, + )} but output did not match expected clean-state markers (${missing.join(", ")})`, + ); + } + } + } + } + } + return failures; +} + +async function buildSuiteResult(params: { + entry: string; + executionMode: CliStartupExecutionMode; + options: CliOptions; + rssHookPath: string; +}): Promise { + const cases = []; + for (const commandCase of params.options.cases) { + const { warmupSamples, samples } = await runCase({ + entry: params.entry, + commandCase, + runs: params.options.runs, + warmup: params.options.warmup, + timeoutMs: params.options.timeoutMs, + runtimeRss: params.options.runtimeRss, + cpuProfDir: params.options.cpuProfDir, + heapProfDir: params.options.heapProfDir, + rssHookPath: params.rssHookPath, + }); + cases.push({ + id: commandCase.id, + name: commandCase.name, + args: commandCase.args, + ...(commandCase.expectedExitCodes && commandCase.expectedExitCodes.some((code) => code !== 0) + ? { expectedExitCodes: [...commandCase.expectedExitCodes] } + : {}), + ...(commandCase.expectedNonzeroOutputIncludes + ? { expectedNonzeroOutputIncludes: [...commandCase.expectedNonzeroOutputIncludes] } + : {}), + contract: + commandCase.firstOutputBudgetMs != null || commandCase.exitBudgetMs != null + ? { + firstOutputBudgetMs: commandCase.firstOutputBudgetMs ?? null, + exitBudgetMs: commandCase.exitBudgetMs ?? null, + } + : null, + warmupSamples, + samples, + summary: summarizeSamples(samples), + }); + } + return { + entry: params.entry, + executionMode: params.executionMode, + ...(params.options.runtimeRss ? { memoryMetric: CLI_RUNTIME_MEMORY_METRIC } : {}), + cases, + }; +} + +function parseOptions(): CliOptions { + const presets = parsePresets(parseFlagValue("--preset")); + const cases = resolveCases({ + presets, + caseIds: parseRepeatableFlag("--case"), + }); + return { + cases, + compareBaseline: parseFlagValue("--compare-baseline"), + compareCandidate: parseFlagValue("--compare-candidate"), + entryPrimary: parseFlagValue("--entry-primary") ?? parseFlagValue("--entry") ?? DEFAULT_ENTRY, + entrySecondary: parseFlagValue("--entry-secondary"), + runs: parsePositiveInt(parseFlagValue("--runs"), DEFAULT_RUNS, "--runs"), + warmup: parseNonNegativeInt(parseFlagValue("--warmup"), DEFAULT_WARMUP, "--warmup"), + timeoutMs: parsePositiveInt(parseFlagValue("--timeout-ms"), DEFAULT_TIMEOUT_MS, "--timeout-ms"), + runtimeRss: hasFlag("--runtime-rss"), + json: hasFlag("--json"), + output: parseFlagValue("--output"), + cpuProfDir: parseFlagValue("--cpu-prof-dir"), + heapProfDir: parseFlagValue("--heap-prof-dir"), + }; +} + +function printUsage(): void { + console.log(`OpenClaw CLI benchmark + +Usage: + pnpm tsx scripts/bench-cli-startup.ts [options] + +Options: + --preset + Command preset to run (default: startup) + --case Specific case id to run; repeatable + --entry Primary entry file (default: openclaw.mjs) + --entry-secondary Secondary entry file for avg delta comparison + --runs Measured runs per case (default: ${DEFAULT_RUNS}) + --warmup Warmup runs per case (default: ${DEFAULT_WARMUP}) + --timeout-ms Per-run timeout (default: ${DEFAULT_TIMEOUT_MS}) + --output Write machine-readable JSON to a file + --compare-baseline Read a saved JSON report as the baseline + --compare-candidate Read a saved JSON report as the candidate and print deltas + --cpu-prof-dir Write V8 CPU profiles for each run + --heap-prof-dir Write V8 heap profiles for each run + --runtime-rss Attribute RSS to the CLI runtime (default: legacy last marker) + --json Emit machine-readable JSON + --help Show this text + +Case ids: + ${COMMAND_CASES.map((commandCase) => `${commandCase.id} (${commandCase.name})`).join("\n ")} +`); +} + +function readBenchmarkReport(filePath: string): BenchmarkReport { + return JSON.parse(readFileSync(filePath, "utf8")) as BenchmarkReport; +} + +function writeJsonOutput(filePath: string, value: unknown): void { + mkdirSync(path.dirname(filePath), { recursive: true }); + writeFileSync(filePath, `${JSON.stringify(value, null, 2)}\n`, "utf8"); +} + +function readBenchmarkComparison( + baselinePath: string, + candidatePath: string, +): BenchmarkComparisonResult { + const baseline = readBenchmarkReport(baselinePath); + const candidate = readBenchmarkReport(candidatePath); + return { + baseline: baseline.primary, + candidate: candidate.primary, + comparison: { + baseline: baselinePath, + candidate: candidatePath, + deltas: buildCaseDeltas(baseline.primary, candidate.primary), + }, + }; +} + +async function main(): Promise { + validateCliArgs(); + if (hasFlag("--help")) { + printUsage(); + return; + } + + const options = parseOptions(); + const transport = sampleTransport(); + if (transport && options.runtimeRss) { + throw new Error("Cross-user runtime RSS sampling is not supported"); + } + if (transport && (options.cpuProfDir || options.heapProfDir)) { + throw new Error( + "Cross-user CLI profiles must be collected through the SUT diagnostic exporter", + ); + } + if (options.compareBaseline || options.compareCandidate) { + if (!options.compareBaseline || !options.compareCandidate) { + throw new Error("--compare-baseline and --compare-candidate must be provided together"); + } + const { baseline, candidate, comparison } = readBenchmarkComparison( + options.compareBaseline, + options.compareCandidate, + ); + if (options.output) { + writeJsonOutput(options.output, comparison); + } + if (options.json) { + console.log(JSON.stringify(comparison, null, 2)); + return; + } + printDelta(baseline, candidate); + return; + } + const tmpDir = mkdtempSync(path.join(os.tmpdir(), "openclaw-cli-bench-")); + const rssHookPath = options.runtimeRss ? buildRuntimeRssHook(tmpDir) : buildRssHook(tmpDir); + try { + const primary = await buildSuiteResult({ + entry: options.entryPrimary, + executionMode: transport ? "transport" : "native", + options, + rssHookPath, + }); + const secondary = options.entrySecondary + ? await buildSuiteResult({ + entry: options.entrySecondary, + executionMode: transport ? "transport" : "native", + options, + rssHookPath, + }) + : undefined; + + const report = { + node: process.version, + runs: options.runs, + warmup: options.warmup, + timeoutMs: options.timeoutMs, + cpuProfDir: options.cpuProfDir ?? null, + heapProfDir: options.heapProfDir ?? null, + primary, + secondary: secondary ?? null, + }; + const failures = [ + ...collectFailedSamples(primary), + ...(secondary ? collectFailedSamples(secondary) : []), + ]; + + if (options.output) { + writeJsonOutput(options.output, report); + } + + if (options.json) { + console.log(JSON.stringify(report, null, 2)); + if (failures.length > 0) { + process.exitCode = 1; + for (const failure of failures) { + console.error(`[startup-bench] ${failure}`); + } + } + return; + } + + console.log(`Node: ${process.version}`); + console.log(`Runs per case: ${options.runs}`); + console.log(`Warmup runs per case: ${options.warmup}`); + console.log(`Timeout: ${options.timeoutMs}ms`); + if (options.cpuProfDir) { + console.log(`CPU profiles: ${options.cpuProfDir}`); + } + if (options.heapProfDir) { + console.log(`Heap profiles: ${options.heapProfDir}`); + } + console.log(""); + + console.log("Primary entry"); + printSuite(primary); + if (secondary) { + console.log("Secondary entry"); + printSuite(secondary); + printDelta(primary, secondary); + } + + if (failures.length > 0) { + process.exitCode = 1; + console.error("\nFailed startup benchmark samples:"); + for (const failure of failures) { + console.error(`- ${failure}`); + } + } + } finally { + rmSync(tmpDir, { recursive: true, force: true }); + } +} + +export const testing = { + buildConfigFixture, + collectFailedSamples, + nodeImportSpecifierForPath, + parseGatewayPortEnv, + parseNonNegativeInt, + parsePositiveInt, + validateCliArgs, +}; + +if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { + await main().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); + }); +} diff --git a/scripts/bench-gateway-concurrency.ts b/scripts/bench-gateway-concurrency.ts new file mode 100644 index 0000000000000000000000000000000000000000..6554fdbb4bacf8fb6f5fe911797e10709192ae57 --- /dev/null +++ b/scripts/bench-gateway-concurrency.ts @@ -0,0 +1,2569 @@ +import { spawn, type ChildProcess, type ChildProcessWithoutNullStreams } from "node:child_process"; +// Bench Gateway Concurrency script measures gateway probes during synthetic streaming turns. +import { randomUUID } from "node:crypto"; +import { + copyFileSync, + cpSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { request } from "node:http"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { performance } from "node:perf_hooks"; +import { pathToFileURL } from "node:url"; +import type { ModelsListResult } from "../packages/gateway-protocol/src/schema/agents-models-skills.js"; +import { asFiniteNumber } from "../packages/normalization-core/src/number-coercion.ts"; +import { isRecord } from "../packages/normalization-core/src/record-coerce.ts"; +import { sliceUtf16Safe } from "../packages/normalization-core/src/utf16-slice.ts"; +import type { SessionsListResult } from "../src/gateway/session-utils.types.js"; +import { applyMockOpenAiModelConfig } from "./e2e/lib/fixtures/mock-openai-config.mjs"; +import { + startGatewayBrowserProbe, + type BrowserSessionClick, + type BrowserSessionTarget, +} from "./lib/gateway-bench-browser.ts"; +import { delay, stopChild } from "./lib/gateway-bench-child.ts"; +import { + type GatewayMemorySample, + type GatewayRpc, + getFreePort, + readGatewayMemory, + readProcessRssMb, +} from "./lib/gateway-bench-probes.ts"; +import { + controlGatewayProfile, + measureGatewayCpuUsage, + readGatewayCpuProfile, + readGatewayCpuUsage, + readGatewayHeapProfile, + type GatewayHeapProfile, + type GatewayCpuProfile, + type GatewayCpuUsage, +} from "./lib/gateway-bench-profile.ts"; +import { + BASE_GATEWAY_BENCH_CONFIG, + buildGatewayBenchChildArgs, + CliArgumentError, + createGatewayBenchEnv, + hasFlag, + hasHelpFlag, + parseFlagValue, + parseNonNegativeInt, + parsePositiveInt, + resolveEntry, + resolveOutputPath, + validateCliArgs, + waitForInitialProbe, + writeGatewayBenchConfig, + writePluginFixtures, +} from "./lib/gateway-bench-runtime.ts"; +import { createGatewayWsClient } from "./lib/gateway-ws-client.ts"; + +type MetricSummary = { + count: number; + max: number; + p50: number; + p95: number; + p99: number; +}; + +type TimedProbe = { + atMs: number; + error: string | null; + latencyMs: number; + ok: boolean; +}; + +type DiagnosticsTimelineSpan = { + durationMs?: number; + name?: string; +}; + +type ReadyProbe = TimedProbe & { + cpuCoreRatio: number | null; + degraded: boolean | null; + degradedSinceMs: number | null; + delayP99Ms: number | null; + delayMaxMs: number | null; + status: number; + utilization: number | null; +}; + +type ControlUiProbe = TimedProbe & { + status: number; +}; + +type GatewaySample = { + controlUi: ControlUiProbe; + readyz: ReadyProbe; + sessionsList: TimedProbe; +}; + +type FreshConnectionProbe = { + error: string | null; + latencyMs: number; + ok: boolean; +}; + +type GatewayChildExit = { + atMonotonicMicros: number; + exitCode: number | null; + signal: string | null; +}; + +type MainProfileArtifacts = { + scope: "main-isolate"; + workersManifestPath: string; +}; + +type BenchmarkRun = { + agentCoverage?: { + configuredAgentIds: string[]; + activeTurnAgentIds: string[]; + beforeLoad: Array<{ + agentId: string; + models: ModelsListResult; + sessions: SessionsListResult[]; + }>; + completedTurns: Array<{ agentId: string; count: number }>; + }; + browser?: { + newPageReadyMs: number; + initialSessionReadyMs: number | null; + historyMessagesPerTarget: number; + inventory: { + seededLoadSessions: number; + seededClickSessions: number; + unarchivedSessions: number; + retainedSessions: number; + }; + clicks: BrowserSessionClick[]; + }; + heapProfile?: GatewayHeapProfile & MainProfileArtifacts; + loadCpuProfile?: GatewayCpuProfile & MainProfileArtifacts; + controlPlane: Array; + controlUi: ControlUiProbe[]; + cpuUsage: GatewayCpuUsage; + durationMs: number; + freshConnection: FreshConnectionProbe; + gatewayExit?: Awaited>; + gatewayProcess?: { + pid: number | undefined; + exitCode: number | null | undefined; + signalCode: string | null | undefined; + exitEvent: GatewayChildExit | undefined; + closeEvent: GatewayChildExit | undefined; + }; + loadWindow?: { startMonotonicMicros: number; endMonotonicMicros: number }; + history: Array; + memory: { after: GatewayMemorySample; before: GatewayMemorySample; peakRssMb: number }; + messageSubscriptions: TimedProbe[]; + messageSubscriptionsDuringLoad: TimedProbe[]; + mockRequests: ReturnType; + turnEvidence: ReturnType["finish"]>; + probeWarmup: { + durationMs: number; + samples: GatewaySample[]; + }; + pluginMetadataScans: ReturnType; + readyz: ReadyProbe[]; + sessionSeedDurationMs: number; + sessionsList: TimedProbe[]; + sessionUpdates: TimedProbe[]; + setupDurationMs: number; + turnCount: number; + turnsDurationMs: number; +}; + +type CliOptions = { + agentCount: number; + browserHistoryMessages: number; + browserSessionClicks: number; + cadenceMs: number; + concurrency: number; + controlPlane: boolean; + cpuProfDir?: string; + loadCpuProfDir?: string; + heapProfDir?: string; + diagnosticsTimeline: boolean; + entry: string; + historyBurst: number; + historyClients: number; + historyMessages: number; + historyMessageChars: number; + json: boolean; + maxControlMs?: number; + maxHandshakeMs?: number; + output?: string; + pluginCount: number; + probeRounds?: number; + runs: number; + sessionCount: number; + sessionUpdateClients: number; + sessionUpdates: number; + streamChunkDelayMs: number; + subscribers: number; + timeoutMs: number; + toolEvents: boolean; + turnsPerSession: number; + visibleObserver: boolean; + warmup: number; + workspaceFanout: boolean; +}; + +const DEFAULT_CADENCE_MS = 100; +const DEFAULT_CONCURRENCY = 8; +const DEFAULT_ENTRY = "dist/entry.js"; +const DEFAULT_RUNS = 1; +const DEFAULT_TIMEOUT_MS = 120_000; +const DEFAULT_WARMUP = 0; +const MOCK_RESPONSE_CHUNK_DELAY_MS = 1_000; +const MAX_AGENT_COUNT = 128; +const MAX_CONCURRENCY = 64; +const MAX_TURNS_PER_SESSION = 100; +const MAX_PLUGIN_COUNT = 100; +const MAX_SESSION_COUNT = 10_000; +const MAX_SESSION_UPDATES = 100_000; +const MAX_SESSION_SEED_CONCURRENCY = 16; +const MAX_RUNS = 20; +const MAX_WARMUP = 10; +const MAX_SAMPLES_PER_RUN = 2_048; +const MAX_HTTP_BODY_BYTES = 1_048_576; +const HTTP_TIMEOUT_MS = 20_000; +const PROBE_WARMUP_TIMEOUT_MS = 60_000; +const PROBE_WARMUP_TARGET_MS = 1_000; +const PROBE_WARMUP_RETRY_DELAY_MS = 100; +const GATEWAY_STDERR_TAIL_LINES = 20; +const AGENT_WAIT_RPC_GRACE_MS = 5_000; +const UTILITY_MODEL_ID = "gateway-bench-utility"; +const OBSERVER_ASSESSMENT = "Synthetic benchmark observation is valid."; +const MOCK_INGRESS_KEYS = ["responses", "chatCompletions", "embeddings", "other"] as const; +const MOCK_SELECTION_KEYS = ["model", "global", "automaticTool", "automaticText"] as const; +type MockRequestSnapshot = { + id: string; + ingress: Record<(typeof MOCK_INGRESS_KEYS)[number], number>; + selections: Record<(typeof MOCK_SELECTION_KEYS)[number], number>; + beforeMs: number; + afterMs: number; +}; +const BOOLEAN_FLAGS = new Set([ + "--help", + "-h", + "--json", + "--control-plane", + "--no-diagnostics-timeline", + "--tool-events", + "--visible-observer", + "--workspace-fanout", +]); +const VALUE_FLAGS = new Set([ + "--agent-count", + "--browser-history-messages", + "--browser-session-clicks", + "--cadence-ms", + "--concurrency", + "--cpu-prof-dir", + "--load-cpu-prof-dir", + "--heap-prof-dir", + "--entry", + "--history-burst", + "--history-clients", + "--history-messages", + "--history-message-chars", + "--max-control-ms", + "--max-handshake-ms", + "--output", + "--plugin-count", + "--probe-rounds", + "--runs", + "--session-count", + "--session-update-clients", + "--session-updates", + "--stream-chunk-delay-ms", + "--subscribers", + "--timeout-ms", + "--turns-per-session", + "--warmup", +]); + +function parseBoundedPositiveInt( + raw: string | undefined, + fallback: number, + label: string, + max: number, +): number { + const value = parsePositiveInt(raw, fallback, label); + if (value > max) { + throw new CliArgumentError(`${label} must be at most ${max}`); + } + return value; +} + +function parseBoundedNonNegativeInt( + raw: string | undefined, + fallback: number, + label: string, + max: number, +): number { + const value = parseNonNegativeInt(raw, fallback, label); + if (value > max) { + throw new CliArgumentError(`${label} must be at most ${max}`); + } + return value; +} + +function parseOptions(argv: string[] = process.argv.slice(2)): CliOptions { + validateCliArgs(argv, { booleanFlags: BOOLEAN_FLAGS, valueFlags: VALUE_FLAGS }); + const options = { + agentCount: parseBoundedPositiveInt( + parseFlagValue(argv, "--agent-count"), + 1, + "--agent-count", + MAX_AGENT_COUNT, + ), + browserHistoryMessages: parseBoundedPositiveInt( + parseFlagValue(argv, "--browser-history-messages"), + 80, + "--browser-history-messages", + 500, + ), + browserSessionClicks: parseBoundedNonNegativeInt( + parseFlagValue(argv, "--browser-session-clicks"), + 0, + "--browser-session-clicks", + 20, + ), + cadenceMs: parseBoundedPositiveInt( + parseFlagValue(argv, "--cadence-ms"), + DEFAULT_CADENCE_MS, + "--cadence-ms", + 5_000, + ), + concurrency: parseBoundedPositiveInt( + parseFlagValue(argv, "--concurrency"), + DEFAULT_CONCURRENCY, + "--concurrency", + MAX_CONCURRENCY, + ), + controlPlane: hasFlag(argv, "--control-plane"), + cpuProfDir: resolveOutputPath(parseFlagValue(argv, "--cpu-prof-dir")), + loadCpuProfDir: resolveOutputPath(parseFlagValue(argv, "--load-cpu-prof-dir")), + heapProfDir: resolveOutputPath(parseFlagValue(argv, "--heap-prof-dir")), + diagnosticsTimeline: !hasFlag(argv, "--no-diagnostics-timeline"), + entry: resolveEntry(parseFlagValue(argv, "--entry"), DEFAULT_ENTRY), + historyBurst: parseBoundedPositiveInt( + parseFlagValue(argv, "--history-burst"), + 5, + "--history-burst", + 32, + ), + historyClients: parseBoundedNonNegativeInt( + parseFlagValue(argv, "--history-clients"), + 0, + "--history-clients", + MAX_CONCURRENCY, + ), + historyMessages: parseBoundedNonNegativeInt( + parseFlagValue(argv, "--history-messages"), + 0, + "--history-messages", + 500, + ), + historyMessageChars: parseBoundedPositiveInt( + parseFlagValue(argv, "--history-message-chars"), + 1_024, + "--history-message-chars", + 65_536, + ), + json: hasFlag(argv, "--json"), + maxControlMs: parseFlagValue(argv, "--max-control-ms") + ? parseBoundedPositiveInt( + parseFlagValue(argv, "--max-control-ms"), + 2_000, + "--max-control-ms", + 30_000, + ) + : undefined, + maxHandshakeMs: parseFlagValue(argv, "--max-handshake-ms") + ? parseBoundedPositiveInt( + parseFlagValue(argv, "--max-handshake-ms"), + 2_000, + "--max-handshake-ms", + 30_000, + ) + : undefined, + output: resolveOutputPath(parseFlagValue(argv, "--output")), + pluginCount: parseBoundedNonNegativeInt( + parseFlagValue(argv, "--plugin-count"), + 0, + "--plugin-count", + MAX_PLUGIN_COUNT, + ), + probeRounds: + parseFlagValue(argv, "--probe-rounds") === undefined + ? undefined + : parseBoundedPositiveInt( + parseFlagValue(argv, "--probe-rounds"), + 1, + "--probe-rounds", + MAX_SAMPLES_PER_RUN, + ), + runs: parseBoundedPositiveInt(parseFlagValue(argv, "--runs"), DEFAULT_RUNS, "--runs", MAX_RUNS), + sessionCount: parseBoundedNonNegativeInt( + parseFlagValue(argv, "--session-count"), + 0, + "--session-count", + MAX_SESSION_COUNT, + ), + sessionUpdateClients: parseBoundedPositiveInt( + parseFlagValue(argv, "--session-update-clients"), + 4, + "--session-update-clients", + MAX_CONCURRENCY, + ), + sessionUpdates: parseBoundedNonNegativeInt( + parseFlagValue(argv, "--session-updates"), + 0, + "--session-updates", + MAX_SESSION_UPDATES, + ), + streamChunkDelayMs: parseBoundedPositiveInt( + parseFlagValue(argv, "--stream-chunk-delay-ms"), + MOCK_RESPONSE_CHUNK_DELAY_MS, + "--stream-chunk-delay-ms", + 30_000, + ), + subscribers: parseBoundedNonNegativeInt( + parseFlagValue(argv, "--subscribers"), + 0, + "--subscribers", + MAX_CONCURRENCY, + ), + timeoutMs: parseBoundedPositiveInt( + parseFlagValue(argv, "--timeout-ms"), + DEFAULT_TIMEOUT_MS, + "--timeout-ms", + 10 * 60_000, + ), + toolEvents: hasFlag(argv, "--tool-events"), + turnsPerSession: parseBoundedPositiveInt( + parseFlagValue(argv, "--turns-per-session"), + 1, + "--turns-per-session", + MAX_TURNS_PER_SESSION, + ), + visibleObserver: hasFlag(argv, "--visible-observer"), + warmup: parseBoundedNonNegativeInt( + parseFlagValue(argv, "--warmup"), + DEFAULT_WARMUP, + "--warmup", + MAX_WARMUP, + ), + workspaceFanout: hasFlag(argv, "--workspace-fanout"), + }; + if (options.loadCpuProfDir && options.heapProfDir) { + throw new CliArgumentError( + "--load-cpu-prof-dir and --heap-prof-dir require separate benchmark runs", + ); + } + if (options.agentCount > Math.max(options.sessionCount, options.concurrency)) { + throw new CliArgumentError("--agent-count must not exceed the total session count"); + } + const historyMessageCount = + Math.max(options.sessionCount, options.concurrency) * options.historyMessages + + (options.browserSessionClicks > 0 + ? (options.browserSessionClicks + 1) * options.browserHistoryMessages + : 0); + if ( + options.probeRounds !== undefined && + options.probeRounds * options.historyClients * options.historyBurst > MAX_SAMPLES_PER_RUN + ) { + throw new CliArgumentError("fixed history workload must not exceed 2048 requests per run"); + } + if ( + historyMessageCount > 100_000 || + historyMessageCount * options.historyMessageChars > 256 * 1024 * 1024 + ) { + throw new CliArgumentError("synthetic history must not exceed 100000 messages or 256 MiB"); + } + return options; +} + +function printUsage(): void { + console.log(`OpenClaw Gateway concurrency benchmark + +Usage: + pnpm test:gateway:concurrency -- [options] + node scripts/bench-gateway-concurrency.ts [options] + +Options: + --agent-count Configured agents sharing the fixed session inventory (default: 1, max: ${MAX_AGENT_COUNT}) + --browser-history-messages Messages per browser click target, independent of inventory history (default: 80, max: 500) + --browser-session-clicks Click n existing sessions and revisit one during load (default: 0, max: 20; requires built UI and Chromium) + --concurrency Concurrent synthetic sessions (default: ${DEFAULT_CONCURRENCY}) + --turns-per-session Serial turns per session (default: 1, max: ${MAX_TURNS_PER_SESSION}) + --control-plane Also probe tasks.list, cron.list, and cron.status during load + --history-messages Inject up to 500 synthetic messages per seeded session + --history-message-chars Synthetic message size (default: 1024, max: 65536) + --cpu-prof-dir

Write Gateway V8 CPU profiles to this directory + --load-cpu-prof-dir

Capture load-phase main/Worker CPU over private IPC, including on Windows + --heap-prof-dir

Sample load-phase main/Worker allocations, including GC-collected objects + --runs Measured gateway runs (default: ${DEFAULT_RUNS}) + --warmup Warmup gateway runs (default: ${DEFAULT_WARMUP}) + --cadence-ms Probe cadence (default: ${DEFAULT_CADENCE_MS}) + --probe-rounds Run exactly n sampler rounds and n bursts per history client + --timeout-ms Per-run cap, excluding probe warmup (default: ${DEFAULT_TIMEOUT_MS}) + --entry Gateway CLI entry file (default: ${DEFAULT_ENTRY}) + --session-count Seed up to ${MAX_SESSION_COUNT} distinct sessions before load + --session-updates Bounded public sessions.patch mutations during load + --session-update-clients Concurrent session mutation clients (default: 4) + --history-clients Concurrent dedicated history-prefetch WebSocket clients + --history-burst Parallel history requests per prefetch client (default: 5) + --subscribers Dedicated session-message subscription clients + --stream-chunk-delay-ms Mock-provider delay between stream chunks (default: ${MOCK_RESPONSE_CHUNK_DELAY_MS}) + --visible-observer Mark subscribed clients visible to exercise session observation + --no-diagnostics-timeline Disable diagnostics timeline file writes + --plugin-count Configure synthetic plugins through plugins.load.paths (default: 0) + --tool-events Make every synthetic turn execute a tool before replying + --workspace-fanout Bind each session to a distinct workspace + --max-control-ms Fail when any load-phase health/control probe exceeds this bound + --max-handshake-ms Fail when a fresh authenticated connection exceeds this bound + --output Write machine-readable JSON to a file + --json Emit machine-readable JSON + --help, -h Show this text +`); +} + +function percentile(sorted: readonly number[], percentileValue: number): number { + const index = Math.max( + 0, + Math.min(sorted.length - 1, Math.ceil((percentileValue / 100) * sorted.length) - 1), + ); + return sorted[index] ?? 0; +} + +function summarizeNumbers(values: readonly number[]): MetricSummary | null { + const sorted = values.filter(Number.isFinite).toSorted((a, b) => a - b); + if (sorted.length === 0) { + return null; + } + return { + count: sorted.length, + max: sorted.at(-1) ?? 0, + p50: percentile(sorted, 50), + p95: percentile(sorted, 95), + p99: percentile(sorted, 99), + }; +} + +function summarizePluginMetadataScans(events: readonly DiagnosticsTimelineSpan[]) { + const durations = events.flatMap((event) => + event.name === "plugins.metadata.scan" && + typeof event.durationMs === "number" && + Number.isFinite(event.durationMs) + ? [event.durationMs] + : [], + ); + return { + count: durations.length, + durationMs: summarizeNumbers(durations), + totalDurationMs: durations.reduce((sum, durationMs) => sum + durationMs, 0), + }; +} + +function readDiagnosticsTimelineSpans( + timelinePath: string, + window?: { from: number; through: number }, +): DiagnosticsTimelineSpan[] { + const contents = readFileSync(timelinePath, "utf8"); + if (!contents.trim() || !contents.endsWith("\n")) { + throw new Error("diagnostics timeline is empty or incomplete"); + } + return contents + .split(/\r?\n/u) + .filter(Boolean) + .flatMap((line) => { + const event: unknown = JSON.parse(line); + if ( + !isRecord(event) || + event.schemaVersion !== "openclaw.diagnostics.v1" || + typeof event.timestamp !== "string" || + !Number.isFinite(Date.parse(event.timestamp)) + ) { + throw new Error("invalid diagnostics timeline record"); + } + const timestamp = Date.parse(event.timestamp); + if (window && (timestamp < window.from || timestamp > window.through)) { + return []; + } + if (event.type !== "span.end") { + return []; + } + if ( + typeof event.name !== "string" || + typeof event.durationMs !== "number" || + !Number.isFinite(event.durationMs) + ) { + throw new Error("invalid diagnostics timeline span"); + } + return [{ name: event.name, durationMs: event.durationMs }]; + }); +} + +function remainingMs(deadlineAt: number): number { + return Math.max(0, deadlineAt - performance.now()); +} + +function requireRemainingMs(deadlineAt: number, label: string): number { + const remaining = remainingMs(deadlineAt); + if (remaining <= 0) { + throw new Error(`benchmark timed out while ${label}`); + } + return remaining; +} + +async function requestHttp(params: { + accept: string; + deadlineAt: number; + path: string; + port: number; +}): Promise<{ body: string; latencyMs: number; status: number }> { + const startedAt = performance.now(); + const requestDeadlineAt = Math.min(params.deadlineAt, startedAt + HTTP_TIMEOUT_MS); + requireRemainingMs(requestDeadlineAt, `requesting ${params.path}`); + return await new Promise((resolve, reject) => { + let settled = false; + const settle = (run: () => void) => { + if (settled) { + return; + } + settled = true; + clearTimeout(timer); + run(); + }; + const fail = (error: Error) => + settle(() => { + req.destroy(); + reject(error); + }); + const req = request( + { + headers: { accept: params.accept }, + host: "127.0.0.1", + method: "GET", + path: params.path, + port: params.port, + }, + (res) => { + const chunks: Buffer[] = []; + let bytes = 0; + res.on("data", (chunk: Buffer) => { + bytes += chunk.length; + if (bytes > MAX_HTTP_BODY_BYTES) { + fail(new Error(`${params.path} response exceeded ${MAX_HTTP_BODY_BYTES} bytes`)); + return; + } + chunks.push(chunk); + }); + res.once("aborted", () => fail(new Error(`${params.path} response aborted`))); + res.once("error", fail); + res.once("end", () => + settle(() => + resolve({ + body: Buffer.concat(chunks).toString("utf8"), + latencyMs: performance.now() - startedAt, + status: res.statusCode ?? 0, + }), + ), + ); + }, + ); + req.once("error", fail); + // Request/socket timeouts measure inactivity; this timer owns the wall-clock deadline. + const timer = setTimeout( + () => fail(new Error(`${params.path} request timed out`)), + Math.max(1, Math.ceil(remainingMs(requestDeadlineAt))), + ); + timer.unref?.(); + req.end(); + }); +} + +function describeProbeError(error: unknown): string { + return sliceUtf16Safe(error instanceof Error ? error.message : String(error), 0, 500); +} + +function formatProbeResult(name: string, probe: TimedProbe & { status?: number }): string { + const status = probe.status === undefined ? "n/a" : probe.status; + return `${name}: ok=${probe.ok} status=${status} latencyMs=${probe.latencyMs.toFixed(1)} error=${probe.error ? JSON.stringify(probe.error) : "none"}`; +} + +function formatProbeFailure(sample: GatewaySample): string { + return [ + "gateway probes did not become fast and healthy before concurrent load", + formatProbeResult("readyz", sample.readyz), + formatProbeResult("sessionsList", sample.sessionsList), + formatProbeResult("controlUi", sample.controlUi), + ].join("\n "); +} + +function tailLines(output: string, lineCount: number): string { + return output.trimEnd().split(/\r?\n/u).slice(-lineCount).join("\n"); +} + +function captureChildOutput(child: ChildProcess): { + readOutput: () => string; + readStderrTail: () => string; +} { + if (!child.stdout || !child.stderr) { + throw new Error("Gateway benchmark children require piped stdout and stderr"); + } + let output = ""; + let stderr = ""; + const appendOutput = (chunk: Buffer) => { + output = sliceUtf16Safe(`${output}${chunk.toString("utf8")}`, -64 * 1_024); + }; + child.stdout.on("data", appendOutput); + child.stderr.on("data", (chunk: Buffer) => { + appendOutput(chunk); + stderr = sliceUtf16Safe(`${stderr}${chunk.toString("utf8")}`, -64 * 1_024); + }); + return { + readOutput: () => output, + readStderrTail: () => tailLines(stderr, GATEWAY_STDERR_TAIL_LINES), + }; +} + +function formatRunFailure( + error: unknown, + gatewayOutput: { readOutput: () => string; readStderrTail: () => string }, + mockOutput: { readOutput: () => string }, +): string { + return [ + error instanceof Error ? error.message : String(error), + gatewayOutput.readStderrTail() + ? `gateway stderr tail:\n${gatewayOutput.readStderrTail()}` + : "gateway stderr tail: (empty)", + gatewayOutput.readOutput() ? `gateway output tail:\n${gatewayOutput.readOutput()}` : "", + mockOutput.readOutput() ? `mock provider output tail:\n${mockOutput.readOutput()}` : "", + ] + .filter(Boolean) + .join("\n"); +} + +async function waitForMockServer(port: number, deadlineAt: number): Promise { + let lastError: unknown; + while (remainingMs(deadlineAt) > 0) { + try { + const result = await requestHttp({ + accept: "application/json", + deadlineAt, + path: "/health", + port, + }); + if (result.status === 200) { + return; + } + } catch (error) { + lastError = error; + } + await delay(Math.min(25, remainingMs(deadlineAt))); + } + const detail = + lastError instanceof Error + ? lastError.message + : typeof lastError === "string" + ? lastError + : "timeout"; + throw new Error(`mock provider did not become healthy: ${detail}`); +} + +async function waitForGatewayDispatchReady( + readOutput: () => string, + deadlineAt: number, +): Promise { + while (remainingMs(deadlineAt) > 0) { + if (readOutput().includes("startup trace: sidecars.ready ")) { + return; + } + await delay(Math.min(25, remainingMs(deadlineAt))); + } + throw new Error("gateway did not finish dispatch-ready sidecars"); +} + +function parseMockRequests(value: unknown, beforeMs: number, afterMs: number): MockRequestSnapshot { + if (!isRecord(value) || typeof value.id !== "string" || !value.id || value.id.length > 64) { + throw new Error("mock request counter identity is missing"); + } + const counts = (record: unknown, keys: readonly K[]): Record => { + if ( + !isRecord(record) || + Object.keys(record).length !== keys.length || + keys.some((key) => !Number.isSafeInteger(record[key]) || (record[key] as number) < 0) + ) { + throw new Error("mock request counters are invalid"); + } + return Object.fromEntries(keys.map((key) => [key, record[key]])) as Record; + }; + return { + id: value.id, + ingress: counts(value.ingress, MOCK_INGRESS_KEYS), + selections: counts(value.selections, MOCK_SELECTION_KEYS), + beforeMs, + afterMs, + }; +} + +async function readMockRequests(port: number, deadlineAt: number): Promise { + const beforeMs = performance.now(); + const response = await requestHttp({ + accept: "application/json", + port, + path: "/health", + deadlineAt, + }); + const afterMs = performance.now(); + if (response.status !== 200) { + throw new Error(`mock request checkpoint failed: HTTP ${response.status}`); + } + return parseMockRequests(JSON.parse(response.body).requests, beforeMs, afterMs); +} + +function summarizeMockRequests(checkpoints: readonly MockRequestSnapshot[]) { + const phases = ["startupAndWarmup", "setup", "loadBracket", "postLoad"] as const; + if (checkpoints.length !== phases.length + 1) { + throw new Error("mock request checkpoints are incomplete"); + } + const first = checkpoints[0]!; + if ( + [...Object.values(first.ingress), ...Object.values(first.selections)].some( + (count) => count !== 0, + ) + ) { + throw new Error("mock request counters were not zero before Gateway startup"); + } + for (let index = 0; index < checkpoints.length; index += 1) { + const current = checkpoints[index]!; + const previous = checkpoints[Math.max(0, index - 1)]!; + if ( + current.id !== first.id || + !Number.isFinite(current.beforeMs) || + !Number.isFinite(current.afterMs) || + current.afterMs < current.beforeMs || + (index > 0 && current.beforeMs < previous.afterMs) || + MOCK_INGRESS_KEYS.some((key) => current.ingress[key] < previous.ingress[key]) || + MOCK_SELECTION_KEYS.some((key) => current.selections[key] < previous.selections[key]) + ) { + throw new Error("mock request checkpoints changed identity or regressed"); + } + } + const difference = (after: MockRequestSnapshot, before: MockRequestSnapshot) => + Object.fromEntries( + MOCK_INGRESS_KEYS.map((key) => [key, after.ingress[key] - before.ingress[key]]), + ) as MockRequestSnapshot["ingress"]; + return { + // Acknowledged HTTP brackets partition ingress, not causal work or exact CPU windows. + checkpoints, + ingress: { + ...Object.fromEntries( + phases.map((phase, index) => [ + phase, + difference(checkpoints[index + 1]!, checkpoints[index]!), + ]), + ), + total: difference(checkpoints.at(-1)!, first), + }, + selections: { ...checkpoints.at(-1)!.selections }, + }; +} + +function createTurnEvidence(toolEvents: boolean) { + const turns = new Map< + string, + { + sessionKey: string; + toolCallId?: string; + toolCompleted: boolean; + final: boolean; + observer: boolean; + } + >(); + let invalid = false; + return { + register(runId: string, sessionKey: string) { + if (turns.has(runId)) { + throw new Error("duplicate benchmark run identity"); + } + turns.set(runId, { sessionKey, toolCompleted: false, final: false, observer: false }); + }, + onEvent(this: void, event: { event: string; payload?: unknown }) { + const payload = event.payload; + if (!isRecord(payload) || typeof payload.runId !== "string") { + return; + } + const turn = turns.get(payload.runId); + if (!turn) { + return; + } + if (event.event === "session.observer") { + if (payload.sessionKey === turn.sessionKey && payload.assessment === OBSERVER_ASSESSMENT) { + turn.observer = true; + } + return; + } + if (!toolEvents || event.event !== "session.tool") { + return; + } + // The WebSocket callback cannot throw into runTurn's promise. Retain bounded + // failure state through client teardown, including late duplicate results. + const data = payload.data; + if ( + payload.sessionKey !== turn.sessionKey || + !isRecord(data) || + data.name !== "exec" || + typeof data.toolCallId !== "string" + ) { + invalid = true; + return; + } + if (data.phase === "start") { + invalid ||= turn.toolCallId !== undefined; + turn.toolCallId = data.toolCallId; + } else if (data.phase === "result") { + const result = isRecord(data.result) ? data.result : {}; + const details = isRecord(result.details) ? result.details : {}; + invalid ||= + turn.toolCompleted || + turn.toolCallId !== data.toolCallId || + data.isError !== false || + details.status !== "completed" || + details.exitCode !== 0 || + typeof details.aggregated !== "string" || + !details.aggregated.includes("openclaw-draft-proof"); + turn.toolCompleted = true; + } + }, + complete(runId: string, terminalReply: unknown) { + const turn = turns.get(runId); + if ( + !turn || + turn.final || + !isRecord(terminalReply) || + terminalReply.disposition !== "visible" || + terminalReply.text !== "OPENCLAW_E2E_DRAFTPROOF" + ) { + throw new Error("benchmark tool turn did not produce its expected visible final reply"); + } + turn.final = true; + }, + finish() { + if ( + invalid || + (toolEvents && + [...turns.values()].some( + (turn) => !turn.toolCallId || !turn.toolCompleted || !turn.final, + )) + ) { + throw new Error( + "benchmark tool lifecycle evidence is missing, duplicated, or unsuccessful", + ); + } + return { + toolTurns: toolEvents ? turns.size : 0, + observerModelDigestTurns: [...turns.values()].filter((turn) => turn.observer).length, + }; + }, + }; +} + +function buildConfig( + root: string, + mockPort: number, + concurrency: number, + pluginCount: number, + browserSessionClicks: number, + agentIds: string[], +): string { + const controlUiRoot = path.join(root, "control-ui"); + mkdirSync(controlUiRoot, { recursive: true }); + const checkoutIndex = path.join(process.cwd(), "ui", "index.html"); + if (browserSessionClicks > 0) { + const builtUi = path.join(process.cwd(), "dist", "control-ui"); + if (!existsSync(path.join(builtUi, "index.html"))) { + throw new Error( + "--browser-session-clicks requires built Control UI assets; run pnpm ui:build", + ); + } + cpSync(builtUi, controlUiRoot, { recursive: true }); + } else { + copyFileSync( + existsSync(checkoutIndex) + ? checkoutIndex + : path.join(process.cwd(), "dist", "control-ui", "index.html"), + path.join(controlUiRoot, "index.html"), + ); + } + + const config = structuredClone(BASE_GATEWAY_BENCH_CONFIG) as Record; + config.gateway = { + ...(config.gateway as Record), + controlUi: { enabled: true, root: controlUiRoot }, + }; + applyMockOpenAiModelConfig(config, { + mockPort, + modelRef: "openai/gpt-5.6-luna", + utilityModelRef: `openai/${UTILITY_MODEL_ID}`, + }); + const agents = config.agents as Record; + agents.defaults = { + ...(agents.defaults as Record), + maxConcurrent: concurrency, + heartbeat: { every: "0m" }, + }; + const pluginFixtures = + pluginCount > 0 ? writePluginFixtures(root, { count: pluginCount }) : undefined; + const agentList = + agentIds.length > 1 + ? agentIds.map((id, index) => { + const workspace = path.join(root, `workspace-${id}`); + mkdirSync(workspace, { recursive: true }); + return { id, default: index === 0, workspace }; + }) + : undefined; + return writeGatewayBenchConfig(root, config, { agentList, pluginFixtures }); +} + +async function readGatewayProtocolVersion(entry: string): Promise { + const protocolPath = path.join( + path.dirname(path.resolve(entry)), + "gateway", + "protocol", + "index.js", + ); + const protocol: unknown = await import(pathToFileURL(protocolPath).href); + if ( + typeof protocol !== "object" || + protocol === null || + !("PROTOCOL_VERSION" in protocol) || + typeof protocol.PROTOCOL_VERSION !== "number" + ) { + throw new Error(`Gateway protocol module is missing PROTOCOL_VERSION: ${protocolPath}`); + } + return protocol.PROTOCOL_VERSION; +} + +async function connectGateway( + port: number, + deadlineAt: number, + protocolVersion: number, + subscribeSessions = true, + onEvent?: Parameters[0]["onEvent"], +) { + let requestDeadlineAt = deadlineAt; + const client = createGatewayWsClient({ + handshakeTimeoutMs: Math.min(8_000, requireRemainingMs(deadlineAt, "connecting WebSocket")), + openTimeoutMs: Math.min(8_000, requireRemainingMs(deadlineAt, "opening WebSocket")), + url: `ws://127.0.0.1:${port}`, + onEvent, + }); + await client.waitOpen(); + + const requestRpc = async ( + method: string, + params: unknown, + requestedTimeoutMs?: number, + ): Promise => { + const response = await client.request( + method, + params, + Math.max( + 1, + Math.min( + requestedTimeoutMs ?? 65_000, + requireRemainingMs(requestDeadlineAt, `waiting for ${method}`), + ), + ), + ); + if (!response.ok) { + const message = + response.error && typeof response.error === "object" && "message" in response.error + ? String(response.error.message) + : JSON.stringify(response.error); + throw new Error(`${method} failed: ${message}`); + } + return response.payload as T; + }; + + await requestRpc("connect", { + minProtocol: protocolVersion, + maxProtocol: protocolVersion, + client: { + id: "gateway-client", + displayName: "gateway-concurrency-benchmark", + version: "1.0.0", + platform: process.platform, + mode: "backend", + }, + role: "operator", + scopes: ["operator.read", "operator.write", "operator.admin"], + caps: [], + }); + if (subscribeSessions) { + await requestRpc("sessions.subscribe", {}); + } + return { + close: client.close, + waitClosed: () => + new Promise((resolve, reject) => { + if (client.ws.readyState === client.ws.CLOSED) { + resolve(); + return; + } + const onClose = () => { + clearTimeout(timer); + resolve(); + }; + const timer = setTimeout(() => { + client.ws.off("close", onClose); + reject(new Error("benchmark event client did not close")); + }, HTTP_TIMEOUT_MS); + timer.unref(); + client.ws.once("close", onClose); + }), + request: requestRpc, + setDeadlineAt: (value: number) => { + requestDeadlineAt = value; + }, + }; +} + +function readGatewayProcessRssMb(pid: number | undefined): number | null { + if (!pid) { + return null; + } + if (process.platform !== "linux") { + return readProcessRssMb(pid); + } + try { + const status = readFileSync(`/proc/${pid}/status`, "utf8"); + const match = /^VmRSS:\s+(\d+)\s+kB$/mu.exec(status); + const rssKb = match ? Number(match[1]) : Number.NaN; + return Number.isFinite(rssKb) && rssKb > 0 ? rssKb / 1024 : null; + } catch { + return null; + } +} + +async function timeRpcProbe( + rpc: GatewayRpc, + method: string, + params: unknown, + runStartedAt: number, +): Promise { + const startedAt = performance.now(); + try { + await rpc(method, params); + return { + atMs: startedAt - runStartedAt, + error: null, + latencyMs: performance.now() - startedAt, + ok: true, + }; + } catch (error) { + return { + atMs: startedAt - runStartedAt, + error: describeProbeError(error), + latencyMs: performance.now() - startedAt, + ok: false, + }; + } +} + +async function runTurn( + rpc: GatewayRpc, + index: number, + deadlineAt: number, + toolEvents = false, + options?: { + onStarted?: () => void; + sessionKey?: string; + evidence?: ReturnType; + }, +): Promise { + const requestedRunId = randomUUID(); + const sessionKey = options?.sessionKey ?? `agent:main:gateway-concurrency-${index + 1}`; + const evidence = options?.evidence ?? createTurnEvidence(toolEvents); + evidence.register(requestedRunId, sessionKey); + const started = await rpc<{ runId?: string; status?: string }>("agent", { + sessionKey, + message: toolEvents + ? `OPENCLAW_E2E_DRAFTPROOF benchmark tool stream ${index + 1}.` + : `Reply with benchmark stream ${index + 1}.`, + deliver: false, + idempotencyKey: requestedRunId, + }); + options?.onStarted?.(); + if (options?.evidence && started.runId !== undefined && started.runId !== requestedRunId) { + throw new Error("agent returned a different benchmark run identity"); + } + if (started.status === "ok" && !toolEvents) { + return; + } + if (started.status !== "accepted" && started.status !== "ok") { + throw new Error(`agent ${index + 1} was not accepted: ${JSON.stringify(started)}`); + } + const remaining = requireRemainingMs(deadlineAt, `waiting for agent ${index + 1} completion`); + // Agent waits share the load deadline; a shorter observation cap would abort + // an active turn while the benchmark still has time to measure completion. + const waitTimeoutMs = Math.max(0, Math.floor(remaining - AGENT_WAIT_RPC_GRACE_MS)); + const rpcTimeoutMs = Math.max(1, Math.ceil(remaining)); + const runId = started.runId ?? requestedRunId; + const completed = await rpc<{ runId?: string; status?: string; terminalReply?: unknown }>( + "agent.wait", + { + runId, + timeoutMs: waitTimeoutMs, + }, + rpcTimeoutMs, + ); + if (completed.status !== "ok") { + throw new Error(`agent ${index + 1} did not complete: ${JSON.stringify(completed)}`); + } + if (completed.runId !== runId) { + throw new Error("agent.wait returned a different or missing benchmark run identity"); + } + if (toolEvents) { + evidence.complete(requestedRunId, completed.terminalReply); + } +} + +async function runSessionTurns( + rpc: GatewayRpc, + index: number, + deadlineAt: number, + options: { + onStarted?: () => void; + sessionKey: string; + toolEvents: boolean; + turnsPerSession: number; + evidence?: ReturnType; + }, +): Promise { + for (let turn = 0; turn < options.turnsPerSession; turn += 1) { + requireRemainingMs(deadlineAt, `starting session ${index + 1} turn ${turn + 1}`); + await runTurn(rpc, index * options.turnsPerSession + turn, deadlineAt, options.toolEvents, { + sessionKey: options.sessionKey, + onStarted: turn === 0 ? options.onStarted : undefined, + evidence: options.evidence, + }); + } + return options.turnsPerSession; +} + +async function sampleGateway(params: { + deadlineAt: number; + port: number; + rpc: GatewayRpc; + runStartedAt: number; + serial?: boolean; +}): Promise { + const atMs = performance.now() - params.runStartedAt; + const safeHttpProbe = async (pathValue: string, accept: string) => { + const startedAt = performance.now(); + try { + return { + ...(await requestHttp({ + accept, + deadlineAt: params.deadlineAt, + path: pathValue, + port: params.port, + })), + error: null, + ok: true, + }; + } catch (error) { + return { + body: "", + error: describeProbeError(error), + latencyMs: performance.now() - startedAt, + ok: false, + status: 0, + }; + } + }; + const probeReadyz = () => safeHttpProbe("/readyz", "application/json"); + const probeControlUi = () => safeHttpProbe("/", "text/html"); + const probeSessions = async () => { + const startedAt = performance.now(); + try { + const payload = await params.rpc( + "sessions.list", + {}, + Math.min(HTTP_TIMEOUT_MS, requireRemainingMs(params.deadlineAt, "probing sessions.list")), + ); + return { error: null, latencyMs: performance.now() - startedAt, ok: true, payload }; + } catch (error) { + return { + error: describeProbeError(error), + latencyMs: performance.now() - startedAt, + ok: false, + payload: null, + }; + } + }; + const [readyz, controlUi, sessions] = params.serial + ? [await probeReadyz(), await probeControlUi(), await probeSessions()] + : await Promise.all([probeReadyz(), probeControlUi(), probeSessions()]); + const readyBody = (() => { + if (readyz.status !== 200) { + return {}; + } + try { + return JSON.parse(readyz.body) as { eventLoop?: Record }; + } catch { + return {}; + } + })(); + const eventLoop = readyBody.eventLoop; + return { + controlUi: { + atMs, + error: + controlUi.error ?? + (controlUi.status === 200 && !controlUi.body.includes(" Promise; + retryDelayMs?: number; + targetMs?: number; +}): Promise<{ durationMs: number; samples: GatewaySample[] }> { + const startedAt = performance.now(); + const samples: GatewaySample[] = []; + const targetMs = params.targetMs ?? PROBE_WARMUP_TARGET_MS; + while (remainingMs(params.deadlineAt) > 0) { + const sample = await params.sample(params.deadlineAt); + samples.push(sample); + const healthy = sample.readyz.ok && sample.sessionsList.ok && sample.controlUi.ok; + const fast = + Math.max( + sample.readyz.latencyMs, + sample.sessionsList.latencyMs, + sample.controlUi.latencyMs, + ) <= targetMs; + const eventLoopSettled = sample.readyz.degraded !== true; + if (healthy && fast && eventLoopSettled) { + return { durationMs: performance.now() - startedAt, samples }; + } + await delay( + Math.min(params.retryDelayMs ?? PROBE_WARMUP_RETRY_DELAY_MS, remainingMs(params.deadlineAt)), + ); + } + const lastSample = samples.at(-1); + throw new Error( + lastSample + ? formatProbeFailure(lastSample) + : "gateway probes did not run before the warmup deadline", + ); +} + +async function runProbeRounds(params: { + rounds?: number; + deadlineAt: number; + cadenceMs: number; + cadenceFrom: "start" | "completion"; + runFirst: boolean; + shouldContinue: () => boolean; + stopped: () => boolean; + runRound: (index: number) => Promise; +}): Promise { + let completed = 0; + const hasWork = () => + !params.stopped() && + (params.rounds === undefined + ? (completed === 0 && params.runFirst) || params.shouldContinue() + : completed < params.rounds); + while (hasWork()) { + requireRemainingMs(params.deadlineAt, "starting gateway probe round"); + const startedAt = performance.now(); + await params.runRound(completed); + completed += 1; + if (!hasWork()) { + break; + } + const elapsed = params.cadenceFrom === "start" ? performance.now() - startedAt : 0; + await delay( + Math.min( + Math.max(0, params.cadenceMs - elapsed), + requireRemainingMs(params.deadlineAt, "pacing gateway probes"), + ), + ); + } + return completed; +} + +async function runGatewaySample(options: { + agentCount: number; + browserHistoryMessages: number; + browserSessionClicks: number; + cadenceMs: number; + concurrency: number; + controlPlane: boolean; + deadlineAt: number; + diagnosticsTimeline: boolean; + entry: string; + cpuProfDir?: string; + loadCpuProfDir?: string; + heapProfDir?: string; + historyBurst: number; + historyClients: number; + historyMessages: number; + historyMessageChars: number; + pluginCount: number; + probeRounds?: number; + sessionCount: number; + sessionUpdateClients: number; + sessionUpdates: number; + streamChunkDelayMs: number; + subscribers: number; + timeoutMs: number; + toolEvents: boolean; + turnsPerSession: number; + visibleObserver: boolean; + workspaceFanout: boolean; +}): Promise { + const root = mkdtempSync(path.join(tmpdir(), "openclaw-gateway-concurrency-")); + const [port, mockPort] = await Promise.all([getFreePort(), getFreePort()]); + const runStartedAt = performance.now(); + const agentIds = Array.from({ length: options.agentCount }, (_, index) => + index === 0 ? "main" : `bench-agent-${index + 1}`, + ); + const timelinePath = path.join(root, "diagnostics-timeline.jsonl"); + const responseControlPath = path.join(root, "mock-provider-responses.json"); + const heapProfilePath = options.heapProfDir + ? path.resolve(options.heapProfDir, `gateway-load-${randomUUID()}.heapprofile`) + : undefined; + const loadCpuProfilePath = options.loadCpuProfDir + ? path.resolve(options.loadCpuProfDir, `gateway-load-${randomUUID()}.cpuprofile`) + : undefined; + const protocolVersion = await readGatewayProtocolVersion(options.entry); + let gateway: ChildProcess | undefined; + let mockProvider: ChildProcessWithoutNullStreams | undefined; + let client: Awaited> | undefined; + let browserProbe: Awaited> | undefined; + const auxiliaryClients: Array>> = []; + let probesStopped = false; + const probeJobs: Promise[] = []; + let gatewayOutput = { readOutput: () => "", readStderrTail: () => "" }; + let mockOutput = { readOutput: () => "", readStderrTail: () => "" }; + let result: Omit; + const mockCheckpoints: MockRequestSnapshot[] = []; + const turnEvidence = createTurnEvidence(options.toolEvents); + let gatewayExit: Awaited> | undefined; + let gatewayExitEvent: GatewayChildExit | undefined; + let gatewayCloseEvent: GatewayChildExit | undefined; + const readGatewayProcess = () => ({ + pid: gateway?.pid, + exitCode: gateway?.exitCode, + signalCode: gateway?.signalCode, + exitEvent: gatewayExitEvent, + closeEvent: gatewayCloseEvent, + }); + let timelineWindow: { from: number; through: number } | undefined; + + try { + try { + const configPath = buildConfig( + root, + mockPort, + options.concurrency, + options.pluginCount, + options.browserSessionClicks, + agentIds, + ); + writeFileSync( + responseControlPath, + JSON.stringify({ + models: { + [UTILITY_MODEL_ID]: { + text: JSON.stringify({ + headline: "Synthetic benchmark work is progressing.", + assessment: OBSERVER_ASSESSMENT, + health: "on-track", + }), + }, + }, + }), + ); + mockProvider = spawn(process.execPath, ["scripts/e2e/mock-openai-server.mjs"], { + cwd: process.cwd(), + detached: process.platform !== "win32", + env: { + LANG: process.env.LANG ?? "en_US.UTF-8", + PATH: process.env.PATH, + MOCK_PORT: String(mockPort), + MOCK_RESPONSE_CONTROL: responseControlPath, + MOCK_RESPONSE_CHUNK_DELAY_MS: String(options.streamChunkDelayMs), + SUCCESS_MARKER: "OpenClaw gateway concurrency benchmark streaming response.", + }, + }); + mockOutput = captureChildOutput(mockProvider); + await waitForMockServer(mockPort, options.deadlineAt); + mockCheckpoints.push(await readMockRequests(mockPort, options.deadlineAt)); + + if (options.cpuProfDir) { + mkdirSync(options.cpuProfDir, { recursive: true }); + } + const gatewayArgs = buildGatewayBenchChildArgs(options.entry, port); + gatewayArgs.unshift( + "--import", + new URL("./lib/gateway-bench-profile-preload.ts", import.meta.url).href, + ); + if (heapProfilePath || loadCpuProfilePath) { + for (const profilePath of [heapProfilePath, loadCpuProfilePath]) { + if (profilePath) { + mkdirSync(path.dirname(profilePath), { recursive: true }); + } + } + } + gateway = spawn( + process.execPath, + options.cpuProfDir + ? ["--cpu-prof", `--cpu-prof-dir=${options.cpuProfDir}`, ...gatewayArgs] + : gatewayArgs, + { + cwd: process.cwd(), + detached: process.platform !== "win32", + stdio: ["pipe", "pipe", "pipe", "ipc"], + env: { + ...createGatewayBenchEnv(root, configPath, { + caseEnv: { + ...(options.diagnosticsTimeline + ? { + OPENCLAW_DIAGNOSTICS: "timeline", + OPENCLAW_DIAGNOSTICS_TIMELINE_PATH: timelinePath, + } + : {}), + OPENCLAW_SKIP_CHANNELS: "1", + }, + }), + OPENAI_API_KEY: "gateway-concurrency-benchmark", + }, + }, + ); + gateway.once("exit", (exitCode, signal) => { + gatewayExitEvent = { + atMonotonicMicros: Number(process.hrtime.bigint() / 1_000n), + exitCode, + signal, + }; + }); + gateway.once("close", (exitCode, signal) => { + gatewayCloseEvent = { + atMonotonicMicros: Number(process.hrtime.bigint() / 1_000n), + exitCode, + signal, + }; + }); + gatewayOutput = captureChildOutput(gateway); + const ready = await waitForInitialProbe({ + deadlineAt: options.deadlineAt, + isDone: () => gateway?.exitCode != null || gateway?.signalCode != null, + path: "/readyz", + port, + startAt: runStartedAt, + }); + if (ready.status !== 200) { + throw new Error(`gateway did not become ready\n${gatewayOutput.readOutput()}`); + } + await waitForGatewayDispatchReady(gatewayOutput.readOutput, options.deadlineAt); + client = await connectGateway( + port, + options.deadlineAt, + protocolVersion, + true, + turnEvidence.onEvent, + ); + const rpc = client.request; + if (options.visibleObserver) { + await rpc("sessions.observer.visibility", { visible: true }); + } + // The first authenticated RPC lazily imports the server-method graph. It measured 6.9s + // on an idle M4 Pro (previously 18.4s) and crossed 20s on Linux; hot probes took 15-40ms. + // Keep that cold work out of the load-phase deadline and latency distributions. + const probeWarmupDeadlineAt = performance.now() + PROBE_WARMUP_TIMEOUT_MS; + client.setDeadlineAt(probeWarmupDeadlineAt); + const probeWarmup = await warmGatewayProbes({ + deadlineAt: probeWarmupDeadlineAt, + sample: (deadlineAt) => + sampleGateway({ + deadlineAt, + port, + rpc, + runStartedAt, + }), + }); + const setupDeadlineAt = performance.now() + options.timeoutMs; + const setupStartedAt = performance.now(); + mockCheckpoints.push(await readMockRequests(mockPort, setupDeadlineAt)); + client.setDeadlineAt(setupDeadlineAt); + const sessionCount = Math.max(options.concurrency, options.sessionCount); + const prepareSessions = + options.agentCount > 1 || + options.browserSessionClicks > 0 || + options.workspaceFanout || + options.sessionCount > 0 || + options.historyClients > 0 || + options.historyMessages > 0 || + options.sessionUpdates > 0 || + options.subscribers > 0; + const sessionAgents = Array.from( + { length: sessionCount }, + (_, index) => agentIds[index % agentIds.length]!, + ); + const sessionKeys = sessionAgents.map( + (agentId, index) => `agent:${agentId}:gateway-concurrency-${index + 1}`, + ); + const sessionSeedStartedAt = performance.now(); + if (prepareSessions) { + let nextSessionIndex = 0; + let seededSessionCount = 0; + await Promise.all( + Array.from({ length: Math.min(MAX_SESSION_SEED_CONCURRENCY, sessionCount) }, async () => { + for (;;) { + const index = nextSessionIndex++; + const sessionKey = sessionKeys[index]; + if (!sessionKey) { + return; + } + const workspaceDir = options.workspaceFanout + ? path.join(root, `workspace-${index + 1}`) + : undefined; + if (workspaceDir) { + mkdirSync(workspaceDir, { recursive: true }); + } + await rpc("sessions.create", { + key: sessionKey, + agentId: sessionAgents[index], + ...(workspaceDir ? { cwd: workspaceDir } : {}), + }); + for ( + let messageIndex = 0; + messageIndex < options.historyMessages; + messageIndex += 1 + ) { + const marker = `Synthetic history ${index + 1}/${messageIndex + 1}. `; + const message = marker + .repeat(Math.ceil(options.historyMessageChars / marker.length)) + .slice(0, options.historyMessageChars); + await rpc("chat.inject", { sessionKey, message }); + } + seededSessionCount += 1; + if (sessionCount >= 250 && seededSessionCount % 250 === 0) { + console.error( + `[bench-gateway-concurrency] seeded ${seededSessionCount}/${sessionCount} sessions`, + ); + } + } + }), + ); + } + const sessionSeedDurationMs = performance.now() - sessionSeedStartedAt; + const browserTargets: BrowserSessionTarget[] = []; + const browserHistoryMessages = options.browserHistoryMessages; + if (options.browserSessionClicks > 0) { + for (let index = 0; index <= options.browserSessionClicks; index += 1) { + const target = { + key: `agent:main:sidebar-click-${index}`, + marker: `Sidebar click history ${index}.`, + }; + await rpc("sessions.create", { key: target.key, agentId: "main" }); + for (let message = 0; message < browserHistoryMessages; message += 1) { + const prefix = `## Synthetic message ${message + 1}\n\n${target.marker}\n\n`; + const messageText = + prefix + + "Synthetic **benchmark** paragraph. " + .repeat(Math.ceil(options.historyMessageChars / 35)) + .slice(0, Math.max(0, options.historyMessageChars - prefix.length)); + await rpc("chat.inject", { sessionKey: target.key, message: messageText }); + } + browserTargets.push(target); + } + browserProbe = await startGatewayBrowserProbe({ + port, + timeoutMs: requireRemainingMs(setupDeadlineAt, "opening browser"), + initial: browserTargets[0]!, + visibleSessions: options.concurrency + browserTargets.length, + }); + } + let browserInventory: NonNullable["inventory"] | undefined; + if (browserProbe) { + const [unarchived, retained] = await Promise.all([ + rpc<{ totalCount: number }>("sessions.list", { limit: 1, archived: false }), + rpc<{ totalCount: number }>("sessions.list", { limit: 1, archived: "all" }), + ]); + browserInventory = { + seededLoadSessions: sessionCount, + seededClickSessions: browserTargets.length, + unarchivedSessions: unarchived.totalCount, + retainedSessions: retained.totalCount, + }; + } + // Normal inventory maintenance can archive older fixture sessions while seeding. + // Active turns and observers use the newest sessions; history still spans the inventory. + const turnSessionKeys = sessionKeys.slice(-options.concurrency); + const turnAgentIds = sessionAgents.slice(-options.concurrency); + const agentCoverage: BenchmarkRun["agentCoverage"] = + options.agentCount > 1 + ? { + configuredAgentIds: agentIds, + activeTurnAgentIds: [...new Set(turnAgentIds)], + beforeLoad: [], + completedTurns: [], + } + : undefined; + if (agentCoverage) { + for (const agentId of agentIds) { + // Require published runtime facts before measuring the configured roster. + const models = await rpc("models.list", { + agentId, + view: "configured", + refresh: false, + }); + if ( + ["gpt-5.6-luna", UTILITY_MODEL_ID].some( + (id) => + !models.models.some((model) => model.provider === "openai" && model.id === id), + ) + ) { + throw new Error(`Configured benchmark model is not published for ${agentId}`); + } + const storePath = path.join( + root, + "state", + "agents", + agentId, + "agent", + "openclaw-agent.sqlite", + ); + const sessions: SessionsListResult[] = []; + let offset = 0; + for (;;) { + const page = await rpc("sessions.list", { + agentId, + archived: "all", + limit: 1_000, + offset, + }); + if (page.path !== storePath || page.count !== page.sessions.length) { + throw new Error(`Benchmark session store or page mismatch for ${agentId}`); + } + sessions.push(page); + if (!page.hasMore) { + break; + } + if ( + !Number.isSafeInteger(page.nextOffset) || + page.nextOffset == null || + page.nextOffset <= offset || + page.nextOffset > sessionCount + browserTargets.length + ) { + throw new Error(`Benchmark session pagination did not advance for ${agentId}`); + } + offset = page.nextOffset; + } + const expectedKeys = sessionKeys.filter((_, index) => sessionAgents[index] === agentId); + const observedRows = sessions.flatMap((page) => + page.sessions.filter((row) => row.key.includes(":gateway-concurrency-")), + ); + const observedKeys = new Set(observedRows.map((row) => row.key)); + if ( + observedRows.length !== expectedKeys.length || + observedKeys.size !== expectedKeys.length || + observedRows.some((row) => row.agentId !== agentId) || + expectedKeys.some((key) => !observedKeys.has(key)) + ) { + throw new Error( + `Benchmark session inventory or distinct store mismatch for ${agentId}`, + ); + } + agentCoverage.beforeLoad.push({ agentId, models, sessions }); + } + } + const messageSubscriptions: TimedProbe[] = []; + for (let index = 0; index < options.subscribers; index += 1) { + const subscriber = await connectGateway(port, setupDeadlineAt, protocolVersion, false); + auxiliaryClients.push(subscriber); + if (options.visibleObserver) { + await subscriber.request("sessions.observer.visibility", { visible: true }); + } + const subscription = await timeRpcProbe( + subscriber.request, + "sessions.messages.subscribe", + { key: turnSessionKeys[index % turnSessionKeys.length] }, + runStartedAt, + ); + messageSubscriptions.push(subscription); + if (!subscription.ok) { + throw new Error(`session message subscription failed: ${subscription.error}`); + } + } + const historyClients = await Promise.all( + Array.from({ length: options.historyClients }, async () => { + const historyClient = await connectGateway(port, setupDeadlineAt, protocolVersion, false); + auxiliaryClients.push(historyClient); + return historyClient; + }), + ); + const sessionUpdateClients = await Promise.all( + Array.from( + { length: options.sessionUpdates > 0 ? options.sessionUpdateClients : 0 }, + async () => { + const updateClient = await connectGateway( + port, + setupDeadlineAt, + protocolVersion, + false, + ); + auxiliaryClients.push(updateClient); + return updateClient; + }, + ), + ); + const subscriptionProbeClient = + options.subscribers > 0 + ? await connectGateway(port, setupDeadlineAt, protocolVersion, false) + : undefined; + if (subscriptionProbeClient) { + auxiliaryClients.push(subscriptionProbeClient); + } + const memoryBefore = await readGatewayMemory(rpc, runStartedAt); + mockCheckpoints.push(await readMockRequests(mockPort, setupDeadlineAt)); + if (loadCpuProfilePath) { + await controlGatewayProfile(gateway, "cpu", "start", loadCpuProfilePath, { + includeWorkers: true, + }); + } + if (heapProfilePath) { + await controlGatewayProfile(gateway, "heap", "start", heapProfilePath, { + includeWorkers: true, + }); + } + const setupDurationMs = performance.now() - setupStartedAt; + // Large session fixtures are setup, not benchmarked load. Every measured + // run therefore gets its complete timeout after all clients are ready. + const loadDeadlineAt = performance.now() + options.timeoutMs; + client.setDeadlineAt(loadDeadlineAt); + for (const auxiliaryClient of auxiliaryClients) { + auxiliaryClient.setDeadlineAt(loadDeadlineAt); + } + const controlPlane: BenchmarkRun["controlPlane"] = []; + const controlUi: ControlUiProbe[] = []; + const history: BenchmarkRun["history"] = []; + const messageSubscriptionsDuringLoad: TimedProbe[] = []; + const readyz: ReadyProbe[] = []; + const sessionsList: TimedProbe[] = []; + const sessionUpdates: TimedProbe[] = []; + let peakRssMb = memoryBefore.rssMb; + let lastRssSampleAt = performance.now(); + let turnsDone = false; + let updatesDone = options.sessionUpdates === 0; + let browserDone = !browserProbe; + const workloadDone = () => turnsDone && updatesDone && browserDone; + let startedTurnCount = 0; + let resolveAllTurnsStarted!: () => void; + const allTurnsStarted = new Promise((resolve) => { + resolveAllTurnsStarted = resolve; + }); + const cpuBefore = await readGatewayCpuUsage(gateway); + const turnsStartedAt = performance.now(); + // Keep the live artifact intact: buffered setup writes can arrive after this boundary. + // Inclusive millisecond timestamps conservatively include events on the boundary. + const timelineFrom = Date.now(); + const loadStartMonotonicMicros = Number(process.hrtime.bigint() / 1_000n); + const turns = Promise.all( + turnSessionKeys.map((sessionKey, index) => + runSessionTurns(rpc, index, loadDeadlineAt, { + onStarted: () => { + startedTurnCount += 1; + if (startedTurnCount === options.concurrency) { + resolveAllTurnsStarted(); + } + }, + sessionKey, + toolEvents: options.toolEvents, + turnsPerSession: options.turnsPerSession, + evidence: turnEvidence, + }), + ), + ).finally(() => { + turnsDone = true; + resolveAllTurnsStarted(); + }); + const freshConnection = allTurnsStarted.then(async (): Promise => { + const startedAt = performance.now(); + try { + const freshClient = await connectGateway(port, loadDeadlineAt, protocolVersion, false); + freshClient.close(); + return { error: null, latencyMs: performance.now() - startedAt, ok: true }; + } catch (error) { + return { + error: describeProbeError(error), + latencyMs: performance.now() - startedAt, + ok: false, + }; + } + }); + const browserClicks = allTurnsStarted.then(async (): Promise => { + try { + if (!browserProbe) { + return []; + } + const clicks: BrowserSessionClick[] = []; + const targets = [...browserTargets.slice(1), browserTargets.at(-2)!]; + for (const [index, target] of targets.entries()) { + clicks.push( + await browserProbe.click( + target, + index === targets.length - 1, + () => !turnsDone, + Math.min(30_000, requireRemainingMs(loadDeadlineAt, "clicking sidebar session")), + ), + ); + } + return clicks; + } finally { + browserDone = true; + } + }); + const sampler = runProbeRounds({ + rounds: options.probeRounds, + deadlineAt: loadDeadlineAt, + cadenceMs: options.cadenceMs, + cadenceFrom: "start", + runFirst: true, + shouldContinue: () => !workloadDone() && readyz.length < MAX_SAMPLES_PER_RUN, + stopped: () => probesStopped, + runRound: async () => { + const subscriptionKey = turnSessionKeys[readyz.length % turnSessionKeys.length]; + const [sample, subscription, controlProbes] = await Promise.all([ + sampleGateway({ + deadlineAt: loadDeadlineAt, + port, + rpc, + runStartedAt, + }), + subscriptionProbeClient && subscriptionKey + ? timeRpcProbe( + subscriptionProbeClient.request, + "sessions.messages.subscribe", + { key: subscriptionKey }, + runStartedAt, + ) + : Promise.resolve(undefined), + options.controlPlane + ? Promise.all( + ["tasks.list", "cron.list", "cron.status"].map(async (method) => + Object.assign(await timeRpcProbe(rpc, method, {}, runStartedAt), { method }), + ), + ) + : Promise.resolve([]), + ]); + controlPlane.push(...controlProbes); + if (subscription) { + messageSubscriptionsDuringLoad.push(subscription); + if (subscription.ok) { + await subscriptionProbeClient?.request("sessions.messages.unsubscribe", { + key: subscriptionKey, + }); + } + } + readyz.push(sample.readyz); + sessionsList.push(sample.sessionsList); + controlUi.push(sample.controlUi); + if (performance.now() - lastRssSampleAt >= 1_000) { + // Linux reads procfs without spawning a process. Non-Linux hosts use + // the shared ps fallback at most once per second to bound perturbation. + peakRssMb = Math.max(peakRssMb, readGatewayProcessRssMb(gateway?.pid) ?? 0); + lastRssSampleAt = performance.now(); + } + }, + }); + probeJobs.push(sampler); + const historyLoad = Promise.all( + historyClients.map((historyClient, clientIndex) => { + let offset = clientIndex * options.historyBurst; + const job = runProbeRounds({ + rounds: options.probeRounds, + deadlineAt: loadDeadlineAt, + cadenceMs: options.cadenceMs, + cadenceFrom: "completion", + runFirst: false, + shouldContinue: () => !workloadDone() && history.length < MAX_SAMPLES_PER_RUN, + stopped: () => probesStopped, + runRound: async () => { + const probes = await Promise.all( + Array.from({ length: options.historyBurst }, (_, index) => { + const sessionKey = sessionKeys[(offset + index) % sessionKeys.length]!; + const probe = timeRpcProbe( + historyClient.request, + "chat.history", + { sessionKey }, + runStartedAt, + ); + return agentCoverage + ? probe.then((sample) => ({ ...sample, sessionKey })) + : probe; + }), + ); + history.push(...probes); + offset += options.historyBurst; + }, + }); + probeJobs.push(job); + return job; + }), + ); + let nextUpdateIndex = 0; + const sessionUpdateLoad = Promise.all( + sessionUpdateClients.map(async (updateClient) => { + for (;;) { + const index = nextUpdateIndex++; + if (index >= options.sessionUpdates) { + return; + } + const sessionKey = sessionKeys[index % sessionKeys.length]; + const update = await timeRpcProbe( + updateClient.request, + "sessions.patch", + { key: sessionKey, label: `Benchmark update ${index + 1}` }, + runStartedAt, + ); + sessionUpdates.push(update); + if (!update.ok) { + throw new Error(`sessions.patch load probe failed: ${update.error}`); + } + } + }), + ).finally(() => { + updatesDone = true; + }); + const [freshConnectionResult, sessionTurnCounts, browserClickResults] = await Promise.all([ + freshConnection, + turns, + browserClicks, + sampler, + historyLoad, + sessionUpdateLoad, + ]); + const cpuAfter = await readGatewayCpuUsage(gateway); + const loadEndMonotonicMicros = Number(process.hrtime.bigint() / 1_000n); + const turnsDurationMs = performance.now() - turnsStartedAt; + const memoryAfter = await readGatewayMemory(rpc, runStartedAt); + timelineWindow = { from: timelineFrom, through: Date.now() }; + let loadCpuProfile: BenchmarkRun["loadCpuProfile"]; + if (loadCpuProfilePath) { + await controlGatewayProfile(gateway, "cpu", "stop", loadCpuProfilePath); + loadCpuProfile = { + ...readGatewayCpuProfile(loadCpuProfilePath), + scope: "main-isolate", + workersManifestPath: `${loadCpuProfilePath}.workers.json`, + }; + } + let heapProfile: BenchmarkRun["heapProfile"]; + if (heapProfilePath) { + await controlGatewayProfile(gateway, "heap", "stop", heapProfilePath); + heapProfile = { + ...readGatewayHeapProfile(heapProfilePath), + scope: "main-isolate", + workersManifestPath: `${heapProfilePath}.workers.json`, + }; + } + if (options.historyClients > 0 && !history.some((sample) => sample.ok)) { + const failure = history[0]?.error ?? "no requests completed before turns finished"; + throw new Error(`all configured chat.history load probes failed: ${failure}`); + } + peakRssMb = Math.max(peakRssMb, memoryAfter.rssMb); + mockCheckpoints.push(await readMockRequests(mockPort, loadDeadlineAt)); + + if (agentCoverage) { + agentCoverage.completedTurns = agentIds.map((agentId) => ({ + agentId, + count: sessionTurnCounts.reduce( + (total, count, index) => total + (turnAgentIds[index] === agentId ? count : 0), + 0, + ), + })); + } + result = { + ...(agentCoverage ? { agentCoverage } : {}), + ...(browserProbe && browserInventory + ? { + browser: { + newPageReadyMs: browserProbe.newPageReadyMs, + initialSessionReadyMs: browserProbe.initialSessionReadyMs, + historyMessagesPerTarget: browserHistoryMessages, + inventory: browserInventory, + clicks: browserClickResults, + }, + } + : {}), + ...(heapProfile ? { heapProfile } : {}), + ...(loadCpuProfile ? { loadCpuProfile } : {}), + controlPlane, + controlUi, + cpuUsage: measureGatewayCpuUsage(cpuBefore, cpuAfter), + durationMs: performance.now() - runStartedAt, + freshConnection: freshConnectionResult, + history, + loadWindow: { + startMonotonicMicros: loadStartMonotonicMicros, + endMonotonicMicros: loadEndMonotonicMicros, + }, + memory: { after: memoryAfter, before: memoryBefore, peakRssMb }, + messageSubscriptions, + messageSubscriptionsDuringLoad, + probeWarmup, + pluginMetadataScans: summarizePluginMetadataScans([]), + readyz, + sessionSeedDurationMs, + sessionsList, + sessionUpdates, + setupDurationMs, + turnCount: sessionTurnCounts.reduce((sum, count) => sum + count, 0), + turnsDurationMs, + }; + } catch (error) { + const detail = formatRunFailure(error, gatewayOutput, mockOutput); + throw new Error(detail, { cause: error }); + } finally { + probesStopped = true; + try { + await browserProbe?.close(); + } finally { + for (const auxiliaryClient of auxiliaryClients) { + auxiliaryClient.close(); + } + try { + if (gateway) { + if (options.cpuProfDir && gateway.exitCode === null && gateway.signalCode === null) { + // V8 flushes the main-isolate CPU profile on its normal interrupt path. + const profileFlushed = new Promise((resolve) => { + gateway!.once("exit", () => { + resolve(); + }); + }); + gateway.kill("SIGINT"); + await Promise.race([profileFlushed, delay(2_000)]); + } + gatewayExit = await stopChild(gateway); + } + // A fatal turn may end Promise.all before fixed probe rounds settle. + // Join their closed-client failures before removing the fixture state. + await Promise.allSettled(probeJobs); + } finally { + // Gateway shutdown drains admitted event dispatches before closing sockets. + // Keep this client alive through that drain so delayed tool evidence survives. + client?.close(); + } + } + } + // close() initiates a handshake; retain late event evidence until it settles. + await client?.waitClosed(); + mockCheckpoints.push(await readMockRequests(mockPort, performance.now() + HTTP_TIMEOUT_MS)); + if (options.diagnosticsTimeline) { + if (!gatewayExit || gatewayExit.exitCode !== 0 || gatewayExit.signal !== null) { + throw new Error( + formatRunFailure( + new Error( + `Gateway did not exit cleanly; diagnostics timeline may be incomplete: ${JSON.stringify({ helper: gatewayExit, child: readGatewayProcess() })}`, + ), + gatewayOutput, + mockOutput, + ), + ); + } + if (gatewayOutput.readOutput().includes("[diagnostics] failed to write timeline event")) { + throw new Error("Gateway reported a diagnostics timeline write failure"); + } + result.pluginMetadataScans = summarizePluginMetadataScans( + readDiagnosticsTimelineSpans(timelinePath, timelineWindow), + ); + } + return { + ...result, + mockRequests: summarizeMockRequests(mockCheckpoints), + turnEvidence: turnEvidence.finish(), + gatewayProcess: readGatewayProcess(), + ...(gatewayExit ? { gatewayExit } : {}), + }; + } finally { + try { + if (mockProvider) { + await stopChild(mockProvider); + } + } finally { + rmSync(root, { force: true, maxRetries: 3, recursive: true, retryDelay: 100 }); + } + } +} + +function summarizeRuns( + runs: readonly BenchmarkRun[], + options: Pick = {}, +) { + const controlPlane = runs.flatMap((run) => run.controlPlane); + const browserClicks = runs.flatMap((run) => run.browser?.clicks ?? []); + const controlUi = runs.flatMap((run) => run.controlUi); + const readyz = runs.flatMap((run) => run.readyz); + const sessionsList = runs.flatMap((run) => run.sessionsList); + const history = runs.flatMap((run) => run.history); + const subscriptions = runs.flatMap((run) => run.messageSubscriptions); + const subscriptionsDuringLoad = runs.flatMap((run) => run.messageSubscriptionsDuringLoad); + const sessionUpdates = runs.flatMap((run) => run.sessionUpdates); + // Setup subscriptions and warmup probes are not load-phase measurements. + const controlMethods = ["tasks.list", "cron.list", "cron.status"]; + const controlMethodProbes = controlMethods.map((method) => ({ + method, + samples: controlPlane.filter((sample) => sample.method === method), + })); + const budgetViolations = [ + ...controlMethodProbes.map(({ method, samples }) => ({ + name: `Gateway ${method} probe`, + maxMs: options.maxControlMs, + samples, + })), + { + name: "fresh Gateway connection", + maxMs: options.maxHandshakeMs, + samples: runs.map((run) => run.freshConnection), + }, + ...( + [ + ["readyz", readyz], + ["Control UI", controlUi], + ["sessions.list", sessionsList], + ["chat.history", history], + ["sessions.messages.subscribe", subscriptionsDuringLoad], + ["sessions.patch", sessionUpdates], + ] as const + ).map(([name, samples]) => ({ + name: `Gateway ${name} probe`, + maxMs: options.maxControlMs, + samples, + })), + ].flatMap(({ name, maxMs, samples }) => { + if (maxMs === undefined) { + return []; + } + const violation = samples.find((sample) => !sample.ok || sample.latencyMs > maxMs); + return violation + ? [ + `${name} exceeded ${maxMs}ms: ok=${violation.ok} ` + + `latencyMs=${violation.latencyMs.toFixed(1)} error=${violation.error ?? "none"}`, + ] + : []; + }); + for (const sample of browserClicks) { + if (sample.error) { + budgetViolations.push(`Browser session ${sample.sessionKey} failed: ${sample.error}`); + } + } + return { + ...(browserClicks.length > 0 + ? { + browserSessionClicks: { + failedSamples: browserClicks.filter((sample) => sample.error !== null).length, + samplesOutsideActiveLoad: browserClicks.filter( + (sample) => !sample.activeLoadAtStart || !sample.activeLoadAtFinish, + ).length, + firstVisitReadyMs: summarizeNumbers( + browserClicks.flatMap((sample) => + !sample.revisit && sample.readyMs !== null ? [sample.readyMs] : [], + ), + ), + revisitReadyMs: summarizeNumbers( + browserClicks.flatMap((sample) => + sample.revisit && sample.readyMs !== null ? [sample.readyMs] : [], + ), + ), + }, + } + : {}), + budgetViolations, + gatewayProcessCpuMs: summarizeNumbers(runs.map((run) => run.cpuUsage.process.totalMs)), + gatewayProcessCpuMsPerTurn: summarizeNumbers( + runs.map((run) => run.cpuUsage.process.totalMs / run.turnCount), + ), + gatewayMainThreadCpuMs: summarizeNumbers(runs.map((run) => run.cpuUsage.mainThread.totalMs)), + gatewayProcessCpuCoreRatio: summarizeNumbers( + runs.map((run) => run.cpuUsage.process.totalMs / run.cpuUsage.wallMs), + ), + gatewaySampledAllocatedBytes: summarizeNumbers( + runs.flatMap((run) => (run.heapProfile ? [run.heapProfile.sampledAllocatedBytes] : [])), + ), + gatewaySampledAllocatedBytesPerTurn: summarizeNumbers( + runs.flatMap((run) => + run.heapProfile ? [run.heapProfile.sampledAllocatedBytes / run.turnCount] : [], + ), + ), + controlPlane: Object.fromEntries( + controlMethodProbes.map(({ method, samples }) => [ + method, + { + failedSamples: samples.filter((sample) => !sample.ok).length, + latencyMs: summarizeNumbers(samples.map((sample) => sample.latencyMs)), + }, + ]), + ), + controlUiFailedSamples: controlUi.filter((sample) => !sample.ok).length, + controlUiLatencyMs: summarizeNumbers(controlUi.map((sample) => sample.latencyMs)), + cpuCoreRatio: summarizeNumbers( + readyz.flatMap((sample) => (sample.cpuCoreRatio == null ? [] : [sample.cpuCoreRatio])), + ), + degradedSamples: readyz.filter((sample) => sample.degraded === true).length, + eventLoopDelayMaxMs: summarizeNumbers( + readyz.flatMap((sample) => (sample.delayMaxMs == null ? [] : [sample.delayMaxMs])), + ), + eventLoopDelayP99Ms: summarizeNumbers( + readyz.flatMap((sample) => (sample.delayP99Ms == null ? [] : [sample.delayP99Ms])), + ), + eventLoopUtilization: summarizeNumbers( + readyz.flatMap((sample) => (sample.utilization == null ? [] : [sample.utilization])), + ), + freshConnectionFailedRuns: runs.filter((run) => !run.freshConnection.ok).length, + freshConnectionLatencyMs: summarizeNumbers(runs.map((run) => run.freshConnection.latencyMs)), + gatewayUncleanExits: runs.filter( + (run) => + run.gatewayExit && (run.gatewayExit.exitCode !== 0 || run.gatewayExit.signal !== null), + ).length, + gatewayExternalGrowthMb: summarizeNumbers( + runs.flatMap((run) => { + const before = run.memory.before.externalMb; + const after = run.memory.after.externalMb; + return before === undefined || after === undefined ? [] : [after - before]; + }), + ), + gatewayExternalMb: summarizeNumbers( + runs.flatMap((run) => + run.memory.after.externalMb === undefined ? [] : [run.memory.after.externalMb], + ), + ), + gatewayArrayBuffersGrowthMb: summarizeNumbers( + runs.flatMap((run) => { + const before = run.memory.before.arrayBuffersMb; + const after = run.memory.after.arrayBuffersMb; + return before === undefined || after === undefined ? [] : [after - before]; + }), + ), + gatewayArrayBuffersMb: summarizeNumbers( + runs.flatMap((run) => + run.memory.after.arrayBuffersMb === undefined ? [] : [run.memory.after.arrayBuffersMb], + ), + ), + gatewayHeapGrowthMb: summarizeNumbers( + runs.map((run) => run.memory.after.heapUsedMb - run.memory.before.heapUsedMb), + ), + gatewayHeapUsedMb: summarizeNumbers(runs.map((run) => run.memory.after.heapUsedMb)), + gatewayPeakRssMb: summarizeNumbers(runs.map((run) => run.memory.peakRssMb)), + gatewayRssGrowthMb: summarizeNumbers( + runs.map((run) => run.memory.after.rssMb - run.memory.before.rssMb), + ), + historyFailedSamples: history.filter((sample) => !sample.ok).length, + historyLatencyMs: summarizeNumbers(history.map((sample) => sample.latencyMs)), + historySampleCount: history.length, + messageSubscriptionFailedSamples: subscriptions.filter((sample) => !sample.ok).length, + messageSubscriptionLatencyMs: summarizeNumbers(subscriptions.map((sample) => sample.latencyMs)), + messageSubscriptionLoadFailedSamples: subscriptionsDuringLoad.filter((sample) => !sample.ok) + .length, + messageSubscriptionLoadLatencyMs: summarizeNumbers( + subscriptionsDuringLoad.map((sample) => sample.latencyMs), + ), + mockRequestIngress: Object.fromEntries( + MOCK_INGRESS_KEYS.map((key) => [ + key, + runs.reduce((count, run) => count + run.mockRequests.ingress.total[key], 0), + ]), + ), + mockResponseSelections: Object.fromEntries( + MOCK_SELECTION_KEYS.map((key) => [ + key, + runs.reduce((count, run) => count + run.mockRequests.selections[key], 0), + ]), + ), + toolTurns: runs.reduce((count, run) => count + run.turnEvidence.toolTurns, 0), + observerModelDigestTurns: runs.reduce( + (count, run) => count + run.turnEvidence.observerModelDigestTurns, + 0, + ), + pluginMetadataScanCount: runs.reduce((sum, run) => sum + run.pluginMetadataScans.count, 0), + pluginMetadataScanTotalDurationMs: runs.reduce( + (sum, run) => sum + run.pluginMetadataScans.totalDurationMs, + 0, + ), + readyzLatencyMs: summarizeNumbers(readyz.map((sample) => sample.latencyMs)), + readyzFailedSamples: readyz.filter((sample) => !sample.ok).length, + sampleCount: readyz.length, + sessionSeedDurationMs: summarizeNumbers(runs.map((run) => run.sessionSeedDurationMs)), + sessionsListLatencyMs: summarizeNumbers(sessionsList.map((sample) => sample.latencyMs)), + sessionsListFailedSamples: sessionsList.filter((sample) => !sample.ok).length, + sessionUpdateFailedSamples: sessionUpdates.filter((sample) => !sample.ok).length, + sessionUpdateLatencyMs: summarizeNumbers(sessionUpdates.map((sample) => sample.latencyMs)), + sessionUpdateSampleCount: sessionUpdates.length, + setupDurationMs: summarizeNumbers(runs.map((run) => run.setupDurationMs)), + turnCount: runs.reduce((sum, run) => sum + run.turnCount, 0), + turnsDurationMs: summarizeNumbers(runs.map((run) => run.turnsDurationMs)), + }; +} + +async function runBenchmarkSamples(params: { + now?: () => number; + onProgress?: (message: string) => void; + options: CliOptions; + runSample?: typeof runGatewaySample; +}): Promise { + const now = params.now ?? performance.now.bind(performance); + const runSample = params.runSample ?? runGatewaySample; + const runs: BenchmarkRun[] = []; + const total = params.options.runs + params.options.warmup; + for (let index = 0; index < total; index += 1) { + // Each sample gets the same budget so earlier runs cannot shrink later agent waits. + // runGatewaySample extends this deadline by its probe warmup before load starts. + const deadlineAt = now() + params.options.timeoutMs; + const run = await runSample({ ...params.options, deadlineAt }); + if (index >= params.options.warmup) { + runs.push(run); + params.onProgress?.( + `[bench-gateway-concurrency] run ${runs.length}/${params.options.runs}: turns=${run.turnCount} samples=${run.readyz.length} duration=${run.durationMs.toFixed(1)}ms`, + ); + } else { + params.onProgress?.( + `[bench-gateway-concurrency] warmup ${index + 1}/${params.options.warmup}: duration=${run.durationMs.toFixed(1)}ms`, + ); + } + } + return runs; +} + +async function main(): Promise { + const argv = process.argv.slice(2); + if (hasHelpFlag(argv)) { + printUsage(); + return; + } + const options = parseOptions(argv); + const runs = await runBenchmarkSamples({ onProgress: console.error, options }); + const payload = { + agentCount: options.agentCount, + browserHistoryMessages: options.browserHistoryMessages, + browserSessionClicks: options.browserSessionClicks, + cadenceMs: options.cadenceMs, + concurrency: options.concurrency, + controlPlane: options.controlPlane, + historyMessages: options.historyMessages, + historyMessageChars: options.historyMessageChars, + diagnosticsTimeline: options.diagnosticsTimeline, + entry: options.entry, + generatedAt: new Date().toISOString(), + historyBurst: options.historyBurst, + historyClients: options.historyClients, + mode: "mock-streaming-agent", + pluginCount: options.pluginCount, + probeWorkload: { + mode: options.probeRounds === undefined ? "adaptive" : "fixed-rounds", + samplerRoundsPerRun: options.probeRounds ?? null, + historyRequestsPerRun: + options.probeRounds === undefined + ? null + : options.probeRounds * options.historyClients * options.historyBurst, + peakRssSampling: "sampler-rounds-and-final-memory", + }, + runs, + sessionCount: Math.max(options.sessionCount, options.concurrency), + sessionUpdateClients: options.sessionUpdates > 0 ? options.sessionUpdateClients : 0, + sessionUpdates: options.sessionUpdates, + streamChunkDelayMs: options.streamChunkDelayMs, + subscribers: options.subscribers, + summary: summarizeRuns(runs, options), + toolEvents: options.toolEvents, + turnsPerSession: options.turnsPerSession, + visibleObserver: options.visibleObserver, + workspaceFanout: options.workspaceFanout, + }; + if (options.output) { + mkdirSync(path.dirname(options.output), { recursive: true }); + writeFileSync(options.output, `${JSON.stringify(payload, null, 2)}\n`); + } + if (options.json || !options.output) { + console.log(JSON.stringify(payload, null, 2)); + } + if (payload.summary.budgetViolations.length > 0) { + throw new Error(payload.summary.budgetViolations.join("\n")); + } +} + +export const testing = { + parseOptions, + parseMockRequests, + summarizeMockRequests, + createTurnEvidence, + formatProbeFailure, + formatRunFailure, + requestHttp, + runBenchmarkSamples, + runProbeRounds, + runSessionTurns, + runTurn, + sampleGateway, + readDiagnosticsTimelineSpans, + summarizePluginMetadataScans, + summarizeNumbers, + summarizeRuns, + tailLines, + warmGatewayProbes, +}; + +if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { + void main() + .catch((error: unknown) => { + console.error(error instanceof CliArgumentError ? error.message : (error as Error)?.stack); + process.exitCode = 1; + }) + .finally(() => { + if (process.exitCode && process.exitCode !== 0) { + console.error(`[bench-gateway-concurrency] FAILED (exit ${process.exitCode})`); + } + }); +} diff --git a/scripts/bench-gateway-startup.ts b/scripts/bench-gateway-startup.ts new file mode 100644 index 0000000000000000000000000000000000000000..fbd1180a857dd4514b3e51d6fb8a090a919c8ccd --- /dev/null +++ b/scripts/bench-gateway-startup.ts @@ -0,0 +1,1119 @@ +// Bench Gateway Startup script supports OpenClaw repository automation. +import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process"; +import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { performance } from "node:perf_hooks"; +import { pathToFileURL } from "node:url"; +import { listBundledPluginPackArtifacts } from "./lib/bundled-plugin-build-entries.mjs"; +import { delay, stopChild } from "./lib/gateway-bench-child.ts"; +import { getFreePort, readProcessRssMb, readProcessTreeCpuMs } from "./lib/gateway-bench-probes.ts"; +import { + BASE_GATEWAY_BENCH_CONFIG, + buildGatewayBenchChildArgs, + classifyGatewayReadyLog, + CliArgumentError, + collectOutputLines, + collectTraceLine, + createGatewayBenchEnv, + flushOutputLineBuffers, + formatMb, + formatMs, + formatStats, + hasFlag, + hasHelpFlag, + parseFlagValue, + parseNonNegativeInt, + parsePositiveInt, + parseRepeatableFlag, + resolveCases as resolveGatewayBenchCases, + resolveEntry as resolveGatewayBenchEntry, + resolveOutputPath, + STALLED_CATALOG_MODEL_ID, + STALLED_CATALOG_PROVIDER_ID, + summarizeNumbers, + summarizeTraceStats, + type InitialProbeResult, + type SummaryStats, + validateCliArgs as validateGatewayBenchCliArgs, + writeGatewayBenchConfig, + writePluginFixtures, + waitForInitialProbe as waitForProbe, +} from "./lib/gateway-bench-runtime.ts"; +import { selectSlowStartupTraceDurations } from "./lib/gateway-startup-trace-ranking.js"; + +type GatewayBenchCase = { + agentTopology?: "incident-scale" | "single" | "shared-eleven-plus-distinct-one"; + completionTracePhase?: string; + config: Record; + env?: Record; + id: string; + incidentFixture?: "combined" | "database" | "null-metadata" | "packaged-plugins" | "workspace"; + name: string; + pluginActivationOnStartup?: boolean; + pluginCount?: number; + providerCatalogStallMs?: number; + providerStaticCatalogModelCount?: number; + providerStaticCatalogStallMs?: number; + runByDefault?: boolean; +}; + +type ProbeResult = InitialProbeResult; + +type GatewaySample = { + completionMs: number | null; + cpuCoreRatio: number | null; + cpuMs: number | null; + exitedBeforeTeardown?: boolean; + exitCode: number | null; + firstOutputMs: number | null; + gatewayReadyLogLine: string | null; + gatewayReadyLogMs: number | null; + healthz: ProbeResult; + httpListenLogLine: string | null; + httpListenLogMs: number | null; + maxRssMb: number | null; + outputTail: string; + readyz: ProbeResult; + signal: string | null; + startupTrace: Record; +}; + +type CaseResult = { + id: string; + name: string; + samples: GatewaySample[]; + summary: { + completionMs: SummaryStats | null; + firstOutputMs: SummaryStats | null; + cpuCoreRatio: SummaryStats | null; + cpuMs: SummaryStats | null; + gatewayReadyLogMs: SummaryStats | null; + healthzMs: SummaryStats | null; + httpListenLogMs: SummaryStats | null; + maxRssMb: SummaryStats | null; + readyzMs: SummaryStats | null; + startupTrace: Record; + }; +}; + +type BenchmarkFailure = { + id: string; + reason: string; + sampleIndex: number; +}; + +type CliOptions = { + cases: GatewayBenchCase[]; + cpuProfDir?: string; + entry: string; + heapProfDir?: string; + json: boolean; + output?: string; + runs: number; + timeoutMs: number; + warmup: number; +}; + +const DEFAULT_RUNS = 5; +const DEFAULT_WARMUP = 1; +const DEFAULT_TIMEOUT_MS = 30_000; +const DEFAULT_ENTRY = "dist/entry.js"; +const INCIDENT_COMBINED_HEALTHZ_P95_MAX_MS = 30_000; +const INCIDENT_COMBINED_READYZ_P95_MAX_MS = 60_000; +const BOOLEAN_FLAGS = new Set(["--help", "-h", "--json"]); +const VALUE_FLAGS = new Set([ + "--case", + "--cpu-prof-dir", + "--entry", + "--heap-prof-dir", + "--output", + "--runs", + "--timeout-ms", + "--warmup", +]); + +const BASE_CONFIG = BASE_GATEWAY_BENCH_CONFIG; +const LARGE_PLUGIN_MODEL_AGENT_COUNT = 256; +const LARGE_PLUGIN_MODEL_COUNT = 58; +const LARGE_PLUGIN_MODEL_PROVIDER_IDS = ["openai", "google", "minimax"] as const; + +function buildLargePluginModelAgentEntries() { + const model = { + primary: `${LARGE_PLUGIN_MODEL_PROVIDER_IDS[0]}/model-01`, + fallbacks: Array.from({ length: LARGE_PLUGIN_MODEL_COUNT - 1 }, (_, offset) => { + const index = offset + 1; + const provider = + index % 3 === 0 + ? LARGE_PLUGIN_MODEL_PROVIDER_IDS[0] + : index % 3 === 1 + ? LARGE_PLUGIN_MODEL_PROVIDER_IDS[1] + : LARGE_PLUGIN_MODEL_PROVIDER_IDS[2]; + return `${provider}/model-${String(index + 1).padStart(2, "0")}`; + }), + }; + return Object.fromEntries( + Array.from({ length: LARGE_PLUGIN_MODEL_AGENT_COUNT }, (_, index) => [ + `agent-${String(index + 1).padStart(4, "0")}`, + { model }, + ]), + ); +} + +const GATEWAY_CASES: readonly GatewayBenchCase[] = [ + { + id: "default", + name: "gateway default", + config: BASE_CONFIG, + }, + { + id: "skipChannels", + name: "gateway, skip channels", + env: { OPENCLAW_SKIP_CHANNELS: "1" }, + config: BASE_CONFIG, + }, + { + id: "preparedRuntimeCatalogStall", + name: "gateway, prepared runtime with CPU-stalling live catalog", + env: { OPENCLAW_SKIP_CHANNELS: "1" }, + providerCatalogStallMs: 2_000, + config: { + ...BASE_CONFIG, + agents: { + defaults: { + model: { primary: `${STALLED_CATALOG_PROVIDER_ID}/${STALLED_CATALOG_MODEL_ID}` }, + models: { + [`${STALLED_CATALOG_PROVIDER_ID}/${STALLED_CATALOG_MODEL_ID}`]: { + agentRuntime: { id: "openclaw" }, + }, + }, + }, + }, + }, + }, + { + id: "preparedRuntimeScaleOne", + name: "gateway, prepared runtime scale with one agent", + agentTopology: "single", + completionTracePhase: "sidecars.ready", + env: { OPENCLAW_SKIP_CHANNELS: "1" }, + providerStaticCatalogModelCount: 64, + providerStaticCatalogStallMs: 100, + config: { + ...BASE_CONFIG, + agents: { + defaults: { + model: { primary: `${STALLED_CATALOG_PROVIDER_ID}/${STALLED_CATALOG_MODEL_ID}` }, + models: { + [`${STALLED_CATALOG_PROVIDER_ID}/${STALLED_CATALOG_MODEL_ID}`]: { + agentRuntime: { id: "openclaw" }, + }, + }, + }, + }, + }, + }, + { + id: "preparedRuntimeScaleMany", + name: "gateway, prepared runtime scale with 11 shared-workspace agents and one distinct", + agentTopology: "shared-eleven-plus-distinct-one", + completionTracePhase: "sidecars.ready", + env: { OPENCLAW_SKIP_CHANNELS: "1" }, + providerStaticCatalogModelCount: 64, + providerStaticCatalogStallMs: 100, + config: { + ...BASE_CONFIG, + agents: { + defaults: { + model: { primary: `${STALLED_CATALOG_PROVIDER_ID}/${STALLED_CATALOG_MODEL_ID}` }, + models: { + [`${STALLED_CATALOG_PROVIDER_ID}/${STALLED_CATALOG_MODEL_ID}`]: { + agentRuntime: { id: "openclaw" }, + }, + }, + }, + }, + }, + }, + { + id: "oneInternalHook", + name: "gateway, one configured internal hook", + env: { OPENCLAW_SKIP_CHANNELS: "1" }, + config: { + ...BASE_CONFIG, + hooks: { + internal: { + entries: { + "session-memory": { enabled: true }, + }, + }, + }, + }, + }, + { + id: "allInternalHooks", + name: "gateway, all internal hooks", + env: { OPENCLAW_SKIP_CHANNELS: "1" }, + config: { + ...BASE_CONFIG, + hooks: { + internal: { + enabled: true, + }, + }, + }, + }, + { + id: "fiftyPlugins", + name: "gateway, 50 manifest plugins", + env: { OPENCLAW_SKIP_CHANNELS: "1" }, + pluginActivationOnStartup: true, + pluginCount: 50, + config: BASE_CONFIG, + }, + { + id: "fiftyStartupLazyPlugins", + name: "gateway, 50 startup-lazy manifest plugins", + env: { OPENCLAW_SKIP_CHANNELS: "1" }, + pluginActivationOnStartup: false, + pluginCount: 50, + config: BASE_CONFIG, + }, + { + id: "largePluginModelConfig", + name: `gateway, ${LARGE_PLUGIN_MODEL_AGENT_COUNT} agents with ${LARGE_PLUGIN_MODEL_COUNT} plugin-owned model refs each`, + completionTracePhase: "config.snapshot.auto-enable", + env: { OPENCLAW_SKIP_CHANNELS: "1" }, + runByDefault: false, + config: { + ...BASE_CONFIG, + agents: { + ownership: "explicit", + entries: buildLargePluginModelAgentEntries(), + }, + plugins: { + enabled: true, + allow: [...LARGE_PLUGIN_MODEL_PROVIDER_IDS], + }, + }, + }, + { + id: "incidentDatabase", + name: "gateway, incident-scale database", + incidentFixture: "database", + runByDefault: false, + config: BASE_CONFIG, + }, + { + id: "incidentNullMetadata", + name: "gateway, incident-scale NULL metadata repair", + incidentFixture: "null-metadata", + runByDefault: false, + config: BASE_CONFIG, + }, + { + id: "incidentWorkspace", + name: "gateway, incident-scale workspace", + agentTopology: "incident-scale", + incidentFixture: "workspace", + runByDefault: false, + config: BASE_CONFIG, + }, + { + id: "incidentPackagedPlugins", + name: "gateway, packaged plugin inventory", + env: { OPENCLAW_DISABLE_BUNDLED_ENTRY_SOURCE_FALLBACK: "1" }, + incidentFixture: "packaged-plugins", + runByDefault: false, + config: BASE_CONFIG, + }, + { + id: "incidentCombined", + name: "gateway, incident-scale combined load", + agentTopology: "incident-scale", + env: { OPENCLAW_DISABLE_BUNDLED_ENTRY_SOURCE_FALLBACK: "1" }, + incidentFixture: "combined", + runByDefault: false, + config: BASE_CONFIG, + }, +] as const; + +function validateCliArgs(argv: string[]): void { + validateGatewayBenchCliArgs(argv, { + booleanFlags: BOOLEAN_FLAGS, + repeatableValueFlags: new Set(["--case"]), + valueFlags: VALUE_FLAGS, + }); +} + +function resolveEntry(raw: string | undefined): string { + return resolveGatewayBenchEntry(raw, DEFAULT_ENTRY); +} + +function resolveCases(caseIds: string[]): GatewayBenchCase[] { + const defaultCases = GATEWAY_CASES.filter((benchCase) => benchCase.runByDefault !== false); + return resolveGatewayBenchCases(caseIds, caseIds.length === 0 ? defaultCases : GATEWAY_CASES, { + allByDefault: true, + validateDuplicatesFirst: true, + }); +} + +function parseOptions(argv: string[] = process.argv.slice(2)): CliOptions { + validateCliArgs(argv); + return { + cases: resolveCases(parseRepeatableFlag(argv, "--case")), + cpuProfDir: parseFlagValue(argv, "--cpu-prof-dir"), + entry: resolveEntry(parseFlagValue(argv, "--entry")), + heapProfDir: parseFlagValue(argv, "--heap-prof-dir"), + json: hasFlag(argv, "--json"), + output: resolveOutputPath(parseFlagValue(argv, "--output")), + runs: parsePositiveInt(parseFlagValue(argv, "--runs"), DEFAULT_RUNS, "--runs"), + timeoutMs: parsePositiveInt( + parseFlagValue(argv, "--timeout-ms"), + DEFAULT_TIMEOUT_MS, + "--timeout-ms", + ), + warmup: parseNonNegativeInt(parseFlagValue(argv, "--warmup"), DEFAULT_WARMUP, "--warmup"), + }; +} + +function printUsage(): void { + console.log(`OpenClaw Gateway startup benchmark + +Usage: + pnpm test:startup:gateway -- [options] + node --import tsx scripts/bench-gateway-startup.ts [options] + +Options: + --case Specific case id to run; repeatable + --entry Gateway CLI entry file (default: ${DEFAULT_ENTRY}) + --runs Measured runs per case (default: ${DEFAULT_RUNS}) + --warmup Warmup runs per case (default: ${DEFAULT_WARMUP}) + --timeout-ms Per-run timeout (default: ${DEFAULT_TIMEOUT_MS}) + --cpu-prof-dir

Write one V8 CPU profile per run + --heap-prof-dir Write one V8 heap profile per run + --output Write machine-readable JSON to a file + --json Emit machine-readable JSON + --help, -h Show this text + +Case ids: + ${GATEWAY_CASES.map((benchCase) => `${benchCase.id} (${benchCase.name})`).join("\n ")} +`); +} + +function summarizeCase(benchCase: GatewayBenchCase, samples: GatewaySample[]): CaseResult { + const startupTrace = summarizeTraceStats(samples, (sample) => sample.startupTrace); + return { + id: benchCase.id, + name: benchCase.name, + samples, + summary: { + completionMs: summarizeNumbers( + samples + .map((sample) => sample.completionMs) + .filter((value): value is number => typeof value === "number"), + ), + firstOutputMs: summarizeNumbers( + samples + .map((sample) => sample.firstOutputMs) + .filter((value): value is number => typeof value === "number"), + ), + cpuCoreRatio: summarizeNumbers( + samples + .map((sample) => sample.cpuCoreRatio) + .filter((value): value is number => typeof value === "number"), + ), + cpuMs: summarizeNumbers( + samples + .map((sample) => sample.cpuMs) + .filter((value): value is number => typeof value === "number"), + ), + gatewayReadyLogMs: summarizeNumbers( + samples + .map((sample) => sample.gatewayReadyLogMs) + .filter((value): value is number => typeof value === "number"), + ), + healthzMs: summarizeNumbers( + samples + .map((sample) => sample.healthz.ms) + .filter((value): value is number => typeof value === "number"), + ), + httpListenLogMs: summarizeNumbers( + samples + .map((sample) => sample.httpListenLogMs) + .filter((value): value is number => typeof value === "number"), + ), + maxRssMb: summarizeNumbers( + samples + .map((sample) => sample.maxRssMb) + .filter((value): value is number => typeof value === "number"), + ), + readyzMs: summarizeNumbers( + samples + .map((sample) => sample.readyz.ms) + .filter((value): value is number => typeof value === "number"), + ), + startupTrace, + }, + }; +} + +function collectResultFailures(results: CaseResult[]): BenchmarkFailure[] { + const failures: BenchmarkFailure[] = []; + for (const result of results) { + result.samples.forEach((sample, index) => { + const missing: string[] = []; + if (sample.healthz.status !== 200 || sample.healthz.ms == null) { + missing.push("/healthz"); + } + if (sample.readyz.status !== 200 || sample.readyz.ms == null) { + missing.push("/readyz"); + } + if (sample.completionMs == null) { + missing.push("completion"); + } + if (sample.cpuMs == null || sample.cpuCoreRatio == null) { + missing.push("cpu"); + } + if (sample.maxRssMb == null) { + missing.push("rss"); + } + if (missing.length > 0) { + failures.push({ + id: result.id, + reason: `missing ${missing.join(", ")}`, + sampleIndex: index + 1, + }); + return; + } + if (sample.exitedBeforeTeardown === true) { + failures.push({ + id: result.id, + reason: + sample.signal == null + ? `child exited ${sample.exitCode ?? "before teardown"}` + : `child exited by ${sample.signal}`, + sampleIndex: index + 1, + }); + } + }); + if (result.id !== "incidentCombined") { + continue; + } + const healthzP95 = result.summary.healthzMs?.p95; + if (healthzP95 == null || healthzP95 >= INCIDENT_COMBINED_HEALTHZ_P95_MAX_MS) { + failures.push({ + id: result.id, + reason: + `/healthz p95 ${healthzP95 == null ? "missing" : formatMs(healthzP95)} ` + + `must be under ${formatMs(INCIDENT_COMBINED_HEALTHZ_P95_MAX_MS)}`, + sampleIndex: 0, + }); + } + const readyzP95 = result.summary.readyzMs?.p95; + if (readyzP95 == null || readyzP95 >= INCIDENT_COMBINED_READYZ_P95_MAX_MS) { + failures.push({ + id: result.id, + reason: + `/readyz p95 ${readyzP95 == null ? "missing" : formatMs(readyzP95)} ` + + `must be under ${formatMs(INCIDENT_COMBINED_READYZ_P95_MAX_MS)}`, + sampleIndex: 0, + }); + } + } + return failures; +} + +function printBenchmarkFailures(failures: BenchmarkFailure[]): void { + if (failures.length === 0) { + return; + } + console.error( + `[gateway-startup-bench] failed: ${failures.length} sample(s) did not produce ready probes or process metrics`, + ); + for (const failure of failures.slice(0, 8)) { + console.error( + `[gateway-startup-bench] ${failure.id} ${failure.sampleIndex ? `run ${failure.sampleIndex}` : "summary"}: ${failure.reason}`, + ); + } + if (failures.length > 8) { + console.error(`[gateway-startup-bench] ${failures.length - 8} more sample failure(s) omitted`); + } +} + +function formatRatio(value: number | null): string { + if (value == null) { + return "n/a"; + } + return value.toFixed(3); +} + +async function waitForStartupTracePhase(params: { + deadlineAt: number; + isDone: () => boolean; + phase: string; + startupTrace: Record; +}): Promise { + const totalKey = `${params.phase}.total`; + while (performance.now() < params.deadlineAt) { + if (Object.hasOwn(params.startupTrace, totalKey)) { + return params.startupTrace[totalKey] ?? null; + } + if (params.isDone()) { + return null; + } + await delay(25); + } + return null; +} + +function buildBenchAgentList( + root: string, + topology: GatewayBenchCase["agentTopology"], +): Array<{ id: string; default?: boolean; workspace: string }> | undefined { + if (!topology) { + return undefined; + } + const sharedWorkspace = path.join(root, "shared-workspace"); + const distinctWorkspace = path.join(root, "distinct-workspace"); + mkdirSync(sharedWorkspace, { recursive: true }); + if (topology === "single") { + return [{ id: "main", default: true, workspace: sharedWorkspace }]; + } + if (topology === "incident-scale") { + return Array.from({ length: 8 }, (_, index) => ({ + id: `incident-agent-${String(index + 1).padStart(2, "0")}`, + default: index === 0, + workspace: path.join(root, "workspaces", `agent-${String(index + 1).padStart(2, "0")}`), + })); + } + mkdirSync(distinctWorkspace, { recursive: true }); + return Array.from({ length: 12 }, (_, index) => ({ + id: `agent-${String(index + 1).padStart(2, "0")}`, + ...(index === 0 ? { default: true } : {}), + workspace: index === 11 ? distinctWorkspace : sharedWorkspace, + })); +} + +const INCIDENT_AUDIT_ROW_COUNT = 200_000; +const INCIDENT_WORKSPACE_FILE_BYTES = 10 * 1024; +const INCIDENT_WORKSPACE_FILE_COUNT = 80_000; +const INCIDENT_WORKSPACE_COUNT = 8; +const INCIDENT_WORKSPACE_FILES_PER_DIRECTORY = 1_000; + +type IncidentFixtureOptions = { + auditRowCount?: number; + kind: NonNullable; + workspaceFileBytes?: number; + workspaceFileCount?: number; +}; + +function fixtureIncludesDatabase(kind: IncidentFixtureOptions["kind"]): boolean { + return kind === "combined" || kind === "database" || kind === "null-metadata"; +} + +function fixtureIncludesWorkspace(kind: IncidentFixtureOptions["kind"]): boolean { + return kind === "combined" || kind === "workspace"; +} + +function fixtureIncludesNullMetadata(kind: IncidentFixtureOptions["kind"]): boolean { + return kind === "combined" || kind === "null-metadata"; +} + +function fixtureIncludesPackagedPlugins(kind: IncidentFixtureOptions["kind"]): boolean { + return kind === "combined" || kind === "packaged-plugins"; +} + +function listIncidentPackagedPluginArtifacts(env = process.env): string[] { + return listBundledPluginPackArtifacts({ + env: { ...env, OPENCLAW_BUNDLED_PLUGIN_BUILD_IDS: undefined }, + }); +} + +function assertIncidentPackagedPluginInventory(): void { + const missing = listIncidentPackagedPluginArtifacts().filter( + (artifact) => !existsSync(path.resolve(process.cwd(), artifact)), + ); + if (missing.length > 0) { + throw new Error( + `Incident packaged-plugin benchmark requires a built plugin inventory; ${missing.length} required artifact(s) are missing. Run pnpm build first.`, + ); + } +} + +function createIncidentWorkspaces(root: string, options: IncidentFixtureOptions): void { + if (!fixtureIncludesWorkspace(options.kind)) { + return; + } + const fileCount = options.workspaceFileCount ?? INCIDENT_WORKSPACE_FILE_COUNT; + const fileBytes = options.workspaceFileBytes ?? INCIDENT_WORKSPACE_FILE_BYTES; + const contents = Buffer.alloc(fileBytes, "x"); + for (let index = 0; index < fileCount; index += 1) { + const workspaceIndex = index % INCIDENT_WORKSPACE_COUNT; + const directoryIndex = Math.floor(index / INCIDENT_WORKSPACE_FILES_PER_DIRECTORY); + const directory = path.join( + root, + "workspaces", + `agent-${String(workspaceIndex + 1).padStart(2, "0")}`, + "incident-artifacts", + `batch-${String(directoryIndex).padStart(3, "0")}`, + ); + mkdirSync(directory, { recursive: true }); + writeFileSync(path.join(directory, `artifact-${String(index).padStart(6, "0")}.bin`), contents); + } +} + +function seedIncidentAuditRows(database: import("node:sqlite").DatabaseSync, count: number): void { + const payload = `fixture:${"x".repeat(1024)}`; + const insert = database.prepare( + `INSERT INTO audit_events ( + event_id, source_id, source_sequence, occurred_at, kind, action, status, actor_type, + actor_id, agent_id, session_key, run_id, channel, target_ref + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + ); + database.exec("BEGIN IMMEDIATE;"); + try { + for (let index = 0; index < count; index += 1) { + const id = `incident-audit-${String(index).padStart(8, "0")}`; + insert.run( + id, + `source-${id}`, + index + 1, + 1_750_000_000_000 + index, + "gateway.admission", + "accepted", + "ok", + "system", + "incident-fixture", + "incident-agent-01", + "agent:incident-agent-01:main", + `run-${id}`, + "fixture", + payload, + ); + } + database.exec("COMMIT;"); + } catch (error) { + database.exec("ROLLBACK;"); + throw error; + } + // Retain half the rows after a large write so the fixture keeps the incident's + // high freelist pressure without storing any customer content. + database.prepare("DELETE FROM audit_events WHERE source_sequence % 2 = 0").run(); +} + +async function writeIncidentFixture(root: string, options: IncidentFixtureOptions): Promise { + createIncidentWorkspaces(root, options); + if (!fixtureIncludesDatabase(options.kind)) { + return; + } + const env = { OPENCLAW_STATE_DIR: path.join(root, "state") }; + const [{ DatabaseSync }, agentDatabase, stateDatabase] = await Promise.all([ + import("node:sqlite"), + import("../src/state/openclaw-agent-db.js"), + import("../src/state/openclaw-state-db.js"), + ]); + const state = stateDatabase.openOpenClawStateDatabase({ env }); + const agents = Array.from({ length: INCIDENT_WORKSPACE_COUNT }, (_, index) => + agentDatabase.openOpenClawAgentDatabase({ + agentId: `incident-agent-${String(index + 1).padStart(2, "0")}`, + env, + }), + ); + const statePath = state.path; + const agentPaths = agents.map((agent) => agent.path); + try { + state.db.exec("PRAGMA wal_checkpoint(TRUNCATE);"); + agents.forEach((agent) => agent.db.exec("PRAGMA wal_checkpoint(TRUNCATE);")); + } finally { + agentDatabase.closeOpenClawAgentDatabasesForTest(); + stateDatabase.closeOpenClawStateDatabaseForTest(); + } + const stateHandle = new DatabaseSync(statePath); + try { + seedIncidentAuditRows(stateHandle, options.auditRowCount ?? INCIDENT_AUDIT_ROW_COUNT); + if (fixtureIncludesNullMetadata(options.kind)) { + stateHandle + .prepare("UPDATE schema_meta SET app_version = NULL WHERE meta_key = 'primary'") + .run(); + } + stateHandle.exec("PRAGMA wal_checkpoint(TRUNCATE);"); + } finally { + stateHandle.close(); + } + if (!fixtureIncludesNullMetadata(options.kind)) { + return; + } + for (const agentPath of agentPaths) { + const agent = new DatabaseSync(agentPath); + try { + agent.prepare("UPDATE schema_meta SET app_version = NULL WHERE meta_key = 'primary'").run(); + agent.exec("PRAGMA wal_checkpoint(TRUNCATE);"); + } finally { + agent.close(); + } + } +} + +async function withGatewayBenchRoot(run: (root: string) => Promise): Promise { + const root = mkdtempSync(path.join(tmpdir(), "openclaw-gateway-bench-")); + try { + return await run(root); + } finally { + rmSync(root, { force: true, maxRetries: 3, recursive: true, retryDelay: 100 }); + } +} + +function writeConfig(root: string, benchCase: GatewayBenchCase): string { + const hasCatalogFixture = + benchCase.providerCatalogStallMs !== undefined || + benchCase.providerStaticCatalogStallMs !== undefined; + const pluginCount = hasCatalogFixture ? 1 : benchCase.pluginCount; + const pluginFixtures = pluginCount + ? writePluginFixtures(root, { + activationOnStartup: hasCatalogFixture ? true : benchCase.pluginActivationOnStartup, + count: pluginCount, + providerCatalogStallMs: benchCase.providerCatalogStallMs, + providerStaticCatalogModelCount: benchCase.providerStaticCatalogModelCount, + providerStaticCatalogStallMs: benchCase.providerStaticCatalogStallMs, + }) + : null; + const agentList = buildBenchAgentList(root, benchCase.agentTopology); + return writeGatewayBenchConfig(root, benchCase.config, { agentList, pluginFixtures }); +} + +function sanitizedEnv( + root: string, + configPath: string, + benchCase: GatewayBenchCase, +): NodeJS.ProcessEnv { + return createGatewayBenchEnv(root, configPath, { caseEnv: benchCase.env }); +} + +function collectStartupTrace(line: string, startupTrace: Record): void { + collectTraceLine(line, "startup trace", startupTrace); +} + +async function runGatewaySample(options: { + benchCase: GatewayBenchCase; + cpuProfDir?: string; + entry: string; + heapProfDir?: string; + sampleIndex: number; + timeoutMs: number; +}): Promise { + return await withGatewayBenchRoot(async (root) => { + const port = await getFreePort(); + const configPath = writeConfig(root, options.benchCase); + const env = sanitizedEnv(root, configPath, options.benchCase); + if (options.benchCase.incidentFixture) { + await writeIncidentFixture(root, { kind: options.benchCase.incidentFixture }); + if (fixtureIncludesPackagedPlugins(options.benchCase.incidentFixture)) { + assertIncidentPackagedPluginInventory(); + } + } + const startAt = performance.now(); + const deadlineAt = startAt + options.timeoutMs; + const startupTrace: Record = {}; + const output: string[] = []; + const outputBuffers: Record<"stderr" | "stdout", string> = { stderr: "", stdout: "" }; + let firstOutputMs: number | null = null; + let gatewayReadyLogLine: string | null = null; + let gatewayReadyLogMs: number | null = null; + let httpListenLogLine: string | null = null; + let httpListenLogMs: number | null = null; + let maxRssMb: number | null = null; + let childExited = false; + let child: ChildProcessWithoutNullStreams | undefined; + let rssTimer: ReturnType | undefined; + + try { + const nodeOptions = [ + ...(options.cpuProfDir + ? [ + "--cpu-prof", + "--cpu-prof-dir", + options.cpuProfDir, + "--cpu-prof-name", + `openclaw-gateway-${options.benchCase.id}-${options.sampleIndex}-${Date.now()}.cpuprofile`, + ] + : []), + ...(options.heapProfDir ? ["--heap-prof", "--heap-prof-dir", options.heapProfDir] : []), + ]; + const childArgs = buildGatewayBenchChildArgs(options.entry, port, nodeOptions); + child = spawn(process.execPath, childArgs, { + cwd: process.cwd(), + detached: process.platform !== "win32", + env, + stdio: ["pipe", "pipe", "pipe"], + }); + const startedChild = child; + const cpuStartMs = readProcessTreeCpuMs(startedChild.pid); + const sampleRss = () => { + const rssMb = readProcessRssMb(startedChild.pid); + if (rssMb != null) { + maxRssMb = maxRssMb == null ? rssMb : Math.max(maxRssMb, rssMb); + } + }; + sampleRss(); + rssTimer = setInterval(sampleRss, 100); + rssTimer.unref?.(); + const childExitPromise = new Promise<{ exitCode: number | null; signal: string | null }>( + (resolve) => { + startedChild.once("exit", (exitCode, signal) => { + childExited = true; + resolve({ exitCode, signal }); + }); + }, + ); + + const onLine = (line: string, nowMs: number) => { + const readyLogKind = classifyGatewayReadyLog(line); + if (readyLogKind === "http-listen" && httpListenLogMs == null) { + httpListenLogMs = nowMs; + httpListenLogLine = line; + } + if (readyLogKind === "gateway-ready" && gatewayReadyLogMs == null) { + gatewayReadyLogMs = nowMs; + gatewayReadyLogLine = line; + } + collectStartupTrace(line, startupTrace); + }; + const onChunk = (stream: "stderr" | "stdout", chunk: Buffer) => { + if (firstOutputMs == null) { + firstOutputMs = performance.now() - startAt; + } + const text = chunk.toString("utf8"); + output.push(text); + if (output.length > 20) { + output.splice(0, output.length - 20); + } + const parsed = collectOutputLines(outputBuffers[stream], text); + outputBuffers[stream] = parsed.carry; + const nowMs = performance.now() - startAt; + for (const line of parsed.lines) { + onLine(line, nowMs); + } + }; + startedChild.stdout.on("data", (chunk: Buffer) => onChunk("stdout", chunk)); + startedChild.stderr.on("data", (chunk: Buffer) => onChunk("stderr", chunk)); + + const [healthz, readyz] = await Promise.all([ + waitForProbe({ + deadlineAt, + isDone: () => childExited, + path: "/healthz", + port, + startAt, + }), + waitForProbe({ + deadlineAt, + isDone: () => childExited, + path: "/readyz", + port, + startAt, + }), + ]); + const completionMs = options.benchCase.completionTracePhase + ? await waitForStartupTracePhase({ + deadlineAt, + isDone: () => childExited, + phase: options.benchCase.completionTracePhase, + startupTrace, + }) + : performance.now() - startAt; + const completedAt = performance.now(); + const cpuEndMs = readProcessTreeCpuMs(startedChild.pid); + const cpuMs = + cpuStartMs == null || cpuEndMs == null ? null : Math.max(0, cpuEndMs - cpuStartMs); + const cpuCoreRatio = cpuMs == null ? null : cpuMs / Math.max(1, completedAt - startAt); + const exit = await stopChild(startedChild); + sampleRss(); + child = undefined; + // stopChild is the bounded teardown wait; the raw exit promise may never settle. + void childExitPromise.catch(() => null); + flushOutputLineBuffers(outputBuffers, onLine, performance.now() - startAt, { + flushPartial: true, + }); + + return { + completionMs, + cpuCoreRatio, + cpuMs, + exitedBeforeTeardown: exit.exitedBeforeTeardown, + exitCode: exit.exitCode, + firstOutputMs, + gatewayReadyLogLine, + gatewayReadyLogMs, + healthz, + httpListenLogLine, + httpListenLogMs, + maxRssMb, + outputTail: output.join("").split(/\r?\n/u).slice(-20).join("\n"), + readyz, + signal: exit.signal, + startupTrace, + }; + } finally { + if (rssTimer) { + clearInterval(rssTimer); + } + if (child) { + await stopChild(child).catch(() => undefined); + } + } + }); +} + +async function runCase(options: { + benchCase: GatewayBenchCase; + cpuProfDir?: string; + entry: string; + heapProfDir?: string; + runs: number; + timeoutMs: number; + warmup: number; +}): Promise { + const samples: GatewaySample[] = []; + const total = options.runs + options.warmup; + for (let index = 0; index < total; index += 1) { + const sample = await runGatewaySample({ + benchCase: options.benchCase, + cpuProfDir: options.cpuProfDir, + entry: options.entry, + heapProfDir: options.heapProfDir, + sampleIndex: index + 1, + timeoutMs: options.timeoutMs, + }); + if (index >= options.warmup) { + samples.push(sample); + const heapUsedMb = sample.startupTrace["memory.ready.heapUsedMb"] ?? null; + console.log( + `[gateway-startup-bench] ${options.benchCase.id} run ${samples.length}/${options.runs}: completion=${formatMs(sample.completionMs)} healthz=${formatMs(sample.healthz.ms)} readyz=${formatMs(sample.readyz.ms)} httpListen=${formatMs(sample.httpListenLogMs)} gatewayReady=${formatMs(sample.gatewayReadyLogMs)} cpu=${formatMs(sample.cpuMs)} cpuCore=${formatRatio(sample.cpuCoreRatio)} rss=${formatMb(sample.maxRssMb)} heap=${formatMb(heapUsedMb)}`, + ); + if ( + sample.outputTail && + (sample.completionMs == null || + sample.healthz.status !== 200 || + sample.readyz.status !== 200) + ) { + console.error( + `[gateway-startup-bench] ${options.benchCase.id} output tail:\n${sample.outputTail}`, + ); + } + } else { + const heapUsedMb = sample.startupTrace["memory.ready.heapUsedMb"] ?? null; + console.log( + `[gateway-startup-bench] ${options.benchCase.id} warmup ${index + 1}/${options.warmup}: healthz=${formatMs(sample.healthz.ms)} readyz=${formatMs(sample.readyz.ms)} cpu=${formatMs(sample.cpuMs)} cpuCore=${formatRatio(sample.cpuCoreRatio)} rss=${formatMb(sample.maxRssMb)} heap=${formatMb(heapUsedMb)}`, + ); + } + } + return summarizeCase(options.benchCase, samples); +} + +function printResult(result: CaseResult): void { + console.log(`\n${result.name} (${result.id})`); + console.log(` completion: ${formatStats(result.summary.completionMs)}`); + console.log(` first output: ${formatStats(result.summary.firstOutputMs)}`); + console.log(` CPU: ${formatStats(result.summary.cpuMs)}`); + console.log(` CPU core: ${formatStats(result.summary.cpuCoreRatio, formatRatio)}`); + console.log(` /healthz: ${formatStats(result.summary.healthzMs)}`); + console.log(` http listen: ${formatStats(result.summary.httpListenLogMs)}`); + console.log(` gateway ready: ${formatStats(result.summary.gatewayReadyLogMs)}`); + console.log(` /readyz: ${formatStats(result.summary.readyzMs)}`); + console.log(` max RSS: ${formatStats(result.summary.maxRssMb, formatMb)}`); + console.log( + ` ready memory: rss=${formatStats(result.summary.startupTrace["memory.ready.rssMb"], formatMb)} heap=${formatStats(result.summary.startupTrace["memory.ready.heapUsedMb"], formatMb)} external=${formatStats(result.summary.startupTrace["memory.ready.externalMb"], formatMb)}`, + ); + console.log( + ` post-ready memory: rss=${formatStats(result.summary.startupTrace["memory.post-ready.rssMb"], formatMb)} heap=${formatStats(result.summary.startupTrace["memory.post-ready.heapUsedMb"], formatMb)} external=${formatStats(result.summary.startupTrace["memory.post-ready.externalMb"], formatMb)}`, + ); + console.log( + ` prepared runtime: agents=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.agentCount"], String)} workspaces=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.workspaceGroupCount"], String)} configuredGroups=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.configuredFactsGroupCount"], String)} configuredModels=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.configuredRuntimeModelCount"], String)} generatedPlugins=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.generatedCatalogPluginCount"], String)} generatedReads=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.generatedCatalogReadCount"], String)} sources=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.catalogSourceCount"], String)} credentials=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.credentialGroupCount"], String)} catalogs=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.catalogGroupCount"], String)} registries=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.runtimeRegistryCount"], String)} workspaceFacts=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.workspaceFactsMs"])} runtimePlugins=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.runtimePluginMs"])} metadata=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.pluginMetadataMs"])} staticProviders=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.staticProviderCatalogMs"])} ambientAuth=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.ambientCredentialsMs"])} agentFacts=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.agentFactsMs"])} configuredProjection=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.configuredProjectionMs"])} sourceMs=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.catalogSourceMs"])} registryMs=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.registryMs"])} sourceLimit=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.sourceConcurrencyLimitCount"], String)} fullCatalogLimit=${formatStats(result.summary.startupTrace["sidecars.model-runtime-build.fullCatalogConcurrencyLimitCount"], String)} staticCatalog=${formatStats(result.summary.startupTrace["benchmark.preparedRuntimeStaticCatalogCallCount"], String)} pluginLoader=${formatStats(result.summary.startupTrace["sidecars.plugin-loader.callsCount"], String)} eventLoopMax=${formatStats(result.summary.startupTrace["sidecars.model-runtime.eventLoopMax"])}`, + ); + const trace = selectSlowStartupTraceDurations(result.summary.startupTrace, 8); + if (trace.length > 0) { + console.log(" trace top:"); + for (const [name, stats] of trace) { + console.log(` ${name}: ${formatStats(stats)}`); + } + } +} + +async function main() { + const argv = process.argv.slice(2); + if (hasHelpFlag(argv)) { + printUsage(); + return; + } + + const options = parseOptions(argv); + if (options.cpuProfDir) { + mkdirSync(options.cpuProfDir, { recursive: true }); + } + if (options.heapProfDir) { + mkdirSync(options.heapProfDir, { recursive: true }); + } + const results: CaseResult[] = []; + for (const benchCase of options.cases) { + results.push( + await runCase({ + benchCase, + cpuProfDir: options.cpuProfDir, + entry: options.entry, + heapProfDir: options.heapProfDir, + runs: options.runs, + timeoutMs: options.timeoutMs, + warmup: options.warmup, + }), + ); + } + + const payload = { + entry: options.entry, + generatedAt: new Date().toISOString(), + results, + }; + if (options.output) { + mkdirSync(path.dirname(options.output), { recursive: true }); + writeFileSync(options.output, `${JSON.stringify(payload, null, 2)}\n`); + } + if (options.json) { + console.log(JSON.stringify(payload, null, 2)); + } else { + for (const result of results) { + printResult(result); + } + } + + const failures = collectResultFailures(results); + if (failures.length > 0) { + printBenchmarkFailures(failures); + process.exitCode = 1; + } +} + +export const testing = { + collectResultFailures, + collectStartupTrace, + listIncidentPackagedPluginArtifacts, + parseOptions, + sanitizedEnv, + summarizeCase, + waitForStartupTracePhase, + withGatewayBenchRoot, + writeIncidentFixture, + writeConfig, +}; + +if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { + main().catch((err: unknown) => { + if (err instanceof CliArgumentError) { + console.error(err.message); + process.exitCode = 1; + return; + } + console.error(err instanceof Error ? err.stack : String(err)); + process.exitCode = 1; + }); +} diff --git a/scripts/bench-task-registry-sqlite-worker.ts b/scripts/bench-task-registry-sqlite-worker.ts new file mode 100644 index 0000000000000000000000000000000000000000..e8bc209748075c0c0766c7b70cf105ef2e76bc30 --- /dev/null +++ b/scripts/bench-task-registry-sqlite-worker.ts @@ -0,0 +1,468 @@ +import path from "node:path"; +import { performance } from "node:perf_hooks"; +import { pathToFileURL } from "node:url"; +import { toErrorObject } from "@openclaw/normalization-core/error-coercion"; +import type { DB as OpenClawStateKyselyDatabase } from "../src/state/openclaw-state-db.generated.js"; +import { + WORKER_RESULT_SENTINEL, + type MemorySample, + type RegistryLifecycleCounts, + type RegistrySnapshot, + type RetainedMemoryMetrics, + type WorkerResult, +} from "./bench-task-registry-sqlite.js"; +import { classifyBoundedUnsignedDecimal } from "./lib/arg-utils.mts"; + +type WorkerOptions = { + size: number; + cycles: number; + warmup: number; + stateDir: string; +}; + +type TimingSample = { + registrationMs: number; + terminalMs: number; + teardownMs: number; + registration: RegistrySnapshot; + terminal: RegistrySnapshot; + teardown: RegistrySnapshot; +}; + +type BenchmarkStateDatabase = Pick< + OpenClawStateKyselyDatabase, + "task_delivery_state" | "task_runs" +>; + +type TaskRecordApi = Pick< + typeof import("../src/tasks/task-registry-record-api.js"), + "createTaskRecord" | "markTaskTerminalById" +>; + +type TaskRegistryQueryApi = Pick< + typeof import("../src/tasks/task-registry-query.js"), + "deleteTaskRecordById" +>; + +function parseInteger(raw: string | undefined, flag: string, min: number, max: number): number { + const result = classifyBoundedUnsignedDecimal(raw, min, max); + if (result.kind === "syntax") { + throw new Error(`${flag} must be an integer`); + } + if (result.kind !== "value") { + throw new Error(`${flag} must be between ${min} and ${max}`); + } + return result.value; +} + +function parseOptions(argv: string[]): WorkerOptions { + const values = new Map(); + for (let index = 0; index < argv.length; index += 2) { + const flag = argv[index]; + const value = argv[index + 1]; + if (!flag?.startsWith("--") || !value || value.startsWith("--")) { + throw new Error(`invalid worker argument near ${flag ?? "end"}`); + } + if (values.has(flag)) { + throw new Error(`${flag} was provided more than once`); + } + values.set(flag, value); + } + const stateDir = values.get("--state-dir"); + if (!stateDir || !path.isAbsolute(stateDir)) { + throw new Error("--state-dir must be an absolute path"); + } + return { + size: parseInteger(values.get("--size"), "--size", 1, 4096), + cycles: parseInteger(values.get("--cycles"), "--cycles", 1, 200), + warmup: parseInteger(values.get("--warmup"), "--warmup", 0, 20), + stateDir, + }; +} + +function processPeakRssBytes(): number { + return Math.max(0, Math.round(process.resourceUsage().maxRSS * 1024)); +} + +function forceGc(): void { + const gc = (globalThis as typeof globalThis & { gc?: () => void }).gc; + if (!gc) { + throw new Error("benchmark worker requires --expose-gc"); + } + gc(); + gc(); +} + +function readMemorySample(cycle: number): MemorySample { + const memory = process.memoryUsage(); + return { + cycle, + heapUsedBytes: memory.heapUsed, + heapTotalBytes: memory.heapTotal, + rssBytes: memory.rss, + externalBytes: memory.external, + arrayBuffersBytes: memory.arrayBuffers, + processPeakRssBytes: processPeakRssBytes(), + }; +} + +function slope(values: number[]): number { + if (values.length < 2) { + return 0; + } + const xMean = (values.length - 1) / 2; + const yMean = values.reduce((sum, value) => sum + value, 0) / values.length; + let numerator = 0; + let denominator = 0; + for (let index = 0; index < values.length; index += 1) { + const xDelta = index - xMean; + numerator += xDelta * ((values[index] ?? 0) - yMean); + denominator += xDelta * xDelta; + } + return denominator === 0 ? 0 : numerator / denominator; +} + +function retainedMemorySlopes(samples: MemorySample[]): RetainedMemoryMetrics { + return { + heapUsedBytes: slope(samples.map((sample) => sample.heapUsedBytes)), + heapTotalBytes: slope(samples.map((sample) => sample.heapTotalBytes)), + rssBytes: slope(samples.map((sample) => sample.rssBytes)), + externalBytes: slope(samples.map((sample) => sample.externalBytes)), + arrayBuffersBytes: slope(samples.map((sample) => sample.arrayBuffersBytes)), + }; +} + +function retainedMemoryDelta( + baseline: MemorySample, + finalSample: MemorySample, +): RetainedMemoryMetrics { + return { + heapUsedBytes: finalSample.heapUsedBytes - baseline.heapUsedBytes, + heapTotalBytes: finalSample.heapTotalBytes - baseline.heapTotalBytes, + rssBytes: finalSample.rssBytes - baseline.rssBytes, + externalBytes: finalSample.externalBytes - baseline.externalBytes, + arrayBuffersBytes: finalSample.arrayBuffersBytes - baseline.arrayBuffersBytes, + }; +} + +function assertLifecycleCounts( + actual: RegistryLifecycleCounts, + expected: RegistryLifecycleCounts, + phase: string, + surface: string, +): void { + for (const field of Object.keys(expected) as Array) { + if (actual[field] !== expected[field]) { + throw new Error( + `${phase} ${surface} invariant failed: ${JSON.stringify({ expected, actual })}`, + ); + } + } +} + +function assertSnapshot( + actual: RegistrySnapshot, + expected: RegistryLifecycleCounts, + phase: string, +): void { + for (const surface of ["memory", "sqlite"] as const) { + assertLifecycleCounts(actual[surface], expected, phase, surface); + } +} + +async function createCountReader() { + const [{ executeSqliteQuerySync, getNodeSqliteKysely }, stateDb, state] = await Promise.all([ + import("../src/infra/kysely-sync.js"), + import("../src/state/openclaw-state-db.js"), + import("../src/tasks/task-registry-state.js"), + ]); + return (): RegistrySnapshot => { + const database = stateDb.openOpenClawStateDatabase(); + const db = getNodeSqliteKysely(database.db); + const taskRows = executeSqliteQuerySync( + database.db, + db.selectFrom("task_runs").select(["status", "delivery_status", "terminal_outcome"]), + ).rows; + const deliveryRows = executeSqliteQuerySync( + database.db, + db.selectFrom("task_delivery_state").select(({ fn }) => fn.countAll().as("count")), + ).rows[0]?.count; + const summarize = ( + taskRecords: Iterable<{ + status: string; + deliveryStatus: string; + terminalOutcome?: string | null; + }>, + deliveryStateCount: number, + ): RegistryLifecycleCounts => { + let taskCount = 0; + let runningTasks = 0; + let succeededTasks = 0; + let pendingDeliveryTasks = 0; + let succeededTerminalOutcomes = 0; + for (const task of taskRecords) { + taskCount += 1; + runningTasks += task.status === "running" ? 1 : 0; + succeededTasks += task.status === "succeeded" ? 1 : 0; + pendingDeliveryTasks += task.deliveryStatus === "pending" ? 1 : 0; + succeededTerminalOutcomes += task.terminalOutcome === "succeeded" ? 1 : 0; + } + return { + taskCount, + deliveryStateCount, + runningTasks, + succeededTasks, + pendingDeliveryTasks, + succeededTerminalOutcomes, + }; + }; + return { + memory: summarize( + [...state.tasks.values()].map((task) => ({ + status: task.status, + deliveryStatus: task.deliveryStatus, + terminalOutcome: task.terminalOutcome, + })), + state.taskDeliveryStates.size, + ), + sqlite: summarize( + taskRows.map((task) => ({ + status: task.status, + deliveryStatus: task.delivery_status, + terminalOutcome: task.terminal_outcome, + })), + deliveryRows ?? 0, + ), + }; + }; +} + +async function runCycle( + size: number, + serial: number, + readSnapshot: () => RegistrySnapshot, + taskRecordApi: TaskRecordApi, + taskRegistryQuery: TaskRegistryQueryApi, +): Promise { + const taskIds: string[] = []; + const startedAt = Date.now(); + const registrationStartedAt = performance.now(); + for (let index = 0; index < size; index += 1) { + const task = taskRecordApi.createTaskRecord({ + runtime: "subagent", + requesterSessionKey: "agent:benchmark:main", + ownerKey: "agent:benchmark:main", + scopeKind: "session", + requesterOrigin: { channel: "benchmark", to: "task-registry" }, + childSessionKey: `agent:benchmark:subagent:${serial}:${index}`, + runId: `benchmark-task-${serial}-${index}`, + task: `durable registration benchmark ${index}`, + status: "running", + deliveryStatus: "pending", + notifyPolicy: "silent", + startedAt, + lastEventAt: startedAt, + }); + if (!task) { + throw new Error(`task registration failed at ${index + 1}/${size}`); + } + taskIds.push(task.taskId); + } + const registrationMs = performance.now() - registrationStartedAt; + const emptyCounts: RegistryLifecycleCounts = { + taskCount: 0, + deliveryStateCount: 0, + runningTasks: 0, + succeededTasks: 0, + pendingDeliveryTasks: 0, + succeededTerminalOutcomes: 0, + }; + const registration = readSnapshot(); + assertSnapshot( + registration, + { + ...emptyCounts, + taskCount: size, + deliveryStateCount: size, + runningTasks: size, + pendingDeliveryTasks: size, + }, + "registration", + ); + + const terminalStartedAt = performance.now(); + for (const taskId of taskIds) { + const endedAt = Date.now(); + const task = taskRecordApi.markTaskTerminalById({ + taskId, + status: "succeeded", + endedAt, + lastEventAt: endedAt, + terminalOutcome: "succeeded", + }); + if (!task) { + throw new Error(`terminal transition failed for task ${taskId}`); + } + } + const terminalMs = performance.now() - terminalStartedAt; + const terminal = readSnapshot(); + assertSnapshot( + terminal, + { + ...emptyCounts, + taskCount: size, + deliveryStateCount: size, + succeededTasks: size, + pendingDeliveryTasks: size, + succeededTerminalOutcomes: size, + }, + "terminal", + ); + + const teardownStartedAt = performance.now(); + for (const taskId of taskIds) { + if (!taskRegistryQuery.deleteTaskRecordById(taskId)) { + throw new Error(`teardown failed for task ${taskId}`); + } + } + const teardownMs = performance.now() - teardownStartedAt; + const teardown = readSnapshot(); + assertSnapshot(teardown, emptyCounts, "teardown"); + return { registrationMs, terminalMs, teardownMs, registration, terminal, teardown }; +} + +async function resetRuntime(persist: boolean): Promise { + const [tasks, stateDb] = await Promise.all([ + import("../src/tasks/task-runtime.test-helpers.js"), + import("../src/state/openclaw-state-db.js"), + ]); + tasks.resetTaskRegistryForTests({ persist }); + stateDb.closeOpenClawStateDatabaseForTest(); +} + +async function runBenchmark(options: WorkerOptions): Promise { + await resetRuntime(true); + // Load lifecycle owners before the baseline so warmup=0 still measures task churn, + // not one-time module initialization retained by the worker. + const [readSnapshot, taskRecordApi, taskRegistryQuery] = await Promise.all([ + createCountReader(), + import("../src/tasks/task-registry-record-api.js"), + import("../src/tasks/task-registry-query.js"), + ]); + const emptyCounts: RegistryLifecycleCounts = { + taskCount: 0, + deliveryStateCount: 0, + runningTasks: 0, + succeededTasks: 0, + pendingDeliveryTasks: 0, + succeededTerminalOutcomes: 0, + }; + assertSnapshot(readSnapshot(), emptyCounts, "initial"); + const timingsMs = { + registration: [] as number[], + terminal: [] as number[], + teardown: [] as number[], + }; + const postGcSamples: MemorySample[] = []; + let lastSample: TimingSample | undefined; + for (let index = 0; index < options.warmup; index += 1) { + await runCycle(options.size, index, readSnapshot, taskRecordApi, taskRegistryQuery); + forceGc(); + } + forceGc(); + const postGcBaseline = readMemorySample(-1); + for (let index = 0; index < options.cycles; index += 1) { + lastSample = await runCycle( + options.size, + options.warmup + index, + readSnapshot, + taskRecordApi, + taskRegistryQuery, + ); + forceGc(); + timingsMs.registration.push(lastSample.registrationMs); + timingsMs.terminal.push(lastSample.terminalMs); + timingsMs.teardown.push(lastSample.teardownMs); + postGcSamples.push(readMemorySample(index)); + } + if (!lastSample) { + throw new Error("benchmark completed without a cycle"); + } + const finalPostGcSample = postGcSamples.at(-1); + if (!finalPostGcSample) { + throw new Error("benchmark completed without a post-GC sample"); + } + return { + size: options.size, + timingsMs, + memory: { + postGcBaseline, + postGcSamples, + retainedSlopesBytesPerCycle: retainedMemorySlopes(postGcSamples), + retainedDeltasBytes: retainedMemoryDelta(postGcBaseline, finalPostGcSample), + processPeakRssBytes: processPeakRssBytes(), + }, + invariant: { + ok: true, + cyclesValidated: options.warmup + options.cycles, + registration: lastSample.registration, + terminal: lastSample.terminal, + teardown: lastSample.teardown, + serializedSharedConnection: true, + }, + }; +} + +async function main(): Promise { + const options = parseOptions(process.argv.slice(2)); + const previousStateDir = process.env.OPENCLAW_STATE_DIR; + const previousNodeEnv = process.env.NODE_ENV; + let result: WorkerResult | undefined; + let failure: unknown; + process.env.OPENCLAW_STATE_DIR = options.stateDir; + process.env.NODE_ENV = "test"; + try { + const { pinRuntimePaths } = await import("../src/config/paths.js"); + pinRuntimePaths(); + result = await runBenchmark(options); + } catch (error) { + failure = error; + } finally { + try { + await resetRuntime(false); + } catch (error) { + failure ??= error; + } finally { + if (previousStateDir === undefined) { + delete process.env.OPENCLAW_STATE_DIR; + } else { + process.env.OPENCLAW_STATE_DIR = previousStateDir; + } + if (previousNodeEnv === undefined) { + delete process.env.NODE_ENV; + } else { + process.env.NODE_ENV = previousNodeEnv; + } + } + } + if (failure) { + throw toErrorObject(failure, "Task registry SQLite benchmark failed"); + } + if (!result) { + throw new Error("benchmark worker completed without a result"); + } + process.stdout.write(`${WORKER_RESULT_SENTINEL}${JSON.stringify(result)}\n`); +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { + try { + await main(); + } catch (error) { + console.error(error instanceof Error ? error.stack : String(error)); + process.exitCode = 1; + } finally { + if (process.exitCode && process.exitCode !== 0) { + console.error(`[bench-task-registry-sqlite-worker] FAILED (exit ${process.exitCode})`); + } + } +} diff --git a/scripts/bench-task-registry-sqlite.ts b/scripts/bench-task-registry-sqlite.ts new file mode 100644 index 0000000000000000000000000000000000000000..aa69298a05d9013e045239f1e4619cae1d63a4cf --- /dev/null +++ b/scripts/bench-task-registry-sqlite.ts @@ -0,0 +1,470 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { isRecord } from "@openclaw/normalization-core/record-coerce"; +import { + emitBenchmarkReport, + parseBenchmarkInteger, + parseBenchmarkIntegerList, + parseBenchmarkOptions, + parseBenchmarkWorkerResult, + runBenchmarkEntrypoint, + runBenchmarkJobs, + runBenchmarkWorker, + summarizeBenchmarkTimings, + type BenchmarkWorkerProcessResult, + type BenchmarkWorkerSpawner, +} from "./lib/benchmark-harness.mts"; + +const DEFAULT_SIZES = [24, 64, 128]; +const WORKER_TIMEOUT_MS = 300_000; +export const WORKER_RESULT_SENTINEL = "[bench-task-registry-sqlite-result] "; + +export type MemorySample = { + cycle: number; + heapUsedBytes: number; + heapTotalBytes: number; + rssBytes: number; + externalBytes: number; + arrayBuffersBytes: number; + processPeakRssBytes: number; +}; + +export type RetainedMemoryMetrics = Pick< + MemorySample, + "heapUsedBytes" | "heapTotalBytes" | "rssBytes" | "externalBytes" | "arrayBuffersBytes" +>; + +export type RegistryLifecycleCounts = { + taskCount: number; + deliveryStateCount: number; + runningTasks: number; + succeededTasks: number; + pendingDeliveryTasks: number; + succeededTerminalOutcomes: number; +}; + +export type RegistrySnapshot = { + memory: RegistryLifecycleCounts; + sqlite: RegistryLifecycleCounts; +}; + +export type WorkerResult = { + size: number; + timingsMs: { + registration: number[]; + terminal: number[]; + teardown: number[]; + }; + memory: { + postGcBaseline: MemorySample; + postGcSamples: MemorySample[]; + retainedSlopesBytesPerCycle: RetainedMemoryMetrics; + retainedDeltasBytes: RetainedMemoryMetrics; + processPeakRssBytes: number; + }; + invariant: { + ok: boolean; + cyclesValidated: number; + registration: RegistrySnapshot; + terminal: RegistrySnapshot; + teardown: RegistrySnapshot; + serializedSharedConnection: boolean; + }; +}; + +type Options = { + sizes: number[]; + cycles: number; + warmup: number; + output?: string; + json: boolean; + help: boolean; +}; + +type WorkerLaunchRuntime = { + spawnWorker?: BenchmarkWorkerSpawner; +}; + +function usage(): string { + return `OpenClaw durable task registry churn benchmark + +Usage: + node --import tsx scripts/bench-task-registry-sqlite.ts [options] + +Options: + --sizes Comma-separated subagent task-record registration burst sizes (default: 24,64,128) + --cycles Measured create/terminal/delete cycles per size (default: 20) + --warmup Warmup cycles per size (default: 3) + --output Write the JSON report to a file + --json Print only the JSON report + --help Show this text +`; +} + +function parseOptions(argv: string[]): Options { + return parseBenchmarkOptions( + argv, + { + sizes: DEFAULT_SIZES, + cycles: 20, + warmup: 3, + json: false, + help: false, + }, + { + "--sizes": (options, value) => { + options.sizes = parseBenchmarkIntegerList(value, "--sizes", 4096); + }, + "--cycles": (options, value) => { + options.cycles = parseBenchmarkInteger(value, "--cycles", 1, 200); + }, + "--warmup": (options, value) => { + options.warmup = parseBenchmarkInteger(value, "--warmup", 0, 20); + }, + "--output": (options, value) => { + options.output = value; + }, + }, + ); +} + +function assertFinite(value: unknown, field: string): asserts value is number { + if (typeof value !== "number" || !Number.isFinite(value)) { + throw new Error(`worker result field ${field} must be finite`); + } +} + +function assertFiniteNonNegative(value: unknown, field: string): asserts value is number { + assertFinite(value, field); + if (value < 0) { + throw new Error(`worker result field ${field} must be nonnegative`); + } +} + +const MEMORY_FIELDS = [ + "heapUsedBytes", + "heapTotalBytes", + "rssBytes", + "externalBytes", + "arrayBuffersBytes", + "processPeakRssBytes", +] as const; + +const RETAINED_MEMORY_FIELDS = [ + "heapUsedBytes", + "heapTotalBytes", + "rssBytes", + "externalBytes", + "arrayBuffersBytes", +] as const; + +const LIFECYCLE_COUNT_FIELDS = [ + "taskCount", + "deliveryStateCount", + "runningTasks", + "succeededTasks", + "pendingDeliveryTasks", + "succeededTerminalOutcomes", +] as const; + +function validateMemorySample( + value: unknown, + expectedCycle: number, + field: string, +): asserts value is MemorySample { + if (!isRecord(value) || value.cycle !== expectedCycle) { + throw new Error(`worker result ${field} has an invalid cycle`); + } + for (const memoryField of MEMORY_FIELDS) { + assertFiniteNonNegative(value[memoryField], `${field}.${memoryField}`); + } +} + +function validateRegistrySnapshot( + value: unknown, + field: string, + expected: RegistryLifecycleCounts, +): asserts value is RegistrySnapshot { + if (!isRecord(value) || !isRecord(value.memory) || !isRecord(value.sqlite)) { + throw new Error(`worker result field ${field} must be a registry snapshot`); + } + for (const surface of ["memory", "sqlite"] as const) { + const counts = surface === "memory" ? value.memory : value.sqlite; + for (const countField of LIFECYCLE_COUNT_FIELDS) { + const count = counts[countField]; + assertFiniteNonNegative(count, `${field}.${surface}.${countField}`); + if (count !== expected[countField]) { + throw new Error(`worker result field ${field}.${surface}.${countField} was unexpected`); + } + } + } +} + +function validateWorkerResult( + value: unknown, + expected: { size: number; cycles: number; warmup: number }, +): WorkerResult { + if (!isRecord(value)) { + throw new Error("worker result must be an object"); + } + if (value.size !== expected.size) { + throw new Error(`worker size ${expected.size} returned mismatched identity`); + } + if (!isRecord(value.timingsMs)) { + throw new Error("worker result timingsMs must be an object"); + } + for (const phase of ["registration", "terminal", "teardown"] as const) { + const timings = value.timingsMs[phase]; + if (!Array.isArray(timings) || timings.length !== expected.cycles) { + throw new Error( + `worker size ${expected.size} returned ${Array.isArray(timings) ? timings.length : "invalid"} ${phase} samples; expected ${expected.cycles}`, + ); + } + timings.forEach((timing, index) => + assertFiniteNonNegative(timing, `timingsMs.${phase}[${index}]`), + ); + } + if (!isRecord(value.memory)) { + throw new Error("worker result memory must be an object"); + } + const postGcBaseline = value.memory.postGcBaseline; + validateMemorySample(postGcBaseline, -1, "memory.postGcBaseline"); + if ( + !Array.isArray(value.memory.postGcSamples) || + value.memory.postGcSamples.length !== expected.cycles + ) { + throw new Error( + `worker size ${expected.size} returned invalid post-GC sample count; expected ${expected.cycles}`, + ); + } + value.memory.postGcSamples.forEach((sample, index) => + validateMemorySample(sample, index, `memory.postGcSamples[${index}]`), + ); + const postGcSamples = value.memory.postGcSamples as MemorySample[]; + if (!isRecord(value.memory.retainedSlopesBytesPerCycle)) { + throw new Error("worker result retained memory slopes must be an object"); + } + if (!isRecord(value.memory.retainedDeltasBytes)) { + throw new Error("worker result retained memory deltas must be an object"); + } + const finalPostGcSample = postGcSamples.at(-1); + if (!finalPostGcSample) { + throw new Error("worker result must include a final post-GC sample"); + } + for (const field of RETAINED_MEMORY_FIELDS) { + assertFinite( + value.memory.retainedSlopesBytesPerCycle[field], + `memory.retainedSlopesBytesPerCycle.${field}`, + ); + assertFinite(value.memory.retainedDeltasBytes[field], `memory.retainedDeltasBytes.${field}`); + const expectedDelta = finalPostGcSample[field] - postGcBaseline[field]; + if (value.memory.retainedDeltasBytes[field] !== expectedDelta) { + throw new Error( + `worker result memory.retainedDeltasBytes.${field} must be end minus baseline`, + ); + } + } + if ("processPeakRssBytes" in value.memory.retainedSlopesBytesPerCycle) { + throw new Error("worker result retained memory slopes must exclude process peak RSS"); + } + if ("processPeakRssBytes" in value.memory.retainedDeltasBytes) { + throw new Error("worker result retained memory deltas must exclude process peak RSS"); + } + assertFiniteNonNegative(value.memory.processPeakRssBytes, "memory.processPeakRssBytes"); + if (!isRecord(value.invariant)) { + throw new Error("worker result invariant must be an object"); + } + assertFiniteNonNegative(value.invariant.cyclesValidated, "invariant.cyclesValidated"); + const emptyCounts: RegistryLifecycleCounts = { + taskCount: 0, + deliveryStateCount: 0, + runningTasks: 0, + succeededTasks: 0, + pendingDeliveryTasks: 0, + succeededTerminalOutcomes: 0, + }; + validateRegistrySnapshot(value.invariant.registration, "invariant.registration", { + ...emptyCounts, + taskCount: expected.size, + deliveryStateCount: expected.size, + runningTasks: expected.size, + pendingDeliveryTasks: expected.size, + }); + validateRegistrySnapshot(value.invariant.terminal, "invariant.terminal", { + ...emptyCounts, + taskCount: expected.size, + deliveryStateCount: expected.size, + succeededTasks: expected.size, + pendingDeliveryTasks: expected.size, + succeededTerminalOutcomes: expected.size, + }); + validateRegistrySnapshot(value.invariant.teardown, "invariant.teardown", emptyCounts); + if ( + value.invariant.ok !== true || + value.invariant.serializedSharedConnection !== true || + value.invariant.cyclesValidated !== expected.cycles + expected.warmup + ) { + throw new Error(`worker size ${expected.size} reported a failed invariant`); + } + return value as WorkerResult; +} + +function parseWorkerProcessResult( + result: BenchmarkWorkerProcessResult, + expected: { size: number; cycles: number; warmup: number }, +): WorkerResult { + return parseBenchmarkWorkerResult({ + result, + label: `size ${expected.size}`, + sentinel: WORKER_RESULT_SENTINEL, + timeoutMs: WORKER_TIMEOUT_MS, + validate: (value) => validateWorkerResult(value, expected), + }); +} + +function runWorker( + options: Options, + size: number, + runtime: WorkerLaunchRuntime = {}, +): WorkerResult { + const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-task-registry-bench-")); + try { + return runBenchmarkWorker({ + args: [ + "--expose-gc", + "--import", + "tsx", + "scripts/bench-task-registry-sqlite-worker.ts", + "--size", + String(size), + "--cycles", + String(options.cycles), + "--warmup", + String(options.warmup), + "--state-dir", + stateDir, + ], + label: `size ${size}`, + sentinel: WORKER_RESULT_SENTINEL, + spawnWorker: runtime.spawnWorker, + timeoutMs: WORKER_TIMEOUT_MS, + validate: (value) => + validateWorkerResult(value, { + size, + cycles: options.cycles, + warmup: options.warmup, + }), + }); + } finally { + fs.rmSync(stateDir, { recursive: true, force: true }); + } +} + +function aggregateWorkerResults(options: Options, workers: WorkerResult[]) { + const bySize = new Map(workers.map((worker) => [worker.size, worker])); + if (bySize.size !== workers.length) { + throw new Error("worker results contain duplicate sizes"); + } + const missing = options.sizes.filter((size) => !bySize.has(size)); + const unexpected = [...bySize.keys()].filter((size) => !options.sizes.includes(size)); + if (missing.length > 0 || unexpected.length > 0) { + throw new Error( + `worker result mismatch: missing=${missing.join(",") || "none"} unexpected=${unexpected.join(",") || "none"}`, + ); + } + const sizes = options.sizes.map((size) => { + const worker = bySize.get(size); + if (!worker) { + throw new Error(`missing worker result for size ${size}`); + } + return { + size, + timingsMs: { + registration: summarizeBenchmarkTimings(worker.timingsMs.registration), + terminal: summarizeBenchmarkTimings(worker.timingsMs.terminal), + teardown: summarizeBenchmarkTimings(worker.timingsMs.teardown), + }, + memory: worker.memory, + invariant: worker.invariant, + }; + }); + const failures = sizes + .filter((entry) => !entry.invariant.ok) + .map((entry) => `size:${entry.size}`); + return { + schemaVersion: 1, + benchmark: "durable-task-registry-churn", + generatedAt: new Date().toISOString(), + runtime: { node: process.version, platform: process.platform, arch: process.arch }, + model: { + unit: "subagent task-record registrations", + execution: + "serialized create, terminal, and delete calls through one process-local shared SQLite connection", + isolation: "fresh --expose-gc worker process per size", + workload: + "all task records start running with pending delivery, transition to succeeded with a succeeded terminal outcome, then delete", + }, + interpretation: { + timings: "advisory only; this is not a concurrent SQLite writer benchmark", + memory: + "post-GC baseline, end-minus-baseline retained deltas, and retained slopes are diagnostic only; they neither claim nor rule out a memory leak", + }, + options: { + sizes: options.sizes, + cycles: options.cycles, + warmup: options.warmup, + }, + memory: { + workerProcessPeakRssBytes: Math.max( + ...workers.map((worker) => worker.memory.processPeakRssBytes), + ), + }, + sizes, + invariants: { + ok: failures.length === 0, + failures, + exactRegistrationTerminalAndTeardownState: failures.length === 0, + zeroRowsAfterEveryTeardown: failures.length === 0, + }, + }; +} + +function benchmark(options: Options) { + const workers = runBenchmarkJobs(options.sizes, { + prefix: "bench-task-registry-sqlite", + describe: (size) => `size=${size}`, + run: (size) => runWorker(options, size), + }); + return aggregateWorkerResults(options, workers); +} + +async function main(argv = process.argv.slice(2)): Promise { + const options = parseOptions(argv); + if (options.help) { + process.stdout.write(usage()); + return; + } + const report = benchmark(options); + emitBenchmarkReport(report, options, (result) => [ + ...result.sizes.map((entry) => { + const registration = entry.timingsMs.registration; + const heapSlope = entry.memory.retainedSlopesBytesPerCycle.heapUsedBytes; + return `size=${entry.size} registration-p50=${registration.p50.toFixed(3)}ms registration-max=${registration.max.toFixed(3)}ms post-gc-heap-slope=${heapSlope.toFixed(1)}B/cycle`; + }), + result.interpretation.timings, + result.interpretation.memory, + ]); +} + +export const testing = { + aggregateWorkerResults, + parseOptions, + parseWorkerProcessResult, + runWorker, +}; + +if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { + await runBenchmarkEntrypoint("bench-task-registry-sqlite", main); +} diff --git a/scripts/build-diffs-viewer-runtime.mts b/scripts/build-diffs-viewer-runtime.mts new file mode 100644 index 0000000000000000000000000000000000000000..ac417b920e6f68778d961edeffa516e71da88ed2 --- /dev/null +++ b/scripts/build-diffs-viewer-runtime.mts @@ -0,0 +1,123 @@ +#!/usr/bin/env node + +// Builds browser runtime bundles for the diffs viewer assets. +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { build, type Plugin } from "esbuild"; +import { writeGeneratedTextAsset } from "./lib/generated-text-asset.mts"; + +const modulePath = fileURLToPath(import.meta.url); +const repoRoot = path.resolve(path.dirname(modulePath), ".."); +const pierreDiffsEmptySideEffectNamespace = "openclaw-diffs-empty-side-effect"; +const pierreDiffsEmptySideEffectPath = "pierre-diffs-parse-decorations-side-effect"; + +const targets = { + curated: { + entry: "extensions/diffs/src/viewer-client.ts", + output: "extensions/diffs/assets/viewer-runtime.js", + shikiAlias: "scripts/diffs-shiki-curated.ts", + languagePackAvailable: false, + }, + full: { + entry: "extensions/diffs/src/viewer-client.ts", + output: "extensions/diffs-language-pack/assets/viewer-runtime.js", + languagePackAvailable: true, + }, +}; + +function toPosixPath(value: string) { + return value.replaceAll("\\", "/"); +} + +/** + * Creates the esbuild plugin that neutralizes Pierre diffs' browser side-effect import. + */ +export function createPierreDiffsSideEffectImportPlugin(): { + name: string; + setup(buildContext: unknown): void; +}; +export function createPierreDiffsSideEffectImportPlugin() { + return { + name: "openclaw-diffs-pierre-side-effect-imports", + setup(buildContext) { + buildContext.onResolve({ filter: /^diff$/ }, (args) => { + const importer = toPosixPath(args.importer); + if (!importer.endsWith("/@pierre/diffs/dist/utils/parseDiffDecorations.js")) { + return undefined; + } + return { + path: pierreDiffsEmptySideEffectPath, + namespace: pierreDiffsEmptySideEffectNamespace, + sideEffects: true, + }; + }); + buildContext.onLoad( + { + filter: /^pierre-diffs-parse-decorations-side-effect$/, + namespace: pierreDiffsEmptySideEffectNamespace, + }, + () => ({ + contents: "export {};\n", + loader: "js", + }), + ); + }, + } satisfies Plugin; +} + +/** + * Builds one configured diffs viewer runtime target. + */ +async function buildDiffsViewerRuntime(targetName: string | undefined) { + const target = Object.entries(targets).find(([name]) => name === targetName)?.[1]; + if (!target) { + throw new Error( + `Usage: node --import tsx scripts/build-diffs-viewer-runtime.mts ${Object.keys(targets).join("|")}`, + ); + } + + const outputPath = path.join(repoRoot, target.output); + const shikiAlias = "shikiAlias" in target ? target.shikiAlias : undefined; + const result = await build({ + entryPoints: [path.join(repoRoot, target.entry)], + bundle: true, + platform: "browser", + target: "es2020", + format: "esm", + minify: true, + define: { + __OPENCLAW_DIFFS_LANGUAGE_PACK__: String(target.languagePackAvailable), + NaN: "Number.NaN", + }, + legalComments: "none", + outfile: outputPath, + write: false, + plugins: [ + createPierreDiffsSideEffectImportPlugin(), + ...(shikiAlias + ? [ + { + name: "openclaw-diffs-curated-shiki", + setup(buildContext) { + buildContext.onResolve({ filter: /^shiki$/ }, () => ({ + path: path.join(repoRoot, shikiAlias), + })); + }, + } satisfies Plugin, + ] + : []), + ], + }); + + const outputFile = result.outputFiles?.[0]; + if (!outputFile) { + throw new Error(`esbuild did not produce ${target.output}`); + } + + const runtime = outputFile.text.replace(/[ \t]+$/gm, ""); + await writeGeneratedTextAsset(outputPath, runtime); +} + +if (process.argv[1] === modulePath) { + await buildDiffsViewerRuntime(process.argv[2]); +} diff --git a/scripts/build-plugin-control-ui.mts b/scripts/build-plugin-control-ui.mts new file mode 100644 index 0000000000000000000000000000000000000000..c6141814037c4f5354547e1567cc1a57b85789e5 --- /dev/null +++ b/scripts/build-plugin-control-ui.mts @@ -0,0 +1,35 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { + buildPluginControlUi, + writePluginBuildManifest, +} from "../src/cli/plugins-control-ui-build.js"; +import { controlUiSource } from "../src/plugins/package-manifest.js"; +import { resolveRepoRoot } from "./lib/repo-root.mjs"; + +const rootDir = process.cwd(); +const packageManifest = JSON.parse(await fs.readFile(path.join(rootDir, "package.json"), "utf8")); +const manifest = JSON.parse(await fs.readFile(path.join(rootDir, "openclaw.plugin.json"), "utf8")); +if (process.argv.includes("--copy")) { + const repoRoot = resolveRepoRoot(import.meta.url); + const pluginDir = path.relative(path.join(repoRoot, "extensions"), rootDir); + if (!pluginDir || pluginDir.includes(path.sep) || pluginDir.startsWith(".")) { + throw new Error("Bundled UI copy must run inside one bundled plugin package."); + } + const output = path.dirname(manifest.controlUi.entry); + if (!/^dist\/control-ui\/[a-f0-9]{64}$/u.test(output)) { + throw new Error("Build the plugin's immutable Control UI assets before copying."); + } + await fs.cp( + path.join(rootDir, output), + path.join(repoRoot, "dist/extensions", pluginDir, output), + { recursive: true }, + ); +} else { + const source = controlUiSource(packageManifest); + if (!source) { + throw new Error("Missing package.json openclaw.controlUi browser entrypoint."); + } + manifest.controlUi = await buildPluginControlUi({ rootDir, source }); + await writePluginBuildManifest(rootDir, manifest); +} diff --git a/scripts/bundle-a2ui.mts b/scripts/bundle-a2ui.mts new file mode 100644 index 0000000000000000000000000000000000000000..09a270e0e5ec59d467990e9ea2711ab1d19c5133 --- /dev/null +++ b/scripts/bundle-a2ui.mts @@ -0,0 +1,9 @@ +#!/usr/bin/env node + +// Runs bundled asset build hooks for the Canvas A2UI runtime. +import { pathToFileURL } from "node:url"; +import { runBundledPluginAssetHooks } from "./bundled-plugin-assets.mts"; + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + await runBundledPluginAssetHooks({ phase: "build", plugins: ["canvas"] }); +} diff --git a/scripts/capture-composer-mic-hover-proof.mts b/scripts/capture-composer-mic-hover-proof.mts new file mode 100644 index 0000000000000000000000000000000000000000..0adb573d00816ed433a2fa1facf7a0c7cf11e994 --- /dev/null +++ b/scripts/capture-composer-mic-hover-proof.mts @@ -0,0 +1,97 @@ +#!/usr/bin/env node +import path from "node:path"; +import { chromium } from "playwright"; +// Captures composer mic-hover proof shots: idle vs hovered talk control, plus +// the mic button's x position in both states (the hover-shift regression). +import { createControlUiE2eArtifactDir } from "../ui/src/test-helpers/control-ui-e2e-artifacts.ts"; +import { + canRunPlaywrightChromium, + installMockGateway, + resolvePlaywrightChromiumExecutablePath, + startControlUiE2eServer, +} from "../ui/src/test-helpers/control-ui-e2e.ts"; +import { readControlUiProofOption } from "./lib/control-ui-proof-args.mts"; + +const outputDir = createControlUiE2eArtifactDir( + "composer-mic-hover-proof", + readControlUiProofOption(process.argv, "output-dir") ?? + ".artifacts/control-ui-e2e/composer-mic-hover-proof", +); +const label = readControlUiProofOption(process.argv, "label") ?? "after"; +const executablePath = resolvePlaywrightChromiumExecutablePath(chromium.executablePath()); +if (!canRunPlaywrightChromium(executablePath)) { + throw new Error(`Playwright Chromium is unavailable at ${executablePath}`); +} + +const server = await startControlUiE2eServer(undefined, { source: true }); +const browser = await chromium.launch({ executablePath }); +const context = await browser.newContext({ + colorScheme: "dark", + viewport: { width: 1280, height: 800 }, +}); +const page = await context.newPage(); +page.setDefaultTimeout(30_000); + +try { + await installMockGateway(page); + await page.goto(`${server.baseUrl}chat`); + const composer = page.locator(".agent-chat__input"); + await composer.waitFor({ state: "visible" }); + const voice = page.getByRole("button", { name: "Start voice input" }); + await voice.waitFor({ state: "visible" }); + + await page.mouse.move(10, 10); + await page.waitForTimeout(400); + const composerBox = await composer.boundingBox(); + if (!composerBox) { + throw new Error("composer has no layout box"); + } + const clip = { + x: Math.max(0, composerBox.x - 12), + y: Math.max(0, composerBox.y - 12), + width: Math.min(1280, composerBox.width + 24), + height: composerBox.height + 24, + }; + const idleVoiceBox = await voice.boundingBox(); + await page.screenshot({ clip, path: path.join(outputDir, `${label}-idle.png`) }); + + await voice.hover(); + await page.waitForTimeout(400); + const hoverVoiceBox = await voice.boundingBox(); + await page.screenshot({ clip, path: path.join(outputDir, `${label}-hover.png`) }); + + const picker = page.getByRole("button", { name: "Microphone input" }); + if (await picker.count()) { + await picker.hover(); + await page.waitForTimeout(400); + await page.screenshot({ clip, path: path.join(outputDir, `${label}-hover-chevron.png`) }); + await picker.click(); + await page.waitForTimeout(500); + await page.screenshot({ + path: path.join(outputDir, `${label}-picker-open.png`), + clip: { ...clip, y: Math.max(0, clip.y - 260), height: clip.height + 260 }, + }); + await page.keyboard.press("Escape"); + } + + if (!idleVoiceBox || !hoverVoiceBox) { + throw new Error("voice button has no layout box"); + } + console.log( + JSON.stringify( + { + label, + idleVoiceX: idleVoiceBox.x, + hoverVoiceX: hoverVoiceBox.x, + hoverShiftPx: hoverVoiceBox.x - idleVoiceBox.x, + outputDir, + }, + null, + 2, + ), + ); +} finally { + await context.close(); + await browser.close(); + await server.close(); +} diff --git a/scripts/capture-injected-turn-notice-proof.mts b/scripts/capture-injected-turn-notice-proof.mts new file mode 100644 index 0000000000000000000000000000000000000000..dfa2b9f92857289ed7776d057629f8e8bb504185 --- /dev/null +++ b/scripts/capture-injected-turn-notice-proof.mts @@ -0,0 +1,117 @@ +#!/usr/bin/env node +import path from "node:path"; +import { chromium } from "playwright"; +// Captures webchat proof that a CLI harness-injected user turn renders as a +// collapsed system notice while the operator's own message keeps its bubble. +// Usage: node --import tsx scripts/capture-injected-turn-notice-proof.mts [--mode after|before] +import { createControlUiE2eArtifactDir } from "../ui/src/test-helpers/control-ui-e2e-artifacts.ts"; +import { + canRunPlaywrightChromium, + installMockGateway, + resolvePlaywrightChromiumExecutablePath, + startControlUiE2eServer, +} from "../ui/src/test-helpers/control-ui-e2e.ts"; +import { readControlUiProofOption } from "./lib/control-ui-proof-args.mts"; + +const mode = readControlUiProofOption(process.argv, "mode") ?? "after"; +if (mode !== "after" && mode !== "before") { + throw new Error(`Expected --mode after|before, received ${mode}`); +} +const outputDir = createControlUiE2eArtifactDir( + "injected-turn-notice-proof", + readControlUiProofOption(process.argv, "output-dir") ?? + ".artifacts/control-ui-e2e/injected-turn-notice-proof", +); + +const baseTime = Date.parse("2026-08-26T20:37:00.000Z"); +const skillBody = [ + "Base directory for this skill: /tmp/skills/autoreview", + "", + "# Auto Review", + "", + "Run the bundled structured review helper as a closeout check.", +].join("\n"); +// Mirrors the chat.history payload proven by the gateway integration test: +// the operator turn is a plain user row; the harness-injected turn carries the +// provenance recorded by the claude-cli importer. Before the fix the importer +// dropped that provenance, so --mode before omits it to reproduce the shipped +// payload. +const historyMessages = [ + { + role: "user", + content: "Run the autoreview skill on this branch.", + timestamp: baseTime, + __openclaw: { id: "user-1", seq: 1 }, + }, + { + role: "user", + content: [{ type: "text", text: skillBody }], + ...(mode === "after" + ? { provenance: { kind: "internal_system", sourceTool: "cli_harness_context" } } + : {}), + timestamp: baseTime + 1_000, + __openclaw: { + id: "skill-meta-1", + importedFrom: "claude-cli", + cliSessionId: "cli-session-1", + externalId: "skill-meta-1", + seq: 2, + }, + }, + { + role: "assistant", + content: "Autoreview finished: no findings.", + timestamp: baseTime + 2_000, + __openclaw: { id: "assistant-1", seq: 3 }, + }, +]; + +const executablePath = resolvePlaywrightChromiumExecutablePath(chromium.executablePath()); +if (!canRunPlaywrightChromium(executablePath)) { + throw new Error(`Playwright Chromium is unavailable at ${executablePath}`); +} + +const server = await startControlUiE2eServer(undefined, { source: true }); +const browser = await chromium.launch({ executablePath }); +try { + const context = await browser.newContext({ + colorScheme: "dark", + locale: "en-US", + reducedMotion: "reduce", + serviceWorkers: "block", + viewport: { width: 1280, height: 900 }, + }); + const page = await context.newPage(); + page.setDefaultTimeout(30_000); + await installMockGateway(page, { historyMessages }); + await page.goto(`${server.baseUrl}chat`); + await page.getByText("Autoreview finished: no findings.").waitFor({ state: "visible" }); + await page.getByText("Run the autoreview skill on this branch.").waitFor({ state: "visible" }); + if (mode === "after") { + // The injected turn renders as a collapsed system notice; expand it so the + // capture shows the disclosure carrying the skill body. + const toggle = page.locator(".chat-notice__toggle"); + await toggle.waitFor({ state: "visible" }); + await toggle.click(); + await page.getByText("Base directory for this skill:", { exact: false }).waitFor({ + state: "visible", + }); + } else { + await page.getByText("Base directory for this skill:", { exact: false }).waitFor({ + state: "visible", + }); + } + await page.evaluate(async () => { + await document.fonts.ready; + await new Promise((resolve) => { + requestAnimationFrame(() => requestAnimationFrame(() => resolve())); + }); + }); + const target = path.join(outputDir, `injected-turn-${mode}.png`); + await page.locator(".chat-thread-inner").screenshot({ animations: "disabled", path: target }); + console.log(`captured ${target}`); + await context.close(); +} finally { + await browser.close(); + await server.close(); +} diff --git a/scripts/capture-where-chip-auto-proof.mts b/scripts/capture-where-chip-auto-proof.mts new file mode 100644 index 0000000000000000000000000000000000000000..64e37fcf2151bd5fae71185b2d2845c17e756e74 --- /dev/null +++ b/scripts/capture-where-chip-auto-proof.mts @@ -0,0 +1,168 @@ +#!/usr/bin/env node +// Captures placement, inventory, and context-meter proof in both themes. +import assert from "node:assert/strict"; +import path from "node:path"; +import { chromium } from "playwright"; +import { createControlUiE2eArtifactDir } from "../ui/src/test-helpers/control-ui-e2e-artifacts.ts"; +import { + canRunPlaywrightChromium, + installMockGateway, + resolvePlaywrightChromiumExecutablePath, + startControlUiE2eServer, +} from "../ui/src/test-helpers/control-ui-e2e.ts"; + +const captureLabel = process.argv[2] ?? "after"; +const outputDir = createControlUiE2eArtifactDir( + "node-slot-pips-proof", + process.argv[3] ?? "/tmp/node-slot-pips-proof", +); +const remoteExec = process.argv[4] === "remote-exec"; +const executablePath = resolvePlaywrightChromiumExecutablePath(chromium.executablePath()); +if (!canRunPlaywrightChromium(executablePath)) { + throw new Error(`Playwright Chromium unavailable at ${executablePath}`); +} +const server = await startControlUiE2eServer(undefined, { source: true }); +const browser = await chromium.launch({ executablePath }); +const environments = [ + { id: "idle", label: "Idle runner", workerSlots: { total: 8, available: 8 } }, + { id: "partial", label: "Busy runner", workerSlots: { total: 8, available: 5 } }, + { id: "full", label: "Full runner", workerSlots: { total: 8, available: 0 } }, + { id: "offline", label: "Offline runner", workerSlots: { total: 8, available: 2 } }, + { id: "large", label: "Large runner", workerSlots: { total: 13, available: 6 } }, + { id: "exec", label: "Terminal runner" }, +].map(({ id, label, workerSlots }) => ({ + id: `node:${id}`, + label, + workerSlots, + type: "node", + status: id === "offline" ? "unavailable" : "available", + sessionHost: true, + platform: "linux", + capabilities: ["codex.exec-server"], + invocableCommands: ["codex.exec-server.stdio.v1"], +})); +try { + for (const mode of ["dark", "light"] as const) { + const context = await browser.newContext({ + colorScheme: mode, + locale: "en-US", + viewport: { width: 1280, height: 900 }, + }); + const page = await context.newPage(); + page.setDefaultTimeout(30_000); + const config = { ui: { prefs: { theme: "claw", themeMode: mode } } }; + await installMockGateway(page, { + ...(remoteExec + ? { + agentModel: "openai/gpt-5.6-luna", + models: [ + { + available: true, + id: "gpt-5.6-luna", + name: "GPT-5.6 Luna", + provider: "openai", + agentRuntime: { + id: "codex", + devicePlacementSupported: true, + source: "model", + devicePlacement: { + requiredNodeCommands: ["codex.exec-server.stdio.v1"], + consumesWorkerSlot: false, + }, + }, + }, + ], + } + : {}), + methodResponses: { + "config.get": { config, hash: "capacity-proof", valid: true, issues: [] }, + "environments.list": { + environments, + profiles: [], + }, + "node.list": { + nodes: environments.map((entry) => ({ + nodeId: entry.id.slice(5), + displayName: entry.label, + platform: entry.platform, + paired: true, + connected: entry.status === "available", + approvalState: "approved", + caps: entry.capabilities, + commands: entry.invocableCommands, + workerSlots: entry.workerSlots, + })), + }, + "sessions.list": { + count: 4, + ts: 0, + path: "", + defaults: {}, + sessions: [ + { label: "Context OK", totalTokens: 40_000 }, + { label: "Context warning", totalTokens: 140_000 }, + { label: "Context danger", totalTokens: 180_000 }, + { label: "Context approximate", totalTokens: 180_000, totalTokensFresh: false }, + ].map(({ label, totalTokens, totalTokensFresh }, index) => ({ + label, + totalTokens, + totalTokensFresh, + key: `agent:main:context-${index}`, + kind: "direct", + updatedAt: 0, + contextTokens: 200_000, + hasActiveRun: false, + })), + }, + }, + }); + await page.goto(`${server.baseUrl}new`); + const chip = page.getByText("Local", { exact: true }).first(); + await chip.waitFor(); + await page.waitForFunction( + (themeMode) => document.documentElement.dataset.themeMode === themeMode, + mode, + ); + await page.evaluate(() => document.fonts.ready); + await chip.click(); + await page.waitForSelector('[data-value="auto-device"]'); + await page.locator('[data-value="device:exec"]').waitFor(); + if (remoteExec) { + assert.equal(await page.locator('[data-value="device:full"]').isEnabled(), true); + assert.equal(await page.locator('[data-value="device:exec"]').isEnabled(), true); + } + await page.screenshot({ + path: path.join(outputDir, `${captureLabel}-${mode}-menu.png`), + animations: "disabled", + }); + await page.locator('[data-value="auto-device"]').click(); + await page.locator('#new-session-where-trigger[data-auto-device="true"]').waitFor(); + if (remoteExec) { + await page.locator("#new-session-where-trigger").click(); + await page.locator('[data-value="device:full"]').click(); + await page.locator('#new-session-where-trigger[data-device-id="full"]').waitFor(); + console.log("SHOT_OK", captureLabel, mode, outputDir); + await context.close(); + continue; + } + await page.goto(`${server.baseUrl}settings/devices`); + await page.getByText("Terminal runner", { exact: true }).waitFor(); + await page.screenshot({ + path: path.join(outputDir, `${captureLabel}-${mode}-devices.png`), + animations: "disabled", + }); + await page.goto(`${server.baseUrl}sessions`); + await page.locator(".session-context-meter").first().waitFor(); + assert.equal(await page.locator(".session-context-meter").count(), 4); + await page.evaluate(() => document.fonts.ready); + await page.locator(".data-table-container").screenshot({ + path: path.join(outputDir, `${captureLabel}-${mode}-sessions.png`), + animations: "disabled", + }); + console.log("SHOT_OK", captureLabel, mode, outputDir); + await context.close(); + } +} finally { + await browser.close(); + await server.close(); +} diff --git a/scripts/capture-workboard-ui-proof.mts b/scripts/capture-workboard-ui-proof.mts new file mode 100644 index 0000000000000000000000000000000000000000..740eb30133275f56fc31a02b8360da2a85009730 --- /dev/null +++ b/scripts/capture-workboard-ui-proof.mts @@ -0,0 +1,81 @@ +#!/usr/bin/env node +import path from "node:path"; +import { chromium } from "playwright"; +import { createControlUiE2eArtifactDir } from "../ui/src/test-helpers/control-ui-e2e-artifacts.ts"; +import { + canRunPlaywrightChromium, + resolvePlaywrightChromiumExecutablePath, +} from "../ui/src/test-helpers/control-ui-e2e.ts"; +import { readControlUiProofOption } from "./lib/control-ui-proof-args.mts"; +const DEFAULT_BASE_URL = "http://127.0.0.1:5187"; +const DEFAULT_OUTPUT_DIR = path.resolve(".artifacts/control-ui-e2e/workboard-proof"); +const WORKBOARD_SESSION_KEY = "agent:main:workboard-proof"; + +const baseUrl = new URL(readControlUiProofOption(process.argv, "base-url") ?? DEFAULT_BASE_URL); +const outputDir = createControlUiE2eArtifactDir( + "workboard-proof", + readControlUiProofOption(process.argv, "output-dir") ?? DEFAULT_OUTPUT_DIR, +); +const executablePath = resolvePlaywrightChromiumExecutablePath(chromium.executablePath()); +if (!canRunPlaywrightChromium(executablePath)) { + throw new Error(`Playwright Chromium is unavailable at ${executablePath}`); +} + +const browser = await chromium.launch({ executablePath }); +const context = await browser.newContext({ + colorScheme: "light", + reducedMotion: "reduce", + viewport: { width: 1440, height: 900 }, +}); +const page = await context.newPage(); +page.setDefaultTimeout(30_000); + +const gatewayUrl = new URL(baseUrl); +gatewayUrl.protocol = gatewayUrl.protocol === "https:" ? "wss:" : "ws:"; +const settingsKey = `openclaw.control.settings.v1:${gatewayUrl.origin}`; +await page.addInitScript( + ({ key, sessionKey }) => { + localStorage.setItem( + key, + JSON.stringify({ + boardSessionViews: { [sessionKey]: { activeTabId: "main" } }, + theme: "light", + }), + ); + }, + { key: settingsKey, sessionKey: WORKBOARD_SESSION_KEY }, +); + +try { + await page.goto(new URL("/workboard/peter-tasks", baseUrl).toString()); + const automationChip = page.locator(".workboard-automation-chip"); + await automationChip.waitFor(); + if ((await automationChip.textContent())?.trim() !== "Automation") { + throw new Error("Workboard automation chip did not render its expected label"); + } + await page.getByText("Prepare launch readiness checklist", { exact: true }).waitFor(); + await page.screenshot({ + animations: "disabled", + path: path.join(outputDir, "workboard-chip.png"), + }); + + await page.goto(new URL("/dashboard", baseUrl).toString()); + const widget = page.locator('[data-test-id="workboard-board-widget"]'); + await widget.waitFor(); + await widget.getByRole("heading", { name: "Validate onboarding flow", exact: true }).waitFor(); + await widget.getByRole("heading", { name: "Review accessibility audit", exact: true }).waitFor(); + await page.locator(".board-session-surface--dock-hidden").waitFor(); + const columnCount = await widget.locator(".workboard-column").count(); + if (columnCount !== 6) { + throw new Error(`Expected 6 Workboard columns, received ${columnCount}`); + } + await page.screenshot({ + animations: "disabled", + path: path.join(outputDir, "workboard-widget.png"), + }); +} finally { + await context.close(); + await browser.close(); +} + +console.log(`[workboard-ui-proof] ${outputDir}`); diff --git a/scripts/changed-lanes.mjs b/scripts/changed-lanes.mjs new file mode 100644 index 0000000000000000000000000000000000000000..837200ec2a219c74f46042264ef774ba1a39acd1 --- /dev/null +++ b/scripts/changed-lanes.mjs @@ -0,0 +1,5 @@ +import { runTsxCliShim } from "./lib/tsx-cli-shim.mjs"; + +await runTsxCliShim(import.meta.url, { + implementation: "./changed-lanes.mts", +}); diff --git a/scripts/check-built-plugin-control-plane-modules.mts b/scripts/check-built-plugin-control-plane-modules.mts new file mode 100644 index 0000000000000000000000000000000000000000..f29bac8145c3e4971a94e2bf8f2fd0ef5c55599f --- /dev/null +++ b/scripts/check-built-plugin-control-plane-modules.mts @@ -0,0 +1,328 @@ +#!/usr/bin/env node + +// Verifies built plugin control-plane artifacts through Node's native require(esm) path. +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import type ts from "typescript"; +import { resolveNodeRuntimeExecutable } from "../src/infra/node-runtime-executable.ts"; +import { collectSourceCheckoutPluginBuildEntries } from "./lib/bundled-plugin-build-entries.mjs"; +import { isRecord } from "./lib/record-shared.mjs"; +import { resolveRepoRoot } from "./lib/repo-root.mjs"; +import { getTypeScript } from "./lib/ts-guard-utils.mts"; + +type BuiltPluginControlPlaneModule = { + pluginId: string; + kind: string; + relativePath: string; +}; + +type BuiltPluginControlPlaneModuleFailure = BuiltPluginControlPlaneModule & { + error: string; +}; + +type BuiltDoctorContractClosureViolation = BuiltPluginControlPlaneModule & { + dependency: string; + importerPath: string; +}; + +type ProbeParams = { + rootDir?: string; + env?: NodeJS.ProcessEnv; + timeoutMs?: number; +}; + +const ROOT = resolveRepoRoot(import.meta.url); +const DIRECT_CONTRACT_ENTRIES = ["contract-api", "doctor-contract-api"]; +const LEGACY_SETUP_PROPERTIES = new Map([ + ["legacyStateMigrations", "channel-legacy-state-migrations"], + ["legacySessionSurface", "channel-legacy-session-surface"], + ["legacySessionSurfaces", "channel-legacy-session-surface"], +]); +const PROBE_RESULT_MARKER = "__OPENCLAW_PLUGIN_CONTROL_PLANE_PROBE__"; +const DEFAULT_TIMEOUT_MS = 120_000; +// Doctor enumeration cold-loads every declaring plugin's contract closure, so a +// doctor artifact must never reach the process-spawn graph. Requiring the artifact +// cannot prove this: plain Node resolves the whole graph fine, and the cost and the +// ESM-only transitive deps (execa -> npm-run-path -> unicorn-magic, which has no +// `require` condition) only surface on source-run hosts whose CJS-flavored resolver +// rejects them. `doctor-contract-closure-guard.test.ts` owns the same invariant over +// sources; bundling can merge runtime code into the artifact behind its back, so the +// built closure is checked here. +const FORBIDDEN_DOCTOR_CONTRACT_DEPENDENCIES = ["execa"]; +const REQUIRE_PROBE_SOURCE = String.raw` +const { createRequire } = require("node:module"); +const path = require("node:path"); +const targets = JSON.parse(Buffer.from(process.argv[1], "base64url").toString("utf8")); +const requireFromRoot = createRequire(path.join(process.cwd(), "package.json")); +const failures = []; +for (const target of targets) { + try { + requireFromRoot(path.resolve(process.cwd(), target.relativePath)); + } catch (error) { + failures.push({ + ...target, + error: error instanceof Error ? (error.stack || error.message) : String(error), + }); + } +} +process.stdout.write("\n${PROBE_RESULT_MARKER}" + JSON.stringify({ failures })); +`; + +function propertyNameText(name: ts.PropertyName) { + const ts = getTypeScript(); + return ts.isIdentifier(name) || ts.isStringLiteralLike(name) ? name.text : ""; +} + +function listLegacySetupModuleSpecifiers(setupEntryPath: string) { + const ts = getTypeScript(); + const source = fs.readFileSync(setupEntryPath, "utf8"); + const sourceFile = ts.createSourceFile(setupEntryPath, source, ts.ScriptTarget.Latest, true); + const specifiers: Array<{ kind: string; specifier: string }> = []; + const visit = (node: ts.Node): void => { + if (ts.isPropertyAssignment(node) && ts.isObjectLiteralExpression(node.initializer)) { + const kind = LEGACY_SETUP_PROPERTIES.get(propertyNameText(node.name)); + if (kind) { + const specifierProperty = node.initializer.properties.find( + (property) => + ts.isPropertyAssignment(property) && propertyNameText(property.name) === "specifier", + ); + if ( + specifierProperty && + ts.isPropertyAssignment(specifierProperty) && + ts.isStringLiteralLike(specifierProperty.initializer) + ) { + specifiers.push({ kind, specifier: specifierProperty.initializer.text }); + } + } + } + ts.forEachChild(node, visit); + }; + visit(sourceFile); + return specifiers; +} + +/** Lists exact built doctor, contract, and channel legacy migration artifacts. */ +export function listBuiltPluginControlPlaneModules( + params: Pick = {}, +) { + const rootDir = path.resolve(params.rootDir ?? ROOT); + const extensionsDir = path.join(rootDir, "dist", "extensions"); + if (!fs.existsSync(extensionsDir)) { + return []; + } + const sourceEntries = new Map( + (fs.existsSync(path.join(rootDir, "extensions")) + ? collectSourceCheckoutPluginBuildEntries({ cwd: rootDir, env: params.env }) + : [] + ).map((entry) => [entry.id, entry]), + ); + const modules = new Map(); + for (const entry of fs + .readdirSync(extensionsDir, { withFileTypes: true }) + .filter((candidate) => candidate.isDirectory()) + .toSorted((left, right) => left.name.localeCompare(right.name))) { + const pluginId = entry.name; + const pluginDir = path.join(extensionsDir, pluginId); + // Packaged core artifacts use ESM; isolated source-checkout plugins use + // the same selected format as their builder and generated metadata. + const extension = sourceEntries.get(pluginId)?.runtimeExtension ?? ".js"; + for (const entryName of DIRECT_CONTRACT_ENTRIES) { + const fileName = `${entryName}${extension}`; + const modulePath = path.join(pluginDir, fileName); + if (fs.existsSync(modulePath)) { + const relativePath = path.relative(rootDir, modulePath).split(path.sep).join("/"); + modules.set(relativePath, { + pluginId, + kind: entryName === "doctor-contract-api" ? "doctor-contract" : "contract", + relativePath, + }); + } + } + const setupEntryPath = path.join(pluginDir, `setup-entry${extension}`); + if (!fs.existsSync(setupEntryPath)) { + continue; + } + for (const { kind, specifier } of listLegacySetupModuleSpecifiers(setupEntryPath)) { + const modulePath = path.resolve(pluginDir, specifier); + const pluginRelativePath = path.relative(pluginDir, modulePath); + if (pluginRelativePath.startsWith(`..${path.sep}`) || path.isAbsolute(pluginRelativePath)) { + throw new Error(`${pluginId} setup entry module escapes the plugin root: ${specifier}`); + } + const relativePath = path.relative(rootDir, modulePath).split(path.sep).join("/"); + modules.set(relativePath, { pluginId, kind, relativePath }); + } + } + return [...modules.values()].toSorted((left, right) => + left.relativePath.localeCompare(right.relativePath), + ); +} + +/** Loads every selected artifact in one timeout-bounded native-require child. */ +export function probeBuiltPluginControlPlaneModules( + modules: BuiltPluginControlPlaneModule[], + params: ProbeParams = {}, +) { + if (modules.length === 0) { + return []; + } + const rootDir = path.resolve(params.rootDir ?? ROOT); + const encodedTargets = Buffer.from(JSON.stringify(modules), "utf8").toString("base64url"); + const result = spawnSync( + resolveNodeRuntimeExecutable() ?? process.execPath, + ["-e", REQUIRE_PROBE_SOURCE, encodedTargets], + { + cwd: rootDir, + encoding: "utf8", + maxBuffer: 8 * 1024 * 1024, + timeout: params.timeoutMs ?? DEFAULT_TIMEOUT_MS, + }, + ); + if (result.error) { + throw new Error( + `built plugin control-plane native-require probe failed: ${result.error.message}`, + ); + } + const markerIndex = result.stdout.lastIndexOf(PROBE_RESULT_MARKER); + if (markerIndex < 0) { + throw new Error( + `built plugin control-plane native-require probe exited ${String(result.status)} without a result`, + ); + } + const payload: unknown = JSON.parse( + result.stdout.slice(markerIndex + PROBE_RESULT_MARKER.length), + ); + if (!isRecord(payload) || !Array.isArray(payload.failures)) { + return []; + } + return payload.failures.filter( + (failure): failure is BuiltPluginControlPlaneModuleFailure => + isRecord(failure) && + typeof failure.pluginId === "string" && + typeof failure.kind === "string" && + typeof failure.relativePath === "string" && + typeof failure.error === "string", + ); +} + +// Follow ESM declarations and eager CJS require calls emitted by isolated builds. +// Dynamic imports and requires inside functions are lazy, not enumeration costs. +function parseStaticModuleSpecifiers(source: string, filePath: string): string[] { + const ts = getTypeScript(); + const sourceFile = ts.createSourceFile(filePath, source, ts.ScriptTarget.Latest, true); + const specifiers: string[] = []; + const visit = (node: ts.Node): void => { + if (ts.isFunctionLike(node)) { + return; + } + const moduleSpecifier = + ts.isImportDeclaration(node) || ts.isExportDeclaration(node) + ? node.moduleSpecifier + : ts.isCallExpression(node) && + ts.isIdentifier(node.expression) && + /^(?:require|_+require\d*)$/u.test(node.expression.text) + ? node.arguments[0] + : undefined; + if (moduleSpecifier && ts.isStringLiteralLike(moduleSpecifier)) { + specifiers.push(moduleSpecifier.text); + } + ts.forEachChild(node, visit); + }; + visit(sourceFile); + return specifiers; +} + +function resolveBuiltChunkPath(importerPath: string, specifier: string): string | undefined { + const target = path.resolve(path.dirname(importerPath), specifier); + // Generated chunk edges carry their exact output suffix, including CJS. + return fs.existsSync(target) && fs.statSync(target).isFile() ? target : undefined; +} + +/** Collects the bare dependencies a built artifact reaches through static imports. */ +function collectBuiltModuleStaticDependencies(entryPath: string): Map { + const dependencies = new Map(); + const visited = new Set(); + const pending: string[] = [entryPath]; + while (pending.length > 0) { + const filePath = pending.pop(); + if (!filePath || visited.has(filePath)) { + continue; + } + visited.add(filePath); + let source: string; + try { + source = fs.readFileSync(filePath, "utf8"); + } catch { + continue; + } + for (const reference of parseStaticModuleSpecifiers(source, filePath)) { + if (reference.startsWith(".") || reference.startsWith("/")) { + const resolved = resolveBuiltChunkPath(filePath, reference); + if (resolved) { + pending.push(resolved); + } + continue; + } + if (!reference.startsWith("node:") && !dependencies.has(reference)) { + dependencies.set(reference, filePath); + } + } + } + return dependencies; +} + +/** Fails when a built doctor artifact statically reaches a forbidden runtime dependency. */ +export function collectBuiltDoctorContractClosureViolations( + modules: BuiltPluginControlPlaneModule[], + params: { rootDir?: string } = {}, +): BuiltDoctorContractClosureViolation[] { + const rootDir = path.resolve(params.rootDir ?? ROOT); + const violations: BuiltDoctorContractClosureViolation[] = []; + for (const module of modules.filter((candidate) => candidate.kind === "doctor-contract")) { + const dependencies = collectBuiltModuleStaticDependencies( + path.join(rootDir, module.relativePath), + ); + for (const dependency of FORBIDDEN_DOCTOR_CONTRACT_DEPENDENCIES) { + const importer = dependencies.get(dependency); + if (importer) { + violations.push({ + ...module, + dependency, + importerPath: path.relative(rootDir, importer).split(path.sep).join("/"), + }); + } + } + } + return violations; +} + +/** Fails the build when a generated plugin control-plane module cannot be required natively. */ +export function verifyBuiltPluginControlPlaneModules(params: ProbeParams = {}) { + const modules = listBuiltPluginControlPlaneModules(params); + const failures = probeBuiltPluginControlPlaneModules(modules, params); + if (failures.length > 0) { + const details = failures.map( + (failure) => + `- ${failure.pluginId} (${failure.kind}) ${failure.relativePath}: ${failure.error}`, + ); + throw new Error(`built plugin control-plane module load failures:\n${details.join("\n")}`); + } + const closureViolations = collectBuiltDoctorContractClosureViolations(modules, params); + if (closureViolations.length > 0) { + const details = closureViolations.map( + (violation) => + `- ${violation.pluginId} ${violation.relativePath} statically reaches ${violation.dependency} through ${violation.importerPath}`, + ); + throw new Error( + `built doctor contract closures reach forbidden runtime dependencies:\n${details.join("\n")}`, + ); + } + console.error( + `[plugin-control-plane-loads] verified ${modules.length} built modules with native require and checked doctor closures`, + ); +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { + verifyBuiltPluginControlPlaneModules(); +} diff --git a/scripts/check-changed.mjs b/scripts/check-changed.mjs new file mode 100644 index 0000000000000000000000000000000000000000..d6ae4b6dd1ac32f71b26bf790601371ac33ca3c6 --- /dev/null +++ b/scripts/check-changed.mjs @@ -0,0 +1,5 @@ +import { runTsxCliShim } from "./lib/tsx-cli-shim.mjs"; + +await runTsxCliShim(import.meta.url, { + implementation: "./check-changed.mts", +}); diff --git a/scripts/check-changelog-attributions.mjs b/scripts/check-changelog-attributions.mjs new file mode 100644 index 0000000000000000000000000000000000000000..5824ba1b1a25a7c98535775a5f2c64e19ca49383 --- /dev/null +++ b/scripts/check-changelog-attributions.mjs @@ -0,0 +1,4 @@ +#!/usr/bin/env node +import { runNodeCliShim } from "./lib/tsx-cli-shim.mjs"; + +await runNodeCliShim(import.meta.url, { implementation: "./check-changelog-attributions.mts" }); diff --git a/scripts/check-channel-agnostic-boundaries.mts b/scripts/check-channel-agnostic-boundaries.mts new file mode 100644 index 0000000000000000000000000000000000000000..e37ce99bf32ebdd724751964922ca10818f6dbd1 --- /dev/null +++ b/scripts/check-channel-agnostic-boundaries.mts @@ -0,0 +1,368 @@ +#!/usr/bin/env node + +// Checks channel-agnostic core surfaces for channel-specific coupling. +import { promises as fs } from "node:fs"; +import path from "node:path"; +import ts from "typescript"; +import { visitModuleSpecifiers } from "./lib/guard-inventory-utils.mjs"; +import { resolveRepoRoot } from "./lib/repo-root.mjs"; +import { + collectTypeScriptFiles, + getPropertyNameText, + runAsScript, + toLine, +} from "./lib/ts-guard-utils.mts"; + +const repoRoot = resolveRepoRoot(import.meta.url); + +const acpCoreProtectedSources = [ + path.join(repoRoot, "src", "acp"), + path.join(repoRoot, "src", "agents", "subagents", "spawn", "acp-spawn.ts"), + path.join(repoRoot, "src", "auto-reply", "reply", "commands-acp"), + path.join(repoRoot, "src", "infra", "outbound", "conversation-id.ts"), +]; + +const channelCoreProtectedSources = [ + path.join(repoRoot, "src", "channels", "thread-bindings-policy.ts"), + path.join(repoRoot, "src", "channels", "thread-bindings-messages.ts"), + path.join(repoRoot, "src", "sessions", "send-policy.ts"), + path.join(repoRoot, "src", "sessions", "session-chat-type-shared.ts"), + path.join(repoRoot, "src", "utils", "delivery-context.shared.ts"), +]; +const acpUserFacingTextSources = [ + path.join(repoRoot, "src", "auto-reply", "reply", "commands-acp"), +]; +const systemMarkLiteralGuardSources = [ + path.join(repoRoot, "src", "auto-reply", "reply", "commands-acp"), + path.join(repoRoot, "src", "auto-reply", "reply", "dispatch-acp.ts"), + path.join(repoRoot, "src", "auto-reply", "reply", "directive-handling.shared.ts"), + path.join(repoRoot, "src", "channels", "thread-bindings-messages.ts"), +]; + +const channelIds = [ + "discord", + "googlechat", + "imessage", + "irc", + "line", + "mattermost", + "matrix", + "msteams", + "nextcloud-talk", + "nostr", + "qqbot", + "signal", + "slack", + "synology-chat", + "telegram", + "tlon", + "twitch", + "web", + "whatsapp", + "zalo", + "zalouser", +]; + +const channelIdSet = new Set(channelIds); +const channelSegmentRe = new RegExp(`(^|[._/-])(?:${channelIds.join("|")})([._/-]|$)`); +const comparisonOperators: ReadonlySet = new Set([ + ts.SyntaxKind.EqualsEqualsEqualsToken, + ts.SyntaxKind.ExclamationEqualsEqualsToken, + ts.SyntaxKind.EqualsEqualsToken, + ts.SyntaxKind.ExclamationEqualsToken, +]); + +const allowedViolations = new Set(); + +type BoundaryViolation = { line: number; reason: string }; +type BoundaryOptions = { + checkModuleSpecifiers?: boolean; + checkConfigPaths?: boolean; + checkChannelComparisons?: boolean; + checkChannelAssignments?: boolean; + moduleSpecifierMatcher?: (specifier: string) => boolean; +}; +type ModuleSpecifierVisit = { + kind: string; + node: ts.Node; + specifier: string; + specifierNode: ts.Node; +}; +function isChannelsPropertyAccess(node: ts.Node) { + if (ts.isPropertyAccessExpression(node)) { + return node.name.text === "channels"; + } + if (ts.isElementAccessExpression(node) && ts.isStringLiteral(node.argumentExpression)) { + return node.argumentExpression.text === "channels"; + } + return false; +} + +function readStringLiteral(node: ts.Node) { + if (ts.isStringLiteral(node)) { + return node.text; + } + if (ts.isNoSubstitutionTemplateLiteral(node)) { + return node.text; + } + return null; +} + +function isChannelLiteralNode(node: ts.Node) { + const text = readStringLiteral(node); + return text ? channelIdSet.has(text) : false; +} + +function matchesChannelModuleSpecifier(specifier: string) { + return channelSegmentRe.test(specifier.replaceAll("\\", "/")); +} + +const userFacingChannelNameRe = + /\b(?:discord|telegram|slack|signal|imessage|whatsapp|google\s*chat|irc|line|zalo|matrix|msteams)\b/i; +const systemMarkLiteral = "⚙️"; + +function isModuleSpecifierStringNode(node: ts.Node) { + const parent = node.parent; + if (ts.isImportDeclaration(parent) || ts.isExportDeclaration(parent)) { + return true; + } + return ( + ts.isCallExpression(parent) && + parent.expression.kind === ts.SyntaxKind.ImportKeyword && + parent.arguments[0] === node + ); +} + +/** + * Finds channel-specific references inside channel-agnostic protected sources. + */ +export function findChannelAgnosticBoundaryViolations( + content: string, + fileName = "source.ts", + options: BoundaryOptions = {}, +) { + const checkModuleSpecifiers = options.checkModuleSpecifiers ?? true; + const checkConfigPaths = options.checkConfigPaths ?? true; + const checkChannelComparisons = options.checkChannelComparisons ?? true; + const checkChannelAssignments = options.checkChannelAssignments ?? true; + const moduleSpecifierMatcher = options.moduleSpecifierMatcher ?? matchesChannelModuleSpecifier; + + const sourceFile = ts.createSourceFile(fileName, content, ts.ScriptTarget.Latest, true); + const violations: BoundaryViolation[] = []; + const moduleViolations = new Map(); + if (checkModuleSpecifiers) { + visitModuleSpecifiers( + ts, + sourceFile, + ({ kind, node, specifier, specifierNode }: ModuleSpecifierVisit) => { + if (moduleSpecifierMatcher(specifier)) { + const verb = + kind === "export" + ? "re-exports" + : kind === "dynamic-import" + ? "dynamically imports" + : "imports"; + moduleViolations.set(node, { + line: toLine(sourceFile, specifierNode), + reason: verb + ' channel module "' + specifier + '"', + }); + } + }, + { includeCommonJs: true, includeImportMetaUrl: true, includeImportTypes: true }, + ); + } + + const visit = (node: ts.Node): void => { + const moduleViolation = moduleViolations.get(node); + if (moduleViolation) { + violations.push(moduleViolation); + } + + if ( + checkConfigPaths && + ts.isPropertyAccessExpression(node) && + channelIdSet.has(node.name.text) + ) { + if (isChannelsPropertyAccess(node.expression)) { + violations.push({ + line: toLine(sourceFile, node.name), + reason: `references config path "channels.${node.name.text}"`, + }); + } + } + + if ( + checkConfigPaths && + ts.isElementAccessExpression(node) && + ts.isStringLiteral(node.argumentExpression) && + channelIdSet.has(node.argumentExpression.text) + ) { + if (isChannelsPropertyAccess(node.expression)) { + violations.push({ + line: toLine(sourceFile, node.argumentExpression), + reason: `references config path "channels[${JSON.stringify(node.argumentExpression.text)}]"`, + }); + } + } + + if ( + checkChannelComparisons && + ts.isBinaryExpression(node) && + comparisonOperators.has(node.operatorToken.kind) + ) { + if (isChannelLiteralNode(node.left) || isChannelLiteralNode(node.right)) { + const leftText = node.left.getText(sourceFile); + const rightText = node.right.getText(sourceFile); + violations.push({ + line: toLine(sourceFile, node.operatorToken), + reason: `compares with channel id literal (${leftText} ${node.operatorToken.getText(sourceFile)} ${rightText})`, + }); + } + } + + if (checkChannelAssignments && ts.isPropertyAssignment(node)) { + const propName = getPropertyNameText(node.name); + if (propName === "channel" && isChannelLiteralNode(node.initializer)) { + violations.push({ + line: toLine(sourceFile, node.initializer), + reason: `assigns channel id literal to "channel" (${node.initializer.getText(sourceFile)})`, + }); + } + } + + ts.forEachChild(node, visit); + }; + + visit(sourceFile); + return violations; +} + +/** + * Finds reverse dependencies from channel core into plugin/runtime surfaces. + */ +export function findChannelCoreReverseDependencyViolations( + content: string, + fileName = "source.ts", +) { + return findChannelAgnosticBoundaryViolations(content, fileName, { + checkModuleSpecifiers: true, + checkConfigPaths: false, + checkChannelComparisons: false, + checkChannelAssignments: false, + moduleSpecifierMatcher: matchesChannelModuleSpecifier, + }); +} + +/** + * Finds user-facing channel names in ACP-owned text sources. + */ +export function findAcpUserFacingChannelNameViolations(content: string, fileName = "source.ts") { + const sourceFile = ts.createSourceFile(fileName, content, ts.ScriptTarget.Latest, true); + const violations: BoundaryViolation[] = []; + + const visit = (node: ts.Node): void => { + const text = readStringLiteral(node); + if (text && userFacingChannelNameRe.test(text) && !isModuleSpecifierStringNode(node)) { + violations.push({ + line: toLine(sourceFile, node), + reason: `user-facing text references channel name (${JSON.stringify(text)})`, + }); + } + ts.forEachChild(node, visit); + }; + + visit(sourceFile); + return violations; +} + +/** + * Finds raw system mark literals where shared constants should be used. + */ +export function findSystemMarkLiteralViolations(content: string, fileName = "source.ts") { + const sourceFile = ts.createSourceFile(fileName, content, ts.ScriptTarget.Latest, true); + const violations: BoundaryViolation[] = []; + + const visit = (node: ts.Node): void => { + const text = readStringLiteral(node); + if (text && text.includes(systemMarkLiteral) && !isModuleSpecifierStringNode(node)) { + violations.push({ + line: toLine(sourceFile, node), + reason: `hardcoded system mark literal (${JSON.stringify(text)})`, + }); + } + ts.forEachChild(node, visit); + }; + + visit(sourceFile); + return violations; +} + +const boundaryRuleSets = [ + { + id: "acp-core", + sources: acpCoreProtectedSources, + scan: findChannelAgnosticBoundaryViolations, + }, + { + id: "channel-core-reverse-deps", + sources: channelCoreProtectedSources, + scan: findChannelCoreReverseDependencyViolations, + }, + { + id: "acp-user-facing-text", + sources: acpUserFacingTextSources, + scan: findAcpUserFacingChannelNameViolations, + }, + { + id: "system-mark-literal-usage", + sources: systemMarkLiteralGuardSources, + scan: findSystemMarkLiteralViolations, + }, +]; + +/** + * Runs all channel-agnostic boundary checks. + */ +export async function main() { + const violations: string[] = []; + for (const ruleSet of boundaryRuleSets) { + const files = ( + await Promise.all( + ruleSet.sources.map( + async (sourcePath) => + await collectTypeScriptFiles(sourcePath, { + ignoreMissing: true, + }), + ), + ) + ).flat(); + for (const filePath of files) { + const relativeFile = path.relative(repoRoot, filePath); + if ( + allowedViolations.has(`${ruleSet.id}:${relativeFile}`) || + allowedViolations.has(relativeFile) + ) { + continue; + } + const content = await fs.readFile(filePath, "utf8"); + for (const violation of ruleSet.scan(content, relativeFile)) { + violations.push(`${ruleSet.id} ${relativeFile}:${violation.line}: ${violation.reason}`); + } + } + } + + if (violations.length === 0) { + return; + } + + console.error("Found channel-specific references in channel-agnostic sources:"); + for (const violation of violations) { + console.error(`- ${violation}`); + } + console.error( + "Move channel-specific logic to channel adapters or add a justified allowlist entry.", + ); + process.exit(1); +} + +runAsScript(import.meta.url, main); diff --git a/scripts/check-cli-startup-memory.mjs b/scripts/check-cli-startup-memory.mjs new file mode 100644 index 0000000000000000000000000000000000000000..d4e528e000d8483bbbe9ea086127179448eb2294 --- /dev/null +++ b/scripts/check-cli-startup-memory.mjs @@ -0,0 +1,610 @@ +#!/usr/bin/env node +// Measures CLI startup memory with an isolated home and an in-process bench entry. +import { spawnSync as defaultSpawnSync } from "node:child_process"; +import { + closeSync, + constants as fsConstants, + fstatSync, + ftruncateSync, + lstatSync, + mkdirSync, + mkdtempSync, + openSync, + readlinkSync, + realpathSync, + rmSync, + rmdirSync, + statSync, + unlinkSync, + writeFileSync, +} from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { resolveRepoRoot } from "./lib/repo-root.mjs"; +const repoRoot = resolveRepoRoot(import.meta.url); +const tmpDir = process.env.TMPDIR || process.env.TEMP || process.env.TMP || os.tmpdir(); +const MAX_RSS_MARKER = "__OPENCLAW_MAX_RSS_KB__="; +const DEFAULT_COMMAND_TIMEOUT_MS = 60_000; +const STARTUP_MEMORY_SAMPLE_COUNT = 3; +const STARTUP_MEMORY_RSS_TOLERANCE_MB = 1; +const COMMAND_TIMEOUT_MS = readPositiveIntEnv( + "OPENCLAW_STARTUP_MEMORY_TIMEOUT_MS", + DEFAULT_COMMAND_TIMEOUT_MS, +); +let tmpHome = null; +let benchEntryPath = null; +const PASS = "pass"; +const FAIL = "fail"; +const REPORT_PATH_FILE_ERROR = "--json and --summary must refer to files"; +const REPORT_PATH_CHANGED_ERROR = "--json or --summary changed during startup benchmarks"; +const REPORT_PATH_ALIAS_ERROR = "--json and --summary must refer to different files"; +function readPositiveIntEnv(name, fallback, env = process.env) { + const value = readPositiveNumberEnv(name, fallback, env); + if (!Number.isSafeInteger(value)) { + throw new Error(`${name} must be a positive integer`); + } + return value; +} +function readPositiveNumberEnv(name, fallback, env = process.env) { + const raw = env[name]; + if (raw === undefined || raw === "") { + return fallback; + } + const text = raw.trim(); + if (!/^(?:\d+(?:\.\d+)?|\.\d+)$/u.test(text)) { + throw new Error(`${name} must be a positive number`); + } + const value = Number(text); + if (!Number.isFinite(value) || value <= 0) { + throw new Error(`${name} must be a positive number`); + } + return value; +} +function readNonEmptyEnv(name) { + const value = process.env[name]; + return value === undefined || value.length === 0 ? null : value; +} +function readRequiredPathOption(argv, index, flag) { + const value = argv[index + 1]; + if (!value || value.startsWith("-")) { + throw new Error(`${flag} requires a path`); + } + return value; +} +function parseArgs(argv) { + const options = { + jsonPath: + readNonEmptyEnv("OPENCLAW_STARTUP_MEMORY_JSON_PATH") ?? + path.join(repoRoot, ".artifacts", "startup-memory", "startup-memory.json"), + summaryPath: + readNonEmptyEnv("OPENCLAW_STARTUP_MEMORY_SUMMARY_PATH") ?? + path.join(repoRoot, ".artifacts", "startup-memory", "summary.md"), + }; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === undefined) { + break; + } + if (arg === "--json") { + const value = readRequiredPathOption(argv, index, "--json"); + options.jsonPath = path.resolve(value); + index += 1; + continue; + } + if (arg === "--summary") { + const value = readRequiredPathOption(argv, index, "--summary"); + options.summaryPath = path.resolve(value); + index += 1; + continue; + } + if (arg === "--help") { + console.log( + "Usage: node scripts/check-cli-startup-memory.mjs [--json ] [--summary ]", + ); + process.exit(0); + } + throw new Error(`Unknown option: ${arg}`); + } + return options; +} +const sameFsObject = (left, right) => left.dev === right.dev && left.ino === right.ino; +function createReportParents(filePath) { + const parentPath = path.dirname(filePath); + const firstCreated = mkdirSync(parentPath, { recursive: true }); + if (!firstCreated) { + return []; + } + const created = []; + for (let current = parentPath; ; current = path.dirname(current)) { + created.push({ identity: lstatSync(current, { bigint: true }), path: current }); + if (current === firstCreated) { + return created; + } + } +} +function resolveReportLeaf(filePath, unresolvedLinks = new Set()) { + const physicalParent = realpathSync.native(path.dirname(filePath)); + const leafPath = path.join(physicalParent, path.basename(filePath)); + const identity = lstatSync(leafPath, { bigint: true, throwIfNoEntry: false }); + if (!identity?.isSymbolicLink()) { + return leafPath; + } + if (unresolvedLinks.has(leafPath)) { + throw Object.assign(new Error("Symlink cycle while resolving report path"), { code: "ELOOP" }); + } + unresolvedLinks.add(leafPath); + const target = readlinkSync(leafPath); + const lastComponent = target.slice(target.lastIndexOf(path.sep) + 1); + if (lastComponent === "" || lastComponent === "." || lastComponent === "..") { + throw new Error(REPORT_PATH_FILE_ERROR); + } + return resolveReportLeaf( + path.isAbsolute(target) ? target : `${physicalParent}${path.sep}${target}`, + unresolvedLinks, + ); +} +function openReport(requestedPath) { + const resolvedPath = resolveReportLeaf(requestedPath); + let created = false; + let fd; + try { + fd = openSync( + resolvedPath, + fsConstants.O_CREAT | fsConstants.O_EXCL | fsConstants.O_RDWR, + 0o600, + ); + created = true; + } catch (error) { + if (error?.code !== "EEXIST") { + throw error; + } + fd = openSync(resolvedPath, fsConstants.O_RDWR | fsConstants.O_NOFOLLOW); + } + const identity = fstatSync(fd, { bigint: true }); + if (!identity.isFile()) { + closeSync(fd); + throw new Error(REPORT_PATH_FILE_ERROR); + } + return { created, fd, identity, path: resolvedPath, requestedPath }; +} +function assertReportReservation(report) { + if (sameFsObject(report.identity, statSync(report.requestedPath, { bigint: true }))) { + return; + } + throw new Error(REPORT_PATH_CHANGED_ERROR); +} +function cleanupReservation({ reports, parents }) { + for (const report of reports.toReversed()) { + try { + if (!report.created) { + continue; + } + const current = lstatSync(report.path, { bigint: true, throwIfNoEntry: false }); + if (current?.isFile() && sameFsObject(report.identity, current)) { + unlinkSync(report.path); + } + } catch {} + } + const deepestFirst = parents.toSorted( + (left, right) => right.path.split(path.sep).length - left.path.split(path.sep).length, + ); + for (const parent of deepestFirst) { + try { + const current = lstatSync(parent.path, { bigint: true, throwIfNoEntry: false }); + if (current?.isDirectory() && sameFsObject(parent.identity, current)) { + rmdirSync(parent.path); + } + } catch {} + } +} +function closeReports(reports, index = reports.length - 1) { + if (index < 0) { + return; + } + try { + closeSync(reports[index].fd); + } finally { + closeReports(reports, index - 1); + } +} +function reserveReports(options) { + const parents = []; + const reports = []; + try { + for (const filePath of [options.jsonPath, options.summaryPath]) { + parents.push(...createReportParents(filePath)); + } + reports.push(openReport(options.jsonPath)); + reports.push(openReport(options.summaryPath)); + reports.forEach(assertReportReservation); + if (sameFsObject(reports[0].identity, reports[1].identity)) { + throw new Error(REPORT_PATH_ALIAS_ERROR); + } + return { parents, reports }; + } catch (error) { + try { + closeReports(reports); + } catch {} + cleanupReservation({ parents, reports }); + throw error; + } +} +function resolveDefaultLimitsMb(platform = process.platform) { + return { + // Linux CI is the tight startup regression signal. macOS consistently reports + // higher RSS for the same launcher path, so keep it supported without hiding + // Linux help-path regressions. + help: platform === "darwin" ? 300 : 100, + // Plugin discovery is heavier than help, but must stay below the doctor/channel + // runtime graph that an empty metadata-only invocation must not import. + pluginsList: platform === "darwin" ? 500 : 400, + // Node 24 status startup reaches ~430 MB on current Linux runner images; + // retain useful regression headroom without failing on allocator variance. + statusJson: 450, + gatewayStatus: 500, + }; +} +const DEFAULT_LIMITS_MB = resolveDefaultLimitsMb(); +const cases = [ + { + id: "help", + label: "--help", + args: ["openclaw.mjs", "--help"], + limitMb: readPositiveNumberEnv("OPENCLAW_STARTUP_MEMORY_HELP_MB", DEFAULT_LIMITS_MB.help), + }, + { + id: "pluginsList", + label: "plugins list --json", + args: ["openclaw.mjs", "plugins", "list", "--json"], + limitMb: readPositiveNumberEnv( + "OPENCLAW_STARTUP_MEMORY_PLUGINS_LIST_MB", + DEFAULT_LIMITS_MB.pluginsList, + ), + }, + { + id: "statusJson", + label: "status --json", + args: ["openclaw.mjs", "status", "--json"], + limitMb: readPositiveNumberEnv( + "OPENCLAW_STARTUP_MEMORY_STATUS_JSON_MB", + DEFAULT_LIMITS_MB.statusJson, + ), + }, + { + id: "gatewayStatus", + label: "gateway status", + args: ["openclaw.mjs", "gateway", "status"], + limitMb: readPositiveNumberEnv( + "OPENCLAW_STARTUP_MEMORY_GATEWAY_STATUS_MB", + DEFAULT_LIMITS_MB.gatewayStatus, + ), + }, +]; +function formatFixGuidance(testCase, details) { + const command = `node ${testCase.args.join(" ")}`; + const guidance = [ + "[startup-memory] Fix guidance", + `Case: ${testCase.label}`, + `Command: ${command}`, + "Next steps:", + `1. Run \`${command}\` locally on the built tree.`, + "2. If this is an RSS overage, compare the startup import graph against the last passing commit and look for newly eager imports, bootstrap side effects, or plugin loading on the command path.", + "3. If this is a non-zero exit, inspect the first transitive import/config error in stderr and fix that root cause before re-checking memory.", + "LLM prompt:", + `"OpenClaw startup-memory CI failed for '${testCase.label}'. Analyze this failure, identify the first runtime/import side effect that makes startup heavier or broken, and propose the smallest safe patch. Failure output:\n${details}"`, + ]; + return `${guidance.join("\n")}\n`; +} +function formatFailure(testCase, message, details = "") { + const trimmedDetails = details.trim(); + const sections = [message]; + if (trimmedDetails) { + sections.push(trimmedDetails); + } + sections.push(formatFixGuidance(testCase, trimmedDetails || message)); + return sections.join("\n\n"); +} +function failResult(report, testCase, error, details = "") { + return { + ...report, + status: FAIL, + error, + failureMessage: formatFailure(testCase, error, details), + }; +} +function parseMaxRssMb(stderr) { + const matches = [...stderr.matchAll(new RegExp(`^${MAX_RSS_MARKER}(\\d+)\\s*$`, "gm"))]; + const lastMatch = matches.at(-1); + if (!lastMatch) { + return null; + } + const maxRssKb = Number(lastMatch[1]); + return Number.isFinite(maxRssKb) && maxRssKb > 0 ? maxRssKb / 1024 : null; +} +function formatMb(value) { + return typeof value === "number" && Number.isFinite(value) ? `${value.toFixed(1)} MB` : "n/a"; +} +function formatCaseCommand(testCase) { + return `node ${testCase.args.join(" ")}`; +} +function nodeImportSpecifierForPath(filePath) { + return pathToFileURL(filePath).href; +} +function buildBenchEnv(homeDir = tmpHome) { + if (!homeDir) { + throw new Error("temporary home is not initialized"); + } + const env = { + HOME: homeDir, + USERPROFILE: homeDir, + XDG_CONFIG_HOME: path.join(homeDir, ".config"), + XDG_DATA_HOME: path.join(homeDir, ".local", "share"), + XDG_CACHE_HOME: path.join(homeDir, ".cache"), + PATH: process.env.PATH ?? "", + TMPDIR: tmpDir, + TEMP: tmpDir, + TMP: tmpDir, + LANG: process.env.LANG ?? "C.UTF-8", + TERM: process.env.TERM ?? "dumb", + }; + if (process.env.LC_ALL) { + env.LC_ALL = process.env.LC_ALL; + } + if (process.env.CI) { + env.CI = process.env.CI; + } + if (process.env.NODE_DISABLE_COMPILE_CACHE) { + env.NODE_DISABLE_COMPILE_CACHE = process.env.NODE_DISABLE_COMPILE_CACHE; + } else { + // Keep the regression check focused on app/runtime startup, not Node's + // one-shot compile cache overhead, which varies across runner builds. + env.NODE_DISABLE_COMPILE_CACHE = "1"; + } + // Keep the benchmark on a single process so RSS reflects the actual command + // path rather than the warning-suppression respawn wrapper. + env.OPENCLAW_NO_RESPAWN = "1"; + return env; +} +function runCaseSample(testCase, sampleIndex, params = {}) { + if (!benchEntryPath) { + throw new Error("bench entry path is not initialized"); + } + if (!tmpHome) { + throw new Error("temporary home is not initialized"); + } + const sampleHome = path.join(tmpHome, "homes", `${testCase.id}-${sampleIndex + 1}`); + mkdirSync(sampleHome, { recursive: true }); + const env = buildBenchEnv(sampleHome); + const spawn = params.spawnSync ?? defaultSpawnSync; + const timeoutMs = params.timeoutMs ?? COMMAND_TIMEOUT_MS; + const result = spawn(process.execPath, [benchEntryPath, ...testCase.args.slice(1)], { + cwd: repoRoot, + env, + encoding: "utf8", + maxBuffer: 20 * 1024 * 1024, + timeout: timeoutMs, + killSignal: "SIGKILL", + }); + const stderr = String(result.stderr ?? ""); + const stdout = String(result.stdout ?? ""); + const maxRssMb = parseMaxRssMb(stderr); + const matrixBootstrapWarning = /matrix: crypto runtime bootstrap failed/i.test(stderr); + const report = { + id: testCase.id, + label: testCase.label, + command: formatCaseCommand(testCase), + limitMb: testCase.limitMb, + rssToleranceMb: STARTUP_MEMORY_RSS_TOLERANCE_MB, + effectiveLimitMb: testCase.limitMb + STARTUP_MEMORY_RSS_TOLERANCE_MB, + maxRssMb, + status: PASS, + exitCode: result.status, + signal: result.signal ?? null, + error: null, + }; + if (result.error) { + const timedOut = result.error.code === "ETIMEDOUT"; + const error = timedOut + ? `${testCase.label} timed out after ${timeoutMs}ms` + : `${testCase.label} failed to start: ${result.error.message}`; + return failResult(report, testCase, error, stderr.trim() || stdout); + } + if (result.status !== 0) { + const exitDetail = result.status ?? result.signal ?? "unknown"; + const error = `${testCase.label} exited with ${String(exitDetail)}`; + return failResult(report, testCase, error, stderr.trim() || stdout); + } + if (maxRssMb == null) { + const error = `${testCase.label} did not report max RSS`; + return failResult(report, testCase, error, stderr); + } + if (matrixBootstrapWarning) { + const error = `${testCase.label} triggered Matrix crypto bootstrap during startup`; + return failResult(report, testCase, error); + } + return report; +} +function median(values) { + const sorted = values.toSorted((left, right) => left - right); + const middle = sorted.at(Math.floor(sorted.length / 2)); + if (middle === undefined) { + throw new Error("cannot compute median of an empty sample set"); + } + return middle; +} +function formatRssSamples(samplesMb) { + return samplesMb.map((value) => value.toFixed(1)).join(", "); +} +function runCase(testCase, params = {}) { + let report = runCaseSample(testCase, 0, params); + if (report.status !== "pass" || report.maxRssMb == null) { + return report; + } + const samples = [report.maxRssMb]; + // Shared CI runners occasionally produce a single allocator/RSS spike. Independent + // homes plus a median keep that outlier from masking regressions; two high samples fail. + for (let sampleIndex = 1; sampleIndex < STARTUP_MEMORY_SAMPLE_COUNT; sampleIndex += 1) { + const sample = runCaseSample(testCase, sampleIndex, params); + if (sample.status !== "pass" || sample.maxRssMb == null) { + return sample; + } + samples.push(sample.maxRssMb); + report = sample; + } + const maxRssMb = median(samples); + const result = { ...report, maxRssMb, rssSamplesMb: samples }; + if (maxRssMb > result.effectiveLimitMb) { + const error = `${testCase.label} median max RSS ${maxRssMb.toFixed(1)} MB exceeded effective ceiling ${result.effectiveLimitMb} MB (base limit ${result.limitMb} MB; RSS tolerance ${result.rssToleranceMb} MB; samples: ${formatRssSamples(samples)} MB)`; + return failResult(result, testCase, error); + } + console.log( + `[startup-memory] ${testCase.label}: ${maxRssMb.toFixed(1)} MB median max RSS ` + + `(base limit ${result.limitMb} MB; RSS tolerance ${result.rssToleranceMb} MB; effective ceiling ${result.effectiveLimitMb} MB; samples ${formatRssSamples(samples)} MB)`, + ); + return result; +} +function writeReport(reports, results) { + const failed = results.filter((result) => result.status !== "pass"); + const report = { + generatedAt: new Date().toISOString(), + platform: process.platform, + repoRoot, + status: failed.length === 0 ? "pass" : "fail", + results: results.map(({ failureMessage: _failureMessage, ...result }) => result), + }; + const lines = [ + "# OpenClaw Startup Memory", + "", + `Generated: ${report.generatedAt}`, + "", + `Status: ${report.status}`, + "", + ...results.map((result) => { + const samples = result.rssSamplesMb + ? `; samples: ${result.rssSamplesMb.map(formatMb).join(", ")}` + : ""; + return `- ${result.label}: ${result.status} median max RSS ${formatMb(result.maxRssMb)} (base limit ${formatMb(result.limitMb)}; RSS tolerance ${formatMb(result.rssToleranceMb)}; effective ceiling ${formatMb(result.effectiveLimitMb)}${samples})`; + }), + "", + ]; + if (failed.length > 0) { + lines.push( + "## Failures", + "", + ...failed.map((result) => `- ${result.label}: ${result.error ?? "unknown failure"}`), + "", + ); + } + for (const [reservation, contents] of [ + [reports[0], `${JSON.stringify(report, null, 2)}\n`], + [reports[1], `${lines.join("\n")}\n`], + ]) { + assertReportReservation(reservation); + ftruncateSync(reservation.fd, 0); + writeFileSync(reservation.fd, contents, "utf8"); + assertReportReservation(reservation); + } + reports.forEach(assertReportReservation); +} +function runStartupMemoryCheck(argv = process.argv.slice(2), params = {}) { + const platform = params.platform ?? process.platform; + if (platform !== "linux" && platform !== "darwin") { + console.log(`[startup-memory] Skipping on unsupported platform: ${platform}`); + return { skipped: true, results: [] }; + } + const options = parseArgs(argv); + const reservation = reserveReports(options); + const results = []; + let operationError; + let published = false; + let outcome; + try { + tmpHome = mkdtempSync(path.join(os.tmpdir(), "openclaw-startup-memory-")); + benchEntryPath = path.join(tmpHome, "bench-entry.mjs"); + // Run the real launcher in-process so peak RSS is self-reported at exit + // without --import/--require flags: the entry declines its dist ESM resolve + // fast path when preload hooks may be registered, so an injected hook would + // measure a slower non-default resolution configuration instead of what a + // plain `node openclaw.mjs ...` invocation pays. + const launcherPath = path.join(repoRoot, "openclaw.mjs"); + writeFileSync( + benchEntryPath, + [ + "process.on('exit', () => {", + " const usage = typeof process.resourceUsage === 'function' ? process.resourceUsage() : null;", + ` if (usage && typeof usage.maxRSS === 'number') console.error('${MAX_RSS_MARKER}' + String(usage.maxRSS));`, + "});", + `const launcherPath = ${JSON.stringify(launcherPath)};`, + "// The launcher and entry expect argv[1] to be the launcher path itself.", + "process.argv[1] = launcherPath;", + `await import(${JSON.stringify(nodeImportSpecifierForPath(launcherPath))});`, + "", + ].join("\n"), + "utf8", + ); + for (const testCase of cases) { + results.push(runCase(testCase, params)); + } + writeReport(reservation.reports, results); + published = true; + const failure = results.find((result) => result.status !== "pass"); + if (failure?.failureMessage) { + throw new Error(failure.failureMessage); + } + outcome = { skipped: false, results }; + } catch (error) { + operationError = error; + } + let cleanupError; + try { + closeReports(reservation.reports); + } catch (error) { + cleanupError = error; + } + if (!published) { + cleanupReservation(reservation); + } + if (tmpHome) { + try { + rmSync(tmpHome, { recursive: true, force: true }); + } catch (error) { + cleanupError ??= error; + } finally { + tmpHome = null; + benchEntryPath = null; + } + } + if (operationError !== undefined) { + throw operationError instanceof Error + ? operationError + : new Error("startup memory check failed", { cause: operationError }); + } + if (cleanupError !== undefined) { + throw cleanupError instanceof Error + ? cleanupError + : new Error("startup memory cleanup failed", { cause: cleanupError }); + } + return outcome; +} +/** + * Test-only access to pure startup memory helper functions. + */ +export const testing = { + cases, + parseArgs, + readPositiveIntEnv, + readPositiveNumberEnv, + repoRoot, + resolveDefaultLimitsMb, + runStartupMemoryCheck, + sampleCount: STARTUP_MEMORY_SAMPLE_COUNT, +}; +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + try { + runStartupMemoryCheck(); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + } +} diff --git a/scripts/check-composite-action-input-interpolation.py b/scripts/check-composite-action-input-interpolation.py new file mode 100644 index 0000000000000000000000000000000000000000..18c52501b85ed2cc0621f3fed1e56f7e9d1f3c43 --- /dev/null +++ b/scripts/check-composite-action-input-interpolation.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import pathlib +import re +import sys + + +INPUT_INTERPOLATION_RE = re.compile(r"\$\{\{\s*inputs\.") +RUN_LINE_RE = re.compile(r"^(\s*)run:\s*(.*)$") +USING_COMPOSITE_RE = re.compile(r"^\s*using:\s*composite\s*$", re.MULTILINE) + + +def indentation(line: str) -> int: + return len(line) - len(line.lstrip(" ")) + + +def scan_file(path: pathlib.Path) -> list[tuple[int, str]]: + text = path.read_text(encoding="utf-8") + if not USING_COMPOSITE_RE.search(text): + return [] + + lines = text.splitlines() + violations: list[tuple[int, str]] = [] + line_count = len(lines) + index = 0 + + while index < line_count: + line = lines[index] + match = RUN_LINE_RE.match(line) + if not match: + index += 1 + continue + + run_indent = len(match.group(1)) + run_value = match.group(2).strip() + line_no = index + 1 + + if run_value and run_value[0] not in ("|", ">"): + if INPUT_INTERPOLATION_RE.search(run_value): + violations.append((line_no, line.strip())) + index += 1 + continue + + index += 1 + while index < line_count: + script_line = lines[index] + if script_line.strip() == "": + index += 1 + continue + if indentation(script_line) <= run_indent: + break + if INPUT_INTERPOLATION_RE.search(script_line): + violations.append((index + 1, script_line.strip())) + index += 1 + + return violations + + +def main() -> int: + root = pathlib.Path(".github/actions") + files = sorted(root.rglob("action.y*ml")) + all_violations: list[tuple[pathlib.Path, int, str]] = [] + + for file_path in files: + for line_no, line in scan_file(file_path): + all_violations.append((file_path, line_no, line)) + + if all_violations: + print("Disallowed direct inputs interpolation in composite run blocks:") + for file_path, line_no, line in all_violations: + print(f"- {file_path}:{line_no}: {line}") + print("Use env: and reference shell variables instead.") + return 1 + + print("No direct inputs interpolation found in composite run blocks.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/check-dependency-pins.mts b/scripts/check-dependency-pins.mts new file mode 100644 index 0000000000000000000000000000000000000000..e3fe984549023262d864d4852fba266992f7cd88 --- /dev/null +++ b/scripts/check-dependency-pins.mts @@ -0,0 +1,186 @@ +#!/usr/bin/env node + +// Audits patched dependency pins for exact versions and drift. +import { execFileSync } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { asRecord } from "@openclaw/normalization-core/record-coerce"; +import YAML from "yaml"; +import { classifyDependencySpec } from "./lib/dependency-spec-policy.mts"; + +const PACKAGE_DEPENDENCY_SECTIONS = ["dependencies", "devDependencies", "optionalDependencies"]; +const WORKSPACE_DEPENDENCY_SECTIONS = ["overrides"]; +const DEFAULT_GIT_TIMEOUT_MS = 60_000; + +type DependencyPinViolation = { + file: string; + section: string; + name: string; + spec: unknown; +}; + +function runGit(cwd: string, args: string[], timeoutMs = DEFAULT_GIT_TIMEOUT_MS): string { + try { + return execFileSync("git", args, { + cwd, + encoding: "utf8", + timeout: timeoutMs, + // A synchronous child that ignores SIGTERM otherwise keeps its parent blocked. + killSignal: "SIGKILL", + }); + } catch (error) { + if (error instanceof Error && "code" in error && error.code === "ETIMEDOUT") { + throw new Error( + `dependency pin guard: git ${args.join(" ")} timed out after ${timeoutMs}ms.`, + { cause: error }, + ); + } + throw error; + } +} + +function listTrackedPackageJsonFiles(cwd: string, timeoutMs = DEFAULT_GIT_TIMEOUT_MS): string[] { + return runGit(cwd, ["ls-files", "-z", "--", "*package.json"], timeoutMs) + .split("\0") + .filter(Boolean) + .toSorted((left, right) => left.localeCompare(right)); +} + +function readJson(filePath: string): Record { + return asRecord(JSON.parse(fs.readFileSync(filePath, "utf8")) as unknown); +} + +function readTrackedJson( + cwd: string, + relativePath: string, + timeoutMs = DEFAULT_GIT_TIMEOUT_MS, +): Record { + const filePath = path.join(cwd, relativePath); + if (fs.existsSync(filePath)) { + return readJson(filePath); + } + return asRecord(JSON.parse(runGit(cwd, ["show", `:${relativePath}`], timeoutMs)) as unknown); +} + +function collectPackageJsonViolations( + cwd: string, + timeoutMs = DEFAULT_GIT_TIMEOUT_MS, +): DependencyPinViolation[] { + const violations: DependencyPinViolation[] = []; + for (const relativePath of listTrackedPackageJsonFiles(cwd, timeoutMs)) { + const packageJson = readTrackedJson(cwd, relativePath, timeoutMs); + for (const section of PACKAGE_DEPENDENCY_SECTIONS) { + for (const [name, spec] of Object.entries(asRecord(packageJson[section]))) { + if (!classifyDependencySpec(spec).allowedPinned) { + violations.push({ file: relativePath, section, name, spec }); + } + } + } + } + return violations; +} + +function collectDependencyMapViolations( + file: string, + section: string, + dependencyMap: unknown, + violations: DependencyPinViolation[], +): void { + for (const [name, spec] of Object.entries(asRecord(dependencyMap))) { + if (section === "overrides" && spec === "-") { + continue; + } + if (!classifyDependencySpec(spec).allowedPinned) { + violations.push({ file, section, name, spec }); + } + } +} + +function collectWorkspaceViolations(cwd: string): DependencyPinViolation[] { + const file = "pnpm-workspace.yaml"; + const workspacePath = path.join(cwd, file); + if (!fs.existsSync(workspacePath)) { + return []; + } + const workspace = asRecord(YAML.parse(fs.readFileSync(workspacePath, "utf8")) as unknown); + const violations: DependencyPinViolation[] = []; + for (const section of WORKSPACE_DEPENDENCY_SECTIONS) { + collectDependencyMapViolations(file, section, workspace?.[section], violations); + } + for (const [packageName, extension] of Object.entries(asRecord(workspace.packageExtensions))) { + collectDependencyMapViolations( + file, + `packageExtensions.${packageName}.dependencies`, + asRecord(extension).dependencies, + violations, + ); + } + return violations; +} + +/** + * Collects dependency pin violations for the current workspace. + */ +export function collectDependencyPinViolations( + cwd = process.cwd(), + { gitTimeoutMs = DEFAULT_GIT_TIMEOUT_MS } = {}, +): DependencyPinViolation[] { + return [...collectPackageJsonViolations(cwd, gitTimeoutMs), ...collectWorkspaceViolations(cwd)]; +} + +/** + * Builds the full dependency pin audit payload. + */ +function collectDependencyPinAudit(cwd = process.cwd()) { + const packageJsonFiles = listTrackedPackageJsonFiles(cwd); + let packageSpecCount = 0; + for (const relativePath of packageJsonFiles) { + const packageJson = readTrackedJson(cwd, relativePath); + for (const section of PACKAGE_DEPENDENCY_SECTIONS) { + packageSpecCount += Object.keys(asRecord(packageJson[section])).length; + } + } + const workspaceViolations = collectWorkspaceViolations(cwd); + const violations = [...collectPackageJsonViolations(cwd), ...workspaceViolations]; + return { + packageManifestCount: packageJsonFiles.length, + packageSpecCount, + violations, + }; +} + +/** + * Runs the dependency pin check. + */ +export async function main() { + const audit = collectDependencyPinAudit(); + const { violations } = audit; + if (violations.length === 0) { + process.stdout.write( + `PASS direct dependency pin guard: checked ${audit.packageSpecCount} directly declared ` + + `dependency specs across ${audit.packageManifestCount} tracked package manifests; ` + + "0 violations.\n", + ); + return; + } + + console.error( + `FAIL direct dependency pin guard: ${violations.length} unpinned directly declared ` + + "dependency specs found. Direct dependency specs must be pinned exactly outside peer " + + "dependency contracts:", + ); + for (const violation of violations) { + console.error( + `- ${violation.file}:${violation.section}:${violation.name} -> ${JSON.stringify(violation.spec)}`, + ); + } + process.exitCode = 1; +} + +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + main().catch((error: unknown) => { + console.error(error); + process.exit(1); + }); +} diff --git a/scripts/check-deprecated-jsdoc.mts b/scripts/check-deprecated-jsdoc.mts new file mode 100644 index 0000000000000000000000000000000000000000..4b2d3c7bb3d51357b6328100ba1c81aa8338155c --- /dev/null +++ b/scripts/check-deprecated-jsdoc.mts @@ -0,0 +1,137 @@ +#!/usr/bin/env node +// Checks deprecated JSDoc blocks for required migration details. +import fs from "node:fs"; +import { createRequire } from "node:module"; +import path from "node:path"; +import type * as Ts from "typescript"; +import { resolveRepoRoot } from "./lib/repo-root.mjs"; + +const require = createRequire(import.meta.url); +const ts: typeof import("typescript") = require("typescript"); + +const repoRoot = resolveRepoRoot(import.meta.url); +const SCAN_ROOTS = ["src", "extensions", "packages"]; +const SOURCE_FILE_RE = /\.(?:ts|tsx)$/; +const SKIP_PATH_RE = + /(?:^|\/)(?:node_modules|dist|build|protocol-generated)(?:\/|$)|(?:\.test|\.spec|\.e2e|\.generated)\.tsx?$/; +const DEPRECATED_SURFACE_COMMENT_RE = + /^(?:back-compat alias|backward-compatible alias(?:es)?|deprecated alias|legacy alias|legacy field|legacy:\s|kept for compatibility with existing imports|keep the legacy helper name exported)\b/i; + +function walk(dir: string, files: string[] = []) { + if (!fs.existsSync(dir)) { + return files; + } + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const filePath = path.join(dir, entry.name); + const relativePath = path.relative(repoRoot, filePath).replaceAll(path.sep, "/"); + if (SKIP_PATH_RE.test(relativePath)) { + continue; + } + if (entry.isDirectory()) { + walk(filePath, files); + } else if (SOURCE_FILE_RE.test(entry.name)) { + files.push(filePath); + } + } + return files; +} + +function leadingCommentText(sourceFile: Ts.SourceFile, node: Ts.Node) { + return (ts.getLeadingCommentRanges(sourceFile.text, node.pos) ?? []) + .map((range) => sourceFile.text.slice(range.pos, range.end)) + .join("\n"); +} + +function normalizeCommentText(comment: string) { + return comment + .replace(/\/\*\*?/g, "") + .replace(/\*\//g, "") + .split("\n") + .map((line) => line.replace(/^\s*(?:\*|\/\/)\s?/, "").trim()) + .filter(Boolean) + .join(" "); +} + +function lineOf(sourceFile: Ts.SourceFile, node: Ts.Node) { + return sourceFile.getLineAndCharacterOfPosition(node.getStart(sourceFile)).line + 1; +} + +function isExported(node: Ts.Node) { + const modifiers = ts.canHaveModifiers(node) ? ts.getModifiers(node) : undefined; + return ( + modifiers?.some((modifier) => modifier.kind === ts.SyntaxKind.ExportKeyword) || + node.parent?.kind === ts.SyntaxKind.SourceFile + ); +} + +type InspectedNode = Ts.Declaration | Ts.VariableStatement; +type Violation = { filePath: string; line: number; name: string }; + +function symbolName(node: InspectedNode) { + const declaration = ts.isVariableStatement(node) ? node.declarationList.declarations[0] : node; + return ts.getNameOfDeclaration(declaration)?.getText() ?? ""; +} + +function shouldInspectNode(node: Ts.Node): node is InspectedNode { + if ( + ts.isFunctionDeclaration(node) || + ts.isClassDeclaration(node) || + ts.isInterfaceDeclaration(node) || + ts.isTypeAliasDeclaration(node) || + ts.isEnumDeclaration(node) || + ts.isVariableStatement(node) + ) { + return isExported(node); + } + return ( + ts.isPropertySignature(node) || + ts.isMethodSignature(node) || + ts.isPropertyDeclaration(node) || + ts.isEnumMember(node) + ); +} + +function collectViolations(filePath: string) { + const sourceText = fs.readFileSync(filePath, "utf8"); + const sourceFile = ts.createSourceFile(filePath, sourceText, ts.ScriptTarget.Latest, true); + const violations: Violation[] = []; + + function visit(node: Ts.Node) { + if (shouldInspectNode(node)) { + const comment = leadingCommentText(sourceFile, node); + const normalizedComment = normalizeCommentText(comment); + if ( + normalizedComment && + DEPRECATED_SURFACE_COMMENT_RE.test(normalizedComment) && + !/@deprecated\b/.test(comment) + ) { + violations.push({ + line: lineOf(sourceFile, node), + name: symbolName(node), + filePath: path.relative(repoRoot, filePath).replaceAll(path.sep, "/"), + }); + } + } + ts.forEachChild(node, visit); + } + + visit(sourceFile); + return violations; +} + +const violations = SCAN_ROOTS.flatMap((root) => + walk(path.join(repoRoot, root)).flatMap(collectViolations), +); + +if (violations.length > 0) { + console.error("Deprecated JSDoc guard failed:"); + for (const violation of violations) { + console.error(`- ${violation.filePath}:${violation.line} ${violation.name}`); + } + console.error( + "Add an @deprecated JSDoc tag or reword the comment if the symbol is not deprecated.", + ); + process.exitCode = 1; +} else { + console.log("deprecated JSDoc guard passed"); +} diff --git a/scripts/check-docker-e2e-boundaries.mts b/scripts/check-docker-e2e-boundaries.mts new file mode 100644 index 0000000000000000000000000000000000000000..58ee5bc42cbb4639b73493e57246aaf4d1285493 --- /dev/null +++ b/scripts/check-docker-e2e-boundaries.mts @@ -0,0 +1,182 @@ +#!/usr/bin/env node +// Cheap guard for Docker E2E test boundaries. +// Docker E2E must test packaged npm tarballs and package-installed images, not +// the source checkout copied or mounted as the app under test. +import fs from "node:fs"; +import path from "node:path"; +import packageJson from "../package.json" with { type: "json" }; +import { laneResources, lanesNeedOpenClawPackage, laneWeight } from "./lib/docker-e2e-plan.mts"; +import { + allReleasePathLanes, + mainLanes, + publicInstallerLanes, + tailLanes, +} from "./lib/docker-e2e-scenarios.mts"; +import { resolveRepoRoot } from "./lib/repo-root.mjs"; +const ROOT_DIR = resolveRepoRoot(import.meta.url); +const errors: string[] = []; +const packageScripts = new Set(Object.keys(packageJson.scripts ?? {})); +// These lanes prove package-installed surfaces against live auth, so they +// intentionally need both live credentials and a package-backed image. +const livePackageBackedLanes = new Set([ + "install-e2e-anthropic", + "install-e2e-openai", + "live-anthropic-cache", + "live-codex-npm-plugin", + "live-mcp-code-mode-gateway", + "live-plugin-tool", + "npm-telegram-live", + "openai-chat-tools", + "openwebui", +]); +// These lanes intentionally build a focused source-checkout image instead of +// consuming the shared package E2E images. +const sourceCheckoutImageLanes = new Set([ + "docker-selected-plugins", + "plugin-binding-command-escape", +]); + +function readText(relativePath: string) { + return fs.readFileSync(path.join(ROOT_DIR, relativePath), "utf8"); +} + +function walk(dir: string, out: string[] = []) { + for (const entry of fs.readdirSync(path.join(ROOT_DIR, dir), { withFileTypes: true })) { + const relativePath = path.join(dir, entry.name); + if (entry.isDirectory()) { + walk(relativePath, out); + } else { + out.push(relativePath); + } + } + return out; +} + +function findRelativeModuleSpecifiers(text: string) { + const specifiers = new Set(); + for (const pattern of [ + /(?:\bfrom\s*|\bimport\s*\()\s*["']([^"']+)["']/gu, + /\bimport\s*["']([^"']+)["']/gu, + ]) { + for (const match of text.matchAll(pattern)) { + const specifier = match[1]; + if (specifier?.startsWith(".")) { + specifiers.add(specifier); + } + } + } + return [...specifiers]; +} + +function isPathWithin(parent: string, candidate: string) { + const relative = path.relative(parent, candidate); + return ( + relative === "" || + (!path.isAbsolute(relative) && !relative.startsWith(`..${path.sep}`) && relative !== "..") + ); +} + +for (const relativePath of walk("scripts/e2e")) { + if (!/\.(?:sh|ts|mts|mjs|js)$/u.test(relativePath)) { + continue; + } + const text = readText(relativePath); + const sourceImport = findRelativeModuleSpecifiers(text).find((specifier) => { + const resolved = path.resolve(ROOT_DIR, path.dirname(relativePath), specifier); + return isPathWithin(path.join(ROOT_DIR, "src"), resolved); + }); + if (sourceImport) { + errors.push( + `${relativePath}: Docker E2E harness must import package exports, not ${sourceImport}`, + ); + } + if (/-v\s+["']?\$ROOT_DIR:\/app(?::|["'\s]|$)/u.test(text)) { + errors.push(`${relativePath}: do not mount the repo root as /app in Docker E2E`); + } +} + +const dockerfile = readText("scripts/e2e/Dockerfile"); +if (/^\s*(?:COPY|ADD)\s+\.\s+\/app(?:\s|$)/imu.test(dockerfile)) { + errors.push("scripts/e2e/Dockerfile: do not copy the source checkout into /app"); +} + +function validateUniqueLanes(label: string, lanes: readonly (typeof mainLanes)[number][]) { + const seen = new Set(); + for (const lane of lanes) { + if (seen.has(lane.name)) { + errors.push(`${label}: duplicate Docker E2E lane '${lane.name}'`); + } + seen.add(lane.name); + } +} + +function validateLane(label: string, lane: (typeof mainLanes)[number]) { + const resources = laneResources(lane); + const sourceCheckoutImageLane = sourceCheckoutImageLanes.has(lane.name); + const needsPackage = lanesNeedOpenClawPackage([lane]); + const localImageBuild = sourceCheckoutImageLane || (needsPackage && !lane.e2eImageKind); + if (!lane.name || typeof lane.name !== "string") { + errors.push(`${label}: Docker E2E lane is missing a string name`); + } + if (!lane.command || typeof lane.command !== "string") { + errors.push(`${label}: Docker E2E lane '${lane.name}' is missing a string command`); + return; + } + if (lane.e2eImageKind && lane.e2eImageKind !== "bare" && lane.e2eImageKind !== "functional") { + const invalidImageKind = + typeof lane.e2eImageKind === "string" + ? lane.e2eImageKind + : (JSON.stringify(lane.e2eImageKind) ?? ""); + errors.push( + `${label}: Docker E2E lane '${lane.name}' has invalid image kind '${invalidImageKind}'`, + ); + } + if (lane.live && needsPackage && !livePackageBackedLanes.has(lane.name)) { + errors.push(`${label}: live Docker E2E lane '${lane.name}' must not require a package image`); + } + if (!lane.live && !needsPackage && !sourceCheckoutImageLane) { + errors.push( + `${label}: package Docker E2E lane '${lane.name}' must request package preparation`, + ); + } + if (localImageBuild && !/\bOPENCLAW_SKIP_DOCKER_BUILD=0\b/u.test(lane.command)) { + errors.push( + `${label}: locally built Docker E2E lane '${lane.name}' must force a local image build`, + ); + } + if (laneWeight(lane) < 1) { + errors.push(`${label}: Docker E2E lane '${lane.name}' must have positive weight`); + } + if (!resources.includes("docker")) { + errors.push(`${label}: Docker E2E lane '${lane.name}' must include the docker resource`); + } + + for (const match of lane.command.matchAll(/\bpnpm\s+([^\s]+)/gu)) { + const script = match[1]; + if (script && !packageScripts.has(script)) { + errors.push( + `${label}: Docker E2E lane '${lane.name}' references missing package script '${script}'`, + ); + } + } +} + +const releasePathLanes = allReleasePathLanes({ includeOpenWebUI: true }); +for (const [label, lanes] of [ + ["release-path", releasePathLanes], + ["public-installer", publicInstallerLanes], + ["main", mainLanes], + ["tail", tailLanes], +] as const) { + validateUniqueLanes(label, lanes); + for (const lane of lanes) { + validateLane(label, lane); + } +} + +if (errors.length > 0) { + console.error(errors.join("\n")); + process.exit(1); +} + +console.log("Docker E2E package boundary/catalog guard passed."); diff --git a/scripts/check-docs-config-examples.mjs b/scripts/check-docs-config-examples.mjs new file mode 100644 index 0000000000000000000000000000000000000000..1deec333f25c9eb921e2ee3c209474c21850f92f --- /dev/null +++ b/scripts/check-docs-config-examples.mjs @@ -0,0 +1,7 @@ +#!/usr/bin/env node +import { runTsxCliShim } from "./lib/tsx-cli-shim.mjs"; + +await runTsxCliShim(import.meta.url, { + implementation: "./check-docs-config-examples.mts", + failureTool: "check-docs-config-examples", +}); diff --git a/scripts/check-docs-mdx.mts b/scripts/check-docs-mdx.mts new file mode 100644 index 0000000000000000000000000000000000000000..25e61db073dabecab53a9c06672a8b9a27c695cb --- /dev/null +++ b/scripts/check-docs-mdx.mts @@ -0,0 +1,477 @@ +#!/usr/bin/env node + +// Validates docs MDX files for syntax and repository-specific conventions. + +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { compile } from "@mdx-js/mdx"; +import { requireOptionArgument } from "./lib/arg-utils.runtime.mjs"; +import { + checkMintlifyAccordionIndentation, + MINTLIFY_ACCORDION_INDENT_MESSAGE, +} from "./lib/mintlify-accordion.mjs"; + +type DocsCheckError = { + type: string; + file: string; + message: string; + line?: number; + column?: number; +}; + +function validationCache(cacheFile: string) { + const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); + const fingerprint = createHash("sha256").update( + JSON.stringify([process.version, process.platform, process.arch, process.execArgv]), + ); + // The publish workflow installs with npm ci before invoking this opt-in cache. + // Include the installed lock as well as the requested lock; never infer a + // successful check from a source commit or translation's source_hash. + for (const input of [ + fileURLToPath(import.meta.url), + fileURLToPath(new URL("./lib/arg-utils.runtime.mjs", import.meta.url)), + fileURLToPath(new URL("./lib/mintlify-accordion.mjs", import.meta.url)), + path.join(root, "package.json"), + path.join(root, "package-lock.json"), + path.join(root, "node_modules", ".package-lock.json"), + ]) { + fingerprint.update(fs.readFileSync(input)).update("\0"); + } + const key = fingerprint.digest("hex"); + let previous = new Map(); + try { + const saved = JSON.parse(fs.readFileSync(cacheFile, "utf8")); + if ( + saved.key === key && + saved.files && + typeof saved.files === "object" && + !Array.isArray(saved.files) + ) { + const entries: [string, string][] = []; + for (const [file, digest] of Object.entries(saved.files)) { + if (typeof digest !== "string" || !/^[a-f0-9]{64}$/.test(digest)) { + throw new Error("Invalid validation cache entry"); + } + entries.push([file, digest]); + } + previous = new Map(entries); + } + } catch { + // Missing, obsolete, or corrupt disposable build artifacts are cold checks. + } + const checked = new Map(); + return { + root, + previous, + checked, + save() { + fs.mkdirSync(path.dirname(path.resolve(cacheFile)), { recursive: true }); + const temporary = `${cacheFile}.${process.pid}.tmp`; + fs.writeFileSync( + temporary, + `${JSON.stringify({ key, files: Object.fromEntries(checked) })}\n`, + ); + fs.renameSync(temporary, cacheFile); + }, + }; +} + +const MINTLIFY_LANGUAGE_CODES = new Set([ + "en", + "cn", + "zh", + "zh-Hans", + "zh-Hant", + "es", + "fr", + "fr-CA", + "fr-ca", + "ja", + "jp", + "ja-jp", + "pt", + "pt-BR", + "de", + "ko", + "it", + "ru", + "ro", + "cs", + "id", + "ar", + "tr", + "hi", + "sv", + "no", + "lv", + "nl", + "uk", + "vi", + "pl", + "uz", + "he", + "ca", + "fi", + "hu", +]); + +const POISON_TEXT_PATTERNS = [ + { + pattern: /\banalysis\s+to=functions\./iu, + message: "Leaked tool-call channel marker.", + }, + { + pattern: /\b(?:commentary|final)\s+to=functions\./iu, + message: "Leaked tool-call channel marker.", + }, + { + pattern: /\bfunctions\.(?:read|write|exec|search|run)\b/iu, + message: "Leaked internal tool name.", + }, + { + pattern: /\b[A-Za-z_\u3400-\u9fff][\w\u3400-\u9fff-]*_input=\{/u, + message: "Leaked tool-call input payload.", + }, + { + pattern: /<\/?openclaw_docs_i18n_input>/iu, + message: "Leaked docs i18n prompt wrapper.", + }, + { + pattern: /\/home\/runner\/work\//u, + message: "Leaked GitHub Actions workspace path.", + }, + { + pattern: /彩神马争霸/u, + message: "Known spam/gambling text from a poisoned translation.", + }, +]; + +function parsePositiveIntegerArg(raw: string | undefined, label: string): number { + const text = raw?.trim() ?? ""; + if (!/^\d+$/u.test(text)) { + throw new Error(`${label} must be a positive integer`); + } + const value = Number(text); + if (!Number.isSafeInteger(value) || value < 1) { + throw new Error(`${label} must be a positive integer`); + } + return value; +} + +/** + * Parses docs MDX check arguments. + */ +export function parseArgs(argv: string[]) { + const roots: string[] = []; + let jsonOut = ""; + let maxErrors = 50; + let cacheFile = ""; + + for (let index = 0; index < argv.length; index += 1) { + const part = argv[index]; + if (part === undefined) { + continue; + } + if (part === "--json-out") { + jsonOut = requireOptionArgument(argv, index, "--json-out"); + index += 1; + continue; + } + if (part === "--cache-file") { + cacheFile = requireOptionArgument(argv, index, "--cache-file"); + index += 1; + continue; + } + if (part === "--max-errors") { + maxErrors = parsePositiveIntegerArg( + requireOptionArgument(argv, index, "--max-errors"), + "--max-errors", + ); + index += 1; + continue; + } + if (part.startsWith("--")) { + throw new Error(`unknown arg: ${part}`); + } + roots.push(part); + } + + return { + roots: roots.length ? roots : ["docs"], + jsonOut, + maxErrors, + ...(cacheFile ? { cacheFile } : {}), + }; +} + +function walkMarkdownFiles(entryPath: string, out: string[] = []): string[] { + const stat = fs.statSync(entryPath); + if (stat.isFile()) { + if (/\.mdx?$/i.test(entryPath)) { + out.push(path.resolve(entryPath)); + } + return out; + } + + for (const entry of fs.readdirSync(entryPath, { withFileTypes: true })) { + if (entry.name === "node_modules" || entry.name === ".git") { + continue; + } + walkMarkdownFiles(path.join(entryPath, entry.name), out); + } + return out; +} + +function stripFrontmatter(raw: string): string { + if (!raw.startsWith("---\n") && !raw.startsWith("---\r\n")) { + return raw; + } + + const lines = raw.split(/\r?\n/u); + for (let index = 1; index < lines.length; index += 1) { + if (lines[index] === "---" || lines[index] === "...") { + // Preserve source line numbers for both MDX and component diagnostics. + return lines.fill("", 0, index + 1).join("\n"); + } + } + return raw; +} + +function errorField(error: unknown, key: string): unknown { + return error && typeof error === "object" && key in error + ? error[key as keyof typeof error] + : undefined; +} + +function formatMdxError(filePath: string, error: unknown): DocsCheckError { + const reason = errorField(error, "reason"); + const message = errorField(error, "message"); + const line = errorField(error, "line"); + const column = errorField(error, "column"); + return { + type: "mdx", + file: filePath, + ...(typeof line === "number" ? { line } : {}), + ...(typeof column === "number" ? { column } : {}), + message: String(reason ?? message ?? error).split("\n")[0] ?? "", + }; +} + +function checkMintlifyMdxStructure(filePath: string, raw: string): DocsCheckError[] { + return checkMintlifyAccordionIndentation(stripFrontmatter(raw)).map((error) => ({ + type: "mintlify-mdx", + file: filePath, + line: error.line, + column: error.column, + message: MINTLIFY_ACCORDION_INDENT_MESSAGE, + })); +} + +function lineColumnForIndex(raw: string, offset: number): { line: number; column: number } { + const prefix = raw.slice(0, offset); + const lines = prefix.split(/\r?\n/u); + return { + line: lines.length, + column: (lines.at(-1) ?? "").length + 1, + }; +} + +function checkPoisonText(filePath: string, raw: string): DocsCheckError[] { + const errors: DocsCheckError[] = []; + for (const { pattern, message } of POISON_TEXT_PATTERNS) { + const match = pattern.exec(raw); + if (!match) { + continue; + } + const location = lineColumnForIndex(raw, match.index); + errors.push({ + type: "poison-text", + file: filePath, + line: location.line, + column: location.column, + message, + }); + } + return errors; +} + +async function checkMdxFile(filePath: string, raw: string): Promise { + const poisonErrors = checkPoisonText(filePath, raw); + if (poisonErrors.length > 0) { + return poisonErrors; + } + const structureErrors = checkMintlifyMdxStructure(filePath, raw); + if (structureErrors.length > 0) { + return structureErrors; + } + await compile({ path: filePath, value: stripFrontmatter(raw) }); + return []; +} + +function findDocsJsonPaths(roots: string[]): string[] { + const paths = new Set(); + for (const root of roots) { + const absolute = path.resolve(root); + if (!fs.existsSync(absolute)) { + continue; + } + const stat = fs.statSync(absolute); + if (stat.isFile() && path.basename(absolute) === "docs.json") { + paths.add(absolute); + continue; + } + if (stat.isDirectory()) { + const docsJsonPath = path.join(absolute, "docs.json"); + if (fs.existsSync(docsJsonPath)) { + paths.add(docsJsonPath); + } + } + } + return [...paths]; +} + +function collectNavigationLanguages(value: unknown, out: string[] = []): string[] { + if (Array.isArray(value)) { + for (const item of value) { + collectNavigationLanguages(item, out); + } + return out; + } + if (!value || typeof value !== "object") { + return out; + } + const record = value as Record; + if (typeof record.language === "string") { + out.push(record.language); + } + for (const child of Object.values(record)) { + if (child && typeof child === "object") { + collectNavigationLanguages(child, out); + } + } + return out; +} + +function checkDocsJson(filePath: string): DocsCheckError[] { + const errors: DocsCheckError[] = []; + let data: unknown; + try { + data = JSON.parse(fs.readFileSync(filePath, "utf8")); + } catch (error) { + return [ + { + type: "docs-json", + file: filePath, + message: `Invalid JSON: ${String(errorField(error, "message") ?? error)}`, + }, + ]; + } + + const navigation = + data && typeof data === "object" && "navigation" in data ? data.navigation : undefined; + const languages = collectNavigationLanguages(navigation); + for (const language of languages) { + if (!MINTLIFY_LANGUAGE_CODES.has(language)) { + errors.push({ + type: "docs-json", + file: filePath, + message: `Unsupported Mintlify navigation language: ${language}`, + }); + } + } + return errors; +} + +function relativize(root: string, filePath: string): string { + const relative = path.relative(root, filePath); + return relative && !relative.startsWith("..") ? relative : filePath; +} + +async function main(): Promise { + const startedAt = Date.now(); + const args = parseArgs(process.argv.slice(2)); + const cwd = process.cwd(); + const roots = args.roots.map((root) => path.resolve(root)); + const cache = args.cacheFile ? validationCache(args.cacheFile) : undefined; + let cacheHits = 0; + const files = [ + ...new Set( + roots.flatMap((root) => { + if (!fs.existsSync(root)) { + throw new Error(`path does not exist: ${root}`); + } + return walkMarkdownFiles(root); + }), + ), + ].toSorted((left, right) => left.localeCompare(right)); + + const errors: DocsCheckError[] = []; + for (const docsJsonPath of findDocsJsonPaths(args.roots)) { + errors.push(...checkDocsJson(docsJsonPath)); + } + + for (const file of files) { + try { + const raw = fs.readFileSync(file); + const relative = cache ? path.relative(cache.root, file).split(path.sep).join("/") : ""; + const cachePath = + relative && !relative.startsWith("../") && !path.isAbsolute(relative) ? relative : ""; + const digest = cachePath ? createHash("sha256").update(raw).digest("hex") : ""; + if (cachePath && cache?.previous.get(cachePath) === digest) { + cache.checked.set(cachePath, digest); + cacheHits += 1; + continue; + } + const pageErrors = await checkMdxFile(file, raw.toString("utf8")); + errors.push(...pageErrors); + if (cachePath && pageErrors.length === 0) { + cache?.checked.set(cachePath, digest); + } + } catch (error) { + errors.push(formatMdxError(file, error)); + if (errors.length >= args.maxErrors) { + break; + } + } + } + + const report = { + files: files.length, + errors: errors.map((error) => Object.assign({}, error, { file: relativize(cwd, error.file) })), + ms: Date.now() - startedAt, + ...(cache ? { cacheHits } : {}), + }; + + if (args.jsonOut) { + fs.mkdirSync(path.dirname(path.resolve(args.jsonOut)), { recursive: true }); + fs.writeFileSync(args.jsonOut, `${JSON.stringify(report, null, 2)}\n`); + } + + if (report.errors.length === 0) { + cache?.save(); + console.log(`Docs MDX check passed (${report.files} files, ${report.ms}ms).`); + if (cache) { + console.log(`Reused ${cacheHits} unchanged successful page check(s).`); + } + return; + } + + console.error(`Docs MDX check failed (${report.errors.length} error(s), ${report.files} files).`); + for (const error of report.errors) { + const location = + error.line && error.column ? `${error.file}:${error.line}:${error.column}` : error.file; + console.error(`- ${location}: ${error.message}`); + } + process.exitCode = 1; +} + +const isMain = process.argv[1] ? fileURLToPath(import.meta.url) === process.argv[1] : false; + +if (isMain) { + main().catch((error: unknown) => { + console.error(errorField(error, "stack") ?? error); + process.exit(1); + }); +} diff --git a/scripts/check-file-utils.ts b/scripts/check-file-utils.ts new file mode 100644 index 0000000000000000000000000000000000000000..8d9878582910fd587540628b14d86ec76063174a --- /dev/null +++ b/scripts/check-file-utils.ts @@ -0,0 +1,131 @@ +// Check File Utils helper supports OpenClaw script workflows. +import { execFileSync } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; + +const DEFAULT_SKIPPED_DIR_NAMES = new Set(["node_modules", "dist", "coverage", ".generated"]); +export const REPO_SCAN_ROOTS = ["src", "test", "extensions", "packages", "ui", "scripts"] as const; +export const REPO_SCAN_SKIPPED_DIR_NAMES: ReadonlySet = new Set([ + ".artifacts", + ".generated", + "coverage", + "dist", + "fixtures", + "node_modules", + "vendor", +]); +// Bound the Git lookup before falling back to direct filesystem traversal. +const GIT_LS_FILES_TIMEOUT_MS = 30_000; +const GIT_LS_FILES_MAX_BUFFER = 64 * 1024 * 1024; + +export function isCodeFile(filePath: string): boolean { + if (filePath.endsWith(".d.ts")) { + return false; + } + return /\.(?:[cm]?ts|[cm]?js|tsx|jsx)$/u.test(filePath); +} + +export function isTestRelatedFile(relativePath: string): boolean { + return ( + /(?:^|[/.])(?:test|spec)\.[cm]?[jt]sx?$/u.test(relativePath) || + /\.(?:e2e|live)\.test\.[cm]?[jt]sx?$/u.test(relativePath) || + /\.(?:test-helpers|test-utils|test-harness|test-support)\.[cm]?[jt]sx?$/u.test(relativePath) || + /-(?:test-helpers|test-utils|test-harness|test-support)\.[cm]?[jt]sx?$/u.test(relativePath) || + /(?:^|\/)(?:test|tests|test-helpers|test-utils|test-harness|test-support)\//u.test( + relativePath, + ) || + relativePath.startsWith("scripts/e2e/") || + /^scripts\/.*-(?:client|e2e|harness|probe|smoke)\.[cm]?[jt]s$/u.test(relativePath) + ); +} + +export function collectFilesSync( + rootDir: string, + options: { + includeFile: (filePath: string) => boolean; + skipDirNames?: ReadonlySet; + }, +): string[] { + const skipDirNames = options.skipDirNames ?? DEFAULT_SKIPPED_DIR_NAMES; + const files: string[] = []; + const stack = [rootDir]; + + while (stack.length > 0) { + const current = stack.pop(); + if (!current) { + continue; + } + let entries: fs.Dirent[]; + try { + entries = fs.readdirSync(current, { withFileTypes: true }); + } catch { + continue; + } + for (const entry of entries) { + const fullPath = path.join(current, entry.name); + if (entry.isDirectory()) { + if (skipDirNames.has(entry.name)) { + continue; + } + stack.push(fullPath); + continue; + } + if (entry.isFile() && options.includeFile(fullPath)) { + files.push(fullPath); + } + } + } + + return files; +} + +export function listRepoFilesSync( + repoRoot: string, + options: { + includeFile: (relativePath: string) => boolean; + roots?: readonly string[]; + skipDirNames?: ReadonlySet; + }, +): string[] { + const roots = options.roots ?? REPO_SCAN_ROOTS; + try { + return execFileSync("git", ["-C", repoRoot, "ls-files", "--", ...roots], { + encoding: "utf8", + stdio: ["ignore", "pipe", "ignore"], + timeout: GIT_LS_FILES_TIMEOUT_MS, + killSignal: "SIGKILL", + maxBuffer: GIT_LS_FILES_MAX_BUFFER, + }) + .split(/\r?\n/u) + .filter(Boolean) + .map(toPosixPath) + .filter(options.includeFile) + .toSorted((left, right) => left.localeCompare(right)); + } catch { + return roots + .flatMap((root) => { + const absoluteRoot = path.join(repoRoot, root); + if (!fs.existsSync(absoluteRoot)) { + return []; + } + return collectFilesSync(absoluteRoot, { + includeFile: (filePath) => + options.includeFile(toPosixPath(path.relative(repoRoot, filePath))), + skipDirNames: options.skipDirNames ?? REPO_SCAN_SKIPPED_DIR_NAMES, + }).map((filePath) => toPosixPath(path.relative(repoRoot, filePath))); + }) + .toSorted((left, right) => left.localeCompare(right)); + } +} + +export function toPosixPath(filePath: string): string { + if (path.sep === "/") { + return filePath; + } + return filePath.replaceAll("\\", "/"); +} + +export function relativeToCwd(filePath: string): string { + const relativePath = path.relative(process.cwd(), filePath) || filePath; + return toPosixPath(relativePath); +} diff --git a/scripts/check-import-cycles.ts b/scripts/check-import-cycles.ts new file mode 100644 index 0000000000000000000000000000000000000000..0ad221acb1dc48fdf63610b3a623c53b2a0ad279 --- /dev/null +++ b/scripts/check-import-cycles.ts @@ -0,0 +1,173 @@ +#!/usr/bin/env node +// Check Import Cycles script supports OpenClaw repository automation. +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import ts from "typescript"; +import { + collectSourceFiles, + collectStronglyConnectedComponents, + formatCycle, +} from "./lib/import-cycle-graph.ts"; + +const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const scanRoots = ["src", "extensions", "scripts"] as const; +const sourceExtensions = [".ts", ".tsx", ".mts", ".cts", ".js", ".mjs", ".cjs"] as const; +const testSourcePattern = /(?:\.test|\.e2e\.test)\.[cm]?[tj]sx?$/; +const generatedSourcePattern = /\.(?:generated|bundle)\.[tj]s$/; +const declarationSourcePattern = /\.d\.[cm]?ts$/; +const ignoredPathPartPattern = + /(^|\/)(node_modules|dist|build|coverage|\.artifacts|\.git|assets)(\/|$)/; + +function shouldSkipRepoPath(repoPath: string): boolean { + return ( + ignoredPathPartPattern.test(repoPath) || + testSourcePattern.test(repoPath) || + generatedSourcePattern.test(repoPath) || + declarationSourcePattern.test(repoPath) + ); +} + +function createSourceResolver(files: readonly string[]) { + const fileSet = new Set(files); + const pathMap = new Map(); + for (const file of files) { + const parsed = path.posix.parse(file); + const extensionless = path.posix.join(parsed.dir, parsed.name); + pathMap.set(extensionless, file); + if (file.endsWith(".ts")) { + pathMap.set(`${extensionless}.js`, file); + } else if (file.endsWith(".tsx")) { + pathMap.set(`${extensionless}.jsx`, file); + } else if (file.endsWith(".mts")) { + pathMap.set(`${extensionless}.mjs`, file); + } else if (file.endsWith(".cts")) { + pathMap.set(`${extensionless}.cjs`, file); + } + } + return (importer: string, specifier: string): string | null => { + if (!specifier.startsWith(".")) { + return null; + } + const base = path.posix.normalize(path.posix.join(path.posix.dirname(importer), specifier)); + if (fileSet.has(base)) { + return base; + } + const mappedBase = pathMap.get(base); + if (mappedBase) { + return mappedBase; + } + const candidates = [ + ...sourceExtensions.map((extension) => `${base}${extension}`), + `${base}/index.ts`, + `${base}/index.tsx`, + `${base}/index.js`, + `${base}/index.mjs`, + ]; + for (const candidate of candidates) { + if (fileSet.has(candidate)) { + return candidate; + } + const mapped = pathMap.get(candidate); + if (mapped) { + return mapped; + } + } + return null; + }; +} + +function importDeclarationHasRuntimeEdge(node: ts.ImportDeclaration): boolean { + if (!node.importClause) { + return true; + } + if (node.importClause.isTypeOnly) { + return false; + } + const bindings = node.importClause.namedBindings; + if (node.importClause.name || !bindings || ts.isNamespaceImport(bindings)) { + return true; + } + return bindings.elements.some((element) => !element.isTypeOnly); +} + +function exportDeclarationHasRuntimeEdge(node: ts.ExportDeclaration): boolean { + if (!node.moduleSpecifier || node.isTypeOnly) { + return false; + } + const clause = node.exportClause; + if (!clause || ts.isNamespaceExport(clause)) { + return true; + } + return clause.elements.some((element) => !element.isTypeOnly); +} + +function collectRuntimeStaticImports( + file: string, + resolveSource: ReturnType, +) { + const sourceFile = ts.createSourceFile( + file, + readFileSync(path.join(repoRoot, file), "utf8"), + ts.ScriptTarget.Latest, + false, + ); + const imports: string[] = []; + const visit = (node: ts.Node) => { + let specifier: string | undefined; + let include = false; + if (ts.isImportDeclaration(node) && ts.isStringLiteral(node.moduleSpecifier)) { + specifier = node.moduleSpecifier.text; + include = importDeclarationHasRuntimeEdge(node); + } else if ( + ts.isExportDeclaration(node) && + node.moduleSpecifier && + ts.isStringLiteral(node.moduleSpecifier) + ) { + specifier = node.moduleSpecifier.text; + include = exportDeclarationHasRuntimeEdge(node); + } + if (include && specifier) { + const resolved = resolveSource(file, specifier); + if (resolved) { + imports.push(resolved); + } + } + ts.forEachChild(node, visit); + }; + visit(sourceFile); + return imports.toSorted((left, right) => left.localeCompare(right)); +} + +function main(): number { + const files = scanRoots.flatMap((root) => + collectSourceFiles(path.join(repoRoot, root), { + repoRoot, + sourceExtensions, + shouldSkipRepoPath, + }), + ); + const resolveSource = createSourceResolver(files); + const graph = new Map( + files.map((file): [string, string[]] => [ + file, + collectRuntimeStaticImports(file, resolveSource), + ]), + ); + const components = collectStronglyConnectedComponents(graph); + + console.log(`Import cycle check: ${components.length} runtime value cycle(s).`); + if (components.length === 0) { + return 0; + } + + console.error("\nRuntime value import cycles:"); + for (const component of components) { + console.error(`\n# component size ${component.length}`); + console.error(formatCycle(component, graph)); + } + console.error("\nBreak the cycle or convert type-only edges to `import type`."); + return 1; +} + +process.exitCode = main(); diff --git a/scripts/check-kysely-guardrails.mts b/scripts/check-kysely-guardrails.mts new file mode 100644 index 0000000000000000000000000000000000000000..cae2b2dd8f663c5b3dfcfbae896ca29f5593be25 --- /dev/null +++ b/scripts/check-kysely-guardrails.mts @@ -0,0 +1,539 @@ +#!/usr/bin/env node + +// Enforces Kysely and SQLite guardrails in infrastructure code. +import { promises as fs } from "node:fs"; +import { createRequire } from "node:module"; +import path from "node:path"; +import type { Expression, ImportDeclaration, Node, SourceFile } from "typescript"; +import { resolveRepoRoot } from "./lib/repo-root.mjs"; +import { + collectTypeScriptFilesFromRoots, + getPropertyNameText, + runAsScript, + toLine, + unwrapExpression, +} from "./lib/ts-guard-utils.mts"; + +const require = createRequire(import.meta.url); +const ts: typeof import("typescript") = require("typescript"); + +const repoRoot = resolveRepoRoot(import.meta.url); +const sourceRoots = [path.join(repoRoot, "src")]; +const nodeSqliteBoundaryRoots = [ + path.join(repoRoot, "src"), + path.join(repoRoot, "extensions"), + path.join(repoRoot, "packages"), +]; + +const nodeSqliteConstructorOwnerPaths = new Set([ + "src/infra/node-sqlite.ts", + "src/infra/sqlite-runtime-version.ts", +]); + +const kyselyRawAllowPaths = new Set(["src/infra/kysely-sync.ts"]); + +const compiledRawAllowPaths = new Set(["src/infra/kysely-node-sqlite.ts"]); + +const rawSqliteAllowPathGroups = { + "native Kysely adapter and sync execution": [ + "src/infra/kysely-node-sqlite.ts", + "src/infra/kysely-sync-cache-state.ts", + "src/infra/kysely-sync.ts", + ], + "SQLite database lifecycle, schema, transactions, and pragmas": [ + "src/infra/node-sqlite.ts", + "src/infra/sqlite-index-schema.ts", + "src/infra/sqlite-integrity.ts", + "src/infra/sqlite-pragma.test-support.ts", + "src/infra/sqlite-schema-contract.ts", + "src/infra/sqlite-strict.ts", + "src/infra/sqlite-transaction.ts", + "src/infra/sqlite-user-version.ts", + "src/infra/sqlite-wal.ts", + "src/state/openclaw-agent-db-maintenance.ts", + "src/state/openclaw-agent-db-registry.ts", + "src/state/openclaw-agent-db-registry-listing.ts", + "src/state/openclaw-agent-db-schema-helpers.ts", + // Existing schema ownership/version preflight, split from schema-helpers for readers. + "src/state/openclaw-agent-db-schema-read.ts", + "src/state/openclaw-agent-db-metadata.ts", + "src/state/openclaw-agent-db-schema.ts", + "src/state/openclaw-agent-db-session-nodes-migration.ts", + "src/state/openclaw-agent-db-session-migrations.ts", + "src/state/openclaw-agent-db-session-provenance.ts", + "src/state/openclaw-agent-db.ts", + "src/state/openclaw-state-db-audit-migration.ts", + "src/state/openclaw-state-db-delivery-queue-backfill.ts", + "src/state/openclaw-state-db-legacy-backfills.ts", + "src/state/openclaw-state-db-maintenance.ts", + "src/state/openclaw-state-db-operator-approval-migration.ts", + "src/state/openclaw-state-db-schema-additive.ts", + "src/state/openclaw-state-db-schema-helpers.ts", + "src/state/openclaw-state-db-schema-repair.ts", + "src/state/openclaw-state-db-schema-v12-foldin.ts", + "src/state/openclaw-state-db-schema-v13-widerow.ts", + "src/state/openclaw-state-db-startup-checkpoint.ts", + "src/state/openclaw-state-db-table-retirements.ts", + "src/state/openclaw-state-db-fast-path.ts", + "src/state/openclaw-state-db.ts", + "src/state/openclaw-state-ownership-operations.ts", + "src/transcripts/sqlite-schema.ts", + "src/state/sqlite-schema-shape.test-support.ts", + ], + "cross-process SQLite coordination locks": ["src/infra/sqlite-coordinator.ts"], + "backup snapshot maintenance": [ + "src/commands/backup-verify.ts", + "src/infra/backup-create.ts", + "src/snapshot/git-backup-codec.ts", + "src/snapshot/local-repository.ts", + ], + "agent auth profile read-only bootstrap": ["src/agents/auth-profiles/sqlite.ts"], + "read-only shared state database access": [ + "src/claws/package-resume.ts", + "src/state/openclaw-agent-db-readonly.ts", + "src/state/openclaw-state-db-readonly.ts", + ], + "cold-process read-only relay lookup avoids the shared state writer lifecycle": [ + "src/agents/harness/native-hook-relay-client.worker.ts", + ], + "read-only schema preflight and integrity verification access": [ + "src/state/openclaw-database-preflight.ts", + "src/state/openclaw-database-verify.worker.ts", + "src/state/openclaw-state-ownership.ts", + ], + "quarantine store must work when other databases are damaged": [ + "src/state/openclaw-quarantine-store.ts", + ], + "read-only SQLite status probes": [ + "src/commands/doctor-db-bloat.ts", + "src/commands/status.scan.shared.ts", + ], + "doctor SQLite maintenance and legacy state migration": [ + "src/commands/doctor-agent-memory-schema.ts", + "src/commands/doctor/cron/legacy-run-log-migration.ts", + "src/commands/doctor/cron/migration-ledger.ts", + "src/commands/doctor-sqlite-compact.ts", + "src/commands/doctor-session-sqlite.ts", + "src/commands/doctor-session-sqlite-readers.ts", + "src/commands/doctor-session-sqlite-transcript-readers.ts", + "src/commands/doctor-session-sqlite-recover-report.ts", + "src/commands/doctor-state-sqlite-compact.ts", + // Disposable import planning only; canonical session tables still use Kysely. + "src/config/sessions/session-accessor.sqlite-import-stage.ts", + "src/infra/state-migrations.task-sidecar-rows.ts", + "src/infra/state-migrations.storage.ts", + "src/infra/state-migrations.cron-run-logs.ts", + "src/infra/state-migrations.debug-proxy.ts", + "src/infra/state-migrations.meeting-transcripts-detection.ts", + "src/infra/state-migrations.meeting-transcripts-files.ts", + "src/infra/state-migrations.meeting-transcripts-verify.ts", + "src/infra/state-migrations.media-persistence.ts", + "src/infra/state-migrations.transcript-directives-archives.ts", + "src/infra/state-migrations.transcript-directives.ts", + // Doctor integrity PRAGMAs and lossless native 64-bit orphan-row preservation. + "src/state/openclaw-state-db-task-delivery-recovery.ts", + ], + "session entry cache connection-local validity counters": [ + "src/config/sessions/session-accessor.sqlite-entry-cache.ts", + ], + "device pairing cache connection-local validity counters": ["src/infra/device-pairing-store.ts"], + "Kysely-backed stores that own a DatabaseSync boundary": [ + "src/acp/event-ledger.ts", + "src/state/user-profiles.ts", + "src/cron/store.ts", + "src/infra/outbound/current-conversation-bindings.ts", + "src/media/store.ts", + "src/plugin-sdk/memory-core-host-engine-storage.ts", + "src/plugins/installed-plugin-index-record-reader.ts", + "src/plugins/installed-plugin-index-store-write.ts", + "src/plugin-state/plugin-state-store.sqlite.ts", + "src/proxy-capture/store.sqlite.ts", + "src/tasks/task-flow-registry.store.sqlite.ts", + "src/tasks/task-registry.store.kernel.ts", + ], +}; + +type GuardViolation = { line: number; message: string }; + +const rawSqliteAllowPathReasons = new Map(); +for (const [reason, paths] of Object.entries(rawSqliteAllowPathGroups)) { + for (const allowedPath of paths) { + if (rawSqliteAllowPathReasons.has(allowedPath)) { + throw new Error(`Duplicate raw SQLite allowlist path: ${allowedPath}`); + } + rawSqliteAllowPathReasons.set(allowedPath, reason); + } +} + +function lineText(sourceFile: SourceFile, node: Node) { + const line = toLine(sourceFile, node); + return sourceFile.text.split("\n")[line - 1] ?? ""; +} + +function hasAllowComment(sourceFile: SourceFile, node: Node, token: string) { + const line = lineText(sourceFile, node); + if (line.includes(token)) { + return true; + } + const leading = ts.getLeadingCommentRanges(sourceFile.text, node.pos) ?? []; + return leading.some((range) => sourceFile.text.slice(range.pos, range.end).includes(token)); +} + +function importSource(node: ImportDeclaration) { + const moduleSpecifier = node.moduleSpecifier; + return ts.isStringLiteral(moduleSpecifier) ? moduleSpecifier.text : ""; +} + +function collectImports(sourceFile: SourceFile) { + const kyselySqlNames = new Set(); + const compiledQueryNames = new Set(); + const syncHelperNames = new Set(); + let hasKyselyContext = false; + let hasSqliteContext = false; + + for (const statement of sourceFile.statements) { + if (!ts.isImportDeclaration(statement)) { + continue; + } + const source = importSource(statement); + const clause = statement.importClause; + const namedBindings = clause?.namedBindings; + + if (source === "kysely") { + hasKyselyContext = true; + if (namedBindings && ts.isNamedImports(namedBindings)) { + for (const element of namedBindings.elements) { + const importedName = element.propertyName?.text ?? element.name.text; + if (importedName === "sql") { + kyselySqlNames.add(element.name.text); + } + if (importedName === "CompiledQuery") { + compiledQueryNames.add(element.name.text); + } + } + } + } + + if (source.endsWith("kysely-sync.js") || source.endsWith("kysely-node-sqlite.js")) { + hasKyselyContext = true; + if (namedBindings && ts.isNamedImports(namedBindings)) { + for (const element of namedBindings.elements) { + const importedName = element.propertyName?.text ?? element.name.text; + if ( + importedName === "executeSqliteQuerySync" || + importedName === "executeSqliteQueryTakeFirstSync" + ) { + syncHelperNames.add(element.name.text); + } + if (importedName === "getNodeSqliteKysely") { + hasKyselyContext = true; + hasSqliteContext = true; + } + } + } + } + + if ( + source === "node:sqlite" || + source.endsWith("node-sqlite.js") || + source.endsWith("sqlite-transaction.js") || + source.endsWith("sqlite-wal.js") || + source.endsWith("openclaw-state-db.js") + ) { + hasSqliteContext = true; + } + } + + return { + compiledQueryNames, + hasKyselyContext, + hasSqliteContext, + kyselySqlNames, + syncHelperNames, + }; +} + +function addViolation( + violations: GuardViolation[], + sourceFile: SourceFile, + node: Node, + message: string, +) { + violations.push({ + line: toLine(sourceFile, node), + message, + }); +} + +function isIdentifierNamed(node: Expression, names: Set) { + const unwrapped = unwrapExpression(node); + return ts.isIdentifier(unwrapped) && names.has(unwrapped.text); +} + +function isTestPath(relativePath: string) { + return ( + /\.(?:test|spec|e2e)\.ts$/u.test(relativePath) || + relativePath.includes(".test-helpers.") || + relativePath.includes(".test-support.") + ); +} + +function isSqliteStorePath(relativePath: string) { + return relativePath.endsWith(".sqlite.ts") || relativePath.includes(".store.sqlite.ts"); +} + +function collectNodeSqliteBoundaryViolations(content: string, relativePath: string) { + if (isTestPath(relativePath) || nodeSqliteConstructorOwnerPaths.has(relativePath)) { + return []; + } + const sourceFile = ts.createSourceFile(relativePath, content, ts.ScriptTarget.Latest, true); + const constructorNames = new Set(); + + function collectConstructorNames(node: Node) { + if (ts.isImportDeclaration(node) && importSource(node) === "node:sqlite") { + const namedBindings = node.importClause?.namedBindings; + if (namedBindings && ts.isNamedImports(namedBindings)) { + for (const element of namedBindings.elements) { + if ((element.propertyName?.text ?? element.name.text) === "DatabaseSync") { + constructorNames.add(element.name.text); + } + } + } + } + if (ts.isVariableDeclaration(node) && ts.isObjectBindingPattern(node.name)) { + for (const element of node.name.elements) { + if ( + !element.dotDotDotToken && + ts.isIdentifier(element.name) && + (element.propertyName ? getPropertyNameText(element.propertyName) : element.name.text) === + "DatabaseSync" + ) { + constructorNames.add(element.name.text); + } + } + } + ts.forEachChild(node, collectConstructorNames); + } + + collectConstructorNames(sourceFile); + const violations: GuardViolation[] = []; + function visit(node: Node) { + if (ts.isNewExpression(node)) { + const expression = unwrapExpression(node.expression); + const isRawConstructor = + (ts.isIdentifier(expression) && constructorNames.has(expression.text)) || + (ts.isPropertyAccessExpression(expression) && + getPropertyNameText(expression.name) === "DatabaseSync"); + if (isRawConstructor) { + addViolation( + violations, + sourceFile, + node, + "production node:sqlite connections must use openNodeSqliteDatabase", + ); + } + } + ts.forEachChild(node, visit); + } + visit(sourceFile); + return violations; +} + +function isLikelySqliteReceiver(expression: Expression) { + const unwrapped = unwrapExpression(expression); + if (ts.isIdentifier(unwrapped)) { + return /^(?:db|database|legacyDb|stateDb|agentDb)$/u.test(unwrapped.text); + } + return ts.isPropertyAccessExpression(unwrapped) && getPropertyNameText(unwrapped.name) === "db"; +} + +function isPersistedRowExpression(expression: Expression) { + const unwrapped = unwrapExpression(expression); + if (ts.isPropertyAccessExpression(unwrapped)) { + const owner = unwrapExpression(unwrapped.expression); + return ts.isIdentifier(owner) && /^(?:row|record|entry)$/u.test(owner.text); + } + if (ts.isElementAccessExpression(unwrapped)) { + const owner = unwrapExpression(unwrapped.expression); + return ts.isIdentifier(owner) && /^(?:row|record|entry)$/u.test(owner.text); + } + return false; +} + +function isPersistedStringCastType(typeText: string) { + return [ + /\bTaskRecord\["(?:runtime|scopeKind|status|deliveryStatus|notifyPolicy|terminalOutcome)"\]/u, + /\bTaskFlowRecord\["(?:status|notifyPolicy)"\]/u, + /\bTaskFlowSyncMode\b/u, + /\bVirtualAgentFsEntryKind\b/u, + /\b[A-Z][A-Za-z0-9]*(?:Status|Kind|Mode|Policy|Runtime|Outcome)\b/u, + ].some((pattern) => pattern.test(typeText)); +} + +/** + * Collects Kysely/raw SQLite violations from one source file. + */ +function collectKyselyGuardrailViolations(content: string, relativePath: string) { + const sourceFile = ts.createSourceFile(relativePath, content, ts.ScriptTarget.Latest, true); + const imports = collectImports(sourceFile); + const violations: GuardViolation[] = []; + + function visit(node: Node) { + if ( + isSqliteStorePath(relativePath) && + (ts.isAsExpression(node) || ts.isTypeAssertionExpression(node)) && + isPersistedStringCastType(node.type.getText(sourceFile)) && + isPersistedRowExpression(node.expression) && + !hasAllowComment(sourceFile, node, "sqlite-allow-persisted-cast") + ) { + addViolation( + violations, + sourceFile, + node, + "persisted SQLite enum-like values must be parsed through closed validators, not cast", + ); + } + + if ( + ts.isCallExpression(node) && + ts.isIdentifier(node.expression) && + imports.syncHelperNames.has(node.expression.text) && + node.typeArguments?.length && + !hasAllowComment(sourceFile, node, "kysely-allow-raw") + ) { + addViolation( + violations, + sourceFile, + node, + "sync helper row generic at call site; let Kysely infer builder result rows", + ); + } + + if ( + ts.isTaggedTemplateExpression(node) && + node.typeArguments?.length && + isIdentifierNamed(node.tag, imports.kyselySqlNames) && + !kyselyRawAllowPaths.has(relativePath) && + !hasAllowComment(sourceFile, node, "kysely-allow-raw") + ) { + addViolation( + violations, + sourceFile, + node, + "typed raw sql snippet needs a small helper or allowlisted boundary", + ); + } + + if ( + ts.isCallExpression(node) && + ts.isPropertyAccessExpression(node.expression) && + isIdentifierNamed(node.expression.expression, imports.kyselySqlNames) && + ["ref", "table", "id", "raw"].includes(getPropertyNameText(node.expression.name) ?? "") && + !hasAllowComment(sourceFile, node, "kysely-allow-raw") + ) { + addViolation( + violations, + sourceFile, + node, + "raw Kysely identifier helper requires a closed-set validator and local allow comment", + ); + } + + if ( + imports.hasKyselyContext && + ts.isPropertyAccessExpression(node) && + getPropertyNameText(node.name) === "dynamic" && + !hasAllowComment(sourceFile, node, "kysely-allow-raw") + ) { + addViolation( + violations, + sourceFile, + node, + "Kysely dynamic refs bypass literal reference checking; use only behind closed unions", + ); + } + + if ( + ts.isCallExpression(node) && + ts.isPropertyAccessExpression(node.expression) && + isIdentifierNamed(node.expression.expression, imports.compiledQueryNames) && + getPropertyNameText(node.expression.name) === "raw" && + !compiledRawAllowPaths.has(relativePath) && + !hasAllowComment(sourceFile, node, "kysely-allow-raw") + ) { + addViolation( + violations, + sourceFile, + node, + "CompiledQuery.raw is only allowed in the native SQLite dialect/test boundary", + ); + } + + if ( + imports.hasSqliteContext && + !isTestPath(relativePath) && + ts.isCallExpression(node) && + ts.isPropertyAccessExpression(node.expression) && + ["prepare", "exec"].includes(getPropertyNameText(node.expression.name) ?? "") && + isLikelySqliteReceiver(node.expression.expression) && + !rawSqliteAllowPathReasons.has(relativePath) && + !hasAllowComment(sourceFile, node, "sqlite-allow-raw") + ) { + addViolation( + violations, + sourceFile, + node, + "new raw node:sqlite access requires Kysely or an explicit raw SQLite allowlist entry", + ); + } + + ts.forEachChild(node, visit); + } + + visit(sourceFile); + return violations; +} + +/** + * Collects Kysely guardrail violations across configured source roots. + */ +async function collectKyselyGuardrails() { + const files = await collectTypeScriptFilesFromRoots(sourceRoots, { includeTests: true }); + const violations: Array = []; + for (const filePath of files) { + const relativePath = path.relative(repoRoot, filePath).split(path.sep).join("/"); + const content = await fs.readFile(filePath, "utf8"); + for (const violation of collectKyselyGuardrailViolations(content, relativePath)) { + violations.push({ path: relativePath, ...violation }); + } + } + const nodeSqliteFiles = await collectTypeScriptFilesFromRoots(nodeSqliteBoundaryRoots, { + includeTests: false, + }); + for (const filePath of nodeSqliteFiles) { + const relativePath = path.relative(repoRoot, filePath).split(path.sep).join("/"); + const content = await fs.readFile(filePath, "utf8"); + for (const violation of collectNodeSqliteBoundaryViolations(content, relativePath)) { + violations.push({ path: relativePath, ...violation }); + } + } + return violations; +} + +/** + * Runs the Kysely guardrail check. + */ +async function main() { + const violations = await collectKyselyGuardrails(); + if (violations.length === 0) { + console.log("Kysely guardrails OK"); + return; + } + console.error("Kysely guardrail violations:"); + for (const violation of violations) { + console.error(`- ${violation.path}:${violation.line}: ${violation.message}`); + } + process.exit(1); +} + +runAsScript(import.meta.url, main); diff --git a/scripts/check-live-cache.ts b/scripts/check-live-cache.ts new file mode 100644 index 0000000000000000000000000000000000000000..7649bb0836041d5757891ea47f414061385a4541 --- /dev/null +++ b/scripts/check-live-cache.ts @@ -0,0 +1,27 @@ +// Check Live Cache script supports OpenClaw repository automation. + +import { LIVE_CACHE_TEST_ENABLED, logLiveCache } from "../src/agents/live-cache-test-support.js"; +import { runLiveCacheRegression } from "../src/agents/test-helpers/live-cache-regression-runner.js"; + +if (!LIVE_CACHE_TEST_ENABLED) { + logLiveCache("skipped; set OPENCLAW_LIVE_TEST=1 and OPENCLAW_LIVE_CACHE_TEST=1"); + process.exit(0); +} + +const result = await runLiveCacheRegression(); +if (result.warnings.length > 0) { + process.stderr.write("\n[live-cache] non-blocking cache observations:\n"); + for (const warning of result.warnings) { + process.stderr.write(`- ${warning}\n`); + } +} +if (result.regressions.length > 0) { + process.stderr.write("\n[live-cache] regressions detected:\n"); + for (const regression of result.regressions) { + process.stderr.write(`- ${regression}\n`); + } + process.exit(1); +} else { + process.stderr.write("\n[live-cache] all regression floors satisfied\n"); + process.exit(0); +} diff --git a/scripts/check-media-download-helper-roundtrip.mts b/scripts/check-media-download-helper-roundtrip.mts new file mode 100644 index 0000000000000000000000000000000000000000..88134f147cb52b50b6f4380df4b37f07e19a1f3e --- /dev/null +++ b/scripts/check-media-download-helper-roundtrip.mts @@ -0,0 +1,67 @@ +#!/usr/bin/env node +// Checks extension media downloads keep helper-read/save calls close together. +import { execFileSync } from "node:child_process"; +import { existsSync, readFileSync } from "node:fs"; + +const WINDOW_LINES = 80; +const READ_HELPER_RE = /\b(?:readRemoteMediaBuffer|fetchRemoteMedia)\s*\(/; +const SAVE_BUFFER_RE = /(?:\.|\b)saveMediaBuffer\s*\(/; + +function listTrackedExtensionSources() { + return execFileSync("git", ["ls-files", "extensions/**/*.ts"], { + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }) + .split("\n") + .filter(Boolean) + .filter((file) => file.includes("/src/")) + .filter((file) => existsSync(file)) + .filter((file) => !isTestOrFixture(file)); +} + +function isTestOrFixture(file: string) { + return ( + file.endsWith(".test.ts") || + file.endsWith(".e2e.test.ts") || + file.endsWith(".test-harness.ts") || + file.endsWith(".test-utils.ts") || + file.endsWith("/test-runtime.ts") || + file.endsWith("/test-helpers.ts") || + file.includes("/test-support/") || + file.includes("/fixtures/") + ); +} + +const findings: Array<{ file: string; line: number; saveLine: number }> = []; + +for (const file of listTrackedExtensionSources()) { + const lines = readFileSync(file, "utf8").split("\n"); + for (let index = 0; index < lines.length; index += 1) { + if (!READ_HELPER_RE.test(lines[index] ?? "")) { + continue; + } + const end = Math.min(lines.length, index + WINDOW_LINES); + for (let nextIndex = index; nextIndex < end; nextIndex += 1) { + if (!SAVE_BUFFER_RE.test(lines[nextIndex] ?? "")) { + continue; + } + findings.push({ + file, + line: index + 1, + saveLine: nextIndex + 1, + }); + break; + } + } +} + +if (findings.length > 0) { + console.error("Avoid remote-media buffer/store round trips in plugin production code."); + console.error( + "Use saveRemoteMedia(...) for URL-to-store or saveResponseMedia(...) for fetched Response objects.", + ); + for (const finding of findings) { + console.error(`- ${finding.file}:${finding.line} -> saveMediaBuffer at ${finding.saveLine}`); + } + process.exitCode = 1; +} diff --git a/scripts/check-memory-fd-repro.mjs b/scripts/check-memory-fd-repro.mjs new file mode 100644 index 0000000000000000000000000000000000000000..267e926e0ec819bc765950aeb2b7221d35183703 --- /dev/null +++ b/scripts/check-memory-fd-repro.mjs @@ -0,0 +1,5 @@ +import { runTsxCliShim } from "./lib/tsx-cli-shim.mjs"; + +await runTsxCliShim(import.meta.url, { + implementation: "./check-memory-fd-repro.mts", +}); diff --git a/scripts/check-native-state-schema-version.mjs b/scripts/check-native-state-schema-version.mjs new file mode 100644 index 0000000000000000000000000000000000000000..d60b99acfae6f80a7596dc06720c6dcc875406e8 --- /dev/null +++ b/scripts/check-native-state-schema-version.mjs @@ -0,0 +1,52 @@ +#!/usr/bin/env node + +import fs from "node:fs"; +import { isDirectRunUrl } from "./lib/direct-run.mjs"; + +const SWIFT_CONTRACT_PATH = + "apps/shared/OpenClawKit/Sources/OpenClawNativeState/OpenClawNativeStateSQLite.swift"; +const TYPESCRIPT_CONTRACT_PATH = "src/state/openclaw-state-db-contract.ts"; + +function extractSingleVersion(source, pattern, label) { + const matches = [...source.matchAll(pattern)]; + if (matches.length !== 1) { + throw new Error(`Expected exactly one ${label} declaration; found ${matches.length}`); + } + return Number(matches[0][1]); +} + +export function compareNativeStateSchemaVersions({ swiftSource, typescriptSource }) { + const swiftVersion = extractSingleVersion( + swiftSource, + /^\s*private static let maximumSupportedSchemaVersion: Int64 = (\d+)\s*$/gmu, + "Swift maximumSupportedSchemaVersion", + ); + const typescriptVersion = extractSingleVersion( + typescriptSource, + /^export const OPENCLAW_STATE_SCHEMA_VERSION = (\d+);\s*$/gmu, + "TypeScript OPENCLAW_STATE_SCHEMA_VERSION", + ); + if (swiftVersion !== typescriptVersion) { + throw new Error( + `Native state schema version drift: Swift supports ${swiftVersion}, TypeScript owns ${typescriptVersion}`, + ); + } + return swiftVersion; +} + +export function checkNativeStateSchemaVersion(readFileSync = fs.readFileSync) { + return compareNativeStateSchemaVersions({ + swiftSource: readFileSync(SWIFT_CONTRACT_PATH, "utf8"), + typescriptSource: readFileSync(TYPESCRIPT_CONTRACT_PATH, "utf8"), + }); +} + +if (isDirectRunUrl(process.argv[1], import.meta.url)) { + try { + const version = checkNativeStateSchemaVersion(); + console.log(`native state schema version guard passed (v${version})`); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + } +} diff --git a/scripts/check-no-conflict-markers.mjs b/scripts/check-no-conflict-markers.mjs new file mode 100644 index 0000000000000000000000000000000000000000..4f7bf84cda50405de054ae246e35165bcf0d2ba9 --- /dev/null +++ b/scripts/check-no-conflict-markers.mjs @@ -0,0 +1,111 @@ +#!/usr/bin/env node + +// Rejects unresolved merge conflict markers in tracked files. +import { spawnSync } from "node:child_process"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const CONFLICT_MARKER_GREP_PATTERN = "^(<<<<<<< |\\|\\|\\|\\|\\|\\|\\| |=======$|>>>>>>> )"; + +/** + * Parses output from `git grep -n -z -o -I -E` into violation records. + * The record format is: + * path\0line-number\0match-text\n + * The path must be read from its NUL delimiter before any newline-based + * record splitting: -z reports paths verbatim, and git allows newlines in + * tracked filenames, so splitting the output on newlines first would cut a + * newline-containing path across two records and silently mangle it. + * Keeping parsing to exact grep match records avoids reading candidate files + * whole and keeps memory bounded regardless of file size. + */ +function parseGitGrepConflictMarkerOutput(stdout) { + const byPath = new Map(); + const output = stdout.toString("utf8"); + let cursor = 0; + + while (cursor < output.length) { + const pathEnd = output.indexOf("\0", cursor); + if (pathEnd === -1) { + break; + } + const lineNumberEnd = output.indexOf("\0", pathEnd + 1); + if (lineNumberEnd === -1) { + break; + } + // With -o the match text is a single-line match, so the record ends at + // the next newline after the line-number field. + const matchEnd = output.indexOf("\n", lineNumberEnd + 1); + const relativePath = output.slice(cursor, pathEnd); + const lineNumber = Number(output.slice(pathEnd + 1, lineNumberEnd)); + cursor = matchEnd === -1 ? output.length : matchEnd + 1; + if (!relativePath || !Number.isFinite(lineNumber) || lineNumber <= 0) { + continue; + } + const existing = byPath.get(relativePath); + if (existing) { + existing.push(lineNumber); + } else { + byPath.set(relativePath, [lineNumber]); + } + } + + const violations = []; + for (const [relativePath, lineNumbers] of byPath) { + lineNumbers.sort((a, b) => a - b); + violations.push({ + filePath: relativePath, + lines: lineNumbers, + }); + } + violations.sort((a, b) => a.filePath.localeCompare(b.filePath)); + return violations; +} + +/** + * Uses git grep to find exact conflict-marker matches in tracked files. + */ +export function findConflictMarkersInTrackedFiles(cwd = process.cwd()) { + const result = spawnSync( + "git", + ["grep", "--no-color", "-n", "-z", "-o", "-I", "-E", CONFLICT_MARKER_GREP_PATTERN, "--", "."], + { + cwd, + encoding: "buffer", + }, + ); + if (result.status === 1) { + return []; + } + if (result.status !== 0) { + const stderr = result.stderr?.toString("utf8").trim(); + throw new Error(stderr || `git grep failed with status ${result.status ?? "unknown"}`); + } + return parseGitGrepConflictMarkerOutput(result.stdout); +} + +/** + * Runs the merge conflict marker check. + */ +export async function main() { + const cwd = process.cwd(); + const violations = findConflictMarkersInTrackedFiles(cwd); + if (violations.length === 0) { + return; + } + + console.error("Found unresolved merge conflict markers:"); + for (const violation of violations) { + // findConflictMarkersInTrackedFiles already returns paths relative to cwd. + console.error(`- ${violation.filePath}:${violation.lines.join(",")}`); + } + process.exitCode = 1; +} + +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + main().catch( + /** @param {unknown} error */ (error) => { + console.error(error); + process.exit(1); + }, + ); +} diff --git a/scripts/check-no-deprecated-channel-access.ts b/scripts/check-no-deprecated-channel-access.ts new file mode 100644 index 0000000000000000000000000000000000000000..cddd9d7c83a5b1a37e5cb5652a0809db3b9e52dc --- /dev/null +++ b/scripts/check-no-deprecated-channel-access.ts @@ -0,0 +1,115 @@ +// Check No Deprecated Channel Access script supports OpenClaw repository automation. +import fs from "node:fs"; +import path from "node:path"; +import { collectFilesSync, isCodeFile, relativeToCwd } from "./check-file-utils.js"; +import { classifyBundledExtensionSourcePath } from "./lib/extension-source-classifier.mts"; + +type Rule = { + label: string; + pattern: RegExp; +}; + +const RULES: Rule[] = [ + { + label: "deprecated channel runtime", + pattern: + /\.channel\.(?:reply\.(?:createReplyDispatcherWithTyping|resolveHumanDelayConfig|dispatchReplyFromConfig|finalizeInboundContext|formatInboundEnvelope)|session\.(?:resolveStorePath|recordInboundSession)|inbound\.(?:runPreparedReply|dispatchReply)|media\.fetchRemoteMedia)\b/u, + }, + { + label: "caller-owned prepared channel dispatch", + pattern: /\b(?:runDispatch|onPreDispatchFailure)\b/u, + }, + { + label: "caller-owned reply dispatcher lifecycle", + pattern: + /\b(?:createReplyDispatcherWithTyping|dispatchInboundMessage(?:WithBufferedDispatcher|WithDispatcher)?|settleReplyDispatcher)\s*\(/u, + }, + { + label: "deprecated channel ingress resolver aliases", + pattern: + /\b(?:resolved|result|directResolved|groupResolved)\.(?:legacyAccess|senderReasonCode|commandAuthorized|shouldBlockControlCommand)\b/u, + }, + { + label: "inline deprecated channel ingress legacyAccess projection", + pattern: /\)\.legacyAccess\b/u, + }, + { + label: "deprecated pairing-store access helper", + pattern: /\breadStoreAllowFromForDmPolicy\b/u, + }, + { + label: "deprecated DM/group access helper", + pattern: /\bresolveDmGroupAccessWith(?:Lists|CommandGate)\b/u, + }, + { + label: "deprecated DM/group access reason constants", + pattern: /\bDM_GROUP_ACCESS_REASON\b/u, + }, + { + label: "deprecated group policy access helper", + pattern: + /\b(?:resolveSenderScopedGroupPolicy|evaluateSenderGroupAccess(?:ForPolicy)?|evaluateGroupRouteAccessForPolicy|evaluateMatchedGroupAccessForPolicy)\b/u, + }, + { + label: "deprecated group access compatibility module", + pattern: /from\s+["']openclaw\/plugin-sdk\/group-access["']/u, + }, + { + label: "deprecated command authorization helper", + pattern: /\bresolveSenderCommandAuthorization(?:WithRuntime)?\b/u, + }, + { + label: "deprecated command auth SDK facade", + pattern: /from\s+["']openclaw\/plugin-sdk\/command-auth["']/u, + }, +]; + +function collectBundledPluginProductionFiles(): string[] { + const extensionsDir = path.join(process.cwd(), "extensions"); + return collectFilesSync(extensionsDir, { + includeFile(filePath) { + if (!isCodeFile(filePath)) { + return false; + } + const repoPath = relativeToCwd(filePath); + const classified = classifyBundledExtensionSourcePath(repoPath); + return classified.isProductionSource; + }, + }).toSorted((left, right) => relativeToCwd(left).localeCompare(relativeToCwd(right))); +} + +function main() { + const offenders: Array<{ file: string; line: number; label: string; text: string }> = []; + for (const file of collectBundledPluginProductionFiles()) { + const content = fs.readFileSync(file, "utf8"); + const lines = content.split(/\r?\n/u); + for (const [index, line] of lines.entries()) { + for (const rule of RULES) { + if (rule.pattern.test(line)) { + offenders.push({ + file: relativeToCwd(file), + line: index + 1, + label: rule.label, + text: line.trim(), + }); + } + } + } + } + + if (offenders.length > 0) { + console.error( + "Bundled plugin production code must use modern channel runtime and access seams.", + ); + for (const offender of offenders) { + console.error(`- ${offender.file}:${offender.line}: ${offender.label}: ${offender.text}`); + } + process.exit(1); + } + + console.log( + "OK: bundled plugin production code avoids deprecated channel runtime and access seams.", + ); +} + +main(); diff --git a/scripts/check-no-monolithic-plugin-sdk-entry-imports.ts b/scripts/check-no-monolithic-plugin-sdk-entry-imports.ts new file mode 100644 index 0000000000000000000000000000000000000000..2a1d7ddeec1f1e7cd2812c32ac1211d9aeba726b --- /dev/null +++ b/scripts/check-no-monolithic-plugin-sdk-entry-imports.ts @@ -0,0 +1,111 @@ +// Check No Monolithic Plugin Sdk Entry Imports script supports OpenClaw repository automation. +import fs from "node:fs"; +import path from "node:path"; +import { discoverOpenClawPlugins } from "../src/plugins/discovery.js"; +import { collectFilesSync, isCodeFile, relativeToCwd } from "./check-file-utils.js"; + +const LEGACY_BROAD_SUBPATH_PATTERNS = [ + { + pattern: /["']openclaw\/plugin-sdk\/config-runtime["']/, + label: "openclaw/plugin-sdk/config-runtime", + }, + { + pattern: /["']openclaw\/plugin-sdk\/infra-runtime["']/, + label: "openclaw/plugin-sdk/infra-runtime", + }, +] as const; + +function findLegacyBroadSubpathImports(content: string): string[] { + return LEGACY_BROAD_SUBPATH_PATTERNS.filter(({ pattern }) => pattern.test(content)).map( + ({ label }) => label, + ); +} + +function collectPluginSourceFiles(rootDir: string): string[] { + const srcDir = path.join(rootDir, "src"); + if (!fs.existsSync(srcDir)) { + return []; + } + return collectFilesSync(srcDir, { + includeFile: (filePath) => isCodeFile(filePath), + skipDirNames: new Set(["node_modules", "dist", ".git", "coverage"]), + }); +} + +function collectSharedExtensionSourceFiles(): string[] { + return collectPluginSourceFiles(path.join(process.cwd(), "extensions", "shared")); +} + +function collectBundledExtensionSourceFiles(): string[] { + const extensionsDir = path.join(process.cwd(), "extensions"); + let entries: fs.Dirent[]; + try { + entries = fs.readdirSync(extensionsDir, { withFileTypes: true }); + } catch { + return []; + } + + const files: string[] = []; + for (const entry of entries) { + if (!entry.isDirectory() || entry.name === "shared") { + continue; + } + for (const srcFile of collectPluginSourceFiles(path.join(extensionsDir, entry.name))) { + files.push(srcFile); + } + } + return files; +} + +function main() { + const discovery = discoverOpenClawPlugins({}); + const bundledCandidates = discovery.candidates.filter((c) => c.origin === "bundled"); + const filesToCheck = new Set(); + for (const candidate of bundledCandidates) { + filesToCheck.add(candidate.source); + for (const srcFile of collectPluginSourceFiles(candidate.rootDir)) { + filesToCheck.add(srcFile); + } + } + for (const sharedFile of collectSharedExtensionSourceFiles()) { + filesToCheck.add(sharedFile); + } + for (const extensionFile of collectBundledExtensionSourceFiles()) { + filesToCheck.add(extensionFile); + } + + const legacyBroadSubpathOffenders = new Map(); + for (const entryFile of filesToCheck) { + let content; + try { + content = fs.readFileSync(entryFile, "utf8"); + } catch { + continue; + } + const legacyBroadSubpaths = findLegacyBroadSubpathImports(content); + if (legacyBroadSubpaths.length > 0) { + legacyBroadSubpathOffenders.set(entryFile, legacyBroadSubpaths); + } + } + + if (legacyBroadSubpathOffenders.size > 0) { + if (legacyBroadSubpathOffenders.size > 0) { + console.error( + "Bundled plugin source files must not import deprecated broad plugin-sdk subpaths.", + ); + for (const [file, labels] of [...legacyBroadSubpathOffenders.entries()].toSorted( + ([left], [right]) => left.localeCompare(right), + )) { + console.error(`- ${relativeToCwd(file)} (${labels.join(", ")})`); + } + } + console.error("Use focused openclaw/plugin-sdk/ subpaths for bundled plugins."); + process.exit(1); + } + + console.log( + `OK: bundled plugin source files avoid deprecated broad plugin-sdk subpaths (${filesToCheck.size} checked).`, + ); +} + +main(); diff --git a/scripts/check-package-patches.mts b/scripts/check-package-patches.mts new file mode 100644 index 0000000000000000000000000000000000000000..2d898b4445d806b98839007320eaadb9740017ed --- /dev/null +++ b/scripts/check-package-patches.mts @@ -0,0 +1,158 @@ +#!/usr/bin/env node + +// Guards pnpm package patches against unapproved additions. +import { execFileSync } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { asRecord } from "@openclaw/normalization-core/record-coerce"; +import YAML from "yaml"; +import { pnpmLockfileDocuments } from "./lib/pnpm-lockfile-documents.mjs"; + +const ALLOWED_PATCHED_DEPENDENCIES = new Map([ + ["@awesome.me/webawesome@3.12.0", "patches/@awesome.me__webawesome@3.12.0.patch"], + ["@novnc/novnc@1.7.0", "patches/@novnc__novnc@1.7.0.patch"], + ["vitest@5.0.0", "patches/vitest@5.0.0.patch"], + ["baileys@7.0.0-rc12", "patches/baileys@7.0.0-rc12.patch"], + ["baileys@7.0.0-rc13", "patches/baileys@7.0.0-rc13.patch"], + ["matrix-js-sdk@42.3.0", "patches/matrix-js-sdk@42.3.0.patch"], +]); + +const ALLOWED_PATCH_FILES = new Set(["patches/.gitkeep", ...ALLOWED_PATCHED_DEPENDENCIES.values()]); + +type PackagePatchViolation = { file: string; kind: string; detail: string }; + +function listTrackedFiles(cwd: string, patterns: string[]) { + return execFileSync("git", ["ls-files", "-z", "--", ...patterns], { + cwd, + encoding: "utf8", + }) + .split("\0") + .filter(Boolean) + .toSorted((left, right) => left.localeCompare(right)); +} + +function readRecordFile(cwd: string, relativePath: string, parse: (source: string) => unknown) { + const filePath = path.join(cwd, relativePath); + if (!fs.existsSync(filePath)) { + return {}; + } + return asRecord(parse(fs.readFileSync(filePath, "utf8"))); +} + +function collectPatchedDependencyViolations( + file: string, + patchedDependencies: unknown, + violations: PackagePatchViolation[], + options: { allowAnyValueForLegacy?: boolean } = {}, +) { + for (const [specifier, patchPathOrHash] of Object.entries(asRecord(patchedDependencies))) { + if ( + options.allowAnyValueForLegacy === true + ? ALLOWED_PATCHED_DEPENDENCIES.has(specifier) + : ALLOWED_PATCHED_DEPENDENCIES.get(specifier) === patchPathOrHash + ) { + continue; + } + violations.push({ + file, + kind: "patchedDependency", + detail: `${specifier} -> ${String(patchPathOrHash)}`, + }); + } +} + +function collectWorkspacePatchViolations(cwd: string, violations: PackagePatchViolation[]) { + const workspace = readRecordFile(cwd, "pnpm-workspace.yaml", YAML.parse); + collectPatchedDependencyViolations( + "pnpm-workspace.yaml", + workspace.patchedDependencies, + violations, + ); +} + +function collectLockfilePatchViolations(cwd: string, violations: PackagePatchViolation[]) { + const filePath = path.join(cwd, "pnpm-lock.yaml"); + if (!fs.existsSync(filePath)) { + return; + } + for (const document of Object.values(pnpmLockfileDocuments(fs.readFileSync(filePath, "utf8")))) { + if (document === null) { + continue; + } + const lockfile = asRecord(YAML.parse(document)); + collectPatchedDependencyViolations("pnpm-lock.yaml", lockfile.patchedDependencies, violations, { + allowAnyValueForLegacy: true, + }); + } +} + +function collectPackageJsonPatchViolations(cwd: string, violations: PackagePatchViolation[]) { + for (const relativePath of listTrackedFiles(cwd, ["*package.json"])) { + const packageJson = readRecordFile(cwd, relativePath, JSON.parse); + const patchedDependencies = asRecord(asRecord(packageJson.pnpm).patchedDependencies); + for (const [specifier, patchPath] of Object.entries(patchedDependencies)) { + violations.push({ + file: relativePath, + kind: "packageJsonPatchedDependency", + detail: `${specifier} -> ${String(patchPath)}`, + }); + } + } +} + +function collectPatchFileViolations(cwd: string, violations: PackagePatchViolation[]) { + for (const relativePath of listTrackedFiles(cwd, ["*.patch"])) { + if (!fs.existsSync(path.join(cwd, relativePath))) { + continue; + } + if (ALLOWED_PATCH_FILES.has(relativePath)) { + continue; + } + violations.push({ + file: relativePath, + kind: "patchFile", + detail: "new package patch file", + }); + } +} + +/** + * Collects disallowed package patch declarations and patch files. + */ +export function collectPackagePatchViolations(cwd = process.cwd()) { + const violations: PackagePatchViolation[] = []; + collectWorkspacePatchViolations(cwd, violations); + collectLockfilePatchViolations(cwd, violations); + collectPackageJsonPatchViolations(cwd, violations); + collectPatchFileViolations(cwd, violations); + return violations; +} + +/** + * Runs the package patch guard. + */ +export async function main() { + const violations = collectPackagePatchViolations(); + if (violations.length === 0) { + process.stdout.write( + `PASS package patch guard: no new pnpm patches; ${ALLOWED_PATCHED_DEPENDENCIES.size} approved patches allowlisted.\n`, + ); + return; + } + + console.error( + "FAIL package patch guard: new pnpm package patches are not allowed. Upstream the fix, publish a new package version, then bump the dependency instead.", + ); + for (const violation of violations) { + console.error(`- ${violation.file}: ${violation.kind}: ${violation.detail}`); + } + process.exitCode = 1; +} + +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + main().catch((error: unknown) => { + console.error(error); + process.exit(1); + }); +} diff --git a/scripts/check-pairing-account-scope.mts b/scripts/check-pairing-account-scope.mts new file mode 100644 index 0000000000000000000000000000000000000000..98970c6776d57a06cfe1f181f25477e0bfc322a4 --- /dev/null +++ b/scripts/check-pairing-account-scope.mts @@ -0,0 +1,94 @@ +#!/usr/bin/env node + +// Checks pairing config logic for account-scoped allowlist handling. +import ts from "typescript"; +import { createPairingGuardContext } from "./lib/pairing-guard-context.mts"; +import { + collectFileViolations, + getPropertyNameText, + runAsScript, + toLine, +} from "./lib/ts-guard-utils.mts"; + +const { repoRoot, sourceRoots } = createPairingGuardContext(import.meta.url); + +function isUndefinedLikeExpression(node: ts.Node) { + if (ts.isIdentifier(node) && node.text === "undefined") { + return true; + } + return node.kind === ts.SyntaxKind.NullKeyword; +} + +function hasRequiredAccountIdProperty(node: ts.Node) { + if (!ts.isObjectLiteralExpression(node)) { + return false; + } + for (const property of node.properties) { + if (ts.isShorthandPropertyAssignment(property) && property.name.text === "accountId") { + return true; + } + if (!ts.isPropertyAssignment(property)) { + continue; + } + if (getPropertyNameText(property.name) !== "accountId") { + continue; + } + if (isUndefinedLikeExpression(property.initializer)) { + return false; + } + return true; + } + return false; +} + +function findViolations(content: string, filePath: string) { + const sourceFile = ts.createSourceFile(filePath, content, ts.ScriptTarget.Latest, true); + const violations: { line: number; reason: string }[] = []; + + const visit = (node: ts.Node): void => { + if (ts.isCallExpression(node) && ts.isIdentifier(node.expression)) { + const callName = node.expression.text; + if (callName === "readChannelAllowFromStore") { + const accountIdArgument = node.arguments[2]; + if (!accountIdArgument || isUndefinedLikeExpression(accountIdArgument)) { + violations.push({ + line: toLine(sourceFile, node), + reason: "readChannelAllowFromStore call must pass explicit accountId as 3rd arg", + }); + } + } else if (callName === "upsertChannelPairingRequest") { + const firstArg = node.arguments[0]; + if (!firstArg || !hasRequiredAccountIdProperty(firstArg)) { + violations.push({ + line: toLine(sourceFile, node), + reason: "upsertChannelPairingRequest call must include accountId in params", + }); + } + } + } + ts.forEachChild(node, visit); + }; + + visit(sourceFile); + return violations; +} + +async function main() { + const violations = await collectFileViolations({ + sourceRoots, + repoRoot, + findViolations, + }); + + if (violations.length === 0) { + return; + } + + console.error("Found unscoped pairing-store calls:"); + for (const violation of violations) { + console.error(`- ${violation.path}:${violation.line} (${violation.reason})`); + } + process.exit(1); +} + +runAsScript(import.meta.url, main); diff --git a/scripts/check-plugin-gateway-gauntlet.mts b/scripts/check-plugin-gateway-gauntlet.mts new file mode 100644 index 0000000000000000000000000000000000000000..823bf6d0c3d717d968ef89a76b684ba97cbac4b5 --- /dev/null +++ b/scripts/check-plugin-gateway-gauntlet.mts @@ -0,0 +1,1187 @@ +#!/usr/bin/env node + +// Runs plugin lifecycle and gateway QA gauntlet probes with timing metrics. +import type { StdioOptions } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import process from "node:process"; +import { fileURLToPath } from "node:url"; +import { + MAX_TIMER_TIMEOUT_MS, + resolveTimerTimeoutMs, +} from "../packages/normalization-core/src/number-coercion.ts"; +import { normalizeCsvOrLooseStringList } from "../packages/normalization-core/src/string-normalization.ts"; +import { stripLeadingPackageManagerSeparator } from "./lib/arg-utils.mts"; +import { hasUnjoinedWork, runManagedCommand } from "./lib/managed-child-process.mts"; +import { + parseNonNegativeInt, + parsePositiveInt, + parsePositiveNumber, +} from "./lib/numeric-options.mjs"; +import { + buildGauntletPrebuildEnv, + collectGatewayCpuObservations, + collectMetricObservations, + collectPluginsWithRequiredEntries, + collectRequiredPluginEntries, + collectQaBaselineRegressionObservations, + detectCommandDiagnosticFailure, + discoverBundledPluginManifests, + readQaSuiteSummary, + selectPluginEntries, +} from "./lib/plugin-gateway-gauntlet.mts"; +// Termination tests import this entrypoint in a child before publishing readiness. +// Keep its record guard on the dependency-light script seam to avoid startup skew. +import { isRecord } from "./lib/record-shared.mjs"; + +const DEFAULT_QA_SCENARIOS = [ + "channel-chat-baseline", + "memory-failure-fallback", + "gateway-restart-inflight-run", +]; +const DEFAULT_CPU_CORE_WARN = 0.9; +const DEFAULT_HOT_WALL_WARN_MS = 30_000; +const DEFAULT_MAX_RSS_WARN_MB = 1536; +const DEFAULT_QA_PLUGIN_CHUNK_SIZE = 12; +const SINGLE_VALUE_FLAGS = new Set([ + "--build-timeout-ms", + "--command-timeout-ms", + "--cpu-core-warn", + "--hot-wall-warn-ms", + "--limit", + "--max-rss-warn-mb", + "--output-dir", + "--qa-cpu-regression-multiplier", + "--qa-plugin-chunk-size", + "--qa-timeout-ms", + "--qa-wall-regression-multiplier", + "--repo-root", + "--rss-anomaly-multiplier", + "--shard-index", + "--shard-total", + "--wall-anomaly-multiplier", +]); +const COMMAND_OUTPUT_MAX_BUFFER_BYTES = 16 * 1024 * 1024; +const ANSI_PATTERN = new RegExp(String.raw`\u001B\[[0-9;]*m`, "gu"); + +type ProcessSignal = `SIG${string}`; +type PluginEntry = ReturnType[number]; +type CommandAlias = PluginEntry["cliCommandAliases"][number]; +export type GauntletMeasuredRow = { + label: string; + phase: string; + pluginId: string | null; + status: number; + diagnosticFailure: string | null; + diagnosticDetail?: string; + signal: ProcessSignal | null; + timedOut: boolean; + spawnError: { code: string | null; message: string } | null; + logPath: string | null; + logWriteError?: string; + qaSummaryPath?: string; + qaMetrics?: unknown; +} & ReturnType; +export type GauntletMeasuredCommandParams = { + args: string[]; + command: string; + consoleOutputMaxBytes?: number; + cwd: string; + env: NodeJS.ProcessEnv; + label: string; + logDir: string; + maxBufferBytes?: number; + phase: string; + pluginId?: string; + spawnOptions?: { stdio?: StdioOptions }; + timeoutKillGraceMs?: number; + timeoutMs: number; + timeMode?: "none"; +}; +type GauntletObservation = Record & { + coldStart?: boolean; + kind?: string; + phase?: string; +}; +type GauntletContext = { + repoRoot: string; + outputDir: string; + env: NodeJS.ProcessEnv; + matrix: PluginEntry[]; + plugins: PluginEntry[]; + rows: GauntletMeasuredRow[]; + commandTimeoutMs: number; + skipSlashHelp: boolean; + includePluginOwnedCliAliases: boolean; + qaBaseline: boolean; + qaScenarios: string[]; + qaPluginChunkSize: number; + qaTimeoutMs: number; +}; +type QaSummary = NonNullable["summary"]>; + +/** + * Parses plugin gateway gauntlet CLI arguments and env defaults. + */ +export function parseArgs(argv: string[]) { + const args = stripLeadingPackageManagerSeparator(argv); + const pluginIds: string[] = []; + const qaScenarios: string[] = []; + const options = { + repoRoot: process.cwd(), + outputDir: path.join( + process.cwd(), + ".artifacts", + "plugin-gateway-gauntlet", + new Date().toISOString().replace(/[:.]/g, "-"), + ), + pluginIds, + shardTotal: readOptionalPositiveIntEnv("OPENCLAW_PLUGIN_GATEWAY_GAUNTLET_TOTAL") ?? 1, + shardIndex: readOptionalNonNegativeIntEnv("OPENCLAW_PLUGIN_GATEWAY_GAUNTLET_INDEX") ?? 0, + limit: undefined as number | undefined, + skipPrebuild: false, + skipLifecycle: false, + skipQa: false, + qaBaseline: false, + skipSlashHelp: false, + qaScenarios, + qaPluginChunkSize: DEFAULT_QA_PLUGIN_CHUNK_SIZE, + cpuCoreWarn: DEFAULT_CPU_CORE_WARN, + hotWallWarnMs: DEFAULT_HOT_WALL_WARN_MS, + maxRssWarnMb: DEFAULT_MAX_RSS_WARN_MB, + wallAnomalyMultiplier: 3, + rssAnomalyMultiplier: 2.5, + qaCpuRegressionMultiplier: 2, + qaWallRegressionMultiplier: 2, + commandTimeoutMs: 120_000, + buildTimeoutMs: 600_000, + qaTimeoutMs: 900_000, + allowEmpty: false, + failOnObservation: process.env.OPENCLAW_PLUGIN_GATEWAY_GAUNTLET_FAIL_ON_OBSERVATION === "1", + keepRunRoot: process.env.OPENCLAW_PLUGIN_GATEWAY_GAUNTLET_KEEP_RUN_ROOT === "1", + }; + const envIds = normalizeCsvOrLooseStringList(process.env.OPENCLAW_PLUGIN_GATEWAY_GAUNTLET_IDS); + options.pluginIds.push(...envIds); + const seenSingleValueFlags = new Set(); + parseArgv: for (let index = 0; index < args.length; index += 1) { + const arg = args[index]; + if (arg === undefined) { + break; + } + if (SINGLE_VALUE_FLAGS.has(arg)) { + if (seenSingleValueFlags.has(arg)) { + throw new Error(`${arg} was provided more than once`); + } + seenSingleValueFlags.add(arg); + } + const readValue = () => { + const value = args[index + 1]; + if (!value || value.startsWith("-")) { + throw new Error(`Missing value for ${arg}`); + } + index += 1; + return value; + }; + switch (arg) { + case "--": + break parseArgv; + case "--repo-root": + options.repoRoot = path.resolve(readValue()); + break; + case "--output-dir": + options.outputDir = path.resolve(readValue()); + break; + case "--plugin": + options.pluginIds.push(readValue()); + break; + case "--shard-total": + options.shardTotal = parsePositiveInt(readValue(), "--shard-total"); + break; + case "--shard-index": + options.shardIndex = parseNonNegativeInt(readValue(), "--shard-index"); + break; + case "--limit": + options.limit = parsePositiveInt(readValue(), "--limit"); + break; + case "--qa-scenario": + options.qaScenarios.push(readValue()); + break; + case "--qa-plugin-chunk-size": + options.qaPluginChunkSize = parsePositiveInt(readValue(), "--qa-plugin-chunk-size"); + break; + case "--qa-baseline": + options.qaBaseline = true; + break; + case "--cpu-core-warn": + options.cpuCoreWarn = parsePositiveNumber(readValue(), "--cpu-core-warn"); + break; + case "--hot-wall-warn-ms": + options.hotWallWarnMs = parsePositiveInt(readValue(), "--hot-wall-warn-ms"); + break; + case "--max-rss-warn-mb": + options.maxRssWarnMb = parsePositiveNumber(readValue(), "--max-rss-warn-mb"); + break; + case "--wall-anomaly-multiplier": + options.wallAnomalyMultiplier = parsePositiveNumber( + readValue(), + "--wall-anomaly-multiplier", + ); + break; + case "--rss-anomaly-multiplier": + options.rssAnomalyMultiplier = parsePositiveNumber(readValue(), "--rss-anomaly-multiplier"); + break; + case "--qa-cpu-regression-multiplier": + options.qaCpuRegressionMultiplier = parsePositiveNumber( + readValue(), + "--qa-cpu-regression-multiplier", + ); + break; + case "--qa-wall-regression-multiplier": + options.qaWallRegressionMultiplier = parsePositiveNumber( + readValue(), + "--qa-wall-regression-multiplier", + ); + break; + case "--command-timeout-ms": + options.commandTimeoutMs = parsePositiveInt(readValue(), "--command-timeout-ms"); + break; + case "--build-timeout-ms": + options.buildTimeoutMs = parsePositiveInt(readValue(), "--build-timeout-ms"); + break; + case "--qa-timeout-ms": + options.qaTimeoutMs = parsePositiveInt(readValue(), "--qa-timeout-ms"); + break; + case "--skip-prebuild": + options.skipPrebuild = true; + break; + case "--skip-lifecycle": + options.skipLifecycle = true; + break; + case "--skip-qa": + options.skipQa = true; + break; + case "--skip-slash-help": + options.skipSlashHelp = true; + break; + case "--keep-run-root": + options.keepRunRoot = true; + break; + case "--allow-empty": + options.allowEmpty = true; + break; + case "--fail-on-observation": + options.failOnObservation = true; + break; + case "--help": + printHelp(); + process.exit(0); + break; + default: + throw new Error(`Unknown argument: ${arg}`); + } + } + if (options.qaScenarios.length === 0) { + options.qaScenarios = [...DEFAULT_QA_SCENARIOS]; + } + assertNoDuplicateValues(options.pluginIds, "--plugin"); + assertNoDuplicateValues(options.qaScenarios, "--qa-scenario"); + return options; +} + +function printHelp() { + console.log(`Usage: pnpm test:plugins:gateway-gauntlet [options] + +Runs a shardable bundled-plugin lifecycle, slash inventory, and QA gateway perf gauntlet. + +Options: + --plugin Plugin id to include, repeatable + --shard-total Total plugin shards (default: env or 1) + --shard-index Zero-based shard index (default: env or 0) + --limit Limit selected plugins after sharding + --output-dir Artifact directory + --qa-scenario QA Lab scenario id, repeatable + --qa-plugin-chunk-size Plugins enabled per QA run (default: 12) + --qa-baseline Run a no-extra-plugin QA baseline before plugin chunks + --cpu-core-warn Hot CPU threshold (default: 0.9) + --hot-wall-warn-ms Minimum wall time for hot CPU observations (default: 30000) + --max-rss-warn-mb Maximum RSS warning threshold (default: 1536) + --wall-anomaly-multiplier Wall-time anomaly multiplier (default: 3) + --rss-anomaly-multiplier RSS anomaly multiplier (default: 2.5) + --qa-cpu-regression-multiplier QA baseline CPU regression multiplier (default: 2) + --qa-wall-regression-multiplier QA baseline wall regression multiplier (default: 2) + --command-timeout-ms Lifecycle/slash command timeout (default: 120000) + --build-timeout-ms Prebuild command timeout (default: 600000) + --qa-timeout-ms QA chunk timeout (default: 900000) + --skip-prebuild Skip the upfront build used to avoid per-command rebuild noise + --skip-lifecycle Skip plugin install/inspect/disable/enable/doctor/uninstall + --skip-qa Skip QA Lab RPC conversation runs + --skip-slash-help Skip CLI help probes for plugin-declared command aliases + --allow-empty Allow zero-command runs when every active phase is skipped + --fail-on-observation Treat RSS/CPU/wall observation rows as guard failures + --keep-run-root Preserve isolated HOME/state/log temp root after success + +Environment: + OPENCLAW_PLUGIN_GATEWAY_GAUNTLET_IDS Comma-separated plugin ids to include + OPENCLAW_PLUGIN_GATEWAY_GAUNTLET_TOTAL Total plugin shards + OPENCLAW_PLUGIN_GATEWAY_GAUNTLET_INDEX Zero-based shard index + OPENCLAW_PLUGIN_GATEWAY_GAUNTLET_FAIL_ON_OBSERVATION=1 + OPENCLAW_PLUGIN_GATEWAY_GAUNTLET_KEEP_RUN_ROOT=1 + OPENCLAW_PLUGIN_GATEWAY_GAUNTLET_QA_SUMMARY_MAX_BYTES QA summary read ceiling +`); +} + +function assertNoDuplicateValues(values: string[], label: string) { + const seen = new Set(); + for (const value of values) { + const normalized = value.trim(); + if (!normalized) { + continue; + } + if (seen.has(normalized)) { + throw new Error(`Duplicate ${label} value: ${normalized}`); + } + seen.add(normalized); + } +} + +function readOptionalPositiveIntEnv(name: string) { + const raw = process.env[name]; + return raw ? parsePositiveInt(raw, name) : undefined; +} + +function readOptionalNonNegativeIntEnv(name: string) { + const raw = process.env[name]; + return raw ? parseNonNegativeInt(raw, name) : undefined; +} + +function shouldPromoteObservationGuardFailure(observation: GauntletObservation) { + // Setup and the first cold work command are still reported, but they are not + // stable enough to fail the gauntlet's steady-state regression guard. + return observation?.phase !== "prebuild" && observation?.coldStart !== true; +} + +export function buildObservationGuardFailures( + observations: GauntletObservation[], + enabled = false, +) { + if (!enabled) { + return []; + } + return observations + .filter((observation) => shouldPromoteObservationGuardFailure(observation)) + .map((observation) => ({ + kind: `observation:${observation.kind ?? "unknown"}`, + message: `Gauntlet observation threshold exceeded: ${observation.kind ?? "unknown"}`, + observation, + })); +} + +/** + * Builds the command that prepares QA runtime artifacts before gauntlet probes. + */ +export function createGauntletPrebuildCommand(repoRoot: string) { + return { + command: process.execPath, + args: ["--import", "tsx", path.join(repoRoot, "scripts", "build-all.mts"), "qaRuntime"], + }; +} + +function openclawCommand(repoRoot: string, args: string[]) { + return { + command: process.execPath, + args: [path.join(repoRoot, "dist", "entry.js"), ...args], + }; +} + +function builtEntryPath(repoRoot: string) { + return path.join(repoRoot, "dist", "entry.js"); +} + +function selectSlashHelpAliases(plugin: PluginEntry, includePluginOwnedCliAliases: boolean) { + return includePluginOwnedCliAliases + ? plugin.cliCommandAliases + : plugin.cliCommandAliases.filter((entry) => !isPluginOwnedCliAlias(entry)); +} + +function requiresBuiltEntry(options: ReturnType, selectedPlugins: PluginEntry[]) { + if (selectedPlugins.length === 0) { + return false; + } + if (!options.skipLifecycle) { + return true; + } + if (options.skipSlashHelp) { + return false; + } + return selectedPlugins.some((plugin) => selectSlashHelpAliases(plugin, true).length > 0); +} + +function sourceOpenclawCommand(repoRoot: string, args: string[]) { + return { + command: process.execPath, + args: [path.join(repoRoot, "scripts", "run-node.mjs"), ...args], + }; +} + +function chunkArray(values: Value[], chunkSize: number) { + const chunks: Value[][] = []; + for (let index = 0; index < values.length; index += chunkSize) { + chunks.push(values.slice(index, index + chunkSize)); + } + return chunks; +} + +/** + * Converts an output path to a repo-relative path, rejecting paths outside the repo. + */ +export function toRepoRelativePath(repoRoot: string, absolutePath: string) { + const relativePath = path.relative(repoRoot, absolutePath); + if (!relativePath || relativePath.startsWith("..") || path.isAbsolute(relativePath)) { + throw new Error(`Output path must stay inside repo root: ${absolutePath}`); + } + return relativePath; +} + +function validateOutputDir(options: ReturnType, repoRoot: string) { + if (!options.skipQa) { + toRepoRelativePath(repoRoot, path.join(options.outputDir, "qa-suite")); + } +} + +function createIsolatedEnv(repoRoot: string, runRoot: string) { + const home = path.join(runRoot, "home"); + const stateDir = path.join(runRoot, "state"); + const configPath = path.join(stateDir, "openclaw.json"); + fs.mkdirSync(home, { recursive: true }); + fs.mkdirSync(stateDir, { recursive: true }); + fs.writeFileSync( + configPath, + `${JSON.stringify({ logging: { file: path.join(runRoot, "logs", "openclaw.log") } })}\n`, + { encoding: "utf8", mode: 0o600, flag: "wx" }, + ); + return { + ...process.env, + HOME: home, + XDG_CONFIG_HOME: path.join(home, ".config"), + XDG_CACHE_HOME: path.join(home, ".cache"), + XDG_DATA_HOME: path.join(home, ".local", "share"), + OPENCLAW_STATE_DIR: stateDir, + OPENCLAW_CONFIG_PATH: configPath, + OPENCLAW_QA_SUITE_PROGRESS: process.env.OPENCLAW_QA_SUITE_PROGRESS ?? "1", + PATH: process.env.PATH, + PWD: repoRoot, + }; +} + +function hasUsrBinTime() { + return fs.existsSync("/usr/bin/time"); +} + +function timeWrapperArgs(command: string, args: string[]) { + if (!hasUsrBinTime()) { + return { command, args, mode: "none" }; + } + if (process.platform === "darwin") { + return { command: "/usr/bin/time", args: ["-l", command, ...args], mode: "bsd" }; + } + return { command: "/usr/bin/time", args: ["-v", command, ...args], mode: "gnu" }; +} + +/** + * Parses `/usr/bin/time` output into wall, CPU, and RSS metrics. + */ +export function parseTimedMetrics(stderr: string, wallMs: number, mode: string) { + let userSeconds: number | null = null; + let systemSeconds: number | null = null; + let maxRssMb: number | null = null; + if (mode === "gnu") { + userSeconds = parseLastFloat(stderr, /^\s*User time \(seconds\):\s*([0-9.]+)\s*$/gmu); + systemSeconds = parseLastFloat(stderr, /^\s*System time \(seconds\):\s*([0-9.]+)\s*$/gmu); + const maxRssKb = parseLastFloat( + stderr, + /^\s*Maximum resident set size \(kbytes\):\s*([0-9.]+)\s*$/gmu, + ); + maxRssMb = maxRssKb == null ? null : maxRssKb / 1024; + } else if (mode === "bsd") { + const cpuLine = parseLastMatch( + stderr, + /^\s*[0-9.]+\s+real\s+([0-9.]+)\s+user\s+([0-9.]+)\s+sys\s*$/gmu, + ); + userSeconds = parseMatchFloat(cpuLine, 1); + systemSeconds = parseMatchFloat(cpuLine, 2); + const maxRssBytes = parseLastFloat(stderr, /^\s*([0-9]+)\s+maximum resident set size\s*$/gmu); + maxRssMb = maxRssBytes == null ? null : maxRssBytes / 1024 / 1024; + } + const cpuMs = + userSeconds == null && systemSeconds == null + ? null + : ((userSeconds ?? 0) + (systemSeconds ?? 0)) * 1000; + return { + wallMs, + cpuMs, + cpuCoreRatio: cpuMs == null || wallMs <= 0 ? null : cpuMs / wallMs, + maxRssMb, + }; +} + +function parseLastMatch(value: string, pattern: RegExp) { + let lastMatch: RegExpMatchArray | null = null; + for (const match of value.matchAll(pattern)) { + lastMatch = match; + } + return lastMatch; +} + +function parseMatchFloat(match: RegExpMatchArray | null, index: number) { + if (!match) { + return null; + } + const parsed = Number(match[index]); + return Number.isFinite(parsed) ? parsed : null; +} + +function parseLastFloat(value: string, pattern: RegExp) { + return parseMatchFloat(parseLastMatch(value, pattern), 1); +} + +function stripAnsi(value: string) { + return value.replace(ANSI_PATTERN, ""); +} + +function resolveOptionalTimerTimeoutMs(valueMs: number | undefined) { + return valueMs === undefined || valueMs <= 0 + ? null + : resolveTimerTimeoutMs(valueMs, MAX_TIMER_TIMEOUT_MS); +} + +function writeCommandLog(params: { + logDir: string; + label: string; + command: string[]; + stdout: string; + stderr: string; +}) { + const { logDir, label, stdout, stderr } = params; + fs.mkdirSync(logDir, { recursive: true }); + const safeLabel = label.replace(/[^a-zA-Z0-9_.-]+/gu, "_"); + const logPath = path.join(logDir, `${safeLabel}.log`); + fs.writeFileSync( + logPath, + [`$ ${params.command.join(" ")}`, "", stripAnsi(stdout), stripAnsi(stderr)].join("\n"), + "utf8", + ); + return logPath; +} + +/** + * Runs one command with optional timing wrapper, bounded output, and log capture. + */ +export async function runMeasuredCommand( + params: GauntletMeasuredCommandParams, +): Promise { + const { command, args, mode } = + params.timeMode === "none" + ? { command: params.command, args: params.args, mode: "none" } + : timeWrapperArgs(params.command, params.args); + const started = performance.now(); + let stdout = ""; + let stderr = ""; + let stdoutBytes = 0; + let stderrBytes = 0; + let stdoutRelayBytes = 0; + let stderrRelayBytes = 0; + let stdoutTruncated = false; + let stderrTruncated = false; + let stdoutRelayTruncated = false; + let stderrRelayTruncated = false; + let spawnError: GauntletMeasuredRow["spawnError"] = null; + let timedOut = false; + let exitStatus: number | null = null; + let exitSignal: ProcessSignal | null = null; + let parentTerminationSignal: NodeJS.Signals | undefined; + let commandError: unknown; + const maxBufferBytes = params.maxBufferBytes ?? COMMAND_OUTPUT_MAX_BUFFER_BYTES; + const maxRelayBytes = params.consoleOutputMaxBytes ?? maxBufferBytes; + const timeoutMs = resolveOptionalTimerTimeoutMs(params.timeoutMs); + const timeoutKillGraceMs = resolveTimerTimeoutMs( + params.timeoutKillGraceMs ?? 5_000, + MAX_TIMER_TIMEOUT_MS, + ); + const appendCapturedOutput = (streamName: "stdout" | "stderr", chunk: string | Uint8Array) => { + const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); + const currentBytes = streamName === "stdout" ? stdoutBytes : stderrBytes; + const alreadyTruncated = streamName === "stdout" ? stdoutTruncated : stderrTruncated; + if (alreadyTruncated) { + return; + } + const remainingBytes = maxBufferBytes - currentBytes; + const appendTruncation = () => { + const message = `\n[${streamName} truncated after ${maxBufferBytes} bytes]\n`; + if (streamName === "stdout") { + stdout += message; + stdoutTruncated = true; + } else { + stderr += message; + stderrTruncated = true; + } + }; + if (remainingBytes <= 0) { + appendTruncation(); + return; + } + const capturedBuffer = + buffer.length > remainingBytes ? buffer.subarray(0, remainingBytes) : buffer; + if (streamName === "stdout") { + stdout += capturedBuffer.toString("utf8"); + stdoutBytes += capturedBuffer.length; + } else { + stderr += capturedBuffer.toString("utf8"); + stderrBytes += capturedBuffer.length; + } + if (buffer.length > remainingBytes) { + appendTruncation(); + } + }; + const relayOutput = (streamName: "stdout" | "stderr", chunk: string | Uint8Array) => { + const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); + const currentBytes = streamName === "stdout" ? stdoutRelayBytes : stderrRelayBytes; + const alreadyTruncated = streamName === "stdout" ? stdoutRelayTruncated : stderrRelayTruncated; + if (alreadyTruncated) { + return; + } + const write = + streamName === "stdout" + ? process.stdout.write.bind(process.stdout) + : process.stderr.write.bind(process.stderr); + const markTruncated = () => { + write(`\n[${streamName} relay truncated after ${maxRelayBytes} bytes]\n`); + if (streamName === "stdout") { + stdoutRelayTruncated = true; + } else { + stderrRelayTruncated = true; + } + }; + const remainingBytes = maxRelayBytes - currentBytes; + if (remainingBytes <= 0) { + markTruncated(); + return; + } + const relayedBuffer = + buffer.length > remainingBytes ? buffer.subarray(0, remainingBytes) : buffer; + if (relayedBuffer.length > 0) { + write(relayedBuffer.toString("utf8")); + } + if (streamName === "stdout") { + stdoutRelayBytes += relayedBuffer.length; + } else { + stderrRelayBytes += relayedBuffer.length; + } + if (buffer.length > remainingBytes) { + markTruncated(); + } + }; + const appendOutput = (streamName: "stdout" | "stderr", chunk: string | Uint8Array) => { + relayOutput(streamName, chunk); + appendCapturedOutput(streamName, chunk); + }; + try { + await runManagedCommand({ + bin: command, + args, + cwd: params.cwd, + env: params.env, + shell: false, + stdio: (mode === "none" ? params.spawnOptions?.stdio : undefined) ?? "pipe", + timeoutMs: timeoutMs ?? undefined, + timeoutKillGraceMs, + signalKillGraceMs: timeoutKillGraceMs, + // Windows retains the managed owner's ordinary completion contract. + requireProcessTreeExit: process.platform !== "win32", + onSignal(signal) { + parentTerminationSignal ??= signal; + }, + onReady(child) { + child.once("exit", (status, signal) => { + exitStatus = status; + exitSignal = signal; + }); + child.stdout?.on("data", (chunk) => appendOutput("stdout", chunk)); + child.stderr?.on("data", (chunk) => appendOutput("stderr", chunk)); + }, + }); + } catch (error) { + commandError = error; + const code = + error && typeof error === "object" && "code" in error && typeof error.code === "string" + ? error.code + : null; + spawnError = { + code, + message: error instanceof Error ? error.message : String(error), + }; + timedOut = code === "ETIMEDOUT"; + } + if (parentTerminationSignal && !hasUnjoinedWork(commandError)) { + // Preserve signal termination after cleanup, without admitting another probe. + const terminationSignal = parentTerminationSignal; + return await new Promise(() => { + process.kill(process.pid, terminationSignal); + }); + } + const wallMs = performance.now() - started; + const finalStatus = + spawnError && !timedOut ? 1 : (exitStatus ?? (exitSignal || spawnError ? 1 : 0)); + const finalStderr = [ + stderr, + spawnError ? `[spawn error] ${spawnError.code ?? "unknown"} ${spawnError.message}` : "", + ] + .filter(Boolean) + .join("\n"); + let logPath: string | null = null; + let logWriteError: string | null = null; + try { + logPath = writeCommandLog({ + logDir: params.logDir, + label: params.label, + command: [params.command, ...params.args], + stdout, + stderr: finalStderr, + }); + } catch (error) { + logWriteError = error instanceof Error ? error.message : String(error); + } + if (hasUnjoinedWork(commandError)) { + throw commandError; + } + const outputDiagnosticFailure = detectCommandDiagnosticFailure(stdout, finalStderr); + const diagnosticFailure = + outputDiagnosticFailure ?? (logWriteError ? "command-log-write-failure" : null); + return { + label: params.label, + phase: params.phase, + pluginId: params.pluginId ?? null, + status: logWriteError ? 1 : finalStatus, + diagnosticFailure, + signal: exitSignal, + timedOut, + spawnError, + logPath, + ...(logWriteError ? { logWriteError } : {}), + ...parseTimedMetrics(finalStderr, wallMs, mode), + }; +} + +/** + * Reports whether gauntlet result rows contain work beyond the prebuild step. + */ +export function hasGauntletWorkRows(rows: Array>) { + return rows.some((row) => row.phase !== "prebuild"); +} + +function isPluginOwnedCliAlias(alias: CommandAlias) { + return alias.kind === "runtime-slash" && alias.cliCommand === alias.name; +} + +function buildSlashHelpProbe( + params: GauntletContext & { + plugin: PluginEntry; + alias: CommandAlias; + }, +) { + const command = params.alias.cliCommand ?? params.alias.name; + return { + cwd: params.repoRoot, + env: params.env, + logDir: path.join(params.outputDir, "logs", "slash-help"), + ...openclawCommand(params.repoRoot, [command, "--help"]), + label: `${params.plugin.id}-slash-${params.alias.name}`, + phase: "slash:help", + pluginId: params.plugin.id, + timeoutMs: params.commandTimeoutMs, + }; +} + +async function runPluginLifecycleCommand( + params: GauntletContext & { + logPluginId: string; + label: string; + phase: string; + args: string[]; + pluginId: string; + }, +) { + process.stderr.write(`[plugin-gauntlet] ${params.logPluginId} ${params.phase}\n`); + params.rows.push( + await runMeasuredCommand({ + cwd: params.repoRoot, + env: params.env, + logDir: path.join(params.outputDir, "logs", "lifecycle"), + ...openclawCommand(params.repoRoot, ["plugins", ...params.args]), + label: params.label, + phase: `lifecycle:${params.phase}`, + pluginId: params.pluginId, + timeoutMs: params.commandTimeoutMs, + }), + ); +} + +async function runPluginLifecycle(params: GauntletContext) { + for (const plugin of params.plugins) { + const requiredPlugins = collectRequiredPluginEntries(params.matrix, [plugin]); + for (const requiredPlugin of requiredPlugins) { + await runPluginLifecycleCommand({ + ...params, + logPluginId: plugin.id, + label: `${plugin.id}-requires-${requiredPlugin.id}-install`, + phase: `requires:${requiredPlugin.id}:install`, + args: ["install", requiredPlugin.id], + pluginId: requiredPlugin.id, + }); + } + type LifecycleCommand = + | { phase: string; args: string[] } + | { phase: string; alias: CommandAlias }; + const commands = [ + { + phase: "install", + args: ["install", plugin.id], + }, + { phase: "inspect", args: ["inspect", plugin.id, "--json"] }, + ...(params.skipSlashHelp + ? [] + : plugin.cliCommandAliases + .filter(isPluginOwnedCliAlias) + .map((alias) => ({ phase: `slash-help:${alias.name}`, alias }))), + { phase: "disable", args: ["disable", plugin.id] }, + ...(plugin.hasRequiredConfigFields ? [] : [{ phase: "enable", args: ["enable", plugin.id] }]), + { phase: "doctor", args: ["doctor"] }, + { phase: "uninstall", args: ["uninstall", plugin.id, "--force"] }, + ] satisfies LifecycleCommand[]; + for (const command of commands) { + if ("alias" in command) { + process.stderr.write(`[plugin-gauntlet] ${plugin.id} ${command.phase}\n`); + params.rows.push( + await runMeasuredCommand({ + ...buildSlashHelpProbe({ + ...params, + plugin, + alias: command.alias, + }), + label: `${plugin.id}-${command.phase}`, + }), + ); + continue; + } + await runPluginLifecycleCommand({ + ...params, + logPluginId: plugin.id, + label: `${plugin.id}-${command.phase}`, + phase: command.phase, + args: command.args, + pluginId: plugin.id, + }); + } + for (const requiredPlugin of requiredPlugins.toReversed()) { + await runPluginLifecycleCommand({ + ...params, + logPluginId: plugin.id, + label: `${plugin.id}-requires-${requiredPlugin.id}-uninstall`, + phase: `requires:${requiredPlugin.id}:uninstall`, + args: ["uninstall", requiredPlugin.id, "--force"], + pluginId: requiredPlugin.id, + }); + } + } +} + +async function runSlashHelpProbes(params: GauntletContext) { + for (const plugin of params.plugins) { + const aliases = selectSlashHelpAliases(plugin, params.includePluginOwnedCliAliases); + for (const alias of aliases) { + process.stderr.write(`[plugin-gauntlet] ${plugin.id} slash-help /${alias.name}\n`); + params.rows.push( + await runMeasuredCommand({ + ...buildSlashHelpProbe({ + ...params, + plugin, + alias, + }), + }), + ); + } + } +} + +async function runQaChunks(params: GauntletContext) { + const chunks = [ + ...(params.qaBaseline ? [{ label: "baseline", plugins: [] }] : []), + ...chunkArray(params.plugins, params.qaPluginChunkSize).map((plugins, index) => ({ + label: `chunk-${String(index).padStart(2, "0")}`, + plugins, + })), + ]; + const summaries: QaSummary[] = []; + for (const [index, chunk] of chunks.entries()) { + const outputDir = path.join(params.outputDir, "qa-suite", chunk.label); + const outputArg = toRepoRelativePath(params.repoRoot, outputDir); + const pluginIds = chunk.plugins.map((plugin) => plugin.id); + const enabledPluginIds = collectPluginsWithRequiredEntries(params.matrix, chunk.plugins).map( + (plugin) => plugin.id, + ); + const pluginIdLabel = pluginIds.length > 0 ? pluginIds.join(",") : ""; + process.stderr.write( + `[plugin-gauntlet] qa chunk ${index + 1}/${chunks.length}: ${pluginIdLabel}\n`, + ); + const row = await runMeasuredCommand({ + cwd: params.repoRoot, + env: params.env, + logDir: path.join(params.outputDir, "logs", "qa-suite"), + ...sourceOpenclawCommand(params.repoRoot, [ + "qa", + "suite", + "--provider-mode", + "mock-openai", + "--concurrency", + "1", + "--output-dir", + outputArg, + ...params.qaScenarios.flatMap((scenario) => ["--scenario", scenario]), + ...enabledPluginIds.flatMap((pluginId) => ["--enable-plugin", pluginId]), + ]), + label: `qa-${chunk.label}`, + phase: "qa:rpc", + timeoutMs: params.qaTimeoutMs, + }); + const summaryPath = path.join(outputDir, "qa-suite-summary.json"); + const qaSummaryResult = readQaSuiteSummary(summaryPath); + const qaDiagnosticFailure = + row.status === 0 && !row.timedOut ? qaSummaryResult.diagnosticFailure : null; + params.rows.push({ + ...row, + pluginId: pluginIdLabel, + qaSummaryPath: summaryPath, + ...(qaDiagnosticFailure ? { diagnosticFailure: qaDiagnosticFailure } : {}), + ...(qaSummaryResult.diagnosticDetail + ? { diagnosticDetail: qaSummaryResult.diagnosticDetail } + : {}), + ...(qaSummaryResult.summary?.metrics ? { qaMetrics: qaSummaryResult.summary.metrics } : {}), + }); + if (qaSummaryResult.summary) { + summaries.push(qaSummaryResult.summary); + } + } + return summaries; +} + +async function main() { + const options = parseArgs(process.argv.slice(2)); + const repoRoot = path.resolve(options.repoRoot); + validateOutputDir(options, repoRoot); + fs.mkdirSync(options.outputDir, { recursive: true }); + const runRoot = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-plugin-gauntlet-")); + let preserveRunRoot = options.keepRunRoot; + try { + const env = createIsolatedEnv(repoRoot, runRoot); + const matrix = discoverBundledPluginManifests(repoRoot); + const selectedPlugins = selectPluginEntries(matrix, { + ids: options.pluginIds, + shardTotal: options.shardTotal, + shardIndex: options.shardIndex, + limit: options.limit, + }); + const selectedPluginsWithRequired = collectPluginsWithRequiredEntries(matrix, selectedPlugins); + const rows: GauntletMeasuredRow[] = []; + const commandEnv = buildGauntletPrebuildEnv(env, { + includePrivateQa: !options.skipQa, + buildIds: selectedPluginsWithRequired.map((plugin) => plugin.buildId), + skipDeclarationBuild: true, + }); + const context = { + repoRoot, + outputDir: options.outputDir, + env: commandEnv, + matrix, + plugins: selectedPlugins, + rows, + commandTimeoutMs: options.commandTimeoutMs, + skipSlashHelp: options.skipSlashHelp, + includePluginOwnedCliAliases: options.skipLifecycle, + qaBaseline: options.qaBaseline, + qaScenarios: options.qaScenarios, + qaPluginChunkSize: options.qaPluginChunkSize, + qaTimeoutMs: options.qaTimeoutMs, + }; + if (!options.skipPrebuild && (selectedPlugins.length > 0 || !options.skipQa)) { + process.stderr.write("[plugin-gauntlet] prebuild\n"); + const prebuildCommand = createGauntletPrebuildCommand(repoRoot); + rows.push( + await runMeasuredCommand({ + cwd: repoRoot, + env: commandEnv, + logDir: path.join(options.outputDir, "logs", "prebuild"), + command: prebuildCommand.command, + args: prebuildCommand.args, + label: "prebuild", + phase: "prebuild", + timeoutMs: options.buildTimeoutMs, + }), + ); + } + const prebuildFailed = rows.some( + (row) => row.phase === "prebuild" && (row.status !== 0 || row.timedOut), + ); + const entryPath = builtEntryPath(repoRoot); + const missingSkippedPrebuildEntry = + selectedPlugins.length > 0 && + options.skipPrebuild && + requiresBuiltEntry(options, selectedPlugins) && + !fs.existsSync(entryPath); + if (!prebuildFailed && !missingSkippedPrebuildEntry && !options.skipLifecycle) { + await runPluginLifecycle(context); + } + if (!prebuildFailed && !missingSkippedPrebuildEntry && !options.skipSlashHelp) { + await runSlashHelpProbes(context); + } + const qaSummaries = options.skipQa || prebuildFailed ? [] : await runQaChunks(context); + const metricObservations = collectMetricObservations(rows, { + cpuCoreWarn: options.cpuCoreWarn, + hotWallWarnMs: options.hotWallWarnMs, + maxRssWarnMb: options.maxRssWarnMb, + wallAnomalyMultiplier: options.wallAnomalyMultiplier, + rssAnomalyMultiplier: options.rssAnomalyMultiplier, + }); + const qaBaselineObservations = collectQaBaselineRegressionObservations(rows, { + cpuRegressionMultiplier: options.qaCpuRegressionMultiplier, + wallRegressionMultiplier: options.qaWallRegressionMultiplier, + }); + const gatewayObservations = qaSummaries.flatMap((qa) => + collectGatewayCpuObservations({ + startup: null, + qa: isRecord(qa.metrics) ? { metrics: qa.metrics } : undefined, + cpuCoreWarn: options.cpuCoreWarn, + hotWallWarnMs: options.hotWallWarnMs, + }), + ); + const failures = rows.filter( + (row) => row.status !== 0 || row.timedOut || row.diagnosticFailure, + ); + const observations = [...metricObservations, ...qaBaselineObservations, ...gatewayObservations]; + const guardFailures: Array<{ kind: string; message: string }> = []; + if (missingSkippedPrebuildEntry) { + guardFailures.push({ + kind: "missing-built-entry", + message: + `${path.relative(repoRoot, entryPath)} is missing; ` + + "run without --skip-prebuild or build the gauntlet runtime first.", + }); + } + if (!hasGauntletWorkRows(rows) && !options.allowEmpty && guardFailures.length === 0) { + guardFailures.push({ + kind: "empty-run", + message: + "No lifecycle, slash-help, or QA gauntlet commands ran; remove a skip flag or pass --allow-empty for intentional dry runs.", + }); + } + guardFailures.push(...buildObservationGuardFailures(observations, options.failOnObservation)); + const hasFailures = failures.length > 0 || guardFailures.length > 0; + preserveRunRoot = preserveRunRoot || hasFailures; + let cleanupError = null; + if (!preserveRunRoot) { + try { + fs.rmSync(runRoot, { recursive: true, force: true }); + } catch (error) { + cleanupError = error instanceof Error ? error.message : String(error); + preserveRunRoot = true; + } + } + const summary = { + generatedAt: new Date().toISOString(), + repoRoot, + outputDir: options.outputDir, + isolatedRunRoot: runRoot, + isolatedRunRootPreserved: preserveRunRoot, + isolatedRunRootCleanupError: cleanupError, + selectedPluginCount: selectedPlugins.length, + totalPluginCount: matrix.length, + options: { + pluginIds: options.pluginIds, + shardTotal: options.shardTotal, + shardIndex: options.shardIndex, + limit: options.limit ?? null, + qaScenarios: options.qaScenarios, + qaPluginChunkSize: options.qaPluginChunkSize, + qaBaseline: options.qaBaseline, + allowEmpty: options.allowEmpty, + failOnObservation: options.failOnObservation, + keepRunRoot: options.keepRunRoot, + skipLifecycle: options.skipLifecycle, + skipQa: options.skipQa, + skipSlashHelp: options.skipSlashHelp, + skipPrebuild: options.skipPrebuild, + thresholds: { + cpuCoreWarn: options.cpuCoreWarn, + hotWallWarnMs: options.hotWallWarnMs, + maxRssWarnMb: options.maxRssWarnMb, + wallAnomalyMultiplier: options.wallAnomalyMultiplier, + rssAnomalyMultiplier: options.rssAnomalyMultiplier, + qaCpuRegressionMultiplier: options.qaCpuRegressionMultiplier, + qaWallRegressionMultiplier: options.qaWallRegressionMultiplier, + }, + }, + matrix, + selectedPlugins, + rows, + observations, + failures, + guardFailures, + }; + const summaryPath = path.join(options.outputDir, "plugin-gateway-gauntlet-summary.json"); + fs.writeFileSync(summaryPath, `${JSON.stringify(summary, null, 2)}\n`, "utf8"); + process.stdout.write(`[plugin-gauntlet] summary: ${summaryPath}\n`); + process.stdout.write( + `[plugin-gauntlet] plugins=${selectedPlugins.length}/${matrix.length} rows=${rows.length} failures=${failures.length} observations=${summary.observations.length}\n`, + ); + if (preserveRunRoot) { + process.stdout.write(`[plugin-gauntlet] isolated run root preserved: ${runRoot}\n`); + } + for (const failure of failures) { + process.stdout.write( + `[plugin-gauntlet] failure phase=${failure.phase} plugin=${failure.pluginId ?? ""} status=${failure.status} timedOut=${failure.timedOut} diagnostic=${failure.diagnosticFailure ?? ""} wallMs=${Math.round(failure.wallMs)} log=${failure.logPath}\n`, + ); + } + for (const failure of guardFailures) { + process.stdout.write(`[plugin-gauntlet] failure ${failure.kind}: ${failure.message}\n`); + } + for (const observation of summary.observations.slice(0, 20)) { + process.stdout.write(`[plugin-gauntlet] observation ${JSON.stringify(observation)}\n`); + } + if (hasFailures) { + process.exitCode = 1; + } + } catch (error) { + const unjoined = hasUnjoinedWork(error); + if (unjoined) { + process.stderr.write(`[plugin-gauntlet] isolated run root preserved: ${runRoot}\n`); + } + if (!options.keepRunRoot && !unjoined) { + try { + fs.rmSync(runRoot, { recursive: true, force: true }); + } catch (cleanupError) { + process.stderr.write( + `[plugin-gauntlet] failed to clean isolated run root ${runRoot}: ${ + cleanupError instanceof Error ? cleanupError.message : String(cleanupError) + }\n`, + ); + } + } + throw error; + } +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === path.resolve(process.argv[1])) { + main().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + }); +} diff --git a/scripts/check-plugin-sdk-wildcard-reexports.mts b/scripts/check-plugin-sdk-wildcard-reexports.mts new file mode 100644 index 0000000000000000000000000000000000000000..7e60ab61abe4a5ec1be963fa4ea1773d4e8674aa --- /dev/null +++ b/scripts/check-plugin-sdk-wildcard-reexports.mts @@ -0,0 +1,31 @@ +#!/usr/bin/env node + +// Rejects wildcard plugin SDK re-exports in extension API barrels. +import { + createExtensionWildcardReexportScanner, + type ExtensionWildcardReexportPolicy, +} from "./lib/extension-wildcard-reexport-scanner.mts"; + +const WILDCARD_PLUGIN_SDK_REEXPORT_PATTERN = + /^\s*export\s+(?:type\s+)?\*\s+(?:as\s+[$\w]+\s+)?from\s+["']openclaw\/plugin-sdk\//u; +const policy = { + // SDK wildcard exposure is only a public extension-root barrel policy. + fileScope: "extension-root-api-files", + pattern: WILDCARD_PLUGIN_SDK_REEXPORT_PATTERN, + successMessage: "No plugin-sdk wildcard re-exports found in extension API barrels.", + findingsMessage: "Found plugin-sdk wildcard re-exports in extension API barrels:", + remediationMessage: "Use explicit named exports from the narrow SDK subpath instead.", +} satisfies ExtensionWildcardReexportPolicy; +const scanner = createExtensionWildcardReexportScanner(policy); + +/** + * Finds wildcard plugin SDK re-export lines in an extension API barrel. + */ +export const findPluginSdkWildcardReexports = scanner.findLines; + +/** + * Runs the plugin SDK wildcard re-export guard. + */ +export const main = scanner.main; + +await scanner.exitIfMain(import.meta.url); diff --git a/scripts/check-policy-config-coverage.ts b/scripts/check-policy-config-coverage.ts new file mode 100644 index 0000000000000000000000000000000000000000..f48a566d0a81621f31e1df47845b6925390372a3 --- /dev/null +++ b/scripts/check-policy-config-coverage.ts @@ -0,0 +1,232 @@ +#!/usr/bin/env node +import fs from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import JSON5 from "json5"; +import { + renderConfigDocBaselineArtifacts, + type ConfigDocBaselineEntry, +} from "../src/config/doc-baseline.js"; + +type ClassificationStatus = "observed" | "ignored" | "out-of-scope" | "deferred"; + +type CoverageClassification = { + readonly pattern: string; + readonly status: ClassificationStatus; + readonly area: string; + readonly policy?: string; + readonly reason: string; + readonly allowNoSchemaPath?: boolean; +}; + +type CoverageConfig = { + readonly monitored: readonly string[]; + readonly classifications: readonly CoverageClassification[]; +}; + +type ConfigDocBaseline = { + readonly coreEntries: readonly ConfigDocBaselineEntry[]; + readonly channelEntries: readonly ConfigDocBaselineEntry[]; + readonly pluginEntries: readonly ConfigDocBaselineEntry[]; +}; + +function flattenConfigDocBaselineEntries( + baseline: ConfigDocBaseline, +): readonly ConfigDocBaselineEntry[] { + return [...baseline.coreEntries, ...baseline.channelEntries, ...baseline.pluginEntries]; +} + +type ClassifiedEntry = { + readonly path: string; + readonly kind: ConfigDocBaselineEntry["kind"]; + readonly classification?: CoverageClassification; +}; + +type UnmatchedMonitoredPattern = { + readonly pattern: string; +}; + +const args = new Set(process.argv.slice(2)); +const json = args.has("--json"); +const check = args.has("--check"); +const showCovered = args.has("--show-covered"); + +if (args.has("--help")) { + console.log(`Usage: pnpm policy:config-coverage [--check] [--json] [--show-covered] + +Internal maintainer report for Policy config coverage. + +Default mode is report-only and exits 0 even when paths are unclassified. +Use --check when a policy maintainer intentionally wants unclassified or stale +coverage entries to fail locally.`); + process.exit(0); +} + +const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const configPath = path.join(repoRoot, "scripts/lib/policy-config-coverage.jsonc"); + +const config = JSON5.parse(await fs.readFile(configPath, "utf8")) as CoverageConfig; +const { baseline } = await renderConfigDocBaselineArtifacts(); +const monitoredEntries = flattenConfigDocBaselineEntries(baseline) + .filter((entry) => !entry.hasChildren) + .filter((entry) => matchesAny(config.monitored, entry.path)) + .toSorted((left, right) => left.path.localeCompare(right.path)); +const leafEntries = flattenConfigDocBaselineEntries(baseline).filter((entry) => !entry.hasChildren); +const unmatchedMonitored = config.monitored + .filter( + (pattern) => + !leafEntries.some((entry) => pathMatchesPattern(pattern, entry.path)) && + !config.classifications.some( + (item) => item.allowNoSchemaPath === true && pathMatchesPattern(item.pattern, pattern), + ), + ) + .map((pattern) => ({ pattern })) + .toSorted((left, right) => left.pattern.localeCompare(right.pattern)); + +const classified: ClassifiedEntry[] = monitoredEntries.map((entry) => ({ + path: entry.path, + kind: entry.kind, + classification: config.classifications.find((item) => + pathMatchesPattern(item.pattern, entry.path), + ), +})); +const unclassified = classified.filter((entry) => entry.classification === undefined); +const stale = config.classifications.filter( + (item) => + item.allowNoSchemaPath !== true && + !monitoredEntries.some((entry) => pathMatchesPattern(item.pattern, entry.path)), +); +const summaryCounts = summarize(classified); + +if (json) { + console.log( + JSON.stringify( + { + ok: unclassified.length === 0 && stale.length === 0 && unmatchedMonitored.length === 0, + monitoredPaths: monitoredEntries.length, + counts: summaryCounts, + unclassified, + unmatchedMonitored, + stale, + }, + null, + 2, + ), + ); +} else { + printTextReport({ + monitoredPaths: monitoredEntries.length, + counts: summaryCounts, + unclassified, + unmatchedMonitored, + stale, + classified, + }); +} + +if (check && (unclassified.length > 0 || stale.length > 0 || unmatchedMonitored.length > 0)) { + process.exit(1); +} + +function printTextReport(input: { + readonly monitoredPaths: number; + readonly counts: Record; + readonly unclassified: readonly ClassifiedEntry[]; + readonly unmatchedMonitored: readonly UnmatchedMonitoredPattern[]; + readonly stale: readonly CoverageClassification[]; + readonly classified: readonly ClassifiedEntry[]; +}): void { + console.log(`Policy config coverage: ${input.monitoredPaths} monitored config leaf paths`); + for (const [key, count] of Object.entries(input.counts).toSorted(([a], [b]) => + a.localeCompare(b), + )) { + console.log(` ${key}: ${count}`); + } + + if (input.unclassified.length > 0) { + console.log("\nUnclassified config paths:"); + for (const entry of input.unclassified) { + console.log(` - ${entry.path} (${entry.kind})`); + } + console.log( + "\nClassify each as observed, ignored, out-of-scope, or deferred in scripts/lib/policy-config-coverage.jsonc.", + ); + } else { + console.log("\nNo unclassified monitored config paths."); + } + + if (input.unmatchedMonitored.length > 0) { + console.log("\nMonitored patterns with no matching config paths:"); + for (const entry of input.unmatchedMonitored) { + console.log(` - ${entry.pattern}`); + } + } else { + console.log("\nNo monitored patterns without matching config paths."); + } + + if (input.stale.length > 0) { + console.log("\nStale coverage classifications:"); + for (const entry of input.stale) { + console.log(` - ${entry.pattern} (${entry.area}, ${entry.status})`); + } + } + + if (showCovered) { + console.log("\nCovered paths:"); + for (const entry of input.classified) { + const classification = entry.classification; + console.log( + ` - ${entry.path}: ${classification?.area ?? "unclassified"} / ${ + classification?.status ?? "unclassified" + }`, + ); + } + } +} + +function summarize(entries: readonly ClassifiedEntry[]): Record { + const counts: Record = {}; + for (const entry of entries) { + const key = + entry.classification === undefined + ? "unclassified" + : `${entry.classification.area}.${entry.classification.status}`; + counts[key] = (counts[key] ?? 0) + 1; + } + return counts; +} + +function matchesAny(patterns: readonly string[], value: string): boolean { + return patterns.some((pattern) => pathMatchesPattern(pattern, value)); +} + +function pathMatchesPattern(pattern: string, value: string): boolean { + const patternParts = pattern.split("."); + const valueParts = value.split("."); + return matchesParts(patternParts, valueParts); +} + +function matchesParts(patternParts: readonly string[], valueParts: readonly string[]): boolean { + if (patternParts.length === 0) { + return valueParts.length === 0; + } + const [head, ...tail] = patternParts; + if (head === "**") { + if (tail.length === 0) { + return true; + } + for (let index = 0; index <= valueParts.length; index += 1) { + if (matchesParts(tail, valueParts.slice(index))) { + return true; + } + } + return false; + } + if (valueParts.length === 0) { + return false; + } + if (head !== "*" && head !== valueParts[0]) { + return false; + } + return matchesParts(tail, valueParts.slice(1)); +} diff --git a/scripts/check-protocol-event-coverage.mts b/scripts/check-protocol-event-coverage.mts new file mode 100644 index 0000000000000000000000000000000000000000..46ae18e91fe5472f68a258c621b6f65f1ba7fee4 --- /dev/null +++ b/scripts/check-protocol-event-coverage.mts @@ -0,0 +1,775 @@ +#!/usr/bin/env node + +// Guards gateway -> mobile-client event coverage drift. +// +// Source of truth for server->client event names is GATEWAY_EVENTS in +// src/gateway/server-methods-list.ts (the catalog advertised to clients in +// hello-ok `features.events`). packages/gateway-protocol only types the event +// frame envelope (`event: NonEmptyString`), so the gateway catalog is the most +// canonical single list of wire event names. +// +// Client "handled" sets are extracted with deliberately simple parsing over +// the mobile app sources: Swift `switch .event { case "..." }` blocks plus +// `.event == "..."` comparisons, and Kotlin `when (event) { "..." -> }` blocks +// plus `event == "..."` comparisons scoped to `fun handle*Event(...)` bodies. +// Swift case labels may use qualified static string constants; those are +// resolved across the scanned source tree. Kotlin labels and comparisons may +// likewise use generated enum `rawValue` constants. Events a client +// intentionally does not consume live in scripts/protocol-event-coverage.allowlist.json. +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +// Dependency-light seam by design: preflight runs this script without +// installed dependencies (the dependency-free manifest contract), so the +// canonical @openclaw/normalization-core import cannot resolve here. +import { isRecord } from "./lib/record-shared.mjs"; + +const GATEWAY_EVENTS_FILE = "src/gateway/server-methods-list.ts"; +const GATEWAY_EVENT_CONSTANTS_FILE = "src/gateway/events.ts"; +const ALLOWLIST_FILE = "scripts/protocol-event-coverage.allowlist.json"; + +// Scan roots per client. The sentinel files are the primary event dispatch +// surfaces; if one moves, the check must fail loudly instead of silently +// passing with an empty handled set. Apple event mapping is shared by iOS and +// macOS, so the iOS coverage owner lives in OpenClawChatUI. +const IOS_SCAN_ROOTS = ["apps/ios/Sources", "apps/shared/OpenClawKit/Sources"]; +const IOS_SENTINEL_FILE = + "apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatGatewayPayloadCodec.swift"; +const ANDROID_SCAN_ROOT = "apps/android/app/src/main/java/ai/openclaw/app"; +const ANDROID_SENTINEL_FILES = [ + "apps/android/app/src/main/java/ai/openclaw/app/gateway/GatewaySession.kt", + "apps/android/app/src/main/java/ai/openclaw/app/chat/ChatController.kt", +]; + +// Minimum plausible catalog size; a partial parse below this means the +// GATEWAY_EVENTS array changed shape and the extractor needs updating. +const MIN_EXPECTED_GATEWAY_EVENTS = 10; + +const GATEWAY_EVENTS_BLOCK_RE = /export const GATEWAY_EVENTS = \[([\s\S]*?)\];/u; +const SWIFT_EVENT_SWITCH_RE = /\bswitch\s+\w+(?:\.\w+)*\.event\s*\{/u; +const SWIFT_CASE_LABEL_RE = /^\s*case\s+(.+?):/u; +const SWIFT_TYPE_DECLARATION_RE = + /^\s*(?:(?:private|fileprivate|internal|public)\s+)?(?:enum|struct|class|actor|extension)\s+([A-Za-z_]\w*)[^{]*\x7b/u; +const SWIFT_STATIC_STRING_CONSTANT_RE = /^\s*static\s+let\s+([A-Za-z_]\w*)\s*=\s*"([^"]+)"/u; +const SWIFT_QUALIFIED_CONSTANT_RE = /\b([A-Za-z_]\w*\.[A-Za-z_]\w*)\b/gu; +const KOTLIN_EVENT_WHEN_RE = /\bwhen\s*\(\s*event\s*\)\s*\{/u; +// Kotlin gateway handlers follow the `handle*Event` naming convention +// (handleEvent, handleGatewayEvent, handleExecApprovalGatewayEvent, ...). +// Handlers named differently surface as loud "unhandled" failures, which is +// the safe direction for a coverage gate. +const KOTLIN_HANDLER_FUN_RE = /\bfun\s+handle\w*Event\s*\(/u; +const KOTLIN_CASE_SEGMENT_RE = /^\s*(.+?)\s*->/u; +const KOTLIN_ENUM_DECLARATION_RE = /^\s*enum\s+class\s+([A-Za-z_]\w*)\s*\(/u; +const KOTLIN_ENUM_STRING_ENTRY_RE = /^\s*([A-Za-z_]\w*)\s*\(\s*"([^"]+)"\s*\)\s*,?/u; +const KOTLIN_STRING_CASE_EXPRESSION_RE = /^"([^"]+)"$/u; +const KOTLIN_RAW_VALUE_EXPRESSION_RE = /^([A-Za-z_]\w*\.[A-Za-z_]\w*)\.rawValue$/u; +const SWIFT_EVENT_COMPARISON_RE = /\.event\s*==\s*"([^"]+)"/gu; +const KOTLIN_EVENT_COMPARISON_RE = /\bevent\s*==\s*"([^"]+)"(?!\s*(?:\.|\?\.|\+|\[|\())/gu; +const KOTLIN_EVENT_CONSTANT_COMPARISON_RE = + /\bevent\s*==\s*([A-Za-z_]\w*\.[A-Za-z_]\w*)\.rawValue\b(?!\s*(?:\.|\?\.|\+|\[|\())/gu; +const STRING_LITERAL_RE = /"([^"]+)"/gu; + +type StringConstants = ReadonlyMap; +type FsImpl = Pick; +type CoverageParams = { + client: string; + serverEvents: string[]; + handledEvents: Set; + allowlist: Record; +}; +type ClientCollectionParams = { + rootDir: string; + roots: string[]; + extension: string; + extract: (source: string, context?: StringConstants) => Set; + buildExtractContext?: (sources: Iterable) => StringConstants; + sentinels: string[]; + fsImpl: FsImpl; +}; +type KotlinLexicalContext = + | { type: "code"; templateDepth: number | null } + | { type: "raw" } + | { type: "quoted" } + | { type: "char" }; + +function requiredCapture(match: RegExpMatchArray, index: number): string { + const value = match[index]; + if (value === undefined) { + throw new Error(`Expected regular expression capture group ${index}.`); + } + return value; +} + +function isMainModule(): boolean { + return process.argv[1] ? path.resolve(process.argv[1]) === fileURLToPath(import.meta.url) : false; +} + +/** + * Extracts the gateway event name list from server-methods-list.ts source. + * Bare identifiers in the array (e.g. GATEWAY_EVENT_UPDATE_AVAILABLE) are + * resolved against constantsSource (src/gateway/events.ts). + */ +export function extractGatewayEventNames(listSource: string, constantsSource: string): string[] { + const block = GATEWAY_EVENTS_BLOCK_RE.exec(listSource); + if (!block) { + throw new Error( + `Could not find the GATEWAY_EVENTS array in ${GATEWAY_EVENTS_FILE}. ` + + "If the catalog moved, update scripts/check-protocol-event-coverage.mjs.", + ); + } + const body = requiredCapture(block, 1) + .replace(/\/\*[\s\S]*?\*\//gu, "") + .replace(/\/\/[^\n]*/gu, ""); + const names: string[] = []; + for (const token of body.matchAll(/"([^"]+)"|([A-Za-z_$][\w$]*)/gu)) { + if (token[1]) { + names.push(token[1]); + continue; + } + const identifier = requiredCapture(token, 2); + const constant = new RegExp(`export const ${identifier} = "([^"]+)"`, "u").exec( + constantsSource, + ); + if (!constant) { + throw new Error( + `Could not resolve GATEWAY_EVENTS identifier "${identifier}" from ${GATEWAY_EVENT_CONSTANTS_FILE}.`, + ); + } + names.push(requiredCapture(constant, 1)); + } + if (names.length < MIN_EXPECTED_GATEWAY_EVENTS) { + throw new Error( + `Extracted only ${names.length} gateway events from ${GATEWAY_EVENTS_FILE}; ` + + "the array shape likely changed. Update the extractor.", + ); + } + return names; +} + +// Neutralizes string literals and line comments so brace counting cannot be +// confused by braces inside strings or comments. +function sanitizeLineForBraces(line: string): string { + return line + .replace(/\\"/gu, "") + .replace(/"[^"]*"/gu, '""') + .replace(/\/\/.*$/u, ""); +} + +// Collects case-label lines at depth 1 of blocks whose header matches +// headerRe, feeding each into extractLabels. Line-based on purpose: this is a +// drift check, not a parser; sources keep one case label per line. +function collectBlockCaseLabels( + source: string, + headerRe: RegExp, + extractLabels: (line: string, sink: string[]) => void, +): string[] { + const names: string[] = []; + const lines = source.split("\n"); + for (let i = 0; i < lines.length; i += 1) { + const headerLine = lines[i]; + if (headerLine === undefined || !headerRe.test(headerLine)) { + continue; + } + let depth = 0; + for (let j = i; j < lines.length; j += 1) { + const line = lines[j]; + if (line === undefined) { + break; + } + if (depth === 1) { + extractLabels(line, names); + } + const braceSource = + j === i + ? sanitizeLineForBraces(line.slice(line.indexOf("{"))) + : sanitizeLineForBraces(line); + for (const char of braceSource) { + if (char === "{") { + depth += 1; + } else if (char === "}") { + depth -= 1; + } + } + if (j > i && depth <= 0) { + break; + } + } + } + return names; +} + +function pushStringLiterals(segment: string, names: string[]): void { + for (const literal of segment.matchAll(STRING_LITERAL_RE)) { + names.push(requiredCapture(literal, 1)); + } +} + +function stripKotlinComments(source: string): string { + let output = ""; + let index = 0; + let blockDepth = 0; + let lineComment = false; + const contexts: KotlinLexicalContext[] = [{ type: "code", templateDepth: null }]; + + while (index < source.length) { + const char = source[index]; + const pair = source.slice(index, index + 2); + const triple = source.slice(index, index + 3); + const context = contexts.at(-1); + if (!context) { + throw new Error("Kotlin lexical context stack became empty"); + } + + if (lineComment) { + output += char === "\n" ? "\n" : " "; + lineComment = char !== "\n"; + index += 1; + continue; + } + + if (blockDepth > 0) { + if (pair === "/*") { + output += " "; + blockDepth += 1; + index += 2; + continue; + } + if (pair === "*/") { + output += " "; + blockDepth -= 1; + index += 2; + continue; + } + output += char === "\n" ? "\n" : " "; + index += 1; + continue; + } + + if (context.type === "raw") { + if (triple === '"""') { + output += triple; + contexts.pop(); + index += 3; + } else if (pair === "${") { + output += pair; + contexts.push({ type: "code", templateDepth: 1 }); + index += 2; + } else { + output += char; + index += 1; + } + continue; + } + + if (context.type === "quoted" || context.type === "char") { + output += char; + if (char === "\\" && index + 1 < source.length) { + output += source[index + 1]; + index += 2; + continue; + } + if (context.type === "quoted" && pair === "${") { + output += source[index + 1]; + contexts.push({ type: "code", templateDepth: 1 }); + index += 2; + continue; + } + const delimiter = context.type === "quoted" ? '"' : "'"; + if (char === delimiter) { + contexts.pop(); + } + index += 1; + continue; + } + + if (pair === "//") { + output += " "; + lineComment = true; + index += 2; + continue; + } + if (pair === "/*") { + output += " "; + blockDepth = 1; + index += 2; + continue; + } + if (triple === '"""') { + output += triple; + contexts.push({ type: "raw" }); + index += 3; + continue; + } + if (char === '"') { + output += char; + contexts.push({ type: "quoted" }); + index += 1; + continue; + } + if (char === "'") { + output += char; + contexts.push({ type: "char" }); + index += 1; + continue; + } + if (context.templateDepth !== null) { + if (char === "{") { + context.templateDepth += 1; + } else if (char === "}") { + context.templateDepth -= 1; + if (context.templateDepth === 0) { + contexts.pop(); + } + } + } + output += char; + index += 1; + } + + return output; +} + +/** + * Extracts qualified static string constants declared at Swift type scope. + * Type qualification avoids resolving unrelated constants that share a short + * member name elsewhere in the app. + */ +export function extractSwiftStaticStringConstants(source: string): Map { + const constants = new Map(); + const lines = source.split("\n"); + for (let i = 0; i < lines.length; i += 1) { + const declarationLine = lines[i]; + if (declarationLine === undefined) { + continue; + } + const declaration = SWIFT_TYPE_DECLARATION_RE.exec(declarationLine); + if (!declaration) { + continue; + } + const typeName = requiredCapture(declaration, 1); + let depth = 0; + for (let j = i; j < lines.length; j += 1) { + const line = lines[j]; + if (line === undefined) { + break; + } + if (depth === 1) { + const constant = SWIFT_STATIC_STRING_CONSTANT_RE.exec(line); + if (constant) { + constants.set( + `${typeName}.${requiredCapture(constant, 1)}`, + requiredCapture(constant, 2), + ); + } + } + const braceSource = sanitizeLineForBraces(line); + for (const char of braceSource) { + if (char === "{") { + depth += 1; + } else if (char === "}") { + depth -= 1; + } + } + if (j > i && depth <= 0) { + break; + } + } + } + return constants; +} + +/** Extracts generated Kotlin enum entries whose constructor stores a wire string. */ +export function extractKotlinEnumStringConstants(source: string): Map { + const constants = new Map(); + const lines = stripKotlinComments(source).split("\n"); + for (let i = 0; i < lines.length; i += 1) { + const declarationLine = lines[i]; + if (declarationLine === undefined) { + continue; + } + const declaration = KOTLIN_ENUM_DECLARATION_RE.exec(declarationLine); + if (!declaration) { + continue; + } + const typeName = requiredCapture(declaration, 1); + let depth = 0; + let opened = false; + for (let j = i; j < lines.length; j += 1) { + const line = lines[j]; + if (line === undefined) { + break; + } + if (opened && depth === 1) { + const entry = KOTLIN_ENUM_STRING_ENTRY_RE.exec(line); + if (entry) { + constants.set(`${typeName}.${requiredCapture(entry, 1)}`, requiredCapture(entry, 2)); + } + } + const sanitized = sanitizeLineForBraces(line); + for (const char of sanitized) { + if (char === "{") { + depth += 1; + opened = true; + } else if (char === "}") { + depth -= 1; + } + } + if (opened && depth <= 0) { + break; + } + } + } + return constants; +} + +/** Extracts Swift gateway-event case labels, including qualified constants. */ +export function extractSwiftHandledEvents( + source: string, + constants: StringConstants = new Map(), +): Set { + const names = collectBlockCaseLabels(source, SWIFT_EVENT_SWITCH_RE, (line, sink) => { + const label = SWIFT_CASE_LABEL_RE.exec(line); + if (label) { + const labelSource = requiredCapture(label, 1); + pushStringLiterals(labelSource, sink); + const constantReferences = sanitizeLineForBraces(labelSource); + for (const reference of constantReferences.matchAll(SWIFT_QUALIFIED_CONSTANT_RE)) { + const value = constants.get(requiredCapture(reference, 1)); + if (value) { + sink.push(value); + } + } + } + }); + for (const comparison of source.matchAll(SWIFT_EVENT_COMPARISON_RE)) { + names.push(requiredCapture(comparison, 1)); + } + return new Set(names); +} + +// Collects the bodies of Kotlin gateway event handler functions +// (`fun handle*Event(...)`). Signatures may span multiple lines, so scan +// forward from the declaration to the first `{` before brace counting. +function extractKotlinHandlerBodies(source: string): string[] { + const bodies: string[] = []; + const lines = source.split("\n"); + const uncommentedLines = stripKotlinComments(source).split("\n"); + for (let i = 0; i < lines.length; i += 1) { + const handlerLine = uncommentedLines[i]; + if (handlerLine === undefined || !KOTLIN_HANDLER_FUN_RE.test(handlerLine)) { + continue; + } + let depth = 0; + let opened = false; + const body: string[] = []; + for (let j = i; j < lines.length; j += 1) { + const line = lines[j]; + const uncommentedLine = uncommentedLines[j]; + if (line === undefined || uncommentedLine === undefined) { + break; + } + const sanitized = sanitizeLineForBraces(uncommentedLine); + if (opened) { + body.push(line); + } + for (const char of sanitized) { + if (char === "{") { + depth += 1; + opened = true; + } else if (char === "}") { + depth -= 1; + } + } + if (opened && depth <= 0) { + break; + } + } + if (opened) { + bodies.push(body.join("\n")); + } + } + return bodies; +} + +/** + * Extracts event names a Kotlin source handles: string-literal case labels of + * `when (event)` blocks plus `event == "..."` comparisons, both scoped to + * `fun handle*Event(...)` bodies. Scoping matters: bare `event == "..."` + * literals also appear in predicate helpers that are not called from the + * dispatch path (e.g. gatewayEventInvalidatesNodesDevices in NodeRuntime.kt), + * and counting those would silently mark events as covered. Swift extraction + * stays tree-wide because Swift consumption always reads `.event` off a + * received EventFrame, which does not have that false-positive shape. + */ +export function extractKotlinHandledEvents( + source: string, + constants: StringConstants = new Map(), +): Set { + const names: string[] = []; + for (const body of extractKotlinHandlerBodies(source)) { + const uncommentedBody = stripKotlinComments(body); + names.push( + ...collectBlockCaseLabels(uncommentedBody, KOTLIN_EVENT_WHEN_RE, (line, sink) => { + const segment = KOTLIN_CASE_SEGMENT_RE.exec(line)?.[1]; + if (!segment) { + return; + } + for (const expression of segment.split(",").map((value) => value.trim())) { + const literal = KOTLIN_STRING_CASE_EXPRESSION_RE.exec(expression); + if (literal) { + sink.push(requiredCapture(literal, 1)); + continue; + } + const reference = KOTLIN_RAW_VALUE_EXPRESSION_RE.exec(expression)?.[1]; + const value = reference ? constants.get(reference) : undefined; + if (value) { + sink.push(value); + } + } + }), + ); + for (const comparison of uncommentedBody.matchAll(KOTLIN_EVENT_COMPARISON_RE)) { + names.push(requiredCapture(comparison, 1)); + } + for (const comparison of uncommentedBody.matchAll(KOTLIN_EVENT_CONSTANT_COMPARISON_RE)) { + const value = constants.get(requiredCapture(comparison, 1)); + if (value) { + names.push(value); + } + } + } + return new Set(names); +} + +/** + * Compares a client's handled events against the gateway catalog and its + * allowlist. Returns human-readable error strings. Client-only names (e.g. the + * client-synthesized "seqGap" pseudo-event) are intentionally ignored; this + * check only guards the server->client direction. + */ +export function compareEventCoverage(params: CoverageParams): string[] { + const { client, serverEvents, handledEvents, allowlist } = params; + const errors: string[] = []; + const serverSet = new Set(serverEvents); + for (const event of serverEvents) { + if (handledEvents.has(event) || event in allowlist) { + continue; + } + errors.push( + `[${client}] gateway event "${event}" has no handler and no allowlist entry. ` + + `Handle it in the ${client} app or add it to ${ALLOWLIST_FILE} with a reason.`, + ); + } + for (const [event, reason] of Object.entries(allowlist)) { + if (typeof reason !== "string" || reason.trim() === "") { + errors.push(`[${client}] allowlist entry "${event}" needs a non-empty reason string.`); + } + if (!serverSet.has(event)) { + errors.push( + `[${client}] allowlist entry "${event}" is not a gateway event anymore; remove it from ${ALLOWLIST_FILE}.`, + ); + } else if (handledEvents.has(event)) { + errors.push( + `[${client}] allowlist entry "${event}" is now handled; remove it from ${ALLOWLIST_FILE}.`, + ); + } + } + return errors; +} + +function listFilesRecursive(rootDir: string, extension: string, fsImpl: FsImpl): string[] { + const files: string[] = []; + const queue = [rootDir]; + while (queue.length > 0) { + const current = queue.pop(); + if (!current) { + continue; + } + let entries; + try { + entries = fsImpl.readdirSync(current, { withFileTypes: true }); + } catch { + continue; + } + for (const entry of entries) { + const fullPath = path.join(current, entry.name); + if (entry.isDirectory()) { + // Test sources match the same event literals but are not product + // handlers; including them would mask real coverage gaps. + if (entry.name === "Tests" || entry.name === ".build" || entry.name === "build") { + continue; + } + queue.push(fullPath); + continue; + } + if (entry.isFile() && entry.name.endsWith(extension)) { + files.push(fullPath); + } + } + } + return files.toSorted((left, right) => left.localeCompare(right)); +} + +function readRequiredFile(rootDir: string, relativePath: string, fsImpl: FsImpl): string { + const fullPath = path.resolve(rootDir, relativePath); + try { + return fsImpl.readFileSync(fullPath, "utf8"); + } catch { + throw new Error( + `Required file ${relativePath} is missing. If it moved, update scripts/check-protocol-event-coverage.mjs.`, + ); + } +} + +function loadAllowlist(rootDir: string, fsImpl: FsImpl) { + const raw = readRequiredFile(rootDir, ALLOWLIST_FILE, fsImpl); + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch (error) { + throw new Error(`${ALLOWLIST_FILE} is not valid JSON: ${String(error)}`, { cause: error }); + } + if (!isRecord(parsed)) { + throw new Error(`${ALLOWLIST_FILE} must contain a JSON object.`); + } + const readClientAllowlist = (client: string) => { + const entries = parsed[client]; + if (!isRecord(entries)) { + throw new Error(`${ALLOWLIST_FILE} must contain an object entry for "${client}".`); + } + return entries; + }; + return { ios: readClientAllowlist("ios"), android: readClientAllowlist("android") }; +} + +function collectClientHandledEvents(params: ClientCollectionParams): Set { + const { rootDir, roots, extension, extract, buildExtractContext, sentinels, fsImpl } = params; + const handled = new Set(); + const sources = new Map(); + for (const root of roots) { + const rootPath = path.resolve(rootDir, root); + if (!fsImpl.existsSync(rootPath)) { + throw new Error( + `Scan root ${root} is missing. If it moved, update scripts/check-protocol-event-coverage.mjs.`, + ); + } + for (const filePath of listFilesRecursive(rootPath, extension, fsImpl)) { + sources.set(filePath, fsImpl.readFileSync(filePath, "utf8")); + } + } + const extractContext = buildExtractContext?.(sources.values()); + for (const source of sources.values()) { + for (const event of extract(source, extractContext)) { + handled.add(event); + } + } + for (const sentinel of sentinels) { + const source = readRequiredFile(rootDir, sentinel, fsImpl); + if (extract(source, extractContext).size === 0) { + throw new Error( + `Sentinel dispatch file ${sentinel} no longer matches any event names; ` + + "its event handling likely moved or changed shape. Update scripts/check-protocol-event-coverage.mjs.", + ); + } + } + return handled; +} + +function collectStringConstants(sources: Iterable, language: "Swift" | "Kotlin") { + const extract = + language === "Swift" ? extractSwiftStaticStringConstants : extractKotlinEnumStringConstants; + const constants = new Map(); + for (const source of sources) { + for (const [name, value] of extract(source)) { + const existing = constants.get(name); + if (existing !== undefined && existing !== value) { + const kind = language === "Swift" ? "string constant" : "enum string"; + throw new Error(`Conflicting ${language} ${kind} values for ${name}.`); + } + constants.set(name, value); + } + } + return constants; +} + +/** + * Runs the full coverage check against a repo checkout and returns error + * strings plus a summary for logging. + */ +function collectProtocolEventCoverageErrors(params: { rootDir?: string; fs?: FsImpl } = {}) { + const rootDir = params.rootDir ?? process.cwd(); + const fsImpl = params.fs ?? fs; + + const serverEvents = extractGatewayEventNames( + readRequiredFile(rootDir, GATEWAY_EVENTS_FILE, fsImpl), + readRequiredFile(rootDir, GATEWAY_EVENT_CONSTANTS_FILE, fsImpl), + ); + const allowlist = loadAllowlist(rootDir, fsImpl); + const clients = [ + { + client: "ios" satisfies keyof typeof allowlist, + handledEvents: collectClientHandledEvents({ + rootDir, + roots: IOS_SCAN_ROOTS, + extension: ".swift", + extract: extractSwiftHandledEvents, + buildExtractContext: (sources) => collectStringConstants(sources, "Swift"), + sentinels: [IOS_SENTINEL_FILE], + fsImpl, + }), + }, + { + client: "android" satisfies keyof typeof allowlist, + handledEvents: collectClientHandledEvents({ + rootDir, + roots: [ANDROID_SCAN_ROOT], + extension: ".kt", + extract: extractKotlinHandledEvents, + buildExtractContext: (sources) => collectStringConstants(sources, "Kotlin"), + sentinels: ANDROID_SENTINEL_FILES, + fsImpl, + }), + }, + ] as const; + + const errors: string[] = []; + const summaryParts: string[] = []; + for (const { client, handledEvents } of clients) { + errors.push( + ...compareEventCoverage({ + client, + serverEvents, + handledEvents, + allowlist: allowlist[client], + }), + ); + const handledCount = serverEvents.filter((event) => handledEvents.has(event)).length; + summaryParts.push( + `${client} handles ${handledCount}, allowlists ${Object.keys(allowlist[client]).length}`, + ); + } + return { + errors, + summary: `${serverEvents.length} gateway events; ${summaryParts.join("; ")}.`, + }; +} + +if (isMainModule()) { + let result; + try { + result = collectProtocolEventCoverageErrors(); + } catch (error) { + console.error( + `Protocol event coverage check failed: ${error instanceof Error ? error.message : String(error)}`, + ); + process.exit(1); + } + if (result.errors.length > 0) { + console.error("Protocol event coverage check failed:"); + for (const error of result.errors) { + console.error(` - ${error}`); + } + process.exit(1); + } + console.log(`Protocol event coverage OK: ${result.summary}`); +} diff --git a/scripts/check-protocol-registry-types.mts b/scripts/check-protocol-registry-types.mts new file mode 100644 index 0000000000000000000000000000000000000000..d90136da73cd1f28c67b9aa85097b6948ebcb674 --- /dev/null +++ b/scripts/check-protocol-registry-types.mts @@ -0,0 +1,118 @@ +import { createRequire } from "node:module"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { isRecord } from "@openclaw/normalization-core/record-coerce"; +import ts from "typescript"; +import { resolveRepoRoot } from "./lib/repo-root.mjs"; + +const packageRoot = path.join(resolveRepoRoot(import.meta.url), "packages/gateway-protocol"); +const fixturePath = path.join(packageRoot, "protocol-registry-mutability.contract.mts"); +const requireFromPackage = createRequire(path.join(packageRoot, "package.json")); +const publicModule: unknown = await import( + pathToFileURL(requireFromPackage.resolve("@openclaw/gateway-protocol/schema")).href +); +if (!isRecord(publicModule) || !isRecord(publicModule.ProtocolSchemas)) { + throw new Error("Public emitted ProtocolSchemas runtime registry was not resolved"); +} +const runtimeKeys = Object.keys(publicModule.ProtocolSchemas).toSorted(); +const writable = new Set([ + "ProgressCardStepStatus", + "ProgressCardStep", + "ProgressCard", + "ProgressCardGetParams", + "ProgressCardGetResult", + "ProgressCardPutParams", + "ProgressCardPutResult", + "ProgressCardChangedEvent", +]); +const prelude = 'import { ProtocolSchemas } from "@openclaw/gateway-protocol/schema";'; + +for (const exactOptionalPropertyTypes of [true, false]) { + const options: ts.CompilerOptions = { + strict: true, + noUncheckedIndexedAccess: true, + exactOptionalPropertyTypes, + target: ts.ScriptTarget.ESNext, + module: ts.ModuleKind.NodeNext, + moduleResolution: ts.ModuleResolutionKind.NodeNext, + noEmit: true, + skipLibCheck: false, + types: [], + }; + let fixture = `${prelude}\ntype Registry = typeof ProtocolSchemas;\n`; + const host = ts.createCompilerHost(options); + const originalGetSourceFile = host.getSourceFile.bind(host); + host.getSourceFile = (fileName, languageVersion, onError, shouldCreateNewSourceFile) => + path.resolve(fileName) === fixturePath + ? ts.createSourceFile(fileName, fixture, languageVersion, true) + : originalGetSourceFile(fileName, languageVersion, onError, shouldCreateNewSourceFile); + const inspect = ts.createProgram([fixturePath], options, host); + const inputErrors = ts.getPreEmitDiagnostics(inspect); + if (inputErrors.length) { + throw new Error( + ts.formatDiagnosticsWithColorAndContext(inputErrors, { + getCanonicalFileName: (fileName) => fileName, + getCurrentDirectory: () => packageRoot, + getNewLine: () => "\n", + }), + ); + } + const source = inspect.getSourceFile(fixturePath); + const declaration = source?.statements.find(ts.isTypeAliasDeclaration); + if (!declaration) { + throw new Error("Public ProtocolSchemas type was not resolved"); + } + const checker = inspect.getTypeChecker(); + const registry = checker.getTypeAtLocation(declaration); + const keys = checker + .getPropertiesOfType(registry) + .map((symbol) => symbol.name) + .toSorted(); + if (JSON.stringify(keys) !== JSON.stringify(runtimeKeys)) { + throw new Error("Public emitted registry keys differ between runtime and declarations"); + } + if ( + !keys.length || + [...writable].some((key) => !keys.includes(key)) || + checker.getIndexInfosOfType(registry).length !== 0 + ) { + throw new Error("Expected named registry properties with the eight writable ProgressCard keys"); + } + + const lines = [prelude]; + const readonlyLines = new Set(); + const expectedReadonly = keys.length - writable.size; + for (const key of keys) { + lines.push( + `ProtocolSchemas[${JSON.stringify(key)}] = ProtocolSchemas[${JSON.stringify(key)}];`, + ); + if (!writable.has(key)) { + readonlyLines.add(lines.length); + } + } + fixture = `${lines.join("\n")}\n`; + // These consumer assignments are compiled in memory and never executed or emitted. + const proof = ts.createProgram([fixturePath], options, host); + const observedReadonly = new Set(); + const unexpected = ts.getPreEmitDiagnostics(proof).filter((diagnostic) => { + if ( + diagnostic.file && + path.resolve(diagnostic.file.fileName) === fixturePath && + diagnostic.start !== undefined + ) { + const line = diagnostic.file.getLineAndCharacterOfPosition(diagnostic.start).line + 1; + if (diagnostic.code === 2540 && readonlyLines.has(line)) { + observedReadonly.add(line); + return false; + } + } + return true; + }); + if (unexpected.length || observedReadonly.size !== expectedReadonly) { + throw new Error( + `Registry mutability mismatch (exactOptionalPropertyTypes=${exactOptionalPropertyTypes}): ` + + `${observedReadonly.size}/${expectedReadonly} readonly assignments rejected; ${unexpected.length} unexpected diagnostics`, + ); + } +} +console.log("protocol registry public mutability contract passed in both optional-property modes"); diff --git a/scripts/check-release-metadata-only.mts b/scripts/check-release-metadata-only.mts new file mode 100644 index 0000000000000000000000000000000000000000..42496da9294ac5ed8dc40659d3d2cf26454ff0d7 --- /dev/null +++ b/scripts/check-release-metadata-only.mts @@ -0,0 +1,210 @@ +#!/usr/bin/env node +// Validates release metadata-only changed scopes for CI routing. +import { execFileSync } from "node:child_process"; +import { existsSync, readFileSync } from "node:fs"; +import path from "node:path"; +import { isRecord } from "@openclaw/normalization-core/record-coerce"; +import { stableStringify } from "../packages/normalization-core/src/stable-stringify.ts"; +import { RELEASE_METADATA_PATHS } from "./changed-lanes.mts"; +import { isReleaseChangelogPath } from "./lib/release-changelog.mjs"; + +const DEFAULT_GIT_TIMEOUT_MS = 60_000; +const MAX_GIT_TIMEOUT_MS = 10 * 60_000; +const GIT_TIMEOUT_ENV = "OPENCLAW_RELEASE_METADATA_GIT_TIMEOUT_MS"; + +const VERSION_ONLY_TEXT_PATHS = new Set([ + "apps/android/Config/Version.properties", + "apps/android/version.json", + "apps/macos/Sources/OpenClaw/Resources/Info.plist", + "apps/mobile/version.json", +]); + +function normalizePath(input: string) { + return input + .trim() + .replaceAll("\\", "/") + .replace(/^\.\/+/u, ""); +} + +function readRefOptionValue(argv: string[], index: number, optionName: string) { + const value = argv[index + 1]; + if (value === undefined || value === "" || value.startsWith("-")) { + throw new Error(`Expected ${optionName} .`); + } + return value; +} + +function resolveGitTimeoutMs(env: NodeJS.ProcessEnv = process.env) { + const raw = env[GIT_TIMEOUT_ENV]?.trim(); + if (!raw) { + return DEFAULT_GIT_TIMEOUT_MS; + } + const parsed = Number(raw); + if (!Number.isFinite(parsed) || parsed <= 0) { + return DEFAULT_GIT_TIMEOUT_MS; + } + return Math.max(1, Math.min(Math.trunc(parsed), MAX_GIT_TIMEOUT_MS)); +} + +export function parseArgs(argv: string[]) { + const separatorIndex = argv.indexOf("--"); + const flagArgv = separatorIndex === -1 ? argv : argv.slice(0, separatorIndex); + const explicitPaths = + separatorIndex === -1 ? [] : argv.slice(separatorIndex + 1).map(normalizePath); + const paths: string[] = []; + const args = { + staged: false, + base: "origin/main", + head: "HEAD", + paths, + }; + for (let index = 0; index < flagArgv.length; index += 1) { + const arg = flagArgv[index]; + if (arg === "--staged") { + args.staged = true; + } else if (arg === "--base") { + args.base = readRefOptionValue(flagArgv, index, arg); + index += 1; + } else if (arg === "--head") { + args.head = readRefOptionValue(flagArgv, index, arg); + index += 1; + } else if (arg?.startsWith("-")) { + throw new Error(`Unknown option: ${arg}`); + } else { + args.paths.push(normalizePath(arg ?? "")); + } + } + args.paths.push(...explicitPaths); + return args; +} + +function git(args: string[]) { + const timeout = resolveGitTimeoutMs(); + try { + return execFileSync("git", args, { + stdio: ["ignore", "pipe", "pipe"], + encoding: "utf8", + maxBuffer: 16 * 1024 * 1024, + timeout, + }); + } catch (error) { + if ( + error instanceof Error && + (("signal" in error && error.signal === "SIGTERM") || + ("code" in error && error.code === "ETIMEDOUT")) + ) { + throw new Error( + `release metadata guard: git ${args.join(" ")} timed out after ${timeout}ms.`, + { cause: error }, + ); + } + throw error; + } +} + +function listChangedPaths(args: ReturnType) { + if (args.paths.length > 0) { + return [...new Set(args.paths.filter(Boolean))].toSorted((left, right) => + left.localeCompare(right), + ); + } + const diffArgs = args.staged + ? ["diff", "--cached", "--name-only", "--diff-filter=ACMR"] + : ["diff", "--name-only", "--diff-filter=ACMR", `${args.base}...${args.head}`]; + return git(diffArgs) + .split("\n") + .map(normalizePath) + .filter(Boolean) + .toSorted((left, right) => left.localeCompare(right)); +} + +function readBlob(ref: string, filePath: string) { + if (ref === "WORKTREE") { + return readFileSync(filePath, "utf8"); + } + return git(["show", `${ref}:${filePath}`]); +} + +function refsFor(args: ReturnType) { + return args.staged ? { before: "HEAD", after: "" } : { before: args.base, after: args.head }; +} + +function readBeforeAfter(args: ReturnType, filePath: string) { + const refs = refsFor(args); + const before = readBlob(refs.before, filePath); + let after = readBlob(refs.after, filePath); + // The worktree overlay covers uncommitted edits; an explicit --head SHA is + // a request for SHA-exact comparison and must not read the checkout. + if (!args.staged && args.head === "HEAD" && existsSync(filePath)) { + const worktree = readBlob("WORKTREE", filePath); + if (worktree !== after) { + after = worktree; + } + } + return { + before, + after, + }; +} + +function stripPackageVersion(raw: string) { + const parsed: unknown = JSON.parse(raw); + if (!isRecord(parsed)) { + throw new Error("package.json must contain an object"); + } + delete parsed.version; + return stableStringify(parsed); +} + +function normalizeVersionText(raw: string) { + return raw + .replace(/\b20\d{2}\.\d{1,2}\.\d{1,2}(?:-beta\.\d+|-\d+)?\b/gu, "") + .replace(/\b20\d{6}(?:\d{2})?\b/gu, ""); +} + +function fail(message: string) { + console.error(`[release-metadata] ${message}`); + process.exitCode = 1; +} + +export function main(argv: string[] = process.argv.slice(2)) { + const args = parseArgs(argv); + const paths = listChangedPaths(args); + + for (const filePath of paths) { + if (!RELEASE_METADATA_PATHS.has(filePath) && !isReleaseChangelogPath(filePath)) { + fail(`${filePath}: not a release metadata path; run the normal changed gate`); + } + } + + if (paths.includes("package.json")) { + const { before, after } = readBeforeAfter(args, "package.json"); + if (stripPackageVersion(before) !== stripPackageVersion(after)) { + fail("package.json changed outside the top-level version field"); + } + } + + for (const filePath of paths) { + if (!VERSION_ONLY_TEXT_PATHS.has(filePath)) { + continue; + } + const { before, after } = readBeforeAfter(args, filePath); + if (normalizeVersionText(before) !== normalizeVersionText(after)) { + fail(`${filePath}: changed outside recognized version/build literals`); + } + } + + if (process.exitCode) { + process.exit(process.exitCode); + } + console.error(`[release-metadata] ok (${paths.length} files)`); +} + +if (process.argv[1] && path.resolve(process.argv[1]) === path.resolve(import.meta.filename)) { + try { + main(); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); + } +} diff --git a/scripts/check-runtime-sidecar-loaders.mts b/scripts/check-runtime-sidecar-loaders.mts new file mode 100644 index 0000000000000000000000000000000000000000..feeff10737b3fdc770e375b845658beaec7d47a4 --- /dev/null +++ b/scripts/check-runtime-sidecar-loaders.mts @@ -0,0 +1,299 @@ +#!/usr/bin/env node + +// Finds hidden local runtime sidecar loaders missing tsdown entries. +import { promises as fs } from "node:fs"; +import path from "node:path"; +import ts from "typescript"; +import { resolveRepoRoot } from "./lib/repo-root.mjs"; +import { + collectTypeScriptFilesFromRoots, + runAsScript, + toLine, + unwrapExpression, +} from "./lib/ts-guard-utils.mts"; + +const repoRoot = resolveRepoRoot(import.meta.url); +const defaultSourceRoots = [path.join(repoRoot, "src"), path.join(repoRoot, "extensions")]; +const localRuntimeSpecifierPattern = /^\.{1,2}\/.*\.runtime\.(?:js|ts)$/; + +export type RuntimeSidecarLoaderViolation = { + line: number; + specifier: string; + sourcePath: string; + reason: string; +}; +type LocatedRuntimeSidecarLoaderViolation = RuntimeSidecarLoaderViolation & { path: string }; + +function toPosixPath(value: string): string { + return value.split(path.sep).join("/"); +} + +function normalizeRelativePath(value: string): string { + return path.posix.normalize(toPosixPath(value).replace(/^\.\//, "")); +} + +function unwrapInitializer(expression: ts.Expression): ts.Expression { + let current = unwrapExpression(expression); + while (ts.isSatisfiesExpression(current)) { + current = unwrapExpression(current.expression); + } + return current; +} + +function readStringLiteral(node: ts.Expression): string | null { + if (ts.isStringLiteral(node) || ts.isNoSubstitutionTemplateLiteral(node)) { + return node.text; + } + return null; +} + +function readArrayStrings(node: ts.Expression): string[] | null { + const expression = unwrapInitializer(node); + if (!ts.isArrayLiteralExpression(expression)) { + return null; + } + const values: string[] = []; + for (const element of expression.elements) { + const value = readStringLiteral(unwrapInitializer(element)); + if (value === null) { + return null; + } + values.push(value); + } + return values; +} + +function isCreateRequireCall(node: ts.Node, createRequireNames: Set): boolean { + return ( + ts.isCallExpression(node) && + ts.isIdentifier(node.expression) && + createRequireNames.has(node.expression.text) + ); +} + +function isLocalRuntimeSpecifier(specifier: string): boolean { + return localRuntimeSpecifierPattern.test(specifier); +} + +function resolveRuntimeSpecifierSource(importerPath: string, specifier: string): string { + const importerDir = path.posix.dirname(normalizeRelativePath(importerPath)); + const resolved = path.posix.normalize(path.posix.join(importerDir, specifier)); + return resolved.replace(/\.js$/, ".ts"); +} + +function readObjectEntrySources(entry: unknown): string[] { + if (!entry || Array.isArray(entry) || typeof entry !== "object") { + return []; + } + return Object.values(entry).filter((value): value is string => typeof value === "string"); +} + +/** + * Collects explicit source entry files from tsdown configuration. + */ +export function collectTsdownEntrySources(config: unknown): Set { + const configs = Array.isArray(config) ? config : [config]; + return new Set( + configs + .flatMap((entry) => + readObjectEntrySources( + entry && typeof entry === "object" && "entry" in entry ? entry.entry : undefined, + ), + ) + .map(normalizeRelativePath), + ); +} + +/** + * Finds local runtime require loaders not represented as explicit tsdown entries. + */ +export function findRuntimeSidecarLoaderViolations( + content: string, + importerPath: string, + explicitEntrySources: Set, +): RuntimeSidecarLoaderViolation[] { + const sourceFile = ts.createSourceFile(importerPath, content, ts.ScriptTarget.Latest, true); + const createRequireNames = new Set(); + const requireNames = new Set(); + const stringConstants = new Map(); + const stringArrays = new Map(); + const forOfRuntimeValues: Array> = []; + const violations: RuntimeSidecarLoaderViolation[] = []; + const seen = new Set(); + + const currentForOfValueMap = (): Map => { + const merged = new Map(); + for (const scope of forOfRuntimeValues) { + for (const [name, values] of scope) { + merged.set(name, values); + } + } + return merged; + }; + + const addSpecifier = (specifier: string, node: ts.Node): void => { + if (!isLocalRuntimeSpecifier(specifier)) { + return; + } + const sourcePath = resolveRuntimeSpecifierSource(importerPath, specifier); + if (explicitEntrySources.has(sourcePath)) { + return; + } + const key = `${sourcePath}:${toLine(sourceFile, node)}`; + if (seen.has(key)) { + return; + } + seen.add(key); + violations.push({ + line: toLine(sourceFile, node), + specifier, + sourcePath, + reason: + `hidden local runtime loader "${specifier}" resolves to ${sourcePath}, ` + + "but that source is not an explicit tsdown entry", + }); + }; + + const readRequireArgumentSpecifiers = (node: ts.CallExpression): string[] => { + const arg = node.arguments[0]; + if (!arg) { + return []; + } + const unwrapped = unwrapInitializer(arg); + const literal = readStringLiteral(unwrapped); + if (literal !== null) { + return [literal]; + } + if (ts.isIdentifier(unwrapped)) { + const loopValues = currentForOfValueMap().get(unwrapped.text); + if (loopValues) { + return loopValues; + } + const constant = stringConstants.get(unwrapped.text); + if (constant !== undefined) { + return [constant]; + } + } + return []; + }; + + const visit = (node: ts.Node): void => { + if (ts.isImportDeclaration(node) && ts.isStringLiteral(node.moduleSpecifier)) { + if (node.moduleSpecifier.text === "node:module") { + const bindings = node.importClause?.namedBindings; + if (bindings && ts.isNamedImports(bindings)) { + for (const element of bindings.elements) { + if ( + element.propertyName?.text === "createRequire" || + element.name.text === "createRequire" + ) { + createRequireNames.add(element.name.text); + } + } + } + } + } + + if (ts.isVariableDeclaration(node) && ts.isIdentifier(node.name) && node.initializer) { + const initializer = unwrapInitializer(node.initializer); + const literal = readStringLiteral(initializer); + if (literal !== null) { + stringConstants.set(node.name.text, literal); + } + const arrayValues = readArrayStrings(initializer); + if (arrayValues) { + stringArrays.set(node.name.text, arrayValues); + } + if (isCreateRequireCall(initializer, createRequireNames)) { + requireNames.add(node.name.text); + } + } + + if (ts.isForOfStatement(node)) { + const initializer = node.initializer; + const expression = unwrapInitializer(node.expression); + if ( + ts.isVariableDeclarationList(initializer) && + initializer.declarations.length === 1 && + initializer.declarations[0] && + ts.isIdentifier(initializer.declarations[0].name) && + ts.isIdentifier(expression) + ) { + const values = stringArrays.get(expression.text); + if (values) { + forOfRuntimeValues.push(new Map([[initializer.declarations[0].name.text, values]])); + ts.forEachChild(node.statement, visit); + forOfRuntimeValues.pop(); + return; + } + } + } + + if ( + ts.isCallExpression(node) && + ts.isIdentifier(node.expression) && + requireNames.has(node.expression.text) + ) { + for (const specifier of readRequireArgumentSpecifiers(node)) { + addSpecifier(specifier, node); + } + } + + ts.forEachChild(node, visit); + }; + + visit(sourceFile); + return violations; +} + +/** + * Collects runtime sidecar loader violations across configured roots. + */ +async function collectRuntimeSidecarLoaderViolations(params: { + repoRoot: string; + sourceRoots: string[]; + explicitEntrySources: Set; +}): Promise { + const files = await collectTypeScriptFilesFromRoots(params.sourceRoots, { + extraTestSuffixes: [".test-support.ts", ".test-helpers.ts"], + }); + const violations: LocatedRuntimeSidecarLoaderViolation[] = []; + for (const filePath of files) { + if (filePath.endsWith(".d.ts")) { + continue; + } + const relativePath = normalizeRelativePath(path.relative(params.repoRoot, filePath)); + const content = await fs.readFile(filePath, "utf8"); + for (const violation of findRuntimeSidecarLoaderViolations( + content, + relativePath, + params.explicitEntrySources, + )) { + violations.push({ path: relativePath, ...violation }); + } + } + return violations; +} + +async function main() { + const { default: tsdownConfig } = await import("../tsdown.config.ts"); + const violations = await collectRuntimeSidecarLoaderViolations({ + repoRoot, + sourceRoots: defaultSourceRoots, + explicitEntrySources: collectTsdownEntrySources(tsdownConfig), + }); + if (violations.length === 0) { + console.log("runtime-sidecar-loaders: local runtime sidecar loaders look OK."); + return; + } + console.error("runtime-sidecar-loaders: hidden local runtime loaders found:"); + for (const violation of violations) { + console.error( + `- ${violation.path}:${violation.line}: ${violation.reason}. ` + + 'Use cached import("./x.runtime.js") or add the sidecar as a stable tsdown entry.', + ); + } + process.exitCode = 1; +} + +runAsScript(import.meta.url, main); diff --git a/scripts/check-script-erasability.mjs b/scripts/check-script-erasability.mjs new file mode 100644 index 0000000000000000000000000000000000000000..c64590289259b0affdb8479979753f1075e71e5f --- /dev/null +++ b/scripts/check-script-erasability.mjs @@ -0,0 +1,119 @@ +#!/usr/bin/env node + +// Verifies that script TypeScript uses Node's transformation-free syntax subset. +import fs from "node:fs"; +import { stripTypeScriptTypes } from "node:module"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const SCRIPT_ROOT = path.dirname(fileURLToPath(import.meta.url)); +const TYPESCRIPT_IMPLEMENTATION_RE = /\.(?:cts|mts|ts)$/u; +const TYPESCRIPT_DECLARATION_RE = /\.d\.(?:cts|mts|ts)$/u; +const SKIPPED_DIRECTORY_NAMES = new Set([ + ".cache", + ".git", + "build", + "coverage", + "dist", + "generated", + "node_modules", + "out", + "vendor", +]); + +function compareNames(left, right) { + return left < right ? -1 : left > right ? 1 : 0; +} + +function listTypeScriptImplementationFiles(rootDir) { + const files = []; + + function visit(directory) { + const entries = fs + .readdirSync(directory, { withFileTypes: true }) + .toSorted((left, right) => compareNames(left.name, right.name)); + for (const entry of entries) { + const entryPath = path.join(directory, entry.name); + if (entry.isDirectory()) { + if (!SKIPPED_DIRECTORY_NAMES.has(entry.name)) { + visit(entryPath); + } + continue; + } + if ( + entry.isFile() && + TYPESCRIPT_IMPLEMENTATION_RE.test(entry.name) && + !TYPESCRIPT_DECLARATION_RE.test(entry.name) + ) { + files.push(entryPath); + } + } + } + + visit(rootDir); + return files; +} + +function diagnosticLine(error, sourceUrl) { + if (!(error instanceof Error) || !error.stack) { + return undefined; + } + const firstLine = error.stack.split("\n", 1)[0]; + const prefix = `${sourceUrl}:`; + if (!firstLine.startsWith(prefix)) { + return undefined; + } + const line = Number.parseInt(firstLine.slice(prefix.length), 10); + return Number.isFinite(line) ? line : undefined; +} + +/** + * Checks each TypeScript implementation file under a scripts tree without resolving imports. + */ +export function checkScriptErasability(rootDir = SCRIPT_ROOT) { + const files = listTypeScriptImplementationFiles(rootDir); + const rootLabel = path.basename(rootDir); + const errors = []; + + for (const filePath of files) { + const relativePath = path.relative(rootDir, filePath).split(path.sep).join("/"); + const sourceUrl = `${rootLabel}/${relativePath}`; + try { + stripTypeScriptTypes(fs.readFileSync(filePath, "utf8"), { + mode: "strip", + sourceUrl, + }); + } catch (error) { + errors.push({ + file: sourceUrl, + line: diagnosticLine(error, sourceUrl), + message: error instanceof Error ? error.message : String(error), + }); + } + } + + return { checkedFiles: files.length, errors }; +} + +export function main() { + const result = checkScriptErasability(); + if (result.errors.length === 0) { + console.log( + `[script-erasability] checked ${result.checkedFiles} TypeScript implementation files`, + ); + return; + } + + console.error( + "TypeScript syntax under scripts/ must be erasable by Node without transformation:", + ); + for (const error of result.errors) { + const location = error.line === undefined ? error.file : `${error.file}:${error.line}`; + console.error(`- ${location}: ${error.message}`); + } + process.exitCode = 1; +} + +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + main(); +} diff --git a/scripts/check-sdk-package-extension-import-boundary.mts b/scripts/check-sdk-package-extension-import-boundary.mts new file mode 100644 index 0000000000000000000000000000000000000000..b12e528ef0997dda14c7bd3fdd92fb413cef6798 --- /dev/null +++ b/scripts/check-sdk-package-extension-import-boundary.mts @@ -0,0 +1,24 @@ +#!/usr/bin/env node + +// Runs the SDK-package extension import boundary checker. +import { createExtensionImportBoundaryChecker } from "./lib/extension-import-boundary-checker.mts"; +import { runAsScript } from "./lib/ts-guard-utils.mts"; + +const checker = createExtensionImportBoundaryChecker({ + roots: ["src/plugin-sdk", "packages"], + boundaryLabel: "sdk/package", + rule: "Rule: src/plugin-sdk/** and packages/** must not import bundled plugin files", + cleanMessage: "No sdk/package import boundary violations found.", + inventoryTitle: "SDK/package extension import boundary inventory:", + skipSourcesWithoutBundledPluginPrefix: true, + shouldSkipFile(relativeFile) { + return relativeFile.startsWith("packages/plugin-sdk/dist/"); + }, +}); + +/** + * Entrypoint for the SDK-package extension import boundary checker. + */ +export const main = checker.main; + +runAsScript(import.meta.url, main); diff --git a/scripts/check-telegram-grammy-types-imports.mts b/scripts/check-telegram-grammy-types-imports.mts new file mode 100644 index 0000000000000000000000000000000000000000..fd3ff67006cb84be6b0ac275faa543e975af3bed --- /dev/null +++ b/scripts/check-telegram-grammy-types-imports.mts @@ -0,0 +1,69 @@ +#!/usr/bin/env node +// Prevents Telegram runtime imports from grammy type-only modules. +import { readdirSync, readFileSync } from "node:fs"; +import path from "node:path"; +import { resolveRepoRoot } from "./lib/repo-root.mjs"; +const repoRoot = resolveRepoRoot(import.meta.url); +const telegramRoot = path.join(repoRoot, "extensions/telegram"); +const importSpecifierPatterns = [ + /\bimport\s+(?:type\s+)?[\s\S]*?\bfrom\s*["']([^"']+)["']/gu, + /\bexport\s+(?:type\s+)?[\s\S]*?\bfrom\s*["']([^"']+)["']/gu, + /\bimport\s*["']([^"']+)["']/gu, + /\bimport\s*\(\s*["']([^"']+)["']\s*\)/gu, + /\brequire\s*\(\s*["']([^"']+)["']\s*\)/gu, +]; + +function isForbiddenTelegramTypeSpecifier(specifier: string): boolean { + return specifier === "@grammyjs/types" || specifier.startsWith("@grammyjs/types/"); +} + +function lineNumberForOffset(source: string, offset: number): number { + return source.slice(0, offset).split(/\r?\n/u).length; +} + +function collectTypeScriptFiles(root: string): string[] { + const files: string[] = []; + for (const entry of readdirSync(root, { withFileTypes: true })) { + if (entry.name === "dist" || entry.name === "node_modules") { + continue; + } + const entryPath = path.join(root, entry.name); + if (entry.isDirectory()) { + files.push(...collectTypeScriptFiles(entryPath)); + continue; + } + if (entry.isFile() && entry.name.endsWith(".ts")) { + files.push(entryPath); + } + } + return files; +} + +const violations: string[] = []; +for (const filePath of collectTypeScriptFiles(telegramRoot)) { + const source = readFileSync(filePath, "utf8"); + for (const pattern of importSpecifierPatterns) { + for (const match of source.matchAll(pattern)) { + const specifier = match[1]; + if (specifier && isForbiddenTelegramTypeSpecifier(specifier)) { + violations.push( + `${path.relative(repoRoot, filePath)}:${lineNumberForOffset(source, match.index ?? 0)}`, + ); + } + } + } +} + +if (violations.length > 0) { + console.error( + [ + "Telegram source must import Telegram Bot API types from grammy/types, not @grammyjs/types.", + "This keeps grammY Context and message/update helper types on the same dependency copy.", + "", + ...violations, + ].join("\n"), + ); + process.exit(1); +} + +console.log("No Telegram direct @grammyjs/types imports found."); diff --git a/scripts/check-tsgo-core-boundary.mts b/scripts/check-tsgo-core-boundary.mts new file mode 100644 index 0000000000000000000000000000000000000000..e540661f50c7f9aafecdb7ce97d5a264e1e35d56 --- /dev/null +++ b/scripts/check-tsgo-core-boundary.mts @@ -0,0 +1,195 @@ +#!/usr/bin/env node + +// Enforces core tsgo project boundaries and sparse-checkout safety. +import path from "node:path"; +import { isDirectRunUrl } from "./lib/direct-run.mjs"; +import { resolveRepoToolBinPath } from "./lib/local-check-runtime.mts"; +import { runManagedCommand, signalExitCode } from "./lib/managed-child-process.mts"; +import { resolveRepoRoot } from "./lib/repo-root.mjs"; +import { + findTsgoCoreTestShardViolations, + TSGO_CORE_TEST_MAX_ROOTS, + TSGO_CORE_GRAPHS, + TSGO_CORE_TEST_SHARDS, +} from "./lib/tsgo-core-test-shards.mts"; +const repoRoot = resolveRepoRoot(import.meta.url); +const tsgoPath = resolveRepoToolBinPath("tsgo", { cwd: repoRoot }); +const canonicalCoreTestConfig = "test/tsconfig/tsconfig.core.test.json"; + +function normalizeFilePath(filePath: string) { + const normalized = filePath.trim().replaceAll("\\", "/"); + const normalizedRoot = repoRoot.replaceAll("\\", "/"); + if (normalized.startsWith(`${normalizedRoot}/`)) { + return normalized.slice(normalizedRoot.length + 1); + } + return normalized; +} + +export class CoreTsgoBoundaryInterruptedError extends Error { + readonly exitCode: number; + + constructor(signal: NodeJS.Signals) { + super(`Core tsgo graph boundary interrupted by ${signal}`); + this.exitCode = signalExitCode(signal); + } +} + +async function runTsgoQuery(config: string, query: string, label: string): Promise { + const outputs: Buffer[][] = [[], []]; + const overflow = new AbortController(); + let outputBytes = 0; + let receivedSignal: NodeJS.Signals | undefined; + let code: number; + try { + code = await runManagedCommand({ + bin: tsgoPath, + args: ["-p", config, "--pretty", "false", query], + cwd: repoRoot, + stdio: ["ignore", "pipe", "pipe"], + signal: overflow.signal, + requireProcessTreeExit: process.platform !== "win32", + onSignal(signal) { + receivedSignal = signal; + }, + onReady(child) { + for (const [index, stream] of [child.stdout!, child.stderr!].entries()) { + stream.on("data", (chunk: Buffer) => { + if (overflow.signal.aborted) { + return; + } + outputBytes += chunk.byteLength; + // Inventory must be complete; preserve spawnSync's bound and fail rather than truncate. + if (outputBytes > 256 * 1024 * 1024) { + overflow.abort(); + return; + } + outputs[index]!.push(chunk); + }); + } + }, + }); + } catch (error) { + if (overflow.signal.aborted) { + throw new Error(`${label} output exceeded 256 MiB`, { cause: error }); + } + throw error; + } + if (receivedSignal) { + throw new CoreTsgoBoundaryInterruptedError(receivedSignal); + } + const [stdout, stderr] = outputs.map((chunks) => Buffer.concat(chunks).toString("utf8")); + if (code !== 0) { + throw new Error( + `${label} failed with exit code ${code}\n${[stdout, stderr].filter(Boolean).join("\n")}`, + ); + } + return stdout!; +} + +async function readGraphConfig(config: string): Promise<{ + compilerOptions?: { tsBuildInfoFile?: string }; + files?: string[]; +}> { + return JSON.parse(await runTsgoQuery(config, "--showConfig", `${config} config expansion`)) as { + compilerOptions?: { tsBuildInfoFile?: string }; + files?: string[]; + }; +} + +export type CoreTsgoGraph = { + name: string; + config: string; + roots: readonly string[]; + files: readonly string[]; +}; + +/** Validates all boundaries and returns this invocation's compiler-resolved inputs. */ +export async function checkCoreTsgoGraphBoundary(): Promise { + const testRootPattern = /\.test\.(?:ts|tsx)$/u; + const canonicalRoots = ((await readGraphConfig(canonicalCoreTestConfig)).files ?? []) + .map(normalizeFilePath) + .filter((file) => testRootPattern.test(file)); + const shardConfigs = []; + for (const shard of TSGO_CORE_TEST_SHARDS) { + shardConfigs.push({ ...shard, expanded: await readGraphConfig(shard.config) }); + } + const shardViolations = findTsgoCoreTestShardViolations({ + canonicalRoots, + shards: shardConfigs.map((shard) => ({ + name: shard.name, + roots: (shard.expanded.files ?? []) + .map(normalizeFilePath) + .filter((file) => testRootPattern.test(file)), + })), + }); + + const buildInfoOwners = new Map(); + for (const shard of shardConfigs) { + const buildInfo = shard.expanded.compilerOptions?.tsBuildInfoFile; + if (!buildInfo) { + shardViolations.push(`${shard.name}: missing compilerOptions.tsBuildInfoFile`); + continue; + } + const owners = buildInfoOwners.get(buildInfo) ?? []; + owners.push(shard.name); + buildInfoOwners.set(buildInfo, owners); + } + for (const [buildInfo, owners] of buildInfoOwners) { + if (owners.length > 1) { + shardViolations.push(`shared tsBuildInfoFile (${owners.join(", ")}): ${buildInfo}`); + } + } + + if (shardViolations.length > 0) { + console.error( + `Core test shards must cover every canonical test root exactly once and stay at or below ${TSGO_CORE_TEST_MAX_ROOTS} roots:`, + ); + for (const violation of shardViolations) { + console.error(`- ${violation}`); + } + throw new Error("Core test graph ownership validation failed"); + } + + const violations: string[] = []; + const graphs: CoreTsgoGraph[] = []; + for (const graph of TSGO_CORE_GRAPHS) { + const files = ( + await runTsgoQuery(graph.config, "--listFilesOnly", `${graph.name} file listing`) + ) + .split(/\r?\n/u) + .map(normalizeFilePath) + .filter(Boolean); + graphs.push({ + ...graph, + files, + roots: (shardConfigs.find((shard) => shard.config === graph.config)?.expanded.files ?? []) + .map((file) => normalizeFilePath(path.resolve(repoRoot, path.dirname(graph.config), file))) + .filter((file) => testRootPattern.test(file)), + }); + const extensionFiles = files.filter((file) => file.startsWith("extensions/")); + for (const file of extensionFiles) { + violations.push(`${graph.name}: ${file}`); + } + } + + if (violations.length > 0) { + console.error("Core tsgo graphs must not include bundled extension files:"); + for (const violation of violations) { + console.error(`- ${violation}`); + } + console.error( + "Move extension-owned behavior behind plugin SDK contracts, public artifacts, or extension-local tests.", + ); + throw new Error("Core tsgo graphs include bundled extension files"); + } + return graphs; +} + +if (isDirectRunUrl(process.argv[1], import.meta.url)) { + try { + await checkCoreTsgoGraphBoundary(); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = error instanceof CoreTsgoBoundaryInterruptedError ? error.exitCode : 1; + } +} diff --git a/scripts/check-web-fetch-provider-boundaries.mts b/scripts/check-web-fetch-provider-boundaries.mts new file mode 100644 index 0000000000000000000000000000000000000000..0c2c807c65b633ab322391d498a2b745ba16779b --- /dev/null +++ b/scripts/check-web-fetch-provider-boundaries.mts @@ -0,0 +1,105 @@ +#!/usr/bin/env node + +// Checks core web-fetch surfaces for provider-owned Firecrawl coupling. +import { resolveRepoRoot } from "./lib/repo-root.mjs"; +import { collectSourceFileContents } from "./lib/source-file-scan-cache.mts"; +import { runAsScript } from "./lib/ts-guard-utils.mts"; +const repoRoot = resolveRepoRoot(import.meta.url); +const scanExtensions = new Set([".ts", ".js", ".mjs", ".cjs"]); +const ignoredDirNames = new Set([ + ".artifacts", + ".git", + ".turbo", + "build", + "coverage", + "dist", + "extensions", + "node_modules", +]); +const allowedFiles = new Set([ + "src/agents/tools/web-fetch.test-harness.ts", + "src/config/legacy-web-fetch.ts", + "src/config/zod-schema.agent-runtime.ts", + "src/secrets/target-registry-data.ts", +]); +const suspiciousPatterns = [ + /fetchFirecrawlContent/, + /firecrawl-fetch-provider\.js/, + /createFirecrawlWebFetchProvider/, + /providerId:\s*"firecrawl"/, + /provider:\s*"firecrawl"/, + /id:\s*"firecrawl"/, +]; + +async function scanWebFetchProviderBoundaryViolations() { + const violations = []; + const files = await collectSourceFileContents({ + repoRoot, + scanRoots: ["src"], + scanExtensions, + ignoredDirNames, + }); + for (const { relativeFile, content } of files) { + if ( + allowedFiles.has(relativeFile) || + relativeFile.includes(".test.") || + relativeFile.includes("test-support") + ) { + continue; + } + const lines = content.split(/\r?\n/); + for (const [index, line] of lines.entries()) { + if (!line.includes("firecrawl") && !line.includes("Firecrawl")) { + continue; + } + if (!suspiciousPatterns.some((pattern) => pattern.test(line))) { + continue; + } + violations.push({ + file: relativeFile, + line: index + 1, + reason: "core web-fetch runtime/tooling contains Firecrawl-specific fetch logic", + }); + } + } + return violations.toSorted( + (left, right) => left.file.localeCompare(right.file) || left.line - right.line, + ); +} + +let webFetchProviderViolationsPromise: + | ReturnType + | undefined; + +/** + * Collects web-fetch provider boundary violations in core source files. + */ +async function collectWebFetchProviderBoundaryViolations() { + if (!webFetchProviderViolationsPromise) { + webFetchProviderViolationsPromise = scanWebFetchProviderBoundaryViolations(); + try { + return await webFetchProviderViolationsPromise; + } catch (error) { + webFetchProviderViolationsPromise = undefined; + throw error; + } + } + return await webFetchProviderViolationsPromise; +} + +/** Runs the web-fetch provider boundary check. */ +async function main() { + const violations = await collectWebFetchProviderBoundaryViolations(); + for (const violation of violations) { + process.stderr.write(`${violation.file}:${violation.line} ${violation.reason}\n`); + } + return violations.length === 0 ? 0 : 1; +} + +runAsScript(import.meta.url, async () => { + const exitCode = await main(); + if (exitCode !== 0) { + process.exit(exitCode); + } + return exitCode; +}); diff --git a/scripts/check-webhook-auth-body-order.mts b/scripts/check-webhook-auth-body-order.mts new file mode 100644 index 0000000000000000000000000000000000000000..350919222942e8202555e04c976478fef1e24675 --- /dev/null +++ b/scripts/check-webhook-auth-body-order.mts @@ -0,0 +1,68 @@ +#!/usr/bin/env node + +// Ensures webhook handlers authenticate before reading request bodies. +import path from "node:path"; +import ts from "typescript"; +import { bundledPluginCallsite, bundledPluginFile } from "./lib/bundled-plugin-paths.mjs"; +import { runCallsiteGuard } from "./lib/callsite-guard.mts"; +import { runAsScript, toLine, unwrapExpression } from "./lib/ts-guard-utils.mts"; + +const sourceRoots = ["extensions"]; +const enforcedFiles = new Set([ + bundledPluginFile("feishu", "src/monitor.transport.ts"), + bundledPluginFile("googlechat", "src/monitor.ts"), + bundledPluginFile("zalo", "src/monitor.webhook.ts"), +]); +const blockedCallees = new Set(["readJsonBodyWithLimit", "readRequestBodyWithLimit"]); +const allowedCallsites = new Set([ + // Feishu signs the exact wire body, so this handler must read raw bytes before parsing JSON. + bundledPluginCallsite("feishu", "src/monitor.transport.ts", 199), +]); + +function getCalleeName(expression: ts.Expression): string | null { + const callee = unwrapExpression(expression); + if (ts.isIdentifier(callee)) { + return callee.text; + } + if (ts.isPropertyAccessExpression(callee)) { + return callee.name.text; + } + return null; +} + +/** + * Finds request body reads that occur before webhook auth validation. + */ +function findBlockedWebhookBodyReadLines(content: string, fileName = "source.ts"): number[] { + const sourceFile = ts.createSourceFile(fileName, content, ts.ScriptTarget.Latest, true); + const lines: number[] = []; + const visit = (node: ts.Node): void => { + if (ts.isCallExpression(node)) { + const calleeName = getCalleeName(node.expression); + if (calleeName && blockedCallees.has(calleeName)) { + lines.push(toLine(sourceFile, node.expression)); + } + } + ts.forEachChild(node, visit); + }; + visit(sourceFile); + return lines; +} + +/** + * Runs the webhook auth/body-order guard. + */ +async function main() { + await runCallsiteGuard({ + importMetaUrl: import.meta.url, + sourceRoots, + findCallLines: findBlockedWebhookBodyReadLines, + skipRelativePath: (relPath) => !enforcedFiles.has(relPath.replaceAll(path.sep, "/")), + allowCallsite: (callsite) => allowedCallsites.has(callsite), + header: "Found forbidden low-level body reads in auth-sensitive webhook handlers:", + footer: + "Use plugin-sdk webhook guards (`readJsonWebhookBodyOrReject` / `readWebhookBodyOrReject`) with explicit pre-auth/post-auth profiles.", + }); +} + +runAsScript(import.meta.url, main); diff --git a/scripts/check-workflows.mts b/scripts/check-workflows.mts new file mode 100644 index 0000000000000000000000000000000000000000..083a689b81e4c6ff2ff959581c23da70c1a3b3f1 --- /dev/null +++ b/scripts/check-workflows.mts @@ -0,0 +1,139 @@ +#!/usr/bin/env node +// Runs local workflow sanity checks. +// Uses installed tools when present, otherwise falls back to pinned hooks where +// possible, then runs repo-specific workflow guards. +import { spawnSync } from "node:child_process"; +import { mkdtempSync, readdirSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +const ACTIONLINT_REVISION = "011a6d15e749bb3f2d771eed9c7aa0e7e3e10ee7"; +const PRE_COMMIT_VERSION = "4.6.2"; +const WORKFLOW_DIR = ".github/workflows"; + +function commandExists(command: string, args: readonly string[] = ["--version"]): boolean { + const result = spawnSync(command, args, { stdio: "ignore" }); + return !result.error && result.status === 0; +} + +function run(command: string, args: readonly string[]): void { + const result = spawnSync(command, args, { stdio: "inherit" }); + if (result.error) { + console.error(`[check-workflows] failed to run ${command}: ${result.error.message}`); + process.exit(1); + } + if (result.status !== 0) { + process.exit(result.status ?? 1); + } +} + +function runChecked(command: string, args: readonly string[]) { + const result = spawnSync(command, args, { stdio: "inherit" }); + if (result.error) { + return { + message: `[check-workflows] failed to run ${command}: ${result.error.message}`, + status: 1, + }; + } + if (result.status !== 0) { + return { + message: null, + status: result.status ?? 1, + }; + } + return null; +} + +function exitWithFailure(failure: NonNullable>): never { + if (failure.message) { + console.error(failure.message); + } + process.exit(failure.status); +} + +function runPreCommitFromTempVenv(hookArgs: string[]): boolean { + if (!commandExists("python3", ["--version"])) { + return false; + } + const venvDir = mkdtempSync(join(tmpdir(), "openclaw-check-workflows-pre-commit-")); + const python = join(venvDir, process.platform === "win32" ? "Scripts/python.exe" : "bin/python"); + let postVenvFailure: ReturnType = null; + try { + const venvFailure = runChecked("python3", ["-m", "venv", venvDir]); + if (venvFailure) { + return false; + } + postVenvFailure = runChecked(python, [ + "-m", + "pip", + "install", + "--disable-pip-version-check", + `pre-commit==${PRE_COMMIT_VERSION}`, + ]); + if (postVenvFailure) { + return false; + } + postVenvFailure = runChecked(python, ["-m", "pre_commit", ...hookArgs]); + if (postVenvFailure) { + return false; + } + return true; + } finally { + rmSync(venvDir, { force: true, recursive: true }); + if (postVenvFailure) { + exitWithFailure(postVenvFailure); + } + } +} + +function workflowFiles(): string[] { + return readdirSync(WORKFLOW_DIR) + .filter((file) => file.endsWith(".yml") || file.endsWith(".yaml")) + .toSorted() + .map((file) => join(WORKFLOW_DIR, file)); +} + +function runPreCommitHook(hook: string, files: string[]): void { + const hookArgs = ["run", "--config", ".pre-commit-config.yaml", hook, "--files", ...files]; + if (commandExists("pre-commit")) { + run("pre-commit", hookArgs); + return; + } + if (commandExists("python3", ["-m", "pre_commit", "--version"])) { + run("python3", ["-m", "pre_commit", ...hookArgs]); + return; + } + if (runPreCommitFromTempVenv(hookArgs)) { + return; + } + + console.error( + `[check-workflows] missing pre-commit runtime for ${hook}: install pre-commit or Python venv support for pre-commit ${PRE_COMMIT_VERSION}.`, + ); + process.exit(1); +} + +const workflows = workflowFiles(); + +if (commandExists("actionlint")) { + run("actionlint", workflows); +} else if (commandExists("go", ["version"])) { + run("go", ["run", `github.com/rhysd/actionlint/cmd/actionlint@${ACTIONLINT_REVISION}`]); +} else if ( + commandExists("pre-commit") || + commandExists("python3", ["-m", "pre_commit", "--version"]) || + commandExists("python3", ["--version"]) +) { + runPreCommitHook("actionlint", workflows); +} else { + console.error( + `[check-workflows] missing workflow linter: install actionlint, Go for actionlint@${ACTIONLINT_REVISION}, or pre-commit.`, + ); + process.exit(1); +} + +runPreCommitHook("zizmor", workflows); + +run("node", ["scripts/generate-ci-git-owner.mts", "--check"]); +run("python3", ["scripts/check-composite-action-input-interpolation.py"]); +run("node", ["scripts/check-no-conflict-markers.mjs"]); diff --git a/scripts/ci-refit-test-timings.mts b/scripts/ci-refit-test-timings.mts new file mode 100644 index 0000000000000000000000000000000000000000..0948d4fff0c36e80889197d0bc5a0c49a7352b3b --- /dev/null +++ b/scripts/ci-refit-test-timings.mts @@ -0,0 +1,321 @@ +#!/usr/bin/env -S node --import tsx +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import path from "node:path"; +import { setTimeout } from "node:timers/promises"; +import { fileURLToPath } from "node:url"; +import { parseArgs } from "node:util"; +import { z } from "zod"; +import { isRetryableGhJsonErrorMessage } from "./ci-run-timings.mjs"; +import { refitTestTimings, type CiTimingRun } from "./lib/ci-test-timings-refit.mts"; +import { ciTestTimingsSchema } from "./lib/ci-test-timings-schema.mts"; +import { parsePositiveInt } from "./lib/numeric-options.mjs"; +import { execPlainGh } from "./lib/plain-gh.mjs"; + +const jobPageSchema = z.object({ + total_count: z.number().int().nonnegative(), + jobs: z.array( + z.object({ + id: z.number().int().positive(), + run_id: z.number().int().positive(), + run_attempt: z.number().int().positive(), + head_sha: z.string().regex(/^[a-f0-9]{40}$/u), + name: z.string(), + status: z.string(), + conclusion: z.string().nullable(), + started_at: z.iso.datetime(), + completed_at: z.iso.datetime().nullable(), + labels: z.array(z.string()), + }), + ), +}); + +async function readGh(args: string[]): Promise { + const retryDelays = [1000, 3000, 6000]; + for (let attempt = 0; ; attempt += 1) { + try { + return execPlainGh(args, { + encoding: "utf8", + timeout: 120_000, + stdio: ["ignore", "pipe", "pipe"], + }); + } catch (error) { + if ( + attempt === retryDelays.length || + !isRetryableGhJsonErrorMessage(error instanceof Error ? error.message : String(error)) + ) { + throw error; + } + await setTimeout(retryDelays[attempt]); + } + } +} + +async function main() { + const { values } = parseArgs({ + args: process.argv.slice(2).filter((arg) => arg !== "--"), + options: { + runs: { type: "string", default: "5" }, + repo: { type: "string", default: "openclaw/openclaw" }, + "dry-run": { type: "boolean", default: false }, + out: { + type: "string", + default: fileURLToPath(new URL("../config/ci-test-timings.json", import.meta.url)), + }, + }, + }); + const count = parsePositiveInt(values.runs, "--runs"); + const repo = z + .string() + .regex(/^[\w.-]+\/[\w.-]+$/u) + .parse(values.repo); + // Freeze both bounds once. Pagination and retries must not move the cohort. + const upper = new Date().toISOString(); + const lower = new Date(Date.parse(upper) - 7 * 86_400_000).toISOString(); + const inWindow = (timestamp: string) => + Date.parse(timestamp) >= Date.parse(lower) && Date.parse(timestamp) <= Date.parse(upper); + const runSchema = z.object({ + id: z.number().int().positive(), + run_attempt: z.number().int().positive(), + created_at: z.iso.datetime().refine(inWindow, "created_at is outside the frozen UTC window"), + status: z.literal("completed"), + conclusion: z.literal("success"), + event: z.string(), + head_branch: z.string().min(1), + head_sha: z.string().regex(/^[a-f0-9]{40}$/u), + }); + console.log(`Frozen UTC window: ${lower} through ${upper} (7 days).\n`); + console.log("Release workflow SHAs identify tooling, not the measured target.\n"); + console.log( + "| Source | Run | Attempt | Workflow SHA | Created (UTC) | Parsed profiles | Timing jobs |\n| --- | ---: | ---: | --- | --- | --- | ---: |", + ); + // New gh versions reject reporter ANSI unless opted in; logs are parsed, never printed. + const logFlags = (await readGh(["api", "--help"])).includes("--allow-escape-sequences") + ? ["--allow-escape-sequences"] + : []; + const runs: CiTimingRun[] = []; + const seenRuns = new Map(); + const seenJobs = new Map(); + async function readRun(run: z.infer, source: "main" | "release") { + const logs: CiTimingRun["logs"] = []; + let pages = 0; + // A partial retry omits successful original jobs. Read every captured attempt, + // then give the refit one run so retries cannot become independent samples. + for (let attempt = 1; attempt <= run.run_attempt; attempt += 1) { + const attemptLogs: CiTimingRun["logs"] = []; + const jobIds = new Set(); + let total: number | undefined; + for (let page = 1; page <= 25; page += 1) { + if (++pages > 25) { + throw new Error(`Job pagination limit reached for run ${run.id}`); + } + const payload = jobPageSchema.parse( + JSON.parse( + await readGh([ + "api", + `repos/${repo}/actions/runs/${run.id}/attempts/${attempt}/jobs?per_page=100&page=${page}`, + ]), + ), + ); + if (total !== undefined && total !== payload.total_count) { + throw new Error(`Job pagination changed for run ${run.id} attempt ${attempt}`); + } + total = payload.total_count; + for (const job of payload.jobs) { + if ( + job.run_id !== run.id || + job.run_attempt !== attempt || + job.head_sha !== run.head_sha + ) { + throw new Error(`Job ${job.id} run_id/run_attempt/head_sha does not match its cohort`); + } + const identity = JSON.stringify(job); + if (seenJobs.has(job.id) && seenJobs.get(job.id) !== identity) { + throw new Error(`Job ${job.id} changed during pagination`); + } + jobIds.add(job.id); + if (seenJobs.has(job.id)) { + continue; + } + seenJobs.set(job.id, identity); + if (job.conclusion !== "success") { + continue; + } + if ( + job.status !== "completed" || + !job.completed_at || + !inWindow(job.started_at) || + !inWindow(job.completed_at) || + Date.parse(job.completed_at) < Date.parse(job.started_at) + ) { + throw new Error( + `Successful job ${job.id} is not completed inside the frozen UTC window`, + ); + } + const kind = + source === "release" + ? /(?:^| \/ )Repo E2E \(Gateway \d+\/\d+\)$/u.test(job.name) + ? "repoE2e" + : undefined + : job.name.startsWith("checks-ui-e2e (") + ? "uiE2e" + : job.name.startsWith("checks-node-compact-") + ? "compact" + : undefined; + if (kind) { + console.error(`[ci-timings] ${run.id} attempt ${attempt}: ${job.name}`); + attemptLogs.push({ + kind, + labels: job.labels, + text: await readGh(["api", `repos/${repo}/actions/jobs/${job.id}/logs`, ...logFlags]), + }); + } + } + if (jobIds.size === total) { + break; + } + if (jobIds.size > total || payload.jobs.length === 0 || page === 25) { + throw new Error(`Job pagination incomplete for run ${run.id} attempt ${attempt}`); + } + } + const { contributingRunIds } = refitTestTimings([ + { id: run.id, createdAt: run.created_at, logs: attemptLogs }, + ]); + const profiles = Object.entries(contributingRunIds) + .filter(([, ids]) => ids.length > 0) + .map(([profile]) => profile); + console.log( + `| ${source} | ${run.id} | ${attempt} | ${run.head_sha} | ${run.created_at} | ${profiles.join(", ") || "none"} | ${attemptLogs.length} |`, + ); + logs.push(...attemptLogs); + } + return { id: run.id, createdAt: run.created_at, logs }; + } + async function sampleWorkflow(workflow: string, source: "main" | "release") { + const pageSchema = z.object({ + total_count: z.number().int().nonnegative(), + workflow_runs: z.array( + runSchema.extend({ + // A dispatch from main can check out target_ref; only push runs prove the measured ref. + event: z.literal(source === "main" ? "push" : "workflow_dispatch"), + head_branch: source === "main" ? z.literal("main") : z.string().min(1), + }), + ), + }); + const pageSize = Math.min(count, 100); + const query = new URLSearchParams({ + ...(source === "main" ? { branch: "main" } : {}), + event: source === "main" ? "push" : "workflow_dispatch", + status: "success", + created: `${lower}..${upper}`, + per_page: String(pageSize), + }); + const listed = new Set(); + let listedRows = 0; + let sampled = 0; + // GitHub caps filtered searches at 1,000 results, even when total_count is larger. + const maxPages = Math.min(25, Math.ceil(1000 / pageSize)); + for (let page = 1; page <= maxPages; page += 1) { + const payload = pageSchema.parse( + JSON.parse( + await readGh([ + "api", + `repos/${repo}/actions/workflows/${workflow}/runs?${query}&page=${page}`, + ]), + ), + ); + listedRows += payload.workflow_runs.length; + if (listedRows > 1000) { + throw new Error(`Run pagination limit reached for ${workflow}; reduce --runs`); + } + for (const run of payload.workflow_runs) { + listed.add(run.id); + const identity = JSON.stringify(run); + if (seenRuns.has(run.id) && seenRuns.get(run.id) !== identity) { + throw new Error(`Run ${run.id} changed during pagination`); + } + if (seenRuns.has(run.id)) { + continue; + } + seenRuns.set(run.id, identity); + const timingRun = await readRun(run, source); + const { contributingRunIds } = refitTestTimings([timingRun]); + const compact = contributingRunIds.blacksmith.length + contributingRunIds.github.length > 0; + if (source === "main" ? compact : contributingRunIds.repoE2e.length > 0) { + runs.push(timingRun); + } + if (source === "release" || compact) { + sampled += 1; + } + if (sampled === count) { + return; + } + } + if (listed.size === payload.total_count) { + return; + } + if (listed.size > payload.total_count || payload.workflow_runs.length === 0) { + throw new Error(`Run pagination incomplete for ${workflow}`); + } + } + throw new Error(`Run pagination limit reached for ${workflow}; reduce --runs`); + } + await sampleWorkflow("ci.yml", "main"); + const fresh = refitTestTimings(runs); + const { blacksmith, github } = fresh.contributingRunIds; + const mainContributors = new Set([...blacksmith, ...github]); + if ( + mainContributors.size < 2 || + (Object.values(fresh.timings.compactGroupSeconds).every( + (profile) => Object.keys(profile).length === 0, + ) && + Object.values(fresh.timings.runtimePlacementTimings).every((profile) => profile.length === 0)) + ) { + throw new Error( + `Found ${mainContributors.size} independent main compact contributors. Need at least two and a newly eligible compact measurement in the frozen UTC window; retry after successful main CI. No timing file written.`, + ); + } + // Release workflows validate their target before Gateway tests. Their head SHA + // binds jobs to tooling, never substitutes for the measured source identity. + for (const workflow of [ + "openclaw-release-checks.yml", + "openclaw-live-and-e2e-checks-reusable.yml", + ]) { + await sampleWorkflow(workflow, "release"); + } + let previous; + try { + previous = ciTestTimingsSchema.parse(JSON.parse(readFileSync(values.out, "utf8"))); + } catch { + // A missing or invalid baseline has no measurements worth preserving. + } + const { timings, changes, runIds, contributingRunIds } = refitTestTimings(runs, previous); + console.log( + `\nIndependent main compact contributors: ${mainContributors.size} (Blacksmith: ${blacksmith.length}; GitHub: ${github.length}). Release Gateway contributors: ${contributingRunIds.repoE2e.length}.\n`, + ); + ciTestTimingsSchema.parse(timings); + console.log(`Sampled successful CI and release-check runs: ${runIds.join(", ")}\n`); + console.log("| Key | Old seconds | New seconds | Delta |\n| --- | ---: | ---: | ---: |"); + for (const change of changes) { + const delta = + change.next === undefined + ? "removed" + : change.old === undefined || change.old === 0 + ? "new" + : `${(((change.next - change.old) / change.old) * 100).toFixed(1)}%`; + console.log(`| ${change.key} | ${change.old ?? "—"} | ${change.next ?? "—"} | ${delta} |`); + } + if (changes.length === 0) { + console.log("\nNo timing changes exceed the 15% write threshold."); + } else if (!values["dry-run"]) { + mkdirSync(path.dirname(values.out), { recursive: true }); + writeFileSync(values.out, `${JSON.stringify(timings, null, 2)}\n`); + } +} + +try { + await main(); +} catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + console.error("[ci-timings] FAILED (exit 1)"); + process.exitCode = 1; +} diff --git a/scripts/ci-run-timings.mjs b/scripts/ci-run-timings.mjs new file mode 100644 index 0000000000000000000000000000000000000000..0536d9ee58ec2ad44b0d61ef5280ea03d5aab4e2 --- /dev/null +++ b/scripts/ci-run-timings.mjs @@ -0,0 +1,1206 @@ +#!/usr/bin/env node + +// Summarizes GitHub Actions run/job timings for CI analysis. +import { execFileSync } from "node:child_process"; +import { mkdirSync, writeFileSync } from "node:fs"; +import path from "node:path"; +import { isDirectRunUrl } from "./lib/direct-run.mjs"; +import { parsePositiveInt } from "./lib/numeric-options.mjs"; +import { execPlainGh } from "./lib/plain-gh.mjs"; + +const DEFAULT_GITHUB_REPOSITORY = "openclaw/openclaw"; +const RUN_JOBS_PAGE_SIZE = 100; +const RUN_JOBS_MAX_PAGES = 25; +const TREND_RUNS_MAX_PAGES = 100; +const DEFAULT_TREND_COMPARE_HOURS = 12; +const DEFAULT_TREND_DETAIL_RUNS = 100; +const GH_JSON_RETRY_DELAYS_MS = [1_000, 3_000, 6_000]; + +function sleepSync(ms) { + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); +} + +function parseJsonCommand(command, args, onAttempt = null, options = {}) { + let lastError; + for (let attempt = 0; attempt <= GH_JSON_RETRY_DELAYS_MS.length; attempt += 1) { + try { + onAttempt?.(); + const stdout = + command === "gh" + ? execPlainGh(args, { + encoding: "utf8", + ...options, + }) + : execFileSync(command, args, { + encoding: "utf8", + ...options, + }); + return JSON.parse(stdout); + } catch (error) { + lastError = error; + const message = error instanceof Error ? error.message : String(error); + const retryable = isRetryableGhJsonErrorMessage(message); + if (!retryable || attempt === GH_JSON_RETRY_DELAYS_MS.length) { + throw error; + } + sleepSync(GH_JSON_RETRY_DELAYS_MS[attempt]); + } + } + throw lastError; +} + +export function isRetryableGhJsonErrorMessage(message) { + return /HTTP 5\d\d|HTTP 429|Server Error|secondary rate limit|abuse detection|ETIMEDOUT|ECONNRESET|EAI_AGAIN/iu.test( + message, + ); +} + +function normalizeRunJob(job) { + return { + completedAt: job.completedAt ?? job.completed_at ?? null, + conclusion: job.conclusion ?? "", + createdAt: job.createdAt ?? job.created_at ?? null, + databaseId: job.databaseId ?? job.id, + labels: Array.isArray(job.labels) ? job.labels : [], + name: job.name, + runnerGroupName: job.runnerGroupName ?? job.runner_group_name ?? null, + runnerName: job.runnerName ?? job.runner_name ?? null, + startedAt: job.startedAt ?? job.started_at ?? null, + status: job.status ?? "", + }; +} + +/** + * Flattens paginated GitHub run job responses. + */ +export function collectRunJobsFromPages(pages) { + return pages.flatMap((page) => (Array.isArray(page.jobs) ? page.jobs.map(normalizeRunJob) : [])); +} + +function parseTime(value) { + if (!value || value === "0001-01-01T00:00:00Z") { + return null; + } + const parsed = Date.parse(value); + return Number.isFinite(parsed) ? parsed : null; +} + +function secondsBetween(start, end) { + return start !== null && end !== null ? Math.round((end - start) / 1000) : null; +} + +function formatSeconds(value) { + return value === null ? "" : `${value}s`; +} + +function percentile(values, percentileValue) { + if (values.length === 0) { + return null; + } + const sorted = [...values].toSorted((left, right) => left - right); + const index = Math.min(sorted.length - 1, Math.ceil(sorted.length * percentileValue) - 1); + return sorted[index]; +} + +function summarizeDistribution(values) { + return { + count: values.length, + max: values.length === 0 ? null : Math.max(...values), + p50: percentile(values, 0.5), + p90: percentile(values, 0.9), + p95: percentile(values, 0.95), + }; +} + +function parseRunList(raw) { + const parsed = JSON.parse(raw); + return Array.isArray(parsed) ? parsed : []; +} + +function isPnpmStoreWarmupGatedJobName(name) { + return ( + name === "build-artifacts" || + name === "check-docs" || + name === "check-guards" || + name === "check-npm-lock" || + name === "check-prod-types" || + name === "check-lint" || + name === "check-dependencies" || + name === "check-test-types" || + name.startsWith("check-additional-") || + name.startsWith("checks-fast-") || + (name.startsWith("checks-node-") && !name.startsWith("checks-node-compat-")) + ); +} + +function collectRunTimingContext(run) { + const created = parseTime(run.createdAt); + const updated = parseTime(run.updatedAt); + const jobs = (run.jobs ?? []) + .filter((job) => !job.name?.startsWith("matrix.")) + .map((job) => { + const started = parseTime(job.startedAt); + const completed = parseTime(job.completedAt); + return { + conclusion: job.conclusion ?? "", + durationSeconds: secondsBetween(started, completed), + name: job.name, + // Actions exposes job start time, but not the split between `needs` + // dependency wait and runner queue. Keep the combined delay honest. + startDelaySeconds: secondsBetween(created, started), + started, + completed, + status: job.status, + }; + }); + + return { created, jobs, updated }; +} + +/** + * Summarizes longest jobs and total timing for a workflow run. + */ +export function summarizeRunTimings(run, limit = 15) { + const { created, jobs, updated } = collectRunTimingContext(run); + if (jobs.length === 0) { + throw new Error("CI run timing summary requires at least one job"); + } + const byDuration = [...jobs] + .filter((job) => job.durationSeconds !== null) + .toSorted((left, right) => right.durationSeconds - left.durationSeconds) + .slice(0, limit); + const byStartDelay = [...jobs] + .filter((job) => job.startDelaySeconds !== null && (job.durationSeconds ?? 0) > 5) + .toSorted((left, right) => right.startDelaySeconds - left.startDelaySeconds) + .slice(0, limit); + const badJobs = jobs.filter( + (job) => job.conclusion && !["success", "skipped", "cancelled"].includes(job.conclusion), + ); + + return { + byDuration, + byStartDelay, + conclusion: run.conclusion ?? "", + status: run.status ?? "", + wallSeconds: secondsBetween(created, updated), + badJobs, + }; +} + +/** + * Summarizes pnpm store warmup overlap near run start. + */ +export function summarizePnpmStoreWarmupBarrier(run, windowSeconds = 5) { + const { jobs } = collectRunTimingContext(run); + const preflight = jobs.find((job) => job.name === "preflight"); + const warmup = jobs.find((job) => job.name === "pnpm-store-warmup"); + if (!warmup?.started || !warmup?.completed) { + return null; + } + + const postWarmupJobs = jobs.filter( + (job) => + job.name !== "preflight" && + job.name !== "security-fast" && + job.name !== "pnpm-store-warmup" && + isPnpmStoreWarmupGatedJobName(job.name) && + job.status === "completed" && + job.conclusion !== "skipped" && + job.started !== null && + job.started >= warmup.completed && + (job.durationSeconds ?? 0) > 5, + ); + const startDelays = postWarmupJobs + .map((job) => secondsBetween(warmup.completed, job.started)) + .filter((delay) => delay !== null); + + return { + activePostWarmupJobCount: postWarmupJobs.length, + firstPostWarmupStartDelaySeconds: startDelays.length === 0 ? null : Math.min(...startDelays), + postWarmupP95StartDelaySeconds: percentile(startDelays, 0.95), + postWarmupStartedWithinWindow: startDelays.filter((delay) => delay <= windowSeconds).length, + preflightToWarmupCompleteSeconds: secondsBetween( + preflight?.completed ?? null, + warmup.completed, + ), + preflightToWarmupStartSeconds: secondsBetween(preflight?.completed ?? null, warmup.started), + warmupDurationSeconds: secondsBetween(warmup.started, warmup.completed), + warmupResult: `${warmup.status}/${warmup.conclusion}`, + windowSeconds, + }; +} + +/** + * Selects the latest main push CI run, optionally matching a head SHA. + * + * @param {Array>} runs + * @param {string | null} [headSha] + */ +export function selectLatestMainPushCiRun(runs, headSha = null) { + const pushRuns = runs.filter((run) => run.event === "push"); + if (headSha) { + const matchingRun = pushRuns.find((run) => run.headSha === headSha); + if (matchingRun) { + return matchingRun; + } + } + return pushRuns[0] ?? null; +} + +function getLatestCiRunId() { + const raw = execPlainGh( + ["run", "list", "--branch", "main", "--workflow", "CI", "--limit", "1", "--json", "databaseId"], + { encoding: "utf8" }, + ); + const runs = JSON.parse(raw); + const runId = runs[0]?.databaseId; + if (!runId) { + throw new Error("No CI runs found on main"); + } + return String(runId); +} + +function getRemoteMainSha() { + const raw = execFileSync("git", ["ls-remote", "origin", "main"], { encoding: "utf8" }).trim(); + const [sha] = raw.split(/\s+/u); + if (!sha) { + throw new Error("Could not resolve origin/main"); + } + return sha; +} + +function getLatestMainPushCiRunId() { + const headSha = getRemoteMainSha(); + const raw = execPlainGh( + [ + "run", + "list", + "--branch", + "main", + "--workflow", + "CI", + "--limit", + "20", + "--json", + "databaseId,headSha,event,status,conclusion", + ], + { encoding: "utf8" }, + ); + const run = selectLatestMainPushCiRun(parseRunList(raw), headSha); + const databaseId = run?.databaseId; + if (typeof databaseId !== "string" && typeof databaseId !== "number") { + throw new Error(`No push CI run found for origin/main ${headSha.slice(0, 10)}`); + } + return String(databaseId); +} + +function listRecentSuccessfulCiRuns(limit) { + const raw = execPlainGh( + [ + "run", + "list", + "--branch", + "main", + "--event", + "push", + "--workflow", + "CI", + "--limit", + String(Math.max(limit * 4, limit)), + "--json", + "databaseId,headSha,event,status,conclusion", + ], + { encoding: "utf8" }, + ); + return JSON.parse(raw) + .filter( + (run) => run.event === "push" && run.status === "completed" && run.conclusion === "success", + ) + .slice(0, limit); +} + +/** + * @param {string | number} runId + * @param {number | null} [runAttempt] + */ +function loadRunJobs(runId, runAttempt = null) { + const repository = process.env.GITHUB_REPOSITORY || DEFAULT_GITHUB_REPOSITORY; + const runPath = runAttempt === null ? `runs/${runId}` : `runs/${runId}/attempts/${runAttempt}`; + const pages = []; + let totalCount = null; + let requestCount = 0; + for (let page = 1; page <= RUN_JOBS_MAX_PAGES; page += 1) { + const payload = parseJsonCommand( + "gh", + [ + "api", + "-X", + "GET", + `repos/${repository}/actions/${runPath}/jobs?per_page=${RUN_JOBS_PAGE_SIZE}&page=${page}`, + ], + () => { + requestCount += 1; + }, + ); + pages.push(payload); + const jobs = Array.isArray(payload.jobs) ? payload.jobs : []; + totalCount = typeof payload.total_count === "number" ? payload.total_count : totalCount; + if ( + jobs.length === 0 || + (totalCount !== null && collectRunJobsFromPages(pages).length >= totalCount) + ) { + break; + } + } + return { jobs: collectRunJobsFromPages(pages), requestCount }; +} + +function loadRun(runId) { + const run = parseJsonCommand("gh", [ + "run", + "view", + runId, + "--json", + "status,conclusion,createdAt,updatedAt", + ]); + return { + ...run, + jobs: loadRunJobs(runId).jobs, + }; +} + +function normalizeTrendRun(run) { + return { + conclusion: run.conclusion ?? "", + createdAt: run.createdAt ?? run.created_at ?? null, + databaseId: run.databaseId ?? run.id, + headSha: run.headSha ?? run.head_sha ?? "", + runAttempt: run.runAttempt ?? run.run_attempt ?? 1, + status: run.status ?? "", + updatedAt: run.updatedAt ?? run.updated_at ?? null, + url: run.url ?? run.html_url ?? "", + }; +} + +function listTrendCiRuns(cutoffMs) { + const repository = process.env.GITHUB_REPOSITORY || DEFAULT_GITHUB_REPOSITORY; + const runs = []; + let requestCount = 0; + for (let page = 1; page <= TREND_RUNS_MAX_PAGES; page += 1) { + const payload = parseJsonCommand( + "gh", + [ + "api", + "-X", + "GET", + `repos/${repository}/actions/workflows/ci.yml/runs?branch=main&event=push&per_page=100&page=${page}`, + ], + () => { + requestCount += 1; + }, + ); + const pageRuns = Array.isArray(payload.workflow_runs) + ? payload.workflow_runs.map(normalizeTrendRun) + : []; + runs.push(...pageRuns); + const oldestCreatedAt = parseTime(pageRuns.at(-1)?.createdAt); + if (pageRuns.length < 100 || (oldestCreatedAt !== null && oldestCreatedAt < cutoffMs)) { + break; + } + } + return { + requestCount, + runs: runs.filter((run) => { + const createdAt = parseTime(run.createdAt); + return createdAt !== null && createdAt >= cutoffMs; + }), + }; +} + +function summarizeJobs(run) { + const { created, jobs, updated } = collectRunTimingContext(run); + if (jobs.length === 0) { + throw new Error("CI run timing summary requires at least one job"); + } + const completedJobs = jobs.filter((job) => job.started !== null && job.completed !== null); + const successfulDurations = jobs + .filter((job) => job.status === "completed" && job.conclusion === "success") + .map((job) => job.durationSeconds) + .filter((duration) => duration !== null); + const firstStart = Math.min(...completedJobs.map((job) => job.started)); + const lastComplete = Math.max(...completedJobs.map((job) => job.completed)); + + return { + avgDurationSeconds: + successfulDurations.length === 0 + ? null + : Math.round( + successfulDurations.reduce((sum, duration) => sum + duration, 0) / + successfulDurations.length, + ), + executionWindowSeconds: + Number.isFinite(firstStart) && Number.isFinite(lastComplete) + ? secondsBetween(firstStart, lastComplete) + : null, + firstStartDelaySeconds: Number.isFinite(firstStart) + ? secondsBetween(created, firstStart) + : null, + jobCount: successfulDurations.length, + maxDurationSeconds: successfulDurations.length === 0 ? null : Math.max(...successfulDurations), + p90DurationSeconds: percentile(successfulDurations, 0.9), + p95DurationSeconds: percentile(successfulDurations, 0.95), + wallSeconds: secondsBetween(created, updated), + }; +} + +function isSyntheticTimingJob(job) { + return job.name?.startsWith("matrix.") || job.name === "ci-timings-summary"; +} + +function isAggregateTimingJob(job) { + return isSyntheticTimingJob(job) || job.name === "openclaw/ci-gate"; +} + +function summarizeTrendRun(run) { + const createdAt = parseTime(run.createdAt); + const updatedAt = parseTime(run.updatedAt); + const jobs = (run.jobs ?? []).filter((job) => !isSyntheticTimingJob(job)); + const createdJobs = jobs + .map((job) => ({ job, createdAt: parseTime(job.createdAt) })) + .filter((entry) => entry.createdAt !== null); + const firstJobCreatedAt = + createdJobs.length === 0 ? null : Math.min(...createdJobs.map((entry) => entry.createdAt)); + const activeJobs = jobs + .map((job) => ({ + completedAt: parseTime(job.completedAt), + createdAt: parseTime(job.createdAt), + job, + startedAt: parseTime(job.startedAt), + })) + .filter( + (entry) => + entry.job.conclusion !== "skipped" && + entry.startedAt !== null && + entry.completedAt !== null, + ); + const jobTimings = activeJobs + .filter((entry) => !isAggregateTimingJob(entry.job)) + .map((entry) => ({ + dependencyGatedSeconds: secondsBetween(firstJobCreatedAt, entry.createdAt), + executionSeconds: secondsBetween(entry.startedAt, entry.completedAt), + labels: entry.job.labels, + name: entry.job.name, + runnerGroupName: entry.job.runnerGroupName, + runnerName: entry.job.runnerName, + runnerQueueSeconds: secondsBetween(entry.createdAt, entry.startedAt), + })); + const completionOrder = activeJobs.toSorted( + (left, right) => + right.completedAt - left.completedAt || + String(left.job.name).localeCompare(String(right.job.name)) || + Number(left.job.databaseId ?? 0) - Number(right.job.databaseId ?? 0), + ); + // The run list keeps the original workflow creation time after a rerun. + // Attempt-specific job data remains useful, but exclude cross-attempt run + // wall/admission metrics rather than mixing it with the latest attempt. + const firstAttempt = run.runAttempt === 1; + + return { + admittedWallSeconds: firstAttempt ? secondsBetween(firstJobCreatedAt, updatedAt) : null, + conclusion: run.conclusion, + createdAt: run.createdAt, + databaseId: run.databaseId, + detailsLoaded: Array.isArray(run.jobs), + headSha: run.headSha, + jobTimings, + lastWorkOwner: + completionOrder.find((entry) => !isAggregateTimingJob(entry.job))?.job.name ?? null, + runAttempt: run.runAttempt, + status: run.status, + terminalOwner: completionOrder[0]?.job.name ?? null, + url: run.url, + wallSeconds: firstAttempt ? secondsBetween(createdAt, updatedAt) : null, + workflowAdmissionSeconds: firstAttempt ? secondsBetween(createdAt, firstJobCreatedAt) : null, + }; +} + +function summarizeOutcomes(runs) { + const counts = { + actionRequired: 0, + cancelled: 0, + failure: 0, + inProgress: 0, + neutral: 0, + other: 0, + pending: 0, + queued: 0, + skipped: 0, + stale: 0, + startupFailure: 0, + success: 0, + timedOut: 0, + total: runs.length, + }; + const conclusionKeys = new Map([ + ["action_required", "actionRequired"], + ["cancelled", "cancelled"], + ["failure", "failure"], + ["neutral", "neutral"], + ["skipped", "skipped"], + ["stale", "stale"], + ["startup_failure", "startupFailure"], + ["success", "success"], + ["timed_out", "timedOut"], + ]); + let completedNonCancelled = 0; + for (const run of runs) { + if (run.status === "completed" && run.conclusion !== "cancelled") { + completedNonCancelled += 1; + } + const key = + run.status === "completed" + ? conclusionKeys.get(run.conclusion) + : run.status === "in_progress" + ? "inProgress" + : run.status; + if (key && Object.hasOwn(counts, key)) { + counts[key] += 1; + } else { + counts.other += 1; + } + } + return { + ...counts, + cancellationRate: counts.total === 0 ? null : counts.cancelled / counts.total, + nonCancelledPassRate: + completedNonCancelled === 0 ? null : counts.success / completedNonCancelled, + }; +} + +function summarizeCriticalOwners(runSummaries) { + const counts = new Map(); + for (const run of runSummaries) { + if (run.lastWorkOwner) { + counts.set(run.lastWorkOwner, (counts.get(run.lastWorkOwner) ?? 0) + 1); + } + } + return [...counts.entries()] + .map(([name, runs]) => ({ name, runs })) + .toSorted((left, right) => right.runs - left.runs || left.name.localeCompare(right.name)); +} + +function summarizeTrendCohort(runs, runSummaries) { + const successfulRuns = runSummaries.filter( + (run) => run.status === "completed" && run.conclusion === "success", + ); + const jobTimings = successfulRuns.flatMap((run) => run.jobTimings); + return { + criticalOwners: summarizeCriticalOwners(successfulRuns), + jobMetrics: { + dependencyGatedSeconds: summarizeDistribution( + jobTimings.map((job) => job.dependencyGatedSeconds).filter((value) => value !== null), + ), + executionSeconds: summarizeDistribution( + jobTimings.map((job) => job.executionSeconds).filter((value) => value !== null), + ), + runnerQueueSeconds: summarizeDistribution( + jobTimings.map((job) => job.runnerQueueSeconds).filter((value) => value !== null), + ), + }, + outcomes: summarizeOutcomes(runs), + samples: { + detailedSuccessfulRuns: successfulRuns.filter((run) => run.detailsLoaded).length, + successfulRuns: successfulRuns.length, + timedJobs: jobTimings.length, + }, + runMetrics: { + admittedWallSeconds: summarizeDistribution( + successfulRuns.map((run) => run.admittedWallSeconds).filter((value) => value !== null), + ), + successfulWallSeconds: summarizeDistribution( + successfulRuns.map((run) => run.wallSeconds).filter((value) => value !== null), + ), + workflowAdmissionSeconds: summarizeDistribution( + successfulRuns.map((run) => run.workflowAdmissionSeconds).filter((value) => value !== null), + ), + }, + }; +} + +function summarizeJobNames(runSummaries, fromMs, toMs) { + const byName = new Map(); + for (const run of runSummaries) { + const createdAt = parseTime(run.createdAt); + if ( + createdAt === null || + createdAt < fromMs || + createdAt >= toMs || + run.status !== "completed" || + run.conclusion !== "success" + ) { + continue; + } + for (const job of run.jobTimings) { + const timings = byName.get(job.name) ?? []; + timings.push(job); + byName.set(job.name, timings); + } + } + return byName; +} + +function summarizeNamedJobComparison(runSummaries, priorWindow, comparisonWindow) { + const prior = summarizeJobNames(runSummaries, priorWindow.fromMs, priorWindow.toMs); + const comparison = summarizeJobNames( + runSummaries, + comparisonWindow.fromMs, + comparisonWindow.toMs, + ); + return [...new Set([...prior.keys(), ...comparison.keys()])] + .map((name) => { + const summarize = (timings) => ({ + executionSeconds: summarizeDistribution( + (timings ?? []).map((job) => job.executionSeconds).filter((value) => value !== null), + ), + runnerQueueSeconds: summarizeDistribution( + (timings ?? []).map((job) => job.runnerQueueSeconds).filter((value) => value !== null), + ), + }); + return { + comparison: summarize(comparison.get(name)), + name, + prior: summarize(prior.get(name)), + }; + }) + .toSorted( + (left, right) => + (right.comparison.executionSeconds.p90 ?? -1) - + (left.comparison.executionSeconds.p90 ?? -1) || left.name.localeCompare(right.name), + ); +} + +function metricDelta(comparison, prior, key) { + const comparisonValue = comparison?.[key] ?? null; + const priorValue = prior?.[key] ?? null; + return comparisonValue === null || priorValue === null ? null : comparisonValue - priorValue; +} + +/** + * Aggregates main CI runs into a baseline, previous comparison window, and latest window. + */ +export function summarizeTrendTimings(runs, options) { + const { compareDurationMs, generatedAtMs, trendDurationMs } = options; + const baselineFromMs = generatedAtMs - trendDurationMs; + const comparisonFromMs = generatedAtMs - compareDurationMs; + const priorFromMs = comparisonFromMs - compareDurationMs; + const inWindow = (run, fromMs, toMs) => { + const createdAt = parseTime(run.createdAt); + return createdAt !== null && createdAt >= fromMs && createdAt < toMs; + }; + const baselineRuns = runs.filter((run) => inWindow(run, baselineFromMs, generatedAtMs)); + const priorRuns = runs.filter((run) => inWindow(run, priorFromMs, comparisonFromMs)); + const comparisonRuns = runs.filter((run) => inWindow(run, comparisonFromMs, generatedAtMs)); + const runSummaries = baselineRuns + .map(summarizeTrendRun) + .toSorted((left, right) => Date.parse(right.createdAt) - Date.parse(left.createdAt)); + const baselineSummaries = runSummaries.filter((run) => + inWindow(run, baselineFromMs, generatedAtMs), + ); + const priorSummaries = runSummaries.filter((run) => inWindow(run, priorFromMs, comparisonFromMs)); + const comparisonSummaries = runSummaries.filter((run) => + inWindow(run, comparisonFromMs, generatedAtMs), + ); + const cohorts = { + baseline: summarizeTrendCohort(baselineRuns, baselineSummaries), + comparison: summarizeTrendCohort(comparisonRuns, comparisonSummaries), + prior: summarizeTrendCohort(priorRuns, priorSummaries), + }; + + return { + changes: { + executionP90Seconds: metricDelta( + cohorts.comparison.jobMetrics.executionSeconds, + cohorts.prior.jobMetrics.executionSeconds, + "p90", + ), + runnerQueueP95Seconds: metricDelta( + cohorts.comparison.jobMetrics.runnerQueueSeconds, + cohorts.prior.jobMetrics.runnerQueueSeconds, + "p95", + ), + successfulWallP50Seconds: metricDelta( + cohorts.comparison.runMetrics.successfulWallSeconds, + cohorts.prior.runMetrics.successfulWallSeconds, + "p50", + ), + successfulWallP90Seconds: metricDelta( + cohorts.comparison.runMetrics.successfulWallSeconds, + cohorts.prior.runMetrics.successfulWallSeconds, + "p90", + ), + workflowAdmissionP95Seconds: metricDelta( + cohorts.comparison.runMetrics.workflowAdmissionSeconds, + cohorts.prior.runMetrics.workflowAdmissionSeconds, + "p95", + ), + }, + cohorts, + jobs: summarizeNamedJobComparison( + runSummaries, + { fromMs: priorFromMs, toMs: comparisonFromMs }, + { fromMs: comparisonFromMs, toMs: generatedAtMs }, + ), + runs: runSummaries, + windows: { + baseline: { + from: new Date(baselineFromMs).toISOString(), + to: new Date(generatedAtMs).toISOString(), + }, + comparison: { + from: new Date(comparisonFromMs).toISOString(), + to: new Date(generatedAtMs).toISOString(), + }, + prior: { + from: new Date(priorFromMs).toISOString(), + to: new Date(comparisonFromMs).toISOString(), + }, + }, + }; +} + +function formatDistribution(summary) { + return [ + `n=${summary.count}`, + `p50=${formatSeconds(summary.p50)}`, + `p90=${formatSeconds(summary.p90)}`, + `p95=${formatSeconds(summary.p95)}`, + `max=${formatSeconds(summary.max)}`, + ].join(" "); +} + +function formatDelta(value) { + if (value === null) { + return ""; + } + return `${value > 0 ? "+" : ""}${formatSeconds(value)}`; +} + +function formatPercent(value) { + return value === null ? "" : `${(value * 100).toFixed(1)}%`; +} + +function printTrendCohort(name, cohort) { + const outcomes = cohort.outcomes; + console.log(`\n${name}`); + console.log( + [ + `runs=${outcomes.total}`, + `success=${outcomes.success}`, + `failure=${outcomes.failure}`, + `timed-out=${outcomes.timedOut}`, + `startup-failure=${outcomes.startupFailure}`, + `action-required=${outcomes.actionRequired}`, + `neutral=${outcomes.neutral}`, + `skipped=${outcomes.skipped}`, + `stale=${outcomes.stale}`, + `cancelled=${outcomes.cancelled}`, + `queued=${outcomes.queued}`, + `pending=${outcomes.pending}`, + `in-progress=${outcomes.inProgress}`, + `other=${outcomes.other}`, + `pass=${formatPercent(outcomes.nonCancelledPassRate)}`, + `cancelled-rate=${formatPercent(outcomes.cancellationRate)}`, + ].join(" "), + ); + console.log( + `successful wall ${formatDistribution(cohort.runMetrics.successfulWallSeconds)}`, + ); + console.log( + `workflow admission ${formatDistribution(cohort.runMetrics.workflowAdmissionSeconds)}`, + ); + console.log( + `dependency gating ${formatDistribution(cohort.jobMetrics.dependencyGatedSeconds)}`, + ); + console.log(`runner queue ${formatDistribution(cohort.jobMetrics.runnerQueueSeconds)}`); + console.log(`job execution ${formatDistribution(cohort.jobMetrics.executionSeconds)}`); + console.log( + `detail sample ${cohort.samples.detailedSuccessfulRuns}/${cohort.samples.successfulRuns} successful runs, ${cohort.samples.timedJobs} jobs`, + ); +} + +function printTrendReport(report) { + const { baseline, comparison, prior } = report.cohorts; + console.log( + `CI trend: ${report.options.trendHours}h baseline; latest ${report.options.compareHours}h vs prior ${report.options.compareHours}h`, + ); + console.log( + `API requests=${report.apiRequests.total} (run-list=${report.apiRequests.runList}, jobs=${report.apiRequests.jobs}); detailed=${report.sampling.detailedSuccessfulRuns}/${report.sampling.eligibleSuccessfulRuns} successful runs`, + ); + printTrendCohort("Baseline", baseline); + printTrendCohort("Prior comparison window", prior); + printTrendCohort("Latest comparison window", comparison); + + console.log("\nLatest minus prior"); + console.log( + [ + `wall-p50=${formatDelta(report.changes.successfulWallP50Seconds)}`, + `wall-p90=${formatDelta(report.changes.successfulWallP90Seconds)}`, + `admission-p95=${formatDelta(report.changes.workflowAdmissionP95Seconds)}`, + `queue-p95=${formatDelta(report.changes.runnerQueueP95Seconds)}`, + `execution-p90=${formatDelta(report.changes.executionP90Seconds)}`, + ].join(" "), + ); + + if (comparison.criticalOwners.length > 0) { + console.log("\nLatest critical-path owners"); + for (const owner of comparison.criticalOwners.slice(0, 15)) { + console.log(`${String(owner.name).padEnd(56)} ${owner.runs} run(s)`); + } + } + + const timedJobs = report.jobs.filter((job) => job.comparison.executionSeconds.count > 0); + if (timedJobs.length > 0) { + console.log("\nLatest job execution p90"); + for (const job of timedJobs.slice(0, 15)) { + console.log( + `${String(job.name).padEnd(56)} latest=${formatSeconds(job.comparison.executionSeconds.p90).padStart(6)} prior=${formatSeconds(job.prior.executionSeconds.p90).padStart(6)}`, + ); + } + } +} + +function printSection(title, jobs, metric) { + console.log(title); + for (const job of jobs) { + console.log( + `${String(job.name).padEnd(48)} ${formatSeconds(job[metric]).padStart(6)} start-delay=${formatSeconds(job.startDelaySeconds).padStart(6)} ${job.status}/${job.conclusion}`, + ); + } +} + +/** + * Parses CI run timing CLI arguments. + */ +export function parseRunTimingArgs(args) { + let compareHours = DEFAULT_TREND_COMPARE_HOURS; + let compareHoursSpecified = false; + let detailRuns = DEFAULT_TREND_DETAIL_RUNS; + let detailRunsSpecified = false; + let explicitRunId; + let json = false; + let limit = 15; + let limitSpecified = false; + let outputPath = null; + let recentLimit = null; + let trendHours = null; + let useLatestMain = false; + + for (let index = 0; index < args.length; index += 1) { + const arg = args[index]; + if (arg === "--") { + continue; + } + if (arg === "--latest-main") { + useLatestMain = true; + continue; + } + if (arg === "--json") { + json = true; + continue; + } + const limitOption = consumePositiveIntFlag(args, index, "--limit"); + if (limitOption) { + limit = limitOption.value; + limitSpecified = true; + index = limitOption.nextIndex; + continue; + } + const recentOption = consumePositiveIntFlag(args, index, "--recent"); + if (recentOption) { + recentLimit = recentOption.value; + index = recentOption.nextIndex; + continue; + } + const trendOption = consumePositiveIntFlag(args, index, "--trend-hours"); + if (trendOption) { + trendHours = trendOption.value; + index = trendOption.nextIndex; + continue; + } + const compareOption = consumePositiveIntFlag(args, index, "--compare-hours"); + if (compareOption) { + compareHours = compareOption.value; + compareHoursSpecified = true; + index = compareOption.nextIndex; + continue; + } + const detailOption = consumePositiveIntFlag(args, index, "--detail-runs"); + if (detailOption) { + detailRuns = detailOption.value; + detailRunsSpecified = true; + index = detailOption.nextIndex; + continue; + } + const outputOption = consumeStringFlag(args, index, "--output"); + if (outputOption) { + outputPath = outputOption.value; + index = outputOption.nextIndex; + continue; + } + if (arg.startsWith("-")) { + throw new Error(`Unknown CI run timing option: ${arg}`); + } + if (explicitRunId) { + throw new Error(`Unexpected CI run id argument: ${arg}`); + } + explicitRunId = arg; + } + + if (recentLimit !== null && (explicitRunId || useLatestMain)) { + throw new Error("--recent cannot be combined with a run id or --latest-main"); + } + if (explicitRunId && useLatestMain) { + throw new Error("A run id cannot be combined with --latest-main"); + } + if (trendHours !== null) { + if (explicitRunId || useLatestMain || recentLimit !== null || limitSpecified) { + throw new Error("--trend-hours cannot be combined with single-run or --recent options"); + } + if (trendHours < compareHours * 2) { + throw new Error("--trend-hours must cover at least two --compare-hours windows"); + } + } else if (compareHoursSpecified || detailRunsSpecified || json || outputPath !== null) { + throw new Error("--compare-hours, --detail-runs, --json, and --output require --trend-hours"); + } + + return { + compareHours, + detailRuns, + explicitRunId, + json, + limit, + outputPath, + recentLimit, + trendHours, + useLatestMain, + }; +} + +function consumePositiveIntFlag(args, index, flag) { + const arg = args[index]; + const inlinePrefix = `${flag}=`; + if (arg.startsWith(inlinePrefix)) { + return { + nextIndex: index, + value: parsePositiveInt(arg.slice(inlinePrefix.length), flag), + }; + } + if (arg !== flag) { + return null; + } + const rawValue = args[index + 1]; + if (!rawValue || rawValue.startsWith("-")) { + throw new Error(`${flag} requires a value`); + } + return { + nextIndex: index + 1, + value: parsePositiveInt(rawValue, flag), + }; +} + +function consumeStringFlag(args, index, flag) { + const arg = args[index]; + const inlinePrefix = `${flag}=`; + if (arg.startsWith(inlinePrefix)) { + const value = arg.slice(inlinePrefix.length); + if (!value) { + throw new Error(`${flag} requires a value`); + } + return { nextIndex: index, value }; + } + if (arg !== flag) { + return null; + } + const value = args[index + 1]; + if (!value || value.startsWith("-")) { + throw new Error(`${flag} requires a value`); + } + return { nextIndex: index + 1, value }; +} + +function selectTrendDetailCandidates(runs, generatedAtMs, compareDurationMs, limit) { + const comparisonFromMs = generatedAtMs - compareDurationMs; + const priorFromMs = comparisonFromMs - compareDurationMs; + const successfulRuns = runs.filter( + (run) => run.status === "completed" && run.conclusion === "success", + ); + const comparison = successfulRuns.filter( + (run) => (parseTime(run.createdAt) ?? 0) >= comparisonFromMs, + ); + const prior = successfulRuns.filter((run) => { + const createdAt = parseTime(run.createdAt) ?? 0; + return createdAt >= priorFromMs && createdAt < comparisonFromMs; + }); + const older = successfulRuns.filter((run) => (parseTime(run.createdAt) ?? 0) < priorFromMs); + const selected = []; + let comparisonIndex = 0; + let priorIndex = 0; + while ( + selected.length < limit && + (comparisonIndex < comparison.length || priorIndex < prior.length) + ) { + if (comparisonIndex < comparison.length && selected.length < limit) { + selected.push(comparison[comparisonIndex]); + comparisonIndex += 1; + } + if (priorIndex < prior.length && selected.length < limit) { + selected.push(prior[priorIndex]); + priorIndex += 1; + } + } + return [ + ...selected, + ...comparison.slice(comparisonIndex), + ...prior.slice(priorIndex), + ...older, + ].slice(0, limit); +} + +function runTrendReport(options) { + const generatedAtMs = Date.now(); + const trendDurationMs = options.trendHours * 60 * 60 * 1000; + const compareDurationMs = options.compareHours * 60 * 60 * 1000; + const listed = listTrendCiRuns(generatedAtMs - trendDurationMs); + const eligibleRuns = listed.runs.filter( + (run) => run.status === "completed" && run.conclusion === "success", + ); + const detailCandidates = selectTrendDetailCandidates( + listed.runs, + generatedAtMs, + compareDurationMs, + options.detailRuns, + ); + console.error( + `[ci-timings] loading job details for ${detailCandidates.length}/${eligibleRuns.length} successful runs; expect at least ${detailCandidates.length} job API requests`, + ); + const detailsByRun = new Map(); + let jobsRequestCount = 0; + for (const run of detailCandidates) { + const loaded = loadRunJobs(run.databaseId, run.runAttempt); + jobsRequestCount += loaded.requestCount; + detailsByRun.set(run.databaseId, loaded.jobs); + } + const runs = listed.runs.map((run) => + detailsByRun.has(run.databaseId) ? { ...run, jobs: detailsByRun.get(run.databaseId) } : run, + ); + const summary = summarizeTrendTimings(runs, { + compareDurationMs, + generatedAtMs, + trendDurationMs, + }); + const repository = process.env.GITHUB_REPOSITORY || DEFAULT_GITHUB_REPOSITORY; + return { + apiRequests: { + jobs: jobsRequestCount, + runList: listed.requestCount, + total: listed.requestCount + jobsRequestCount, + }, + generatedAt: new Date(generatedAtMs).toISOString(), + options: { + compareHours: options.compareHours, + detailRuns: options.detailRuns, + trendHours: options.trendHours, + }, + repository, + sampling: { + detailedSuccessfulRuns: detailCandidates.length, + eligibleSuccessfulRuns: eligibleRuns.length, + }, + ...summary, + }; +} + +async function main() { + const options = parseRunTimingArgs(process.argv.slice(2)); + const { explicitRunId, limit, recentLimit, useLatestMain } = options; + if (options.trendHours !== null) { + const report = runTrendReport(options); + const reportJson = `${JSON.stringify(report, null, 2)}\n`; + if (options.outputPath) { + mkdirSync(path.dirname(options.outputPath), { recursive: true }); + writeFileSync(options.outputPath, reportJson); + } + if (options.json) { + process.stdout.write(reportJson); + } else { + printTrendReport(report); + if (options.outputPath) { + console.log(`\nJSON report: ${options.outputPath}`); + } + } + return; + } + if (recentLimit !== null) { + for (const run of listRecentSuccessfulCiRuns(recentLimit)) { + const summary = summarizeJobs(loadRun(run.databaseId)); + console.log( + [ + `CI run ${run.databaseId}`, + run.headSha.slice(0, 10), + `wall=${formatSeconds(summary.wallSeconds)}`, + `exec=${formatSeconds(summary.executionWindowSeconds)}`, + `firstStartDelay=${formatSeconds(summary.firstStartDelaySeconds)}`, + `jobs=${summary.jobCount}`, + `avg=${formatSeconds(summary.avgDurationSeconds)}`, + `p90=${formatSeconds(summary.p90DurationSeconds)}`, + `p95=${formatSeconds(summary.p95DurationSeconds)}`, + `max=${formatSeconds(summary.maxDurationSeconds)}`, + ].join(" "), + ); + } + return; + } + const runId = explicitRunId ?? (useLatestMain ? getLatestMainPushCiRunId() : getLatestCiRunId()); + const run = loadRun(runId); + const summary = summarizeRunTimings(run, limit); + const warmupBarrier = summarizePnpmStoreWarmupBarrier(run); + + console.log( + `CI run ${runId}: ${summary.status}/${summary.conclusion} wall=${formatSeconds(summary.wallSeconds)}`, + ); + if (warmupBarrier) { + console.log("\npnpm-store-warmup barrier"); + console.log( + [ + `result=${warmupBarrier.warmupResult}`, + `preflight->start=${formatSeconds(warmupBarrier.preflightToWarmupStartSeconds)}`, + `duration=${formatSeconds(warmupBarrier.warmupDurationSeconds)}`, + `preflight->complete=${formatSeconds(warmupBarrier.preflightToWarmupCompleteSeconds)}`, + ].join(" "), + ); + console.log( + [ + `active-post-warmup-jobs=${warmupBarrier.activePostWarmupJobCount}`, + `first-start-delay=${formatSeconds(warmupBarrier.firstPostWarmupStartDelaySeconds)}`, + `p95-start-delay=${formatSeconds(warmupBarrier.postWarmupP95StartDelaySeconds)}`, + `started-within-${warmupBarrier.windowSeconds}s=${warmupBarrier.postWarmupStartedWithinWindow}`, + ].join(" "), + ); + } + printSection("\nSlowest jobs", summary.byDuration, "durationSeconds"); + printSection( + "\nLongest start delays (dependencies + runner queue)", + summary.byStartDelay, + "startDelaySeconds", + ); + if (summary.badJobs.length > 0) { + console.log("\nFailed jobs"); + for (const job of summary.badJobs) { + console.log(`${job.name} ${job.status}/${job.conclusion}`); + } + } +} + +if (isDirectRunUrl(process.argv[1], import.meta.url)) { + await main(); +} diff --git a/scripts/clawhub-prepared-artifact.mjs b/scripts/clawhub-prepared-artifact.mjs new file mode 100644 index 0000000000000000000000000000000000000000..2441254b17bf8f180047bb905b677a2e46716a42 --- /dev/null +++ b/scripts/clawhub-prepared-artifact.mjs @@ -0,0 +1,651 @@ +#!/usr/bin/env node + +import { + lstatSync, + mkdirSync, + mkdtempSync, + readdirSync, + renameSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { dirname, join } from "node:path"; +import { pathToFileURL } from "node:url"; +import { isDeepStrictEqual, parseArgs } from "node:util"; +import { + CLAWHUB_CHILD_WORKFLOW, + readPackedClawHubTransaction, +} from "./clawhub-parent-authorization.mjs"; +import { + downloadExactActionsArtifactArchive, + inspectActionsArtifactZipWithPolicy, + readBoundedRegularFile, + validateActionsArtifactBinding, + validateActionsArtifactProducerJob, +} from "./lib/actions-artifact-archive.mjs"; +import { readBoundedResponseText } from "./lib/bounded-response.mjs"; +import { isRecord } from "./lib/record-shared.mjs"; +import { runReleaseToolingGh, verifyReleaseToolingIdentity } from "./release-tooling-identity.mjs"; + +const REPOSITORY = "openclaw/openclaw"; +const MANIFEST_FILE = "prepared-clawhub.json"; +const MAX_MANIFEST_BYTES = 1024 * 1024; +const MAX_PACKAGE_BYTES = 120 * 1024 * 1024; +const MAX_PACKAGE_ZIP_BYTES = 130 * 1024 * 1024; +const SHA = /^[a-f0-9]{40}$/u; +const DIGEST = /^[a-f0-9]{64}$/u; +const ARTIFACT = /^[A-Za-z0-9][A-Za-z0-9_.-]{0,254}$/u; +const VERSION = + /^[0-9]{4}\.[1-9][0-9]*\.[1-9][0-9]*(?:-(?:alpha|beta)\.[1-9][0-9]*|-[1-9][0-9]*)?$/u; +const PRODUCER_KEYS = + "repository runId runAttempt workflowPath workflowEvent workflowHeadBranch workflowSha".split( + " ", + ); +const ARTIFACT_KEYS = "artifactId artifactName artifactDigest artifactSizeBytes".split(" "); +const PACKAGE_KEYS = + "packageName packageDir version publishTag tarballName tarballSha256 tarballSizeBytes inventoryDigest".split( + " ", + ); + +function exactKeys(value, keys, label) { + if ( + !isRecord(value) || + Object.keys(value).length !== keys.length || + keys.some((key) => !Object.hasOwn(value, key)) + ) { + throw new Error(`${label} fields are invalid.`); + } +} +function matches(value, pattern, label) { + if (typeof value !== "string" || !pattern.test(value)) { + throw new Error(`${label} is invalid.`); + } + return value; +} +function positive(value, maximum, label) { + if (!Number.isSafeInteger(value) || value < 1 || value > maximum) { + throw new Error(`${label} is invalid.`); + } +} +function same(actual, expected, label) { + if (!isDeepStrictEqual(actual, expected)) { + throw new Error(`${label} mismatch.`); + } +} +function pick(value, keys) { + return Object.fromEntries(keys.map((key) => [key, value[key]])); +} +function producerFullRef(producer) { + if (producer.workflowHeadBranch === "main") { + return "refs/heads/main"; + } + const ref = matches( + producer.workflowHeadBranch, + /^release-publish\/[a-f0-9]{12}-[1-9][0-9]*$/u, + "Prepared ClawHub tooling ref", + ); + if (!ref.startsWith(`release-publish/${producer.workflowSha.slice(0, 12)}-`)) { + throw new Error("Prepared ClawHub tooling ref does not match its SHA."); + } + return `refs/tags/${ref}`; +} +function validateProducer(value) { + exactKeys(value, PRODUCER_KEYS, "Prepared ClawHub producer"); + if ( + value.repository !== REPOSITORY || + value.workflowPath !== CLAWHUB_CHILD_WORKFLOW || + value.workflowEvent !== "workflow_dispatch" + ) { + throw new Error("Prepared ClawHub producer repository or workflow mismatch."); + } + positive(value.runId, Number.MAX_SAFE_INTEGER, "Prepared ClawHub run ID"); + positive(value.runAttempt, Number.MAX_SAFE_INTEGER, "Prepared ClawHub run attempt"); + matches(value.workflowSha, SHA, "Prepared ClawHub tooling SHA"); + producerFullRef(value); + return value; +} +function validateArtifact(value) { + positive(value.artifactId, Number.MAX_SAFE_INTEGER, "Prepared ClawHub artifact ID"); + positive(value.artifactSizeBytes, MAX_PACKAGE_ZIP_BYTES, "Prepared ClawHub artifact size"); + matches(value.artifactName, ARTIFACT, "Prepared ClawHub artifact name"); + matches(value.artifactDigest, /^sha256:[a-f0-9]{64}$/u, "Prepared ClawHub artifact digest"); +} +function validateDescriptor(value, toolingSha) { + exactKeys(value, [...PRODUCER_KEYS, ...ARTIFACT_KEYS], "Prepared ClawHub artifact descriptor"); + validateProducer(pick(value, PRODUCER_KEYS)); + validateArtifact(value); + if (toolingSha !== undefined) { + same(value.workflowSha, toolingSha, "Prepared ClawHub tooling SHA"); + } + return value; +} +function selectionEntry(entry) { + matches(entry.packageName, /^@openclaw\/[a-z0-9][a-z0-9._-]*$/u, "Prepared ClawHub package name"); + matches( + entry.packageDir, + /^extensions\/[a-z0-9][a-z0-9._-]*$/u, + "Prepared ClawHub package directory", + ); + matches(entry.version, VERSION, "Prepared ClawHub package version"); + matches(entry.publishTag, /^(alpha|beta|latest)$/u, "Prepared ClawHub publication tag"); + return pick(entry, ["packageName", "packageDir", "version", "publishTag"]); +} +function validatePackage(entry) { + exactKeys(entry, [...PACKAGE_KEYS, ...ARTIFACT_KEYS], "Prepared ClawHub package"); + selectionEntry(entry); + validateArtifact(entry); + matches(entry.tarballName, /^[A-Za-z0-9][A-Za-z0-9._-]*\.tgz$/u, "Prepared ClawHub tarball name"); + matches(entry.tarballSha256, DIGEST, "Prepared ClawHub tarball digest"); + matches(entry.inventoryDigest, DIGEST, "Prepared ClawHub inventory digest"); + positive(entry.tarballSizeBytes, MAX_PACKAGE_BYTES, "Prepared ClawHub tarball size"); + return entry; +} +function orderedSelection(matrix) { + if (!Array.isArray(matrix) || matrix.length > 512) { + throw new Error("Prepared ClawHub package selection is invalid."); + } + const selection = matrix + .map(selectionEntry) + .toSorted((left, right) => left.packageName.localeCompare(right.packageName)); + if (new Set(selection.map((entry) => entry.packageName)).size !== selection.length) { + throw new Error("Prepared ClawHub package selection contains duplicates."); + } + return selection; +} +function preparedPackageArtifactName(entry, producer) { + const packageName = entry.packageName.slice(1).replace("/", "-"); + return `clawhub-package-${packageName}-${entry.version}-${producer.runId}-${producer.runAttempt}`; +} +export function preparedClawHubArtifactName(candidateSha, producer) { + matches(candidateSha, SHA, "Prepared ClawHub candidate SHA"); + validateProducer(producer); + return `clawhub-prepared-${candidateSha.slice(0, 12)}-${producer.runId}-${producer.runAttempt}`; +} +export function validatePreparedClawHubManifest(value, { descriptor, candidateSha }) { + exactKeys( + value, + ["schemaVersion", "kind", "candidateSha", "producer", "selectionMode", "packages"], + "Prepared ClawHub manifest", + ); + if (value.schemaVersion !== 1 || value.kind !== "openclaw-clawhub-prepared") { + throw new Error("Unsupported prepared ClawHub manifest."); + } + matches(value.selectionMode, /^(selected|all-publishable)$/u, "Prepared ClawHub selection mode"); + validateDescriptor(descriptor); + matches(value.candidateSha, SHA, "Prepared ClawHub candidate SHA"); + same(value.candidateSha, candidateSha, "Prepared ClawHub candidate SHA"); + validateProducer(value.producer); + same(value.producer, pick(descriptor, PRODUCER_KEYS), "Prepared ClawHub producer"); + same( + descriptor.artifactName, + preparedClawHubArtifactName(candidateSha, value.producer), + "Prepared ClawHub manifest artifact name", + ); + if (!Array.isArray(value.packages) || value.packages.length > 512) { + throw new Error("Prepared ClawHub manifest package count is invalid."); + } + value.packages.forEach((entry) => { + validatePackage(entry); + same( + entry.artifactName, + preparedPackageArtifactName(entry, value.producer), + "Prepared ClawHub package artifact producer attempt", + ); + }); + const selection = orderedSelection(value.packages); + same( + value.packages.map((entry) => entry.packageName), + selection.map((entry) => entry.packageName), + "Prepared ClawHub package order", + ); + if ( + new Set(value.packages.map((entry) => entry.artifactId)).size !== value.packages.length || + new Set(value.packages.map((entry) => entry.artifactName)).size !== value.packages.length + ) { + throw new Error("Prepared ClawHub package artifacts are duplicated."); + } + return value; +} + +function api(path) { + const raw = runReleaseToolingGh(["api", `repos/${REPOSITORY}/${path}`, "--method", "GET"]); + if (Buffer.byteLength(raw) > 4 * 1024 * 1024) { + throw new Error("Prepared ClawHub GitHub metadata exceeds its limit."); + } + return JSON.parse(raw); +} +function verifyProducerTooling(producer, runGhJson) { + verifyReleaseToolingIdentity({ + repository: REPOSITORY, + workflowRef: producer.workflowHeadBranch, + workflowFullRef: producerFullRef(producer), + workflowSha: producer.workflowSha, + runGh: (args) => JSON.stringify(runGhJson(args[1].replace(`repos/${REPOSITORY}/`, ""))), + }); +} +function workflowRunForProducer(run, producer) { + const [workflowPath, ...qualifiedRef] = String(run.path).split("@"); + if (qualifiedRef.length > 0 && qualifiedRef.join("@") !== producerFullRef(producer)) { + throw new Error("Prepared ClawHub producer full ref mismatch."); + } + return { ...run, path: workflowPath }; +} +async function downloadBoundArtifact({ + descriptor, + token, + fetchImpl, + runGhJson = api, + maxBytes, + archivePath, +}) { + validateDescriptor(descriptor); + const run = runGhJson(`actions/runs/${descriptor.runId}/attempts/${descriptor.runAttempt}`); + const artifact = runGhJson(`actions/artifacts/${descriptor.artifactId}`); + validateActionsArtifactBinding({ + expected: { ...descriptor, runStatePolicy: "completed-success" }, + artifactMetadata: artifact, + workflowRun: workflowRunForProducer(run, descriptor), + }); + return await downloadExactActionsArtifactArchive({ + expected: { ...descriptor, artifactExpiresAt: artifact.expires_at }, + token, + fetchImpl, + archivePath, + maxArchiveBytes: maxBytes, + }); +} +export async function downloadPreparedClawHubRelease(options) { + const { + descriptor, + candidateSha, + toolingSha, + selectionMode, + plugins, + token, + fetchImpl, + runGhJson = api, + archivePath, + } = options; + validateDescriptor(descriptor, toolingSha); + verifyProducerTooling(descriptor, runGhJson); + const { archiveBytes } = await downloadBoundArtifact({ + descriptor, + token, + fetchImpl, + runGhJson, + archivePath, + maxBytes: MAX_MANIFEST_BYTES, + }); + const files = inspectActionsArtifactZipWithPolicy(archiveBytes, { + expectedEntries: [MANIFEST_FILE], + maxArchiveBytes: MAX_MANIFEST_BYTES, + maxExpandedBytes: MAX_MANIFEST_BYTES, + maxEntryBytes: () => MAX_MANIFEST_BYTES, + }); + const manifest = validatePreparedClawHubManifest( + JSON.parse(files.get(MANIFEST_FILE).toString("utf8")), + { descriptor, candidateSha }, + ); + matches(selectionMode, /^(selected|all-publishable)$/u, "Requested ClawHub selection mode"); + same(manifest.selectionMode, selectionMode, "Prepared ClawHub selection mode"); + const requested = ( + typeof plugins === "string" ? plugins.split(",").filter(Boolean) : (plugins ?? []) + ) + .map((name) => matches(name, /^@openclaw\/[a-z0-9][a-z0-9._-]*$/u, "Requested ClawHub package")) + .toSorted((left, right) => left.localeCompare(right)); + if (selectionMode === "all-publishable") { + if (requested.length > 0) { + throw new Error("All-publishable ClawHub promotion must not narrow the prepared selection."); + } + } else { + same( + requested, + manifest.packages.map((entry) => entry.packageName), + "Prepared ClawHub full package selection", + ); + } + return manifest; +} +export function isPreparedClawHubTrustedPublisher(publisher) { + return ( + publisher?.provider === "github-actions" && + publisher?.repository === REPOSITORY && + publisher?.workflowFilename === "plugin-clawhub-release.yml" && + publisher?.environment == null + ); +} + +async function preparedPackageIsPublished(entry, fetchImpl) { + const packageUrl = `https://clawhub.ai/api/v1/packages/${encodeURIComponent(entry.packageName)}`; + const publisherRepair = + "Complete bootstrap or trusted-publisher repair through the existing Plugin ClawHub New owner before preparing again."; + const signal = AbortSignal.timeout(30_000); + const publisherResponse = await fetchImpl(`${packageUrl}/trusted-publisher`, { signal }); + if (!publisherResponse.ok) { + await publisherResponse.body?.cancel().catch(() => undefined); + throw new Error( + `Prepared ClawHub trusted publisher is unavailable for ${entry.packageName}: HTTP ${publisherResponse.status}.${publisherResponse.status === 404 ? ` ${publisherRepair}` : ""}`, + ); + } + const publisher = JSON.parse( + await readBoundedResponseText( + publisherResponse, + "Prepared ClawHub trusted publisher", + 64 * 1024, + { signal }, + ), + ).trustedPublisher; + if (!isPreparedClawHubTrustedPublisher(publisher)) { + throw new Error( + `Prepared ClawHub trusted publisher is missing or mismatched for ${entry.packageName}. ${publisherRepair}`, + ); + } + const response = await fetchImpl(`${packageUrl}/versions/${encodeURIComponent(entry.version)}`, { + signal, + }); + await response.body?.cancel().catch(() => undefined); + if (response.status === 404) { + return false; + } + if (response.status === 200) { + return true; + } + throw new Error( + `Cannot resolve prepared ClawHub version ${entry.packageName}@${entry.version}: HTTP ${response.status}.`, + ); +} +export async function resolvePreparedClawHubMatrix(options) { + const manifest = await downloadPreparedClawHubRelease(options); + const matrix = []; + for (let index = 0; index < manifest.packages.length; index += 8) { + const entries = await Promise.allSettled( + manifest.packages.slice(index, index + 8).map(async (entry) => + Object.assign(selectionEntry(entry), { + artifactName: entry.artifactName, + alreadyPublished: await preparedPackageIsPublished(entry, options.fetchImpl ?? fetch), + prepared: entry, + }), + ), + ); + for (const result of entries) { + if (result.status === "fulfilled") { + matrix.push(result.value); + } + } + const failure = entries.find((result) => result.status === "rejected"); + if (failure) { + throw failure.reason; + } + } + return matrix; +} +function expectedTransaction(entry, artifactName) { + return { + name: entry.packageName, + version: entry.version, + inventoryDigest: entry.inventoryDigest, + artifactName, + artifactSha256: entry.tarballSha256, + artifactSize: entry.tarballSizeBytes, + }; +} +function verifyRestoredPackage(directory, entry, artifactName) { + const actual = readPackedClawHubTransaction({ + artifactDir: directory, + packageName: entry.packageName, + version: entry.version, + artifactName, + }); + same( + actual, + expectedTransaction(entry, artifactName), + "Restored ClawHub package bytes and inventory", + ); +} +export async function restorePreparedClawHubPackage(options) { + const { descriptor, entry, outputDir, token, fetchImpl, runGhJson = api } = options; + validateDescriptor(descriptor, options.toolingSha); + validatePackage(entry); + const artifactName = options.artifactName ?? entry.artifactName; + matches(artifactName, ARTIFACT, "Restored ClawHub artifact name"); + verifyProducerTooling(descriptor, runGhJson); + const source = { ...pick(descriptor, PRODUCER_KEYS), ...pick(entry, ARTIFACT_KEYS) }; + mkdirSync(dirname(outputDir), { recursive: true }); + const { archiveBytes } = await downloadBoundArtifact({ + descriptor: source, + token, + fetchImpl, + runGhJson, + archivePath: options.archivePath, + maxBytes: MAX_PACKAGE_ZIP_BYTES, + }); + const files = inspectActionsArtifactZipWithPolicy(archiveBytes, { + expectedEntries: [entry.tarballName], + maxArchiveBytes: MAX_PACKAGE_ZIP_BYTES, + maxExpandedBytes: MAX_PACKAGE_BYTES, + maxEntryBytes: () => MAX_PACKAGE_BYTES, + }); + // Publish only after a complete, checked transfer. A failed download never + // leaves a tarball in the directory consumed by the upload/publication step. + let existing; + try { + existing = lstatSync(outputDir); + } catch (error) { + if (!error || typeof error !== "object" || error.code !== "ENOENT") { + throw error; + } + } + if (existing) { + if (!existing.isDirectory() || existing.isSymbolicLink()) { + throw new Error("Restored ClawHub output must be a regular directory."); + } + verifyRestoredPackage(outputDir, entry, artifactName); + return { + packageName: entry.packageName, + tarballSha256: entry.tarballSha256, + inventoryDigest: entry.inventoryDigest, + }; + } + const staging = mkdtempSync(`${outputDir}.download-`); + try { + writeFileSync(join(staging, entry.tarballName), files.get(entry.tarballName), { + flag: "wx", + mode: 0o600, + }); + verifyRestoredPackage(staging, entry, artifactName); + renameSync(staging, outputDir); + } finally { + rmSync(staging, { recursive: true, force: true }); + } + return { + packageName: entry.packageName, + tarballSha256: entry.tarballSha256, + inventoryDigest: entry.inventoryDigest, + }; +} +export function createPreparedClawHubManifest({ + candidateSha, + producer, + selectionMode, + matrix, + directory, + artifacts, + workflowRun, + workflowJobs, +}) { + validateProducer(producer); + matches(selectionMode, /^(selected|all-publishable)$/u, "Prepared ClawHub selection mode"); + const packages = orderedSelection(matrix).map((selected) => { + const entry = matrix.find((item) => item.packageName === selected.packageName); + if (entry.artifactName !== preparedPackageArtifactName(entry, producer)) { + throw new Error( + "Prepared ClawHub package artifact does not bind its exact producer attempt; rerun all preparation jobs.", + ); + } + const artifactMatches = artifacts.filter((artifact) => artifact.name === entry.artifactName); + if (artifactMatches.length !== 1) { + throw new Error(`Prepared ClawHub artifact is missing or ambiguous: ${entry.artifactName}.`); + } + const artifact = artifactMatches[0]; + // Failed-job reruns retain the run ID/SHA and may reuse older matrix outputs. + // Both the artifact name and its successful pack job must bind this attempt. + const expected = { + ...producer, + artifactId: artifact.id, + artifactName: artifact.name, + artifactDigest: artifact.digest, + artifactSizeBytes: artifact.size_in_bytes, + runStatePolicy: "same-run-producer-success", + consumerRunAttempt: producer.runAttempt, + producerJobName: `Pack ClawHub package (${entry.packageName})`, + }; + validateActionsArtifactBinding({ + expected, + artifactMetadata: artifact, + workflowRun: workflowRunForProducer(workflowRun, producer), + }); + validateActionsArtifactProducerJob({ expected, workflowJobs }); + const artifactDir = join(directory, entry.artifactName); + const transaction = readPackedClawHubTransaction({ + artifactDir, + artifactName: entry.artifactName, + packageName: entry.packageName, + version: entry.version, + }); + return validatePackage({ + ...selected, + artifactId: artifact.id, + artifactName: artifact.name, + artifactDigest: artifact.digest, + artifactSizeBytes: artifact.size_in_bytes, + tarballName: readdirSync(artifactDir)[0], + tarballSha256: transaction.artifactSha256, + tarballSizeBytes: transaction.artifactSize, + inventoryDigest: transaction.inventoryDigest, + }); + }); + return { + schemaVersion: 1, + kind: "openclaw-clawhub-prepared", + candidateSha: matches(candidateSha, SHA, "Prepared ClawHub candidate SHA"), + producer, + selectionMode, + packages, + }; +} +function readJson(file) { + return JSON.parse( + readBoundedRegularFile(file, { label: file, maxBytes: MAX_MANIFEST_BYTES }).toString("utf8"), + ); +} +function listRunInventory(path, field) { + const entries = []; + for (let page = 1; page <= 20; page++) { + const result = api(`${path}?per_page=100&page=${page}`); + if ( + !Array.isArray(result[field]) || + !Number.isSafeInteger(result.total_count) || + result.total_count < 0 || + result.total_count > 2000 + ) { + throw new Error(`Prepared ClawHub ${field} inventory is invalid.`); + } + entries.push(...result[field]); + if (entries.length === result.total_count) { + return { total_count: result.total_count, [field]: entries }; + } + if (result[field].length === 0 || entries.length > result.total_count) { + break; + } + } + throw new Error(`Prepared ClawHub ${field} inventory is incomplete.`); +} +async function main() { + const { values, positionals } = parseArgs({ + allowPositionals: true, + options: Object.fromEntries( + ["descriptor", "matrix", "directory", "output", "package"].map((key) => [ + key, + { type: "string" }, + ]), + ), + }); + const env = process.env; + let result; + if (positionals[0] === "seal") { + const producer = validateProducer({ + repository: env.GITHUB_REPOSITORY, + runId: Number(env.GITHUB_RUN_ID), + runAttempt: Number(env.GITHUB_RUN_ATTEMPT), + workflowPath: CLAWHUB_CHILD_WORKFLOW, + workflowEvent: env.GITHUB_EVENT_NAME, + workflowHeadBranch: env.GITHUB_REF_NAME, + workflowSha: env.GITHUB_WORKFLOW_SHA, + }); + if ( + env.DRY_RUN !== "true" || + env.GITHUB_WORKFLOW_REF !== + `${REPOSITORY}/${CLAWHUB_CHILD_WORKFLOW}@${producerFullRef(producer)}` + ) { + throw new Error("Prepared ClawHub artifacts must be sealed by the validation-only workflow."); + } + result = createPreparedClawHubManifest({ + candidateSha: env.TARGET_SHA, + producer, + selectionMode: env.PUBLISH_SCOPE, + matrix: readJson(values.matrix), + directory: values.directory, + artifacts: listRunInventory(`actions/runs/${producer.runId}/artifacts`, "artifacts") + .artifacts, + workflowRun: api(`actions/runs/${producer.runId}/attempts/${producer.runAttempt}`), + workflowJobs: listRunInventory( + `actions/runs/${producer.runId}/attempts/${producer.runAttempt}/jobs`, + "jobs", + ), + }); + writeFileSync( + env.GITHUB_OUTPUT, + `artifact_name=${preparedClawHubArtifactName(env.TARGET_SHA, producer)}\n`, + { flag: "a" }, + ); + } else if (positionals[0] === "resolve") { + result = await resolvePreparedClawHubMatrix({ + descriptor: readJson(values.descriptor), + candidateSha: env.TARGET_SHA, + toolingSha: env.GITHUB_WORKFLOW_SHA, + selectionMode: env.PUBLISH_SCOPE, + plugins: env.RELEASE_PLUGINS, + token: env.GH_TOKEN, + }); + } else if (positionals[0] === "restore") { + const selected = readJson(values.package); + same( + selectionEntry(selected), + selectionEntry(selected.prepared), + "Prepared ClawHub package selection", + ); + result = await restorePreparedClawHubPackage({ + descriptor: readJson(values.descriptor), + entry: selected.prepared, + artifactName: selected.artifactName, + toolingSha: env.GITHUB_WORKFLOW_SHA, + outputDir: values.directory, + token: env.GH_TOKEN, + }); + } else { + throw new Error("Expected seal, resolve, or restore."); + } + if (values.output) { + mkdirSync(dirname(values.output), { recursive: true }); + writeFileSync(values.output, `${JSON.stringify(result)}\n`, { flag: "wx" }); + } +} +if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { + try { + await main(); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + console.error("[clawhub-prepared-artifact] FAILED (exit 1)"); + process.exitCode = 1; + } +} diff --git a/scripts/clawlog.sh b/scripts/clawlog.sh new file mode 100644 index 0000000000000000000000000000000000000000..a0fe068374b220a15ba8b23a243fca547eaf2dae --- /dev/null +++ b/scripts/clawlog.sh @@ -0,0 +1,369 @@ +#!/bin/bash + +# VibeTunnel Logging Utility +# Simplifies access to VibeTunnel logs using macOS unified logging system + +set -euo pipefail + +# Configuration +SUBSYSTEM="ai.openclaw" +DEFAULT_LEVEL="info" + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Function to handle sudo password errors +handle_sudo_error() { + echo -e "\n${RED}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" + echo -e "${YELLOW}⚠️ Password Required for Log Access${NC}" + echo -e "${RED}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}\n" + echo -e "clawlog needs to use sudo to show complete log data (Apple hides sensitive info by default)." + echo -e "\nTo avoid password prompts, configure passwordless sudo for the log command:" + echo -e "See: ${BLUE}apple/docs/logging-private-fix.md${NC}\n" + echo -e "Quick fix:" + echo -e " 1. Run: ${GREEN}sudo visudo${NC}" + echo -e " 2. Add: ${GREEN}$(whoami) ALL=(ALL) NOPASSWD: /usr/bin/log${NC}" + echo -e " 3. Save and exit (:wq)\n" + echo -e "${RED}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}\n" + exit 1 +} + +# Default values +STREAM_MODE=false +TIME_RANGE="5m" # Default to last 5 minutes +CATEGORY="" +LOG_LEVEL="$DEFAULT_LEVEL" +SEARCH_TEXT="" +OUTPUT_FILE="" +ERRORS_ONLY=false +TAIL_LINES=50 # Default number of lines to show +SHOW_TAIL=true +STYLE_JSON=false +LIST_CATEGORIES=false + +# Function to show usage +show_usage() { + cat << EOF +clawlog - OpenClaw Logging Utility + +USAGE: + clawlog [OPTIONS] + +DESCRIPTION: + View OpenClaw logs with full details (bypasses Apple's privacy redaction). + Requires sudo access configured for /usr/bin/log command. + +LOG FLOW ARCHITECTURE: + OpenClaw logs flow through the macOS unified log (subsystem: ai.openclaw). + +LOG CATEGORIES (examples): + • voicewake - Voice wake detection/test harness + • gateway - Gateway process manager + • xpc - XPC service calls + • notifications - Notification helper + • screenshot - Screenshotter + • shell - ShellExecutor + +QUICK START: + clawlog -n 100 Show last 100 lines from all components + clawlog -f Follow logs in real-time + clawlog -e Show only errors + clawlog -c ServerManager Show logs from ServerManager only + +OPTIONS: + -h, --help Show this help message + -f, --follow Stream logs continuously (like tail -f) + -n, --lines NUM Number of lines (or JSON records) to show (default: 50) + -l, --last TIME Time range to search (default: 5m) + Examples: 5m, 1h, 2d, 1w + -c, --category CAT Filter by category (e.g., ServerManager, SessionService) + -e, --errors Show only error messages + -d, --debug Show debug level logs (more verbose) + -s, --search TEXT Search for specific text in log messages + -o, --output FILE Export logs to file + --server Show only server output logs + --all Show all logs without tail limit + --list-categories List all available log categories (not with --json) + --json Output one JSON array (not with --follow or --list-categories) + +EXAMPLES: + clawlog Show last 50 lines from past 5 minutes (default) + clawlog -f Stream logs continuously + clawlog -n 100 Show last 100 lines + clawlog -e Show only recent errors + clawlog -l 30m -n 200 Show last 200 lines from past 30 minutes + clawlog -c ServerManager Show recent ServerManager logs + clawlog -s "fail" Search for "fail" in recent logs + clawlog --server -e Show recent server errors + clawlog -f -d Stream debug logs continuously + +CATEGORIES: + Common categories include: + - ServerManager - Server lifecycle and configuration + - SessionService - Terminal session management + - TerminalManager - Terminal spawning and control + - GitRepository - Git integration features + - ScreencapService - Screen capture functionality + - WebRTCManager - WebRTC connections + - UnixSocket - Unix socket communication + - WindowTracker - Window tracking and focus + - NgrokService - Ngrok tunnel management + - ServerOutput - Node.js server output + +TIME FORMATS: + - 5m = 5 minutes - 1h = 1 hour + - 2d = 2 days - 1w = 1 week + +EOF +} + +# Function to list categories +list_categories() { + echo -e "${BLUE}Fetching VibeTunnel log categories from the last hour...${NC}\n" + + # Get unique categories from recent logs + log show --predicate "subsystem == \"$SUBSYSTEM\"" --last 1h 2>/dev/null | \ + grep -E "category: \"[^\"]+\"" | \ + sed -E 's/.*category: "([^"]+)".*/\1/' | \ + sort | uniq | \ + while read -r cat; do + echo " • $cat" + done + + echo -e "\n${YELLOW}Note: Only categories with recent activity are shown${NC}" +} + +# Escape user input embedded in macOS log predicate string literals. +escape_predicate_literal() { + local value="$1" + value="${value//\\/\\\\}" + value="${value//\"/\\\"}" + printf '%s' "$value" +} + +# Parse command line arguments +while [[ $# -gt 0 ]]; do + case $1 in + -h|--help) + show_usage + exit 0 + ;; + -f|--follow) + STREAM_MODE=true + SHOW_TAIL=false + shift + ;; + -n|--lines) + if [[ $# -lt 2 ]]; then + echo -e "${RED}Error: $1 requires a value${NC}" >&2 + exit 1 + fi + TAIL_LINES="$2" + shift 2 + ;; + -l|--last) + if [[ $# -lt 2 ]]; then + echo -e "${RED}Error: $1 requires a value${NC}" >&2 + exit 1 + fi + TIME_RANGE="$2" + shift 2 + ;; + -c|--category) + if [[ $# -lt 2 ]]; then + echo -e "${RED}Error: $1 requires a value${NC}" >&2 + exit 1 + fi + CATEGORY="$2" + shift 2 + ;; + -e|--errors) + ERRORS_ONLY=true + shift + ;; + -d|--debug) + LOG_LEVEL="debug" + shift + ;; + -s|--search) + if [[ $# -lt 2 ]]; then + echo -e "${RED}Error: $1 requires a value${NC}" >&2 + exit 1 + fi + SEARCH_TEXT="$2" + shift 2 + ;; + -o|--output) + if [[ $# -lt 2 ]]; then + echo -e "${RED}Error: $1 requires a value${NC}" >&2 + exit 1 + fi + OUTPUT_FILE="$2" + shift 2 + ;; + --server) + CATEGORY="ServerOutput" + shift + ;; + --list-categories) + LIST_CATEGORIES=true + shift + ;; + --json) + STYLE_JSON=true + shift + ;; + --all) + SHOW_TAIL=false + shift + ;; + *) + echo -e "${RED}Unknown option: $1${NC}" + echo "Use -h or --help for usage information" + exit 1 + ;; + esac +done + +if [[ "$STYLE_JSON" == true ]] && [[ "$STREAM_MODE" == true || "$LIST_CATEGORIES" == true ]]; then + echo "Error: --json cannot be combined with --follow or --list-categories" >&2 + exit 1 +fi + +if [[ "$LIST_CATEGORIES" == true ]]; then + list_categories + exit 0 +fi + +# Build the predicate +PREDICATE="subsystem == \"$SUBSYSTEM\"" + +# Add category filter if specified +if [[ -n "$CATEGORY" ]]; then + ESCAPED_CATEGORY=$(escape_predicate_literal "$CATEGORY") + PREDICATE="$PREDICATE AND category == \"$ESCAPED_CATEGORY\"" +fi + +# Add error filter if specified +if [[ "$ERRORS_ONLY" == true ]]; then + PREDICATE="$PREDICATE AND (eventType == \"error\" OR messageType == \"error\" OR eventMessage CONTAINS \"ERROR\" OR eventMessage CONTAINS \"[31m\")" +fi + +# Add search filter if specified +if [[ -n "$SEARCH_TEXT" ]]; then + ESCAPED_SEARCH_TEXT=$(escape_predicate_literal "$SEARCH_TEXT") + PREDICATE="$PREDICATE AND eventMessage CONTAINS[c] \"$ESCAPED_SEARCH_TEXT\"" +fi + +# Build the command as argv array to avoid shell eval injection +LOG_CMD=(sudo log) +if [[ "$STREAM_MODE" == true ]]; then + # Streaming mode + LOG_CMD+=(stream --predicate "$PREDICATE" --level "$LOG_LEVEL" --info) + + echo -e "${GREEN}Streaming VibeTunnel logs continuously...${NC}" + echo -e "${YELLOW}Press Ctrl+C to stop${NC}\n" +else + # Show mode + LOG_CMD+=(show --predicate "$PREDICATE") + + # Add log level for show command + if [[ "$LOG_LEVEL" == "debug" ]]; then + LOG_CMD+=(--debug) + else + LOG_CMD+=(--info) + fi + + # Add time range + LOG_CMD+=(--last "$TIME_RANGE") + + if [[ "$STYLE_JSON" == false ]]; then + if [[ "$SHOW_TAIL" == true ]]; then + echo -e "${GREEN}Showing last $TAIL_LINES log lines from the past $TIME_RANGE${NC}" + else + echo -e "${GREEN}Showing all logs from the past $TIME_RANGE${NC}" + fi + + # Show applied filters + if [[ "$ERRORS_ONLY" == true ]]; then + echo -e "${RED}Filter: Errors only${NC}" + fi + if [[ -n "$CATEGORY" ]]; then + echo -e "${BLUE}Category: $CATEGORY${NC}" + fi + if [[ -n "$SEARCH_TEXT" ]]; then + echo -e "${YELLOW}Search: \"$SEARCH_TEXT\"${NC}" + fi + echo "" # Empty line for readability + fi +fi + +if [[ "$STYLE_JSON" == true ]]; then + LOG_CMD+=(--style ndjson) + sudo -n /usr/bin/log show --last 1s >/dev/null || exit $? + + if [[ -n "$OUTPUT_FILE" ]]; then + if [[ -d "$OUTPUT_FILE" ]]; then + echo "Error: output path is a directory: $OUTPUT_FILE" >&2 + exit 1 + fi + STAGED_OUTPUT=$(mktemp -- "${OUTPUT_FILE}.tmp.XXXXXX") + else + STAGED_OUTPUT=$(mktemp "${TMPDIR:-/tmp}/clawlog.XXXXXX") + fi + trap 'rm -f -- "$STAGED_OUTPUT"' EXIT + + TAIL_ARGS=(-n "$TAIL_LINES") + if [[ "$SHOW_TAIL" == false ]]; then + TAIL_ARGS=(-n +1) + fi + + # macOS appends a documented "finished" metadata record after NDJSON events. + if "${LOG_CMD[@]}" | sed '$d' | tail "${TAIL_ARGS[@]}" | \ + awk 'BEGIN { printf "[" } NR > 1 { printf "," } { printf "%s", $0 } END { print "]" }' > "$STAGED_OUTPUT"; then + if [[ -n "$OUTPUT_FILE" ]]; then + mv -f -- "$STAGED_OUTPUT" "$OUTPUT_FILE" + else + cat "$STAGED_OUTPUT" + fi + else + exit $? + fi + exit 0 +fi + +# First check if sudo works without password for the log command. +if sudo -n /usr/bin/log show --last 1s 2>&1 | grep -q "password"; then + handle_sudo_error +fi + +# Execute the command +if [[ -n "$OUTPUT_FILE" ]]; then + echo -e "${BLUE}Exporting logs to: $OUTPUT_FILE${NC}\n" + if [[ "$SHOW_TAIL" == true ]] && [[ "$STREAM_MODE" == false ]]; then + "${LOG_CMD[@]}" 2>&1 | tail -n "$TAIL_LINES" > "$OUTPUT_FILE" + else + "${LOG_CMD[@]}" > "$OUTPUT_FILE" 2>&1 + fi + + # Check if file was created and has content + if [[ -s "$OUTPUT_FILE" ]]; then + LINE_COUNT=$(wc -l < "$OUTPUT_FILE" | tr -d ' ') + echo -e "${GREEN}✓ Exported $LINE_COUNT lines to $OUTPUT_FILE${NC}" + else + echo -e "${YELLOW}⚠ No logs found matching the criteria${NC}" + fi +else + # Run interactively + if [[ "$SHOW_TAIL" == true ]] && [[ "$STREAM_MODE" == false ]]; then + # Apply tail for non-streaming mode + "${LOG_CMD[@]}" 2>&1 | tail -n "$TAIL_LINES" + echo -e "\n${YELLOW}Showing last $TAIL_LINES lines. Use --all or -n to see more.${NC}" + else + "${LOG_CMD[@]}" + fi +fi diff --git a/scripts/close-duplicate-prs-after-merge.mjs b/scripts/close-duplicate-prs-after-merge.mjs new file mode 100644 index 0000000000000000000000000000000000000000..92928bfe1cbe8099b6a99f89d8ccd8ac63bb2037 --- /dev/null +++ b/scripts/close-duplicate-prs-after-merge.mjs @@ -0,0 +1,323 @@ +// Finds duplicate PRs after merge and closes overlapping candidates. +import { execFileSync } from "node:child_process"; +import { pathToFileURL } from "node:url"; +import { isRecord } from "./lib/record-shared.mjs"; +function normalizeDuplicatePrListInput(value) { + if ( + typeof value !== "string" && + typeof value !== "number" && + typeof value !== "boolean" && + typeof value !== "bigint" + ) { + return undefined; + } + const normalized = String(value).trim(); + return normalized || undefined; +} +const DEFAULT_LABELS = ["duplicate", "close:duplicate", "dedupe:child"]; +// Duplicate PR closure performs multiple sequential gh API reads and writes. +// Keep enough headroom for GitHub latency while preventing one stalled request +// from blocking the surrounding workflow job. +const GH_COMMAND_TIMEOUT_MS = 60_000; +function usage() { + return `Usage: node scripts/close-duplicate-prs-after-merge.mjs --landed-pr --duplicates [--repo owner/repo] [--apply] + +Closes explicit duplicate PRs after a landed PR, after verifying the landed PR is merged and +each duplicate has either a shared referenced issue or overlapping changed hunks. Defaults to dry-run.`; +} +/** + * Parses comma-separated PR numbers from CLI/env input. + */ +export function parsePrNumberList(value) { + const text = normalizeDuplicatePrListInput(value) ?? ""; + return [ + ...new Set( + text + .split(/[\s,]+/u) + .map((part) => part.trim().replace(/^#/u, "")) + .filter(Boolean) + .map((part) => { + if (!/^\d+$/u.test(part)) { + throw new Error(`Invalid PR number: ${part}`); + } + return Number(part); + }), + ), + ]; +} +/** + * Parses duplicate PR close workflow arguments. + */ +export function parseArgs(argv, env = process.env) { + let help = false; + const args = { + apply: false, + duplicates: [], + labels: DEFAULT_LABELS, + landedPr: undefined, + repo: env.GITHUB_REPOSITORY || "openclaw/openclaw", + }; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + const next = () => { + index += 1; + if (index >= argv.length) { + throw new Error(`Missing value for ${arg}`); + } + return argv[index] ?? ""; + }; + if (arg === "--apply") { + args.apply = true; + } else if (arg === "--dry-run") { + args.apply = false; + } else if (arg === "--repo") { + args.repo = next(); + } else if (arg === "--landed-pr") { + args.landedPr = parsePrNumberList(next())[0]; + } else if (arg === "--duplicates") { + args.duplicates = parsePrNumberList(next()); + } else if (arg === "--labels") { + args.labels = next() + .split(/[\s,]+/u) + .map((label) => label.trim()) + .filter(Boolean); + } else if (arg === "--help" || arg === "-h") { + help = true; + } else { + throw new Error(`Unknown argument: ${arg}`); + } + } + if (!help && !args.landedPr) { + throw new Error("--landed-pr is required"); + } + if (!help && args.duplicates.length === 0) { + throw new Error("--duplicates is required"); + } + return help ? Object.assign(args, { help: true }) : args; +} +export function defaultRunGh(args, options = {}, params = {}) { + const execFileSyncImpl = params.execFileSyncImpl ?? execFileSync; + return execFileSyncImpl("gh", args, { + encoding: "utf8", + killSignal: "SIGKILL", + stdio: options.input ? ["pipe", "pipe", "inherit"] : ["ignore", "pipe", "inherit"], + timeout: GH_COMMAND_TIMEOUT_MS, + ...(options.input ? { input: options.input } : {}), + }); +} +function issueRefsFromPr(pr) { + const refs = new Set(); + const issues = Array.isArray(pr.closingIssuesReferences) ? pr.closingIssuesReferences : []; + for (const issue of issues) { + if (isRecord(issue) && typeof issue.number === "number") { + refs.add(issue.number); + } + } + const text = `${pr.title ?? ""}\n${pr.body ?? ""}`; + for (const match of text.matchAll(/(?:close[sd]?|fix(?:e[sd])?|resolve[sd]?)\s+#(\d+)/giu)) { + refs.add(Number(match[1])); + } + return refs; +} +function intersectSets(left, right) { + return [...left].filter((value) => right.has(value)); +} +/** + * Parses changed hunk ranges from unified diff text. + */ +export function parseUnifiedDiffRanges(diffText) { + const ranges = new Map(); + let currentPath = null; + const text = typeof diffText === "string" ? diffText : ""; + for (const line of text.split("\n")) { + const pathMatch = /^diff --git a\/.+ b\/(.+)$/u.exec(line); + const changedPath = pathMatch?.[1]; + if (changedPath) { + currentPath = changedPath; + if (!ranges.has(currentPath)) { + ranges.set(currentPath, []); + } + continue; + } + const hunkMatch = /^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@/u.exec(line); + if (!hunkMatch || !currentPath) { + continue; + } + const start = Number(hunkMatch[1]); + const length = hunkMatch[2] === undefined ? 1 : Number(hunkMatch[2]); + const end = Math.max(start, start + Math.max(length, 1) - 1); + ranges.get(currentPath)?.push({ start, end }); + } + return ranges; +} +/** + * Reports whether two PR diffs touch overlapping hunks. + */ +function hasOverlappingHunks(leftRanges, rightRanges) { + for (const [path, left] of leftRanges) { + const right = rightRanges.get(path) ?? []; + for (const leftRange of left) { + for (const rightRange of right) { + if (leftRange.start <= rightRange.end && rightRange.start <= leftRange.end) { + return true; + } + } + } + } + return false; +} +function filePaths(pr) { + const files = Array.isArray(pr.files) ? pr.files : []; + return new Set( + files.flatMap((file) => (isRecord(file) && typeof file.path === "string" ? [file.path] : [])), + ); +} +function formatEvidence(evidence) { + const parts = []; + if (evidence.sharedIssues.length > 0) { + parts.push(`shared issue(s): ${evidence.sharedIssues.map((issue) => `#${issue}`).join(", ")}`); + } + if (evidence.overlappingHunks) { + parts.push("overlapping changed hunks"); + } + if (evidence.sharedFiles.length > 0) { + parts.push(`shared file(s): ${evidence.sharedFiles.join(", ")}`); + } + return parts.join("; "); +} +/** + * Builds the close/skip plan for duplicate PR candidates. + */ +export function buildDuplicateClosePlan({ candidates, diffs, landed, repo }) { + if (landed.state !== "MERGED" || !landed.mergedAt) { + throw new Error(`#${landed.number} is not merged`); + } + const landedIssues = issueRefsFromPr(landed); + const landedFiles = filePaths(landed); + const landedRanges = parseUnifiedDiffRanges(diffs.get(landed.number) ?? ""); + const [owner, name] = repo.split("/"); + const commit = isRecord(landed.mergeCommit) ? landed.mergeCommit.oid : undefined; + const commitRef = + commit && owner && name + ? `https://github.com/${owner}/${name}/commit/${commit}` + : "the merge commit"; + return candidates.map((candidate) => { + if (candidate.state !== "OPEN") { + return { + action: "skip", + candidate, + reason: `#${candidate.number} is ${candidate.state}`, + }; + } + const sharedFiles = intersectSets(landedFiles, filePaths(candidate)).toSorted((left, right) => + left.localeCompare(right), + ); + const sharedIssues = intersectSets(landedIssues, issueRefsFromPr(candidate)).toSorted( + (left, right) => left - right, + ); + const overlappingHunks = hasOverlappingHunks( + landedRanges, + parseUnifiedDiffRanges(diffs.get(candidate.number) ?? ""), + ); + const evidence = { overlappingHunks, sharedFiles, sharedIssues }; + if (sharedIssues.length === 0 && !overlappingHunks) { + throw new Error( + `Refusing to close #${candidate.number}: no shared issue and no overlapping changed hunks with #${landed.number}`, + ); + } + return { + action: "close", + candidate, + comment: `Thanks for the fix. This is now covered by the landed #${landed.number} / commit ${commitRef}. + +Evidence: ${formatEvidence(evidence)}. + +Closing #${candidate.number} as a duplicate.`, + evidence, + }; + }); +} +function loadPr(repo, number, runGh) { + const pullRequest = JSON.parse( + runGh([ + "pr", + "view", + String(number), + "--repo", + repo, + "--json", + "number,title,body,state,mergedAt,mergeCommit,closingIssuesReferences,files,url", + ]), + ); + if (!isRecord(pullRequest)) { + throw new Error(`Invalid GitHub response for PR #${number}`); + } + const prNumber = pullRequest.number; + const state = pullRequest.state; + if (typeof prNumber !== "number" || typeof state !== "string") { + throw new Error(`Invalid GitHub response for PR #${number}`); + } + return Object.assign({}, pullRequest, { number: prNumber, state }); +} +function loadDiff(repo, number, runGh) { + return runGh(["pr", "diff", String(number), "--repo", repo, "--color=never"]); +} +/** + * Applies labels/comments/closes for planned duplicate PR actions. + */ +export function applyClosePlan({ labels = DEFAULT_LABELS, plan, repo, runGh }) { + for (const item of plan) { + if (!("comment" in item) || typeof item.comment !== "string") { + continue; + } + const number = String(item.candidate.number); + const labelArgs = labels.flatMap((label) => ["--add-label", label]); + if (labelArgs.length > 0) { + runGh(["pr", "edit", number, "--repo", repo, ...labelArgs]); + } + runGh(["pr", "comment", number, "--repo", repo, "--body", item.comment]); + runGh(["pr", "close", number, "--repo", repo]); + } +} +/** + * Runs the duplicate PR close workflow. + */ +export function runDuplicateCloseWorkflow(args, runGh = defaultRunGh) { + if (!args.landedPr) { + throw new Error("--landed-pr is required"); + } + const landed = loadPr(args.repo, args.landedPr, runGh); + const candidates = args.duplicates.map((number) => loadPr(args.repo, number, runGh)); + const diffs = new Map([[landed.number, loadDiff(args.repo, landed.number, runGh)]]); + for (const candidate of candidates) { + diffs.set(candidate.number, loadDiff(args.repo, candidate.number, runGh)); + } + const plan = buildDuplicateClosePlan({ candidates, diffs, landed, repo: args.repo }); + for (const item of plan) { + if ("reason" in item) { + console.log(`skip #${item.candidate.number}: ${item.reason}`); + } else { + console.log(`close #${item.candidate.number}: ${formatEvidence(item.evidence)}`); + } + } + if (!args.apply) { + console.log("dry-run only; pass --apply to label/comment/close duplicate PRs"); + return plan; + } + applyClosePlan({ labels: args.labels, plan, repo: args.repo, runGh }); + return plan; +} +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + try { + const args = parseArgs(process.argv.slice(2)); + if ("help" in args) { + console.log(usage()); + process.exit(0); + } + runDuplicateCloseWorkflow(args); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + console.error(usage()); + process.exit(1); + } +} diff --git a/scripts/codespell-ignore.txt b/scripts/codespell-ignore.txt new file mode 100644 index 0000000000000000000000000000000000000000..3a5019f6fbaef70f4f0403c9f1be810451e4c422 --- /dev/null +++ b/scripts/codespell-ignore.txt @@ -0,0 +1,24 @@ +iTerm +FO +Nam +Lins +Vai +OptionA +CAF +overlayed +re-use +AKS +Ballance +optIn +Brining +wit +allowIn +planText +nwe +smoe +thirdParty +doubleClick +ND +CNA +unparseable +afterAll diff --git a/scripts/connect.sh b/scripts/connect.sh new file mode 100644 index 0000000000000000000000000000000000000000..600a66f013693962ed7e4e9fb47c35209599beb5 --- /dev/null +++ b/scripts/connect.sh @@ -0,0 +1,220 @@ +#!/usr/bin/env bash +# Bash 5.3+ can deadlock writing heredoc pipes on macOS before the reader starts. +if [[ ${OSTYPE:-} == darwin* && $BASH != /bin/bash ]] && ((BASH_VERSINFO[0] > 5 || (BASH_VERSINFO[0] == 5 && BASH_VERSINFO[1] >= 3))); then + exec /bin/bash "$0" "$@" +fi +set -eEuo pipefail + +# Dormant fresh-machine session-host bootstrap. Public website activation is a +# separate release/publish step; this repository wrapper is safe to invoke directly. + +umask 077 + +VERSION="" +PREFIX="${OPENCLAW_PREFIX:-}" +DISPLAY_NAME="" +JOIN_TARGET="" +TEMP_DIR="" +FAILURE_CONTEXT="Connect setup failed. Review the preceding output and retry." +ERROR_REPORTED=0 + +print_usage() { + cat <<'EOF' +Usage: connect.sh --version [--prefix ] [--display-name ] + +Installs an exact OpenClaw CLI version, connects the machine as a worker-session +host, and installs the node service. The join target is handed to OpenClaw through +a private temporary file, never as a child-process argument. + +Options: + --version Required exact version, for example 2026.8.1 + --prefix Install prefix (default: ~/.openclaw or $OPENCLAW_PREFIX) + --display-name Override the node display name + -h, --help Show this help + +Environment: + OPENCLAW_INSTALL_CLI_URL HTTPS installer URL, file:// URL, or local installer path +EOF +} + +cleanup() { + if [[ -n "$TEMP_DIR" && -d "$TEMP_DIR" ]]; then + rm -rf -- "$TEMP_DIR" + fi +} + +on_exit() { + local status=$? + cleanup + if [[ $status -ne 0 && $ERROR_REPORTED -eq 0 ]]; then + printf 'ERROR: %s\n' "$FAILURE_CONTEXT" >&2 + fi +} +trap on_exit EXIT + +fail() { + ERROR_REPORTED=1 + printf 'ERROR: %s\n' "$1" >&2 + exit 1 +} + +require_value() { + local option="$1" + local value="${2:-}" + if [[ -z "$value" || "$value" == --* ]]; then + fail "${option} requires a value. Run connect.sh --help for usage." + fi +} + +is_exact_version() { + local value="$1" + local number='(0|[1-9][0-9]*)' + local prerelease='(0|[1-9][0-9]*|[0-9A-Za-z-]*[A-Za-z-][0-9A-Za-z-]*)' + local pattern="^${number}\\.${number}\\.${number}(-${prerelease}(\\.${prerelease})*)?(\\+[0-9A-Za-z-]+(\\.[0-9A-Za-z-]+)*)?$" + [[ "$value" =~ $pattern ]] +} + +require_home_for_prefix() { + local message="$1" + local home="${HOME:-}" + if [[ -z "$home" || "$home" == "/" || ! -d "$home" ]]; then + fail "$message" + fi +} + +resolve_prefix() { + case "$PREFIX" in + \~) + require_home_for_prefix "Cannot expand prefix '~': HOME is unavailable. Pass an absolute --prefix or set OPENCLAW_PREFIX." + PREFIX="$HOME" + ;; + \~/*) + require_home_for_prefix "Cannot expand prefix '${PREFIX}': HOME is unavailable. Pass an absolute --prefix or set OPENCLAW_PREFIX." + PREFIX="${HOME}${PREFIX:1}" + ;; + /*) ;; + *) PREFIX="${PWD}/${PREFIX}" ;; + esac +} + +download_installer() { + local source="$1" + local destination="$2" + local local_path="" + + if [[ -f "$source" ]]; then + cp -- "$source" "$destination" + return + fi + case "$source" in + file://*) + local_path="${source#file://}" + [[ -f "$local_path" ]] || fail "Installer override does not exist: ${local_path}" + cp -- "$local_path" "$destination" + ;; + https://*) + if command -v curl >/dev/null 2>&1; then + curl -fsSL --proto '=https' --tlsv1.2 \ + --speed-limit 1 --speed-time 30 \ + --retry 3 --retry-delay 1 --retry-connrefused \ + -o "$destination" -- "$source" + elif command -v wget >/dev/null 2>&1; then + wget -q --https-only --secure-protocol=TLSv1_2 --tries=3 --timeout=20 \ + -O "$destination" -- "$source" + else + fail "Missing downloader. Install curl or wget, then retry." + fi + ;; + *) + fail "OPENCLAW_INSTALL_CLI_URL must be HTTPS or a readable local installer path." + ;; + esac +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --version) + require_value "$1" "${2:-}" + VERSION="$2" + shift 2 + ;; + --prefix) + require_value "$1" "${2:-}" + PREFIX="$2" + shift 2 + ;; + --display-name) + require_value "$1" "${2:-}" + DISPLAY_NAME="$2" + shift 2 + ;; + -h|--help) + print_usage + exit 0 + ;; + --*) + fail "Unknown option: $1. Run connect.sh --help for usage." + ;; + *) + if [[ -n "$JOIN_TARGET" ]]; then + fail "Exactly one join target is required. Run connect.sh --help for usage." + fi + JOIN_TARGET="$1" + shift + ;; + esac +done + +[[ -n "$VERSION" ]] || fail "--version is required and must name an exact published version." +if ! is_exact_version "$VERSION"; then + fail "Invalid --version '${VERSION}'. Use an exact registry version such as 2026.8.1; leading v, moving tags, ranges, and wildcards are not allowed." +fi +[[ -n "$JOIN_TARGET" ]] || fail "A join target is required. Mint one with 'openclaw devices join-code' and retry." +if [[ -z "$PREFIX" ]]; then + require_home_for_prefix "Cannot resolve the default install prefix: pass --prefix, set OPENCLAW_PREFIX, or provide an existing HOME directory." + PREFIX="${HOME}/.openclaw" +fi + +resolve_prefix +TEMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/openclaw-connect.XXXXXX")" +chmod 0700 "$TEMP_DIR" +INSTALLER_PATH="${TEMP_DIR}/install-cli.sh" +TARGET_FILE="${TEMP_DIR}/join-target" +INSTALLER_SOURCE="${OPENCLAW_INSTALL_CLI_URL:-https://openclaw.ai/install-cli.sh}" + +FAILURE_CONTEXT="Could not obtain the OpenClaw CLI installer. Check network access or OPENCLAW_INSTALL_CLI_URL, then retry." +download_installer "$INSTALLER_SOURCE" "$INSTALLER_PATH" +[[ -s "$INSTALLER_PATH" ]] || fail "The OpenClaw CLI installer was empty. Check the installer source and retry." +chmod 0700 "$INSTALLER_PATH" + +FAILURE_CONTEXT="OpenClaw CLI installation failed. Verify the exact version and install prefix, then retry." +bash "$INSTALLER_PATH" --version "$VERSION" --prefix "$PREFIX" --no-onboard + +OPENCLAW_BIN="${PREFIX}/bin/openclaw" +[[ -x "$OPENCLAW_BIN" ]] || fail "Installed OpenClaw CLI is missing at ${OPENCLAW_BIN}. Check the installer output and retry." + +CAPABILITY_ERROR="The selected exact version ${VERSION} does not support session-host onboarding. Choose a newer supporting exact version and retry." +if ! CONNECT_HELP="$("$OPENCLAW_BIN" connect --help 2>&1)"; then + fail "$CAPABILITY_ERROR" +fi +for required_flag in --target-file --service --session-host; do + if ! grep -Eq -- "^[[:space:]]+${required_flag}([[:space:]=<]|$)" <<<"$CONNECT_HELP"; then + fail "$CAPABILITY_ERROR" + fi +done +unset CONNECT_HELP + +: >"$TARGET_FILE" +chmod 0600 "$TARGET_FILE" +printf '%s\n' "$JOIN_TARGET" >"$TARGET_FILE" +JOIN_TARGET="" + +CONNECT_ARGS=(connect --target-file "$TARGET_FILE" --service --session-host) +if [[ -n "$DISPLAY_NAME" ]]; then + CONNECT_ARGS+=(--display-name "$DISPLAY_NAME") +fi + +FAILURE_CONTEXT="OpenClaw could not connect or install the session-host service. Mint a fresh join target, verify Gateway reachability, and retry." +"$OPENCLAW_BIN" "${CONNECT_ARGS[@]}" + +printf 'OpenClaw session-host service installed.\n' diff --git a/scripts/control-ui-i18n.ts b/scripts/control-ui-i18n.ts new file mode 100644 index 0000000000000000000000000000000000000000..23ea2ad14730e49ba6fe83499598f4655c201764 --- /dev/null +++ b/scripts/control-ui-i18n.ts @@ -0,0 +1,1475 @@ +// Control Ui I18N script supports OpenClaw repository automation. +import { spawn, spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { existsSync } from "node:fs"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { createLlmRuntime, type AssistantMessage, type Model } from "@openclaw/ai"; +import { registerBuiltInApiProviders } from "@openclaw/ai/providers"; +import { formatErrorMessage } from "@openclaw/normalization-core/error-coercion"; +import { expectDefined } from "../packages/normalization-core/src/expect.js"; +import { sliceUtf16Safe } from "../packages/normalization-core/src/utf16-slice.ts"; +import { formatDurationCompact } from "../src/infra/format-time/format-duration.ts"; +import { + extractTranslationPlaceholders, + syncControlUiCatalogFallbackBaseline, + verifyControlUiGeneratedCatalogs, + verifyRuntimeLocaleConfig, +} from "./control-ui-i18n-verify.ts"; +import { isStrictAffirmativeValue } from "./lib/arg-utils.mts"; +import { + hashControlUiTranslationText, + loadControlUiTranslationMemory, + materializeControlUiLocaleCatalog, +} from "./lib/control-ui-i18n-catalog-values.ts"; +import { + loadControlUiSourceCatalog, + readControlUiSourceCatalog, +} from "./lib/control-ui-i18n-catalog.ts"; +import { CONTROL_UI_LOCALE_ENTRIES } from "./lib/control-ui-i18n-config.ts"; +import { syncControlUiRawCopyBaseline } from "./lib/control-ui-i18n-raw-copy.ts"; +import { + compareStringArrays, + createControlUiLocaleSyncPlan, + flattenTranslations, + type GlossaryEntry, + type LocaleEntry, + type LocaleMeta, + type TranslationBatchItem, +} from "./lib/control-ui-i18n-sync-plan.ts"; +import { escapeRegExp } from "./lib/regexp.mjs"; +import { sleep } from "./lib/sleep.mjs"; +import { resolveWindowsTaskkillPath } from "./lib/windows-taskkill.mjs"; + +// Translation is standalone tooling: Gateway host hooks open operator state +// and log model identifiers before this script can redact provider failures. +const translationRuntime = createLlmRuntime(); +registerBuiltInApiProviders(translationRuntime.registry); + +type RunProcessParentSignalState = { + done: boolean; + signal: NodeJS.Signals | null; +}; + +const CONTROL_UI_I18N_WORKFLOW = 1; +const DEFAULT_OPENAI_MODEL = "gpt-6-astra"; +const DEFAULT_ANTHROPIC_MODEL = "claude-opus-4-6"; +const DEFAULT_PROVIDER = "openai"; +const HERE = path.dirname(fileURLToPath(import.meta.url)); +const ROOT = path.resolve(HERE, ".."); +const I18N_ASSETS_DIR = path.join(ROOT, "ui", "src", "i18n", ".i18n"); +const SOURCE_LOCALE = "en"; +const MAX_BATCH_ITEMS = 20; +const DEFAULT_BATCH_CHAR_BUDGET = 2_000; +const TRANSLATE_MAX_ATTEMPTS = 2; +const TRANSLATE_BASE_DELAY_MS = 15_000; +const DEFAULT_PROMPT_TIMEOUT_MS = 120_000; +const RUN_PROCESS_OUTPUT_MAX_CHARS = 1024 * 1024; +const RUN_PROCESS_TIMEOUT_MS = 120_000; +const RUN_PROCESS_KILL_GRACE_MS = 5_000; +const activeRunProcessParentSignals = new Set(); +const PROGRESS_HEARTBEAT_MS = 30_000; +const ENV_PROVIDER = "OPENCLAW_CONTROL_UI_I18N_PROVIDER"; +const ENV_MODEL = "OPENCLAW_CONTROL_UI_I18N_MODEL"; +const ENV_FALLBACK_MODEL = "OPENCLAW_I18N_FALLBACK_MODEL"; +const ENV_THINKING = "OPENCLAW_CONTROL_UI_I18N_THINKING"; +const ENV_BATCH_CHAR_BUDGET = "OPENCLAW_CONTROL_UI_I18N_BATCH_CHAR_BUDGET"; +const ENV_PROMPT_TIMEOUT = "OPENCLAW_CONTROL_UI_I18N_PROMPT_TIMEOUT"; +const ENV_AUTH_OPTIONAL = "OPENCLAW_CONTROL_UI_I18N_AUTH_OPTIONAL"; + +type TranslationProvider = "openai" | "anthropic"; + +const TRANSLATION_PROVIDER_DEFAULTS: Record> = { + openai: { + api: "openai-responses", + provider: "openai", + baseUrl: "https://api.openai.com/v1", + reasoning: true, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 400_000, + maxTokens: 32_000, + }, + anthropic: { + api: "anthropic-messages", + provider: "anthropic", + baseUrl: "https://api.anthropic.com", + reasoning: true, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 200_000, + maxTokens: 32_000, + }, +}; + +const LOCALE_ENTRIES: readonly LocaleEntry[] = CONTROL_UI_LOCALE_ENTRIES; + +const DEFAULT_GLOSSARY: readonly GlossaryEntry[] = [ + { source: "OpenClaw", target: "OpenClaw" }, + { source: "Gateway", target: "Gateway" }, + { source: "Control UI", target: "Control UI" }, + { source: "Skills", target: "Skills" }, + { source: "Tailscale", target: "Tailscale" }, + { source: "WhatsApp", target: "WhatsApp" }, + { source: "Telegram", target: "Telegram" }, + { source: "Discord", target: "Discord" }, + { source: "Signal", target: "Signal" }, + { source: "iMessage", target: "iMessage" }, +]; + +function usage(): never { + console.error( + [ + "Usage:", + " node --import tsx scripts/control-ui-i18n.ts check", + " node --import tsx scripts/control-ui-i18n.ts sync [--write] [--locale ] [--force]", + " node --import tsx scripts/control-ui-i18n.ts sync --write --locale --refresh-key [--refresh-key ...]", + ].join("\n"), + ); + process.exit(2); +} + +function parseArgs(argv: string[]) { + const [command, ...rest] = argv; + if (command !== "check" && command !== "sync") { + usage(); + } + + let localeFilter: string | null = null; + let write = false; + let force = false; + const refreshKeys = new Set(); + + for (let index = 0; index < rest.length; index += 1) { + const part = rest[index]; + switch (part) { + case "--locale": + localeFilter = rest[index + 1] ?? null; + index += 1; + break; + case "--write": + write = true; + break; + case "--force": + force = true; + break; + case "--refresh-key": { + const key = rest[index + 1]; + if (!key || key.startsWith("--")) { + throw new Error("--refresh-key requires a catalog key"); + } + refreshKeys.add(key); + if (refreshKeys.size > 64) { + throw new Error("--refresh-key accepts at most 64 distinct keys"); + } + index += 1; + break; + } + default: + usage(); + } + } + + if (command === "check" && write) { + usage(); + } + if (refreshKeys.size > 0 && (command !== "sync" || !write || !localeFilter || force)) { + throw new Error( + "--refresh-key requires sync --write --locale and cannot be combined with --force", + ); + } + + return { + command, + force, + localeFilter, + refreshKeys, + write, + }; +} + +function prettyLanguageLabel(locale: string): string { + switch (locale) { + case "en": + return "English"; + case "zh-CN": + return "Simplified Chinese"; + case "zh-TW": + return "Traditional Chinese"; + case "pt-BR": + return "Brazilian Portuguese"; + case "ja-JP": + return "Japanese"; + case "ko": + return "Korean"; + case "fr": + return "French"; + case "hi": + return "Hindi"; + case "ar": + return "Arabic"; + case "it": + return "Italian"; + case "tr": + return "Turkish"; + case "uk": + return "Ukrainian"; + case "id": + return "Indonesian"; + case "pl": + return "Polish"; + case "th": + return "Thai"; + case "vi": + return "Vietnamese"; + case "nl": + return "Dutch"; + case "fa": + return "Persian"; + case "ru": + return "Russian"; + case "sv": + return "Swedish"; + case "de": + return "German"; + case "es": + return "Spanish"; + default: + return locale; + } +} + +function resolveConfiguredProvider(): string { + const configured = process.env[ENV_PROVIDER]?.trim(); + if (configured) { + return configured; + } + if (process.env.OPENAI_API_KEY?.trim()) { + return "openai"; + } + if (process.env.ANTHROPIC_API_KEY?.trim()) { + return "anthropic"; + } + return DEFAULT_PROVIDER; +} + +function resolveConfiguredModel(): string { + const configured = process.env[ENV_MODEL]?.trim(); + if (configured) { + return configured; + } + return resolveConfiguredProvider() === "anthropic" + ? DEFAULT_ANTHROPIC_MODEL + : DEFAULT_OPENAI_MODEL; +} + +function hasTranslationProvider(): boolean { + return Boolean(process.env.OPENAI_API_KEY?.trim() || process.env.ANTHROPIC_API_KEY?.trim()); +} + +function resolveKnownTranslationProvider(): TranslationProvider { + const provider = resolveConfiguredProvider(); + if (provider === "openai" || provider === "anthropic") { + return provider; + } + throw new Error(`Unsupported translation provider: ${provider}`); +} + +function sha256(input: string | Uint8Array): string { + return createHash("sha256").update(input).digest("hex"); +} + +function cacheNamespace(): string { + return `wf=${CONTROL_UI_I18N_WORKFLOW}|engine=openclaw-llm`; +} + +function cacheKey(segmentId: string, textHash: string, targetLocale: string): string { + return sha256([cacheNamespace(), SOURCE_LOCALE, targetLocale, segmentId, textHash].join("|")); +} + +function glossaryPath(entry: LocaleEntry): string { + return path.join(I18N_ASSETS_DIR, `glossary.${entry.locale}.json`); +} + +function metaPath(entry: LocaleEntry): string { + return path.join(I18N_ASSETS_DIR, `${entry.locale}.meta.json`); +} + +function tmPath(entry: LocaleEntry): string { + return path.join(I18N_ASSETS_DIR, `${entry.locale}.tm.jsonl`); +} + +type PlaceholderMismatch = { + key: string; + locale: string; + sourcePlaceholders: string[]; + translatedPlaceholders: string[]; +}; + +export function findPlaceholderMismatches( + sourceFlat: ReadonlyMap, + translatedFlat: ReadonlyMap, + locale: string, +): PlaceholderMismatch[] { + const mismatches: PlaceholderMismatch[] = []; + for (const [key, sourceText] of sourceFlat.entries()) { + const sourcePlaceholders = extractTranslationPlaceholders(sourceText); + const translatedPlaceholders = extractTranslationPlaceholders(translatedFlat.get(key) ?? ""); + if (!compareStringArrays(sourcePlaceholders, translatedPlaceholders)) { + mismatches.push({ + key, + locale, + sourcePlaceholders, + translatedPlaceholders, + }); + } + } + return mismatches; +} + +export function filterPlaceholderCompatibleTranslations( + sourceFlat: ReadonlyMap, + translatedFlat: ReadonlyMap, +): Map { + return new Map( + [...translatedFlat].filter(([key, translated]) => { + const source = sourceFlat.get(key); + return ( + source !== undefined && + compareStringArrays( + extractTranslationPlaceholders(source), + extractTranslationPlaceholders(translated), + ) + ); + }), + ); +} + +function assertPlaceholderParity( + sourceFlat: ReadonlyMap, + translatedFlat: ReadonlyMap, + locale: string, +) { + const mismatches = findPlaceholderMismatches(sourceFlat, translatedFlat, locale); + if (mismatches.length === 0) { + return; + } + + const details = mismatches + .slice(0, 20) + .map( + (mismatch) => + `${mismatch.locale}:${mismatch.key} expected {${mismatch.sourcePlaceholders.join("},{")}} got {${mismatch.translatedPlaceholders.join("},{")}}`, + ) + .join("\n"); + throw new Error( + [ + `control-ui-i18n placeholder mismatch detected for ${locale}.`, + details, + mismatches.length > 20 ? `...and ${mismatches.length - 20} more` : "", + ] + .filter(Boolean) + .join("\n"), + ); +} + +async function loadGlossary(filePath: string): Promise { + if (!existsSync(filePath)) { + return []; + } + const raw = await readFile(filePath, "utf8"); + const parsed = JSON.parse(raw) as GlossaryEntry[]; + return Array.isArray(parsed) ? parsed : []; +} + +async function loadMeta(filePath: string): Promise { + if (!existsSync(filePath)) { + return null; + } + const raw = await readFile(filePath, "utf8"); + return JSON.parse(raw) as LocaleMeta; +} + +function buildGlossaryPrompt(glossary: readonly GlossaryEntry[]): string { + if (glossary.length === 0) { + return ""; + } + return [ + "Required terminology (use exactly when the source term matches):", + ...glossary + .filter((entry) => entry.source.trim() && entry.target.trim()) + .map((entry) => `- ${entry.source} -> ${entry.target}`), + ].join("\n"); +} + +function buildSystemPrompt(targetLocale: string, glossary: readonly GlossaryEntry[]): string { + const glossaryBlock = buildGlossaryPrompt(glossary); + const lines = [ + "You are a translation function, not a chat assistant.", + `Translate UI strings from ${prettyLanguageLabel(SOURCE_LOCALE)} to ${prettyLanguageLabel(targetLocale)}.`, + "", + "Rules:", + "- Output ONLY valid JSON.", + "- The JSON must be an object whose keys exactly match the provided ids.", + "- Translate all English prose; keep code, URLs, product names, CLI commands, config keys, and env vars in English.", + "- Preserve placeholders exactly, including {count}, {time}, {shown}, {total}, and similar tokens.", + "- Preserve Swift interpolation expressions such as \\(name) exactly, including the backslash and parentheses.", + "- Preserve Kotlin interpolation expressions such as $name and ${value} exactly.", + "- Use natural target-language punctuation and spacing in translated prose. Keep ellipses and arrows unchanged when they are UI indicators.", + "- Preserve exact syntax, punctuation, and casing in code, URLs, commands, placeholders, identifiers, and clearly identified literal third-party UI labels.", + "- Preserve Markdown, inline code, HTML tags, and slash commands when present.", + "- Use fluent, neutral product UI language.", + "- Do not add explanations, comments, or extra keys.", + "- Never return an empty string for a key; if unsure, return the source text unchanged.", + ]; + if (glossaryBlock) { + lines.push("", glossaryBlock); + } + return lines.join("\n"); +} + +function buildBatchPayload(items: readonly TranslationBatchItem[]) { + return Object.fromEntries( + items.map( + (item) => + [ + item.key, + item.sourcePath + ? { + text: item.text, + sourcePath: item.sourcePath, + sourceContext: item.sourceContext, + } + : item.text, + ] as const, + ), + ); +} + +export function buildBatchPrompt( + items: readonly TranslationBatchItem[], + validationError?: string, +): string { + const payload = buildBatchPayload(items); + const lines = ["Translate this JSON object.", "Return ONLY a JSON object with the same keys."]; + if (items.some((item) => item.sourcePath)) { + lines.push( + "For object values, translate only text. Use sourcePath and the bounded sourceContext excerpt to understand the native UI owner and disambiguate its meaning; these fields are context, not text to translate.", + "Preserve the source order and meaning of unnumbered printf arguments. Rephrase surrounding prose rather than swapping the roles of argument values. Preserve literal percent escapes exactly.", + "Return each id mapped directly to its translated string, without the context fields.", + ); + } + if (validationError) { + lines.push( + "", + "Your previous response failed validation. Correct that exact failure in the new response:", + validationError, + ); + } + lines.push("", JSON.stringify(payload, null, 2)); + return lines.join("\n"); +} + +function formatDuration(ms: number): string { + return formatDurationCompact(ms, { spaced: true }) ?? "0ms"; +} + +function logProgress(message: string) { + process.stdout.write(`control-ui-i18n: ${message}\n`); +} + +function isPromptTimeoutError(error: Error): boolean { + return error.message.toLowerCase().includes("timed out"); +} + +export function isProviderAuthError(error: Error): boolean { + const message = error.message.toLowerCase(); + return ( + message.includes("401") || + message.includes("authentication_error") || + message.includes("incorrect api key") || + message.includes("invalid x-api-key") + ); +} + +function isProviderAuthOptional(): boolean { + return isStrictAffirmativeValue(process.env[ENV_AUTH_OPTIONAL]); +} + +function resolvePromptTimeoutMs(): number { + const raw = process.env[ENV_PROMPT_TIMEOUT]?.trim(); + if (!raw) { + return DEFAULT_PROMPT_TIMEOUT_MS; + } + const parsed = Number(raw); + return Number.isFinite(parsed) && parsed > 0 ? parsed : DEFAULT_PROMPT_TIMEOUT_MS; +} + +function resolveThinkingLevel(): "low" | "high" { + return process.env[ENV_THINKING]?.trim().toLowerCase() === "high" ? "high" : "low"; +} + +function resolveBatchCharBudget(): number { + const raw = process.env[ENV_BATCH_CHAR_BUDGET]?.trim(); + if (!raw) { + return DEFAULT_BATCH_CHAR_BUDGET; + } + const parsed = Number(raw); + return Number.isFinite(parsed) && parsed > 0 ? parsed : DEFAULT_BATCH_CHAR_BUDGET; +} + +function estimateBatchChars(items: readonly TranslationBatchItem[]): number { + return JSON.stringify(buildBatchPayload(items)).length; +} + +type RunProcessOptions = { + cwd?: string; + input?: string; + killGraceMs?: number; + maxOutputChars?: number; + rejectOnFailure?: boolean; + timeoutMs?: number; +}; + +type ProcessOutputCapture = { + text: string; + truncatedChars: number; +}; + +function resolveRunProcessOutputLimit(options: RunProcessOptions): number { + const value = options.maxOutputChars; + if (typeof value !== "number" || !Number.isFinite(value) || value <= 0) { + return RUN_PROCESS_OUTPUT_MAX_CHARS; + } + return Math.max(1, Math.floor(value)); +} + +export function appendBoundedProcessOutput( + capture: ProcessOutputCapture, + chunk: unknown, + maxChars: number, +): ProcessOutputCapture { + const nextText = capture.text + String(chunk); + if (nextText.length <= maxChars) { + return { text: nextText, truncatedChars: capture.truncatedChars }; + } + const text = sliceUtf16Safe(nextText, -maxChars); + const truncatedChars = capture.truncatedChars + nextText.length - text.length; + return { text, truncatedChars }; +} + +function formatProcessOutput(capture: ProcessOutputCapture): string { + if (capture.truncatedChars === 0) { + return capture.text; + } + return `[output truncated ${capture.truncatedChars} chars; showing tail]\n${capture.text}`; +} + +function maybeReraiseRunProcessParentSignal(signal: NodeJS.Signals): void { + for (const state of activeRunProcessParentSignals) { + if (state.signal === null || !state.done) { + return; + } + } + process.kill(process.pid, signal); +} + +export async function runProcess( + executable: string, + args: string[], + options: RunProcessOptions = {}, +): Promise<{ code: number; stderr: string; stdout: string }> { + return await new Promise((resolve, reject) => { + const useProcessGroup = process.platform !== "win32"; + const child = spawn(executable, args, { + cwd: options.cwd ?? ROOT, + detached: useProcessGroup, + env: process.env, + stdio: ["pipe", "pipe", "pipe"], + }); + + const maxOutputChars = resolveRunProcessOutputLimit(options); + const timeoutMs = options.timeoutMs ?? RUN_PROCESS_TIMEOUT_MS; + const killGraceMs = options.killGraceMs ?? RUN_PROCESS_KILL_GRACE_MS; + let stdout: ProcessOutputCapture = { text: "", truncatedChars: 0 }; + let stderr: ProcessOutputCapture = { text: "", truncatedChars: 0 }; + let timedOut = false; + let settled = false; + let waitingForKillGrace = false; + let childClosedResult: { code: number | null; signal: NodeJS.Signals | null } | null = null; + let killTimer: ReturnType | undefined; + let parentSignalPending: NodeJS.Signals | null = null; + const parentSignalState: RunProcessParentSignalState = { done: false, signal: null }; + activeRunProcessParentSignals.add(parentSignalState); + const parentSignalHandlers: { handler: () => void; signal: NodeJS.Signals }[] = []; + const cleanupParentSignalHandlers = () => { + for (const { signal, handler } of parentSignalHandlers) { + process.off(signal, handler); + } + parentSignalHandlers.length = 0; + }; + const signalWindowsProcessTree = (force: boolean): boolean => { + if (process.platform !== "win32" || typeof child.pid !== "number") { + return false; + } + const taskkillArgs = ["/PID", String(child.pid), "/T"]; + if (force) { + taskkillArgs.push("/F"); + } + const result = spawnSync(resolveWindowsTaskkillPath(), taskkillArgs, { stdio: "ignore" }); + return result.status === 0; + }; + const signalChild = (signal: NodeJS.Signals) => { + if (useProcessGroup && typeof child.pid === "number") { + try { + process.kill(-child.pid, signal); + return; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ESRCH") { + stderr = appendBoundedProcessOutput( + stderr, + `failed to send ${signal} to process group: ${error instanceof Error ? error.message : String(error)}\n`, + maxOutputChars, + ); + } + } + } + if (process.platform === "win32") { + const force = signal === "SIGKILL"; + if (signalWindowsProcessTree(force) || (!force && signalWindowsProcessTree(true))) { + return; + } + } + child.kill(signal); + }; + const relayParentSignal = (signal: NodeJS.Signals) => { + const handler = () => { + parentSignalPending = signal; + parentSignalState.signal = signal; + signalChild(signal); + cleanupParentSignalHandlers(); + if (!processGroupIsAlive()) { + parentSignalState.done = true; + maybeReraiseRunProcessParentSignal(signal); + return; + } + if (killTimer) { + clearTimeout(killTimer); + } + waitingForKillGrace = true; + // Keep this timer ref'ed so parent signal relay can force-kill stubborn + // process groups before re-raising the original signal. + killTimer = setTimeout(() => { + waitingForKillGrace = false; + killTimer = undefined; + signalChild("SIGKILL"); + parentSignalState.done = true; + maybeReraiseRunProcessParentSignal(signal); + }, killGraceMs); + }; + parentSignalHandlers.push({ handler, signal }); + process.once(signal, handler); + }; + const relayedSignals: NodeJS.Signals[] = + process.platform === "win32" ? ["SIGINT", "SIGTERM"] : ["SIGINT", "SIGTERM", "SIGHUP"]; + for (const signal of relayedSignals) { + relayParentSignal(signal); + } + const processGroupIsAlive = () => { + if (!useProcessGroup || typeof child.pid !== "number") { + return false; + } + try { + process.kill(-child.pid, 0); + return true; + } catch (error) { + return (error as NodeJS.ErrnoException).code === "EPERM"; + } + }; + const settle = (callback: () => void) => { + if (settled) { + return; + } + settled = true; + clearTimeout(timeout); + if (!parentSignalPending && killTimer) { + clearTimeout(killTimer); + } + if (!parentSignalPending) { + activeRunProcessParentSignals.delete(parentSignalState); + } + cleanupParentSignalHandlers(); + callback(); + }; + const finishClose = (code: number | null, signal: NodeJS.Signals | null) => { + settle(() => { + const stdoutText = formatProcessOutput(stdout); + const stderrText = formatProcessOutput(stderr); + if (timedOut) { + reject(new Error(`${executable} ${args.join(" ")} timed out after ${timeoutMs}ms`)); + return; + } + if ((code ?? 1) !== 0 && options.rejectOnFailure) { + reject( + new Error( + `${executable} ${args.join(" ")} failed: ${ + stderrText.trim() || stdoutText.trim() || (signal ? `terminated by ${signal}` : "") + }`, + ), + ); + return; + } + if ((code ?? 1) === 0 && stdout.truncatedChars > 0) { + reject( + new Error( + `${executable} ${args.join(" ")} produced more than ${maxOutputChars} stdout chars`, + ), + ); + return; + } + resolve({ code: code ?? 1, stderr: stderrText, stdout: stdout.text }); + }); + }; + const scheduleKill = () => { + if (waitingForKillGrace) { + return; + } + waitingForKillGrace = true; + killTimer = setTimeout(() => { + waitingForKillGrace = false; + killTimer = undefined; + signalChild("SIGKILL"); + if (childClosedResult) { + finishClose(childClosedResult.code, childClosedResult.signal); + } + }, killGraceMs); + }; + const timeout = setTimeout(() => { + timedOut = true; + signalChild("SIGTERM"); + scheduleKill(); + }, timeoutMs); + child.stdout.on("data", (chunk) => { + stdout = appendBoundedProcessOutput(stdout, chunk, maxOutputChars); + }); + child.stderr.on("data", (chunk) => { + stderr = appendBoundedProcessOutput(stderr, chunk, maxOutputChars); + }); + child.once("error", (error) => { + settle(() => { + reject(error); + }); + }); + if (options.input !== undefined) { + child.stdin.end(options.input); + } else { + child.stdin.end(); + } + child.once("close", (code, signal) => { + if (parentSignalPending) { + if (processGroupIsAlive()) { + childClosedResult = { code, signal }; + return; + } + if (killTimer) { + clearTimeout(killTimer); + killTimer = undefined; + } + parentSignalState.done = true; + maybeReraiseRunProcessParentSignal(parentSignalPending); + return; + } + if (waitingForKillGrace && processGroupIsAlive()) { + childClosedResult = { code, signal }; + return; + } + finishClose(code, signal); + }); + }); +} + +type LocaleRunContext = { + localeCount: number; + localeIndex: number; +}; + +type TranslationBatchContext = LocaleRunContext & { + batchCount: number; + batchIndex: number; + locale: string; + splitDepth?: number; + segmentLabel?: string; + validateTranslation?: TranslationValidator; +}; + +type TranslationValidator = (source: string, target: string, key: string, locale: string) => void; + +type ClientAccess = { + getClient: () => Promise; + resetClient: () => Promise; +}; + +function createTranslationClientAccess( + targetLocale: string, + glossary: readonly GlossaryEntry[], +): ClientAccess { + let client: TranslationClient | null = null; + return { + async getClient() { + client ??= await TranslationClient.create(buildSystemPrompt(targetLocale, glossary)); + return client; + }, + async resetClient() { + await client?.close(); + client = null; + }, + }; +} + +function formatLocaleLabel(locale: string, context: LocaleRunContext): string { + return `[${context.localeIndex}/${context.localeCount}] ${locale}`; +} + +function formatBatchLabel(context: TranslationBatchContext): string { + const suffix = context.segmentLabel ? `.${context.segmentLabel}` : ""; + return `${formatLocaleLabel(context.locale, context)} batch ${context.batchIndex}/${context.batchCount}${suffix}`; +} + +function buildTranslationBatches(items: readonly TranslationBatchItem[]): TranslationBatchItem[][] { + const batches: TranslationBatchItem[][] = []; + const budget = resolveBatchCharBudget(); + let current: TranslationBatchItem[] = []; + let currentChars = 2; + + for (const item of items) { + const itemChars = estimateBatchChars([item]); + const wouldOverflow = current.length > 0 && currentChars + itemChars > budget; + const reachedMaxItems = current.length >= MAX_BATCH_ITEMS; + if (wouldOverflow || reachedMaxItems) { + batches.push(current); + current = []; + currentChars = 2; + } + current.push(item); + currentChars += itemChars; + } + + if (current.length > 0) { + batches.push(current); + } + + return batches; +} + +export function resolveTranslationModel(): Model { + const provider = resolveKnownTranslationProvider(); + const modelId = resolveConfiguredModel(); + return { + ...TRANSLATION_PROVIDER_DEFAULTS[provider], + id: modelId, + name: modelId, + }; +} + +class TranslationClient { + private closed = false; + private sequence: Promise = Promise.resolve(); + private model: Model; + private readonly systemPrompt: string; + + private constructor(systemPrompt: string) { + this.systemPrompt = systemPrompt; + this.model = resolveTranslationModel(); + } + + static async create(systemPrompt: string): Promise { + return new TranslationClient(systemPrompt); + } + + async prompt(message: string, label: string): Promise { + const result = this.sequence.then(async () => { + if (this.closed) { + throw new Error("translation runtime unavailable"); + } + + const timeoutMs = resolvePromptTimeoutMs(); + const startedAt = Date.now(); + const controller = new AbortController(); + + return await new Promise((resolve, reject) => { + const heartbeat = setInterval(() => { + logProgress( + `${label}: still waiting (${formatDuration(Date.now() - startedAt)} / ${formatDuration(timeoutMs)})`, + ); + }, PROGRESS_HEARTBEAT_MS); + const timer = setTimeout(() => { + clearInterval(heartbeat); + controller.abort(); + reject(new Error(`${label}: translation prompt timed out after ${timeoutMs}ms`)); + }, timeoutMs); + + const complete = () => + translationRuntime + .completeSimple( + this.model, + { + systemPrompt: this.systemPrompt, + messages: [{ role: "user", content: message, timestamp: Date.now() }], + }, + { + maxTokens: 4096, + reasoning: resolveThinkingLevel(), + signal: controller.signal, + timeoutMs, + }, + ) + .then(extractTranslationResult); + complete() + .catch(async (error: unknown) => { + const fallback = process.env[ENV_FALLBACK_MODEL]?.trim(); + if ( + error instanceof TranslationProviderError && + error.code === "model_not_found" && + !controller.signal.aborted && + !this.closed && + this.model.provider === "openai" && + fallback && + fallback !== this.model.id + ) { + logProgress(`${label}: primary model unavailable; using configured fallback`); + this.model = { ...this.model, id: fallback, name: fallback }; + return await complete(); + } + throw error; + }) + .then((translation) => { + clearTimeout(timer); + clearInterval(heartbeat); + resolve(translation); + }) + .catch((error: unknown) => { + clearTimeout(timer); + clearInterval(heartbeat); + reject( + error instanceof TranslationProviderError + ? error + : new TranslationProviderError("provider_error"), + ); + }); + }); + }); + + this.sequence = result.catch(() => undefined); + return await result; + } + + async close() { + if (this.closed) { + return; + } + this.closed = true; + } +} + +class TranslationProviderError extends Error { + readonly code: "model_not_found" | "authentication_error" | "provider_error"; + + constructor(code: TranslationProviderError["code"]) { + super(`translation provider failed (${code}); check the private provider configuration`); + this.code = code; + } +} + +function extractTranslationResult(message: AssistantMessage): string { + if (message.errorMessage || message.stopReason === "error") { + // Provider prose can contain private model names. Only the explicit model + // availability code authorizes fallback; all public errors use fixed text. + throw new TranslationProviderError( + message.errorCode === "model_not_found" + ? "model_not_found" + : isProviderAuthError(new Error(message.errorMessage)) + ? "authentication_error" + : "provider_error", + ); + } + const text = message.content + .map((block) => (block.type === "text" ? block.text : "")) + .join("") + .trim(); + if (!text) { + throw new Error("assistant translation not found"); + } + return text; +} + +// Models intermittently wrap the JSON reply in a Markdown code fence even +// when told not to; strip it instead of burning a retry on a parse error. +function parseTranslationReply(raw: string): Record { + const trimmed = raw.trim(); + const fenced = /^```(?:json)?\s*\n([\s\S]*?)\n```\s*$/.exec(trimmed); + const json = fenced ? expectDefined(fenced[1], "fenced translation JSON body") : trimmed; + try { + return JSON.parse(json); + } catch { + throw new Error("translation provider returned invalid JSON"); + } +} + +export function parseTranslationBatchReply( + raw: string, + items: readonly TranslationBatchItem[], + locale: string, + validateTranslation?: TranslationValidator, +): Map { + const parsed = parseTranslationReply(raw); + const translated = new Map(); + for (const item of items) { + const value = parsed[item.key]; + if (typeof value !== "string" || !value.trim()) { + throw new Error(`missing translation for ${item.key}`); + } + const privateModels = [process.env[ENV_MODEL], process.env[ENV_FALLBACK_MODEL]]; + if ( + privateModels.some( + (model) => model?.trim() && value.toLowerCase().includes(model.trim().toLowerCase()), + ) + ) { + throw new TranslationProviderError("provider_error"); + } + validateTranslation?.(item.text, value, item.key, locale); + translated.set(item.key, value); + } + assertPlaceholderParity(new Map(items.map((item) => [item.key, item.text])), translated, locale); + return translated; +} + +async function translateBatch( + clientAccess: ClientAccess, + items: readonly TranslationBatchItem[], + context: TranslationBatchContext, +): Promise> { + const batchLabel = formatBatchLabel(context); + const splitDepth = context.splitDepth ?? 0; + let lastError: Error | null = null; + let validationError: string | undefined; + for (let attempt = 0; attempt < TRANSLATE_MAX_ATTEMPTS; attempt += 1) { + const attemptNumber = attempt + 1; + const attemptLabel = `${batchLabel} attempt ${attemptNumber}/${TRANSLATE_MAX_ATTEMPTS}`; + const startedAt = Date.now(); + logProgress(`${attemptLabel}: start keys=${items.length}`); + let promptCompleted = false; + try { + const raw = await ( + await clientAccess.getClient() + ).prompt(buildBatchPrompt(items, validationError), attemptLabel); + promptCompleted = true; + const translated = parseTranslationBatchReply( + raw, + items, + context.locale, + context.validateTranslation, + ); + logProgress(`${attemptLabel}: done (${formatDuration(Date.now() - startedAt)})`); + return translated; + } catch (error) { + lastError = error instanceof Error ? error : new Error(String(error)); + if (promptCompleted) { + validationError = lastError.message; + } + await clientAccess.resetClient(); + logProgress( + `${attemptLabel}: failed after ${formatDuration(Date.now() - startedAt)}: ${lastError.message}`, + ); + if (isPromptTimeoutError(lastError) && items.length > 1) { + const midpoint = Math.ceil(items.length / 2); + logProgress( + `${batchLabel}: splitting timed out batch into ${midpoint} + ${items.length - midpoint} keys`, + ); + const left = await translateBatch(clientAccess, items.slice(0, midpoint), { + ...context, + splitDepth: splitDepth + 1, + segmentLabel: `${context.segmentLabel ?? ""}a`, + }); + const right = await translateBatch(clientAccess, items.slice(midpoint), { + ...context, + splitDepth: splitDepth + 1, + segmentLabel: `${context.segmentLabel ?? ""}b`, + }); + return new Map([...left, ...right]); + } + if (isPromptTimeoutError(lastError)) { + break; + } + if (attempt + 1 < TRANSLATE_MAX_ATTEMPTS) { + const delayMs = TRANSLATE_BASE_DELAY_MS * attemptNumber; + logProgress(`${attemptLabel}: retrying in ${formatDuration(delayMs)}`); + await sleep(delayMs); + } + } + } + throw lastError ?? new Error("translation failed"); +} + +type NativeTranslationEntry = { + id: string; + source: string; + sourcePath: string; + sourceContext?: string; +}; + +export async function translateNativeEntries( + entries: readonly NativeTranslationEntry[], + targetLocale: string, + glossary: readonly GlossaryEntry[] = [], + validateTranslation?: TranslationValidator, +): Promise> { + if (!hasTranslationProvider()) { + throw new Error("native app translation requires OPENAI_API_KEY or ANTHROPIC_API_KEY"); + } + const pending = entries.map((entry) => ({ + cacheKey: cacheKey(entry.id, hashControlUiTranslationText(entry.source), targetLocale), + key: entry.id, + text: entry.source, + textHash: hashControlUiTranslationText(entry.source), + sourcePath: entry.sourcePath, + sourceContext: entry.sourceContext, + })); + const batches = buildTranslationBatches(pending); + const clientAccess = createTranslationClientAccess(targetLocale, glossary); + try { + const translated = new Map(); + for (const [batchIndex, batch] of batches.entries()) { + const result = await translateBatch(clientAccess, batch, { + locale: targetLocale, + localeCount: 1, + localeIndex: 1, + batchCount: batches.length, + batchIndex: batchIndex + 1, + validateTranslation, + }); + for (const [id, value] of result) { + translated.set(id, value); + } + } + return translated; + } finally { + await clientAccess.resetClient(); + } +} + +type SyncOutcome = { + changed: boolean; + fallbackCount: number; + locale: string; + wrote: boolean; +}; + +export function assertNoControlUiFallbacks( + outcomes: ReadonlyArray>, +) { + const fallbackLocales = outcomes.filter((outcome) => outcome.fallbackCount > 0); + if (fallbackLocales.length === 0) { + return; + } + throw new Error( + [ + "control-ui-i18n generated locales still contain English fallbacks.", + ...fallbackLocales.map( + (outcome) => `${outcome.locale}: ${outcome.fallbackCount} fallback keys`, + ), + ].join("\n"), + ); +} + +async function syncLocale( + entry: LocaleEntry, + options: { + allowTranslate: boolean; + checkOnly: boolean; + force: boolean; + write: boolean; + refreshKeys: ReadonlySet; + }, + context: LocaleRunContext, +) { + const localeLabel = formatLocaleLabel(entry.locale, context); + const localeStartedAt = Date.now(); + const sourceRaw = await readControlUiSourceCatalog(); + const sourceHash = sha256(sourceRaw); + const sourceMap = loadControlUiSourceCatalog(); + const sourceFlat = flattenTranslations(sourceMap); + const tm = loadControlUiTranslationMemory(tmPath(entry)); + const existingMap = materializeControlUiLocaleCatalog(sourceFlat, tm); + const existingFlat = flattenTranslations(existingMap); + // Placeholder changes invalidate the old translation even when the key stays + // stable. Treat it as pending so the locale bot can repair source-only PRs. + const reusableExistingFlat = filterPlaceholderCompatibleTranslations(sourceFlat, existingFlat); + const previousMeta = await loadMeta(metaPath(entry)); + const glossaryFilePath = glossaryPath(entry); + const glossary = await loadGlossary(glossaryFilePath); + const allowTranslate = options.allowTranslate; + const plan = createControlUiLocaleSyncPlan({ + allowTranslate, + cacheKeyFor: (key, textHash) => cacheKey(key, textHash, entry.locale), + entry, + existingFlat: reusableExistingFlat, + force: options.force, + refreshKeys: options.refreshKeys, + hashText: hashControlUiTranslationText, + previousMeta, + sourceFlat, + sourceHash, + translationMemory: tm, + }); + if (options.refreshKeys.size > 0 && !allowTranslate) { + throw new Error("--refresh-key requires a configured translation provider"); + } + + // Writing NEW English fallbacks trips the shipped-fallback CI gate + // (test/scripts/control-ui-i18n.test.ts), and post-merge translation is owned + // by the control-ui-locale-refresh workflow. An unauthenticated local sync + // must fail here instead of silently recording fallback bundles; refreshing + // already-recorded fallback copy (force mode) stays allowed. + if (!allowTranslate && options.write && !options.checkOnly && !isProviderAuthOptional()) { + if (plan.newFallbackCount > 0) { + throw new Error( + `${localeLabel}: ${plan.newFallbackCount} new key(s) need translation but no provider is configured. ` + + `Commit only locales/en.ts and let the control-ui-locale-refresh workflow translate after merge, ` + + `or export ANTHROPIC_API_KEY/OPENAI_API_KEY and rerun. ` + + `Set ${ENV_AUTH_OPTIONAL}=1 to record English fallbacks anyway.`, + ); + } + } + + if (allowTranslate && plan.pending.length > 0) { + const batches = buildTranslationBatches(plan.pending); + const batchCount = batches.length; + logProgress( + `${localeLabel}: start keys=${sourceFlat.size} pending=${plan.pending.length} batches=${batchCount} thinking=${resolveThinkingLevel()} timeout=${formatDuration(resolvePromptTimeoutMs())} batch_chars=${resolveBatchCharBudget()}`, + ); + const clientAccess = createTranslationClientAccess(entry.locale, glossary); + try { + for (const [batchIndex, batch] of batches.entries()) { + const translated = await translateBatch(clientAccess, batch, { + ...context, + batchCount, + batchIndex: batchIndex + 1, + locale: entry.locale, + }); + plan.recordTranslations(batch, translated, { + sourceLocale: SOURCE_LOCALE, + updatedAt: () => new Date().toISOString(), + }); + } + } catch (error) { + const failure = error instanceof Error ? error : new Error(String(error)); + if ( + options.refreshKeys.size === 0 && + isProviderAuthOptional() && + isProviderAuthError(failure) + ) { + logProgress(`${localeLabel}: translation provider auth failed; skipping refresh`); + return { + changed: false, + fallbackCount: previousMeta?.fallbackKeys.length ?? 0, + locale: entry.locale, + wrote: false, + } satisfies SyncOutcome; + } + throw failure; + } finally { + await clientAccess.resetClient(); + } + } else if (allowTranslate) { + logProgress( + `${localeLabel}: no translation work needed (all keys reused from cache or existing files)`, + ); + } else { + logProgress(`${localeLabel}: no provider configured, using English fallback for pending keys`); + } + + // Do not infer fallback state from source-text equality alone. + // Product names, config keys, and other intentional carry-through strings may + // legitimately stay identical to English. Track fallback keys from actual + // fallback decisions and previous fallback metadata instead. + + const artifacts = plan.render({ + defaultGlossary: DEFAULT_GLOSSARY, + generatedAt: new Date().toISOString(), + glossary, + workflow: CONTROL_UI_I18N_WORKFLOW, + }); + assertPlaceholderParity(sourceFlat, artifacts.nextFlat, entry.locale); + + const expectedMeta = artifacts.meta; + const expectedGlossary = artifacts.glossary; + const expectedTm = artifacts.translationMemory; + + const currentMeta = existsSync(metaPath(entry)) ? await readFile(metaPath(entry), "utf8") : ""; + const currentGlossary = existsSync(glossaryFilePath) + ? await readFile(glossaryFilePath, "utf8") + : ""; + const currentTm = existsSync(tmPath(entry)) ? await readFile(tmPath(entry), "utf8") : ""; + + const changed = + currentMeta !== expectedMeta || + currentGlossary !== expectedGlossary || + currentTm !== expectedTm; + + if ( + !changed || + (previousMeta?.sourceHash === sourceHash && + !options.force && + !options.checkOnly && + !options.write) + ) { + logProgress( + `${localeLabel}: done changed=${changed} fallbacks=${artifacts.fallbackCount} elapsed=${formatDuration(Date.now() - localeStartedAt)}`, + ); + return { + changed, + fallbackCount: artifacts.fallbackCount, + locale: entry.locale, + wrote: false, + } satisfies SyncOutcome; + } + + if (!options.checkOnly && options.write) { + await mkdir(I18N_ASSETS_DIR, { recursive: true }); + await writeFile(metaPath(entry), expectedMeta, "utf8"); + await writeFile(glossaryFilePath, expectedGlossary, "utf8"); + if (expectedTm) { + await writeFile(tmPath(entry), expectedTm, "utf8"); + } else if (existsSync(tmPath(entry))) { + await writeFile(tmPath(entry), "", "utf8"); + } + } + + logProgress( + `${localeLabel}: done changed=${changed} fallbacks=${artifacts.fallbackCount} elapsed=${formatDuration(Date.now() - localeStartedAt)}${!options.checkOnly && options.write && changed ? " wrote" : ""}`, + ); + return { + changed, + fallbackCount: artifacts.fallbackCount, + locale: entry.locale, + wrote: !options.checkOnly && options.write && changed, + } satisfies SyncOutcome; +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + if (args.command === "check") { + await verifyControlUiGeneratedCatalogs({ + checkOnly: true, + write: false, + }); + } else { + await verifyRuntimeLocaleConfig(); + } + if (args.command === "sync" && args.write && !args.localeFilter) { + await syncControlUiRawCopyBaseline({ + checkOnly: false, + write: args.write, + }); + } + + const entries = args.localeFilter + ? LOCALE_ENTRIES.filter((entry) => entry.locale === args.localeFilter) + : [...LOCALE_ENTRIES]; + + if (entries.length === 0) { + throw new Error(`unknown locale: ${args.localeFilter}`); + } + + const allowTranslate = args.command === "sync" && hasTranslationProvider(); + logProgress( + `command=${args.command} locales=${entries.length} translation=${allowTranslate ? "enabled" : "disabled"} thinking=${allowTranslate ? resolveThinkingLevel() : "n/a"} timeout=${formatDuration(resolvePromptTimeoutMs())} batch_chars=${resolveBatchCharBudget()}`, + ); + const outcomes: SyncOutcome[] = []; + for (const [index, entry] of entries.entries()) { + const outcome = await syncLocale( + entry, + { + allowTranslate, + checkOnly: args.command === "check", + force: args.force, + refreshKeys: args.refreshKeys, + write: args.write, + }, + { + localeCount: entries.length, + localeIndex: index + 1, + }, + ); + outcomes.push(outcome); + } + + const changed = outcomes.filter((outcome) => outcome.changed); + const summary = outcomes + .map( + (outcome) => + `${outcome.locale}: ${outcome.changed ? "dirty" : "clean"} (fallbacks=${outcome.fallbackCount}${outcome.wrote ? ", wrote" : ""})`, + ) + .join("\n"); + process.stdout.write(`${summary}\n`); + + if (args.command === "sync" && args.write) { + await syncControlUiCatalogFallbackBaseline({ + // A scoped matrix worker can observe unsynced sibling locales. The final + // aggregate sync still rebuilds and validates the complete catalog. + allowCatalogDrift: Boolean(args.localeFilter), + checkOnly: false, + write: true, + }); + } + + if (args.command === "check") { + assertNoControlUiFallbacks(outcomes); + if (changed.length > 0) { + throw new Error( + [ + "control-ui-i18n drift detected.", + "Run `node --import tsx scripts/control-ui-i18n.ts sync --write` and commit the results.", + ].join("\n"), + ); + } + } + + if (args.command === "sync" && !args.write && changed.length > 0) { + process.stdout.write( + "dry-run only. re-run with `node --import tsx scripts/control-ui-i18n.ts sync --write` to update files.\n", + ); + } +} + +function isCliEntrypoint() { + const entrypoint = process.argv[1]; + return Boolean(entrypoint && import.meta.url === pathToFileURL(path.resolve(entrypoint)).href); +} + +if (isCliEntrypoint()) { + await main().catch((error: unknown) => { + console.error( + formatErrorMessage(error, { + // Keep failure reporting independent of Gateway logging configuration. + redact: (text) => { + let redacted = text; + for (const name of [ + ENV_MODEL, + ENV_FALLBACK_MODEL, + "OPENAI_API_KEY", + "ANTHROPIC_API_KEY", + ]) { + const secret = process.env[name]?.trim(); + if (secret) { + redacted = redacted.replaceAll(new RegExp(escapeRegExp(secret), "gi"), "[redacted]"); + } + } + return redacted; + }, + }), + ); + process.exit(1); + }); +} diff --git a/scripts/control-ui-mock-channels.ts b/scripts/control-ui-mock-channels.ts new file mode 100644 index 0000000000000000000000000000000000000000..7f8a9fc78bc020c8e537219b85d432fb97a835eb --- /dev/null +++ b/scripts/control-ui-mock-channels.ts @@ -0,0 +1,201 @@ +// Channels-page fixtures for the Control UI mock dev harness: a deterministic +// channels.status snapshot plus a scripted setup-wizard step sequence. + +export function buildChannelsStatusMock(baseTime: number) { + const channelMeta = [ + { id: "whatsapp", label: "WhatsApp", detailLabel: "WhatsApp Web" }, + { id: "telegram", label: "Telegram", detailLabel: "Telegram Bot" }, + { id: "discord", label: "Discord", detailLabel: "Discord Bot" }, + { id: "slack", label: "Slack", detailLabel: "Slack App" }, + { id: "signal", label: "Signal", detailLabel: "signal-cli" }, + { id: "imessage", label: "iMessage", detailLabel: "macOS Messages" }, + { id: "googlechat", label: "Google Chat", detailLabel: "Chat API" }, + { id: "nostr", label: "Nostr", detailLabel: "Nostr relays" }, + ]; + const account = (params: { + accountId?: string; + configured?: boolean; + running?: boolean; + connected?: boolean; + lastInboundAt?: number; + lastError?: string; + }) => ({ + accountId: params.accountId ?? "default", + enabled: params.configured ?? true, + configured: params.configured ?? true, + running: params.running ?? false, + connected: params.connected ?? null, + ...(params.lastInboundAt ? { lastInboundAt: params.lastInboundAt } : {}), + ...(params.lastError ? { lastError: params.lastError } : {}), + }); + return { + ts: baseTime, + channelOrder: channelMeta.map((entry) => entry.id), + channelLabels: Object.fromEntries(channelMeta.map((entry) => [entry.id, entry.label])), + channelDetailLabels: Object.fromEntries( + channelMeta.map((entry) => [entry.id, entry.detailLabel]), + ), + channelMeta, + channels: { + whatsapp: { + configured: true, + linked: true, + running: true, + connected: true, + lastConnectedAt: baseTime - 90_000, + lastMessageAt: baseTime - 120_000, + authAgeMs: 6 * 24 * 60 * 60 * 1000, + }, + telegram: { + configured: true, + running: true, + connected: true, + lastStartAt: baseTime - 600_000, + }, + discord: { + configured: true, + running: false, + lastError: "Disallowed intents: enable Message Content in the developer portal.", + }, + }, + channelAccounts: { + whatsapp: [account({ running: true, connected: true, lastInboundAt: baseTime - 120_000 })], + telegram: [account({ running: true, connected: true, lastInboundAt: baseTime - 45_000 })], + discord: [ + account({ + running: false, + lastError: "Disallowed intents: enable Message Content in the developer portal.", + }), + ], + }, + channelDefaultAccountId: { + whatsapp: "default", + telegram: "default", + discord: "default", + }, + }; +} + +export function buildChannelsPairingMock(baseTime: number) { + const iso = (offsetMs: number) => new Date(baseTime + offsetMs).toISOString(); + return { + accounts: [ + { + channel: "telegram", + channelLabel: "Telegram", + accountId: "default", + accountLabel: "Telegram Bot", + notifySupported: true, + }, + { + channel: "whatsapp", + channelLabel: "WhatsApp", + accountId: "default", + accountLabel: "WhatsApp Web", + notifySupported: false, + }, + ], + requests: [ + { + requestId: "pairing-req-1", + channel: "telegram", + channelLabel: "Telegram", + accountId: "default", + accountLabel: "Telegram Bot", + senderId: "telegram-demo-user", + senderLabel: "Mira Delgado (@miradelgado)", + metadata: { username: "miradelgado", firstName: "Mira", lastName: "Delgado" }, + createdAt: iso(-14 * 60_000), + lastSeenAt: iso(-2 * 60_000), + expiresAt: iso(46 * 60_000), + notifySupported: true, + }, + { + requestId: "pairing-req-2", + channel: "whatsapp", + channelLabel: "WhatsApp", + accountId: "default", + accountLabel: "WhatsApp Web", + senderId: "whatsapp-demo-user", + senderLabel: "Unknown sender", + createdAt: iso(-3 * 60_000), + lastSeenAt: iso(-60_000), + expiresAt: iso(57 * 60_000), + notifySupported: false, + }, + ], + commandOwnerConfigured: false, + limits: { pendingPerAccount: 3, ttlMs: 3_600_000 }, + }; +} + +export function buildChannelWizardMocks() { + const channelOptions = [ + { value: "telegram", label: "Telegram", hint: "bot via @BotFather" }, + { value: "slack", label: "Slack", hint: "socket mode app" }, + { value: "signal", label: "Signal", hint: "signal-cli link" }, + { value: "imessage", label: "iMessage", hint: "macOS Messages" }, + ]; + const channelSelectStep = { + id: "mock-wizard-step-channel", + type: "select", + message: "Which channel do you want to set up?", + options: [...channelOptions, { value: "__skip__", label: "Skip for now" }], + initialValue: "telegram", + executor: "client", + }; + const tokenStep = { + id: "mock-wizard-step-token", + type: "text", + message: "Paste the bot token from @BotFather", + placeholder: "paste the token here", + sensitive: true, + executor: "client", + }; + return { + start: { + cases: [ + ...channelOptions.map(({ value, label }) => ({ + match: { flow: "channels", channel: value }, + response: { + sessionId: "mock-wizard-session", + done: false, + status: "running", + step: { + id: `mock-wizard-step-${value}`, + type: "note", + message: `Continue to configure ${label}.`, + executor: "client", + }, + }, + })), + { + match: { flow: "channels" }, + response: { + sessionId: "mock-wizard-session", + done: false, + status: "running", + step: channelSelectStep, + }, + }, + ], + }, + next: { + cases: [ + ...channelOptions.map(({ value }) => ({ + match: { + answer: { stepId: `mock-wizard-step-${value}`, value: null }, + }, + response: { done: true, status: "done", channels: [value] }, + })), + { + match: { + answer: { stepId: "mock-wizard-step-channel", value: "telegram" }, + }, + response: { done: false, status: "running", step: tokenStep }, + }, + { response: { done: true, status: "done", channels: ["telegram"] } }, + ], + }, + }; +} diff --git a/scripts/control-ui-mock-cron.ts b/scripts/control-ui-mock-cron.ts new file mode 100644 index 0000000000000000000000000000000000000000..07a76d4548afa4e37512e783112887426cd2fdce --- /dev/null +++ b/scripts/control-ui-mock-cron.ts @@ -0,0 +1,386 @@ +import type { + CronJob, + CronJobsListResult, + CronRunLogEntry, + CronRunsResult, + CronStatus, +} from "../ui/src/api/types.ts"; + +const CRON_LIST_SNAPSHOT_REVISION = "control-ui-mock-cron"; + +function listResult( + jobs: CronJob[], + options: { total?: number; limit?: number; offset?: number } = {}, +): CronJobsListResult { + const total = options.total ?? jobs.length; + const limit = options.limit ?? 50; + const offset = options.offset ?? 0; + const nextOffset = offset + jobs.length; + const hasMore = nextOffset < total; + return { + jobs, + snapshotRevision: CRON_LIST_SNAPSHOT_REVISION, + total, + offset, + limit, + hasMore, + nextOffset: hasMore ? nextOffset : null, + }; +} + +function runsResult(entries: CronRunLogEntry[]): CronRunsResult { + return { + entries, + total: entries.length, + offset: 0, + limit: 50, + hasMore: false, + nextOffset: null, + }; +} + +function singleJobListCases(jobs: CronJob[], match: Record) { + return jobs.map((job, offset) => ({ + match: { ...match, offset }, + response: listResult([job], { total: jobs.length, limit: 1, offset }), + })); +} + +export function buildCronMocks( + baseTime: number, + options: { richAttention?: boolean; secondAgentId?: string } = {}, +) { + const richAttention = options.richAttention === true; + const minute = 60_000; + const hour = 60 * minute; + const day = 24 * hour; + const failedJob: CronJob = { + id: "mock-cron-calendar-sync", + agentId: "main", + name: "Sync team calendar", + description: "Refresh the shared calendar cache for the morning briefing.", + enabled: true, + createdAtMs: baseTime - 30 * day, + updatedAtMs: baseTime - 4 * minute, + schedule: { kind: "cron", expr: "0 */6 * * *", tz: "America/New_York" }, + sessionTarget: "isolated", + wakeMode: "now", + payload: { + kind: "agentTurn", + message: "Sync the team calendar and summarize schedule conflicts.", + }, + delivery: { mode: "announce", channel: "telegram", to: "@operations" }, + failureAlert: { + after: 2, + channel: "telegram", + to: "@operations", + cooldownMs: 3_600_000, + includeSkipped: false, + mode: "announce", + }, + state: { + nextRunAtMs: baseTime + 5 * hour, + lastRunAtMs: baseTime - 5 * minute, + lastRunStatus: "error", + lastError: + "OAuth refresh failed: invalid_grant. The provider rejected the stored refresh token because it was revoked or expired. Reconnect Google Calendar before the next scheduled sync.", + lastDurationMs: 8_420, + consecutiveErrors: 2, + lastDeliveryStatus: "not-requested", + }, + }; + const extraFailedJobs: CronJob[] = richAttention + ? [ + { + id: "mock-cron-release-notify", + agentId: "main", + name: "Notify release stakeholders about deployment readiness and rollback constraints", + description: + "Send the release decision, deploy window, rollback owner, and incident contact to every stakeholder group.", + enabled: true, + createdAtMs: baseTime - 24 * day, + updatedAtMs: baseTime - 7 * minute, + schedule: { kind: "every", everyMs: 20 * minute, anchorMs: baseTime - 24 * day }, + sessionTarget: "isolated", + wakeMode: "now", + payload: { + kind: "agentTurn", + message: + "Prepare the release notification, verify the rollback owner, and publish the final deployment readiness summary.", + }, + delivery: { mode: "announce", channel: "slack", to: "#release-operations" }, + state: { + nextRunAtMs: baseTime + 20 * minute, + lastRunAtMs: baseTime - 9 * minute, + lastRunStatus: "error", + lastError: + "Delivery failed after the provider accepted the request but closed the stream before the final acknowledgement. The retry queue retained the payload, the release channel has not been notified, and the notification fan-out must be reconciled before another deployment attempt.", + lastDurationMs: 18_640, + consecutiveErrors: 3, + lastDeliveryStatus: "not-delivered", + }, + }, + { + id: "mock-cron-backup-verify", + agentId: "main", + name: "Verify encrypted backup rotation before the retention window closes", + description: + "Check the latest encrypted backup, key rotation receipt, and restore manifest before retention pruning.", + enabled: true, + createdAtMs: baseTime - 42 * day, + updatedAtMs: baseTime - 11 * minute, + schedule: { kind: "cron", expr: "15 * * * *", tz: "UTC" }, + sessionTarget: "isolated", + wakeMode: "now", + payload: { + kind: "agentTurn", + message: + "Verify backup rotation and report any missing restore manifest or key receipt.", + }, + delivery: { mode: "none" }, + state: { + nextRunAtMs: baseTime + 15 * minute, + lastRunAtMs: baseTime - 13 * minute, + lastRunStatus: "error", + lastError: + "Restore verification could not read the encrypted manifest: checksum mismatch after the object store returned a partial range. Keep the current backup, do not prune the retention window, and retry after the storage replica is healthy.", + lastDurationMs: 42_900, + consecutiveErrors: 4, + lastDeliveryStatus: "not-requested", + }, + }, + ] + : []; + const overdueJob: CronJob = { + id: "mock-cron-inbox-triage", + agentId: "main", + name: "Triage support inbox", + description: "Classify new support mail and prepare the daily response queue.", + enabled: true, + createdAtMs: baseTime - 18 * day, + updatedAtMs: baseTime - 35 * minute, + schedule: { kind: "every", everyMs: 15 * minute, anchorMs: baseTime - 18 * day }, + sessionTarget: "isolated", + wakeMode: "now", + payload: { kind: "agentTurn", message: "Triage the support inbox." }, + delivery: { mode: "none" }, + state: { + nextRunAtMs: baseTime - 20 * minute, + lastRunAtMs: baseTime - 35 * minute, + lastRunStatus: "ok", + lastDurationMs: 24_180, + lastDeliveryStatus: "not-requested", + }, + }; + const extraOverdueJobs: CronJob[] = richAttention + ? [ + { + id: "mock-cron-security-digest", + agentId: "main", + name: "Prepare the daily security digest", + description: "Summarize new security advisories and unresolved remediation work.", + enabled: true, + createdAtMs: baseTime - 31 * day, + updatedAtMs: baseTime - 50 * minute, + schedule: { kind: "every", everyMs: hour, anchorMs: baseTime - 31 * day }, + sessionTarget: "isolated", + wakeMode: "now", + payload: { kind: "agentTurn", message: "Prepare the daily security digest." }, + delivery: { mode: "none" }, + state: { + nextRunAtMs: baseTime - 42 * minute, + lastRunAtMs: baseTime - 102 * minute, + lastRunStatus: "ok", + lastDurationMs: 38_420, + lastDeliveryStatus: "not-requested", + }, + }, + ] + : []; + const healthyJob: CronJob = { + id: "mock-cron-release-digest", + agentId: options.secondAgentId ?? "main", + name: "Publish release digest", + description: "Summarize merged changes for the engineering channel.", + enabled: true, + createdAtMs: baseTime - 9 * day, + updatedAtMs: baseTime - 30 * minute, + schedule: { kind: "cron", expr: "30 9 * * 1-5", tz: "America/New_York" }, + sessionTarget: "isolated", + wakeMode: "now", + payload: { kind: "agentTurn", message: "Draft and publish the release digest." }, + delivery: { mode: "announce", channel: "slack", to: "#engineering" }, + state: { + nextRunAtMs: baseTime + 30 * minute, + lastRunAtMs: baseTime - 30 * minute, + lastRunStatus: "ok", + lastDurationMs: 51_230, + lastDelivered: true, + lastDeliveryStatus: "delivered", + }, + }; + const jobs = [overdueJob, ...extraOverdueJobs, healthyJob, failedJob, ...extraFailedJobs]; + const failedJobs = [failedJob, ...extraFailedJobs]; + const failedRuns: CronRunLogEntry[] = failedJobs.map((job, index) => ({ + ts: baseTime - (5 + index * 4) * minute, + runAtMs: baseTime - (5 + index * 4) * minute, + jobId: job.id, + jobName: job.name, + action: "finished", + status: "error", + durationMs: job.state?.lastDurationMs, + error: job.state?.lastError, + deliveryStatus: "not-requested", + model: index === 1 ? "claude-sonnet-4-6" : "gpt-5", + provider: index === 1 ? "anthropic" : "openai", + })); + const runs: CronRunLogEntry[] = [ + ...failedRuns, + { + ts: baseTime - 30 * minute, + runAtMs: baseTime - 30 * minute, + jobId: healthyJob.id, + jobName: healthyJob.name, + action: "finished", + status: "ok", + durationMs: healthyJob.state?.lastDurationMs, + summary: "Published a digest covering 14 merged changes.", + delivered: true, + deliveryStatus: "delivered", + model: "claude-sonnet-4-6", + provider: "anthropic", + }, + { + ts: baseTime - 35 * minute, + runAtMs: baseTime - 35 * minute, + jobId: overdueJob.id, + jobName: overdueJob.name, + action: "finished", + status: "ok", + durationMs: overdueJob.state?.lastDurationMs, + summary: "Classified 23 messages and prepared 6 replies.", + deliveryStatus: "not-requested", + deliverySuppressionReason: "Delivery mode is none for this inbox-only automation.", + model: "gpt-5", + provider: "openai", + }, + ]; + const queuedRuns: Array<{ runId: string; entry: CronRunLogEntry }> = jobs.map((job, index) => ({ + runId: `mock-cron-manual-${job.id}`, + entry: { + ts: baseTime + index, + runAtMs: baseTime + index, + jobId: job.id, + jobName: job.name, + action: "finished", + status: "ok", + durationMs: 42_000 + index * 2_500, + summary: `Completed an on-demand run for ${job.name}.`, + deliveryStatus: "not-requested", + model: "gpt-5", + provider: "openai", + }, + })); + const status: CronStatus = { + enabled: true, + triggersEnabled: true, + jobs: jobs.length, + nextWakeAtMs: Math.min( + ...jobs.flatMap((job) => + job.state?.nextRunAtMs === undefined ? [] : [job.state.nextRunAtMs], + ), + ), + }; + const runByJobId = new Map(runs.map((entry) => [entry.jobId, entry])); + const sortedJobLists = [ + { + match: { sortBy: "nextRunAtMs", sortDir: "asc" }, + jobs: jobs.toSorted( + (left, right) => (left.state?.nextRunAtMs ?? 0) - (right.state?.nextRunAtMs ?? 0), + ), + }, + { + match: { sortBy: "nextRunAtMs", sortDir: "desc" }, + jobs: jobs.toSorted( + (left, right) => (right.state?.nextRunAtMs ?? 0) - (left.state?.nextRunAtMs ?? 0), + ), + }, + { + match: { sortBy: "updatedAtMs", sortDir: "asc" }, + jobs: jobs.toSorted((left, right) => (left.updatedAtMs ?? 0) - (right.updatedAtMs ?? 0)), + }, + { + match: { sortBy: "updatedAtMs", sortDir: "desc" }, + jobs: jobs.toSorted((left, right) => (right.updatedAtMs ?? 0) - (left.updatedAtMs ?? 0)), + }, + { + match: { sortBy: "name", sortDir: "asc" }, + jobs: jobs.toSorted((left, right) => left.name.localeCompare(right.name)), + }, + { + match: { sortBy: "name", sortDir: "desc" }, + jobs: jobs.toSorted((left, right) => right.name.localeCompare(left.name)), + }, + ]; + + return { + "cron.status": status, + "cron.list": { + // Cases mirror the concrete queries today's Cron UI issues. Unknown combinations fall back + // to the full fixture list; dynamic evaluation is intentionally out of scope because the + // scenario is JSON-serialized into the page rather than installed as a live responder. + cases: [ + { + match: { enabled: "enabled", lastRunStatus: "error" }, + response: listResult(failedJobs, { limit: failedJobs.length }), + }, + { match: { enabled: "disabled" }, response: listResult([]) }, + ...singleJobListCases(jobs, { + enabled: "enabled", + sortBy: "nextRunAtMs", + sortDir: "asc", + limit: 1, + }), + ...singleJobListCases(jobs, { includeDisabled: true, limit: 1 }), + ...sortedJobLists.map((entry) => ({ + match: entry.match, + response: listResult(entry.jobs), + })), + { response: listResult(jobs) }, + ], + }, + "cron.runs": { + cases: [ + ...queuedRuns.map((run) => ({ + match: { runId: run.runId }, + response: runsResult([run.entry]), + })), + ...jobs.flatMap((job) => { + const jobRun = runByJobId.get(job.id); + return [ + { + match: { scope: "job", id: job.id, statuses: ["error"] }, + response: runsResult(jobRun?.status === "error" ? [jobRun] : []), + }, + { + match: { scope: "job", id: job.id }, + response: runsResult(jobRun ? [jobRun] : []), + }, + ]; + }), + { match: { statuses: ["error"] }, response: runsResult(failedRuns) }, + { response: runsResult(runs) }, + ], + }, + // Writes acknowledge the UI action but intentionally keep the fixture snapshot immutable. + "cron.add": { id: "mock-cron-created" }, + "cron.update": { ok: true }, + "cron.remove": { ok: true }, + "cron.run": { + cases: queuedRuns.map((run) => ({ + match: { id: run.entry.jobId }, + response: { ok: true, enqueued: true, runId: run.runId }, + })), + }, + }; +} diff --git a/scripts/control-ui-mock-dev.ts b/scripts/control-ui-mock-dev.ts new file mode 100644 index 0000000000000000000000000000000000000000..8b3bb94f2bbec760a0fee9255a5c4db5f262bbdf --- /dev/null +++ b/scripts/control-ui-mock-dev.ts @@ -0,0 +1,3936 @@ +// Control Ui Mock Dev script supports OpenClaw repository automation. +import { createHash } from "node:crypto"; +import fs, { rmSync } from "node:fs"; +import { mkdir, mkdtemp } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import qrcode from "qrcode"; +import { createServer, type Plugin, type ViteDevServer } from "vite"; +import type { + SystemAgentChatHistoryResult, + SystemChangesListResult, + UserProfile, +} from "../packages/gateway-protocol/src/index.js"; +import { expectDefined } from "../packages/normalization-core/src/expect.js"; +import { applySharedChannelFieldHelp } from "../src/config/schema.channel-field-help.js"; +import { buildBaseHints } from "../src/config/schema.hints.js"; +import { applyConfigTierHints, applyResolvedConfigTierHints } from "../src/config/schema.tiers.js"; +import { CONTROL_UI_BOOTSTRAP_CONFIG_PATH } from "../src/gateway/control-ui-contract.js"; +import { + controlUiPluginAssetPrefix, + controlUiPluginAssetRoot, +} from "../src/gateway/control-ui-plugin-assets-contract.js"; +import { buildUpdateRestartSentinelPayload } from "../src/infra/update-restart-sentinel-payload.js"; +import type { UpdateRunResult } from "../src/infra/update-runner.js"; +import { buildPluginLoaderAliasMap } from "../src/plugins/sdk-alias.js"; +import { buildNewAgentWelcome } from "../src/system-agent/new-agent-welcome.js"; +import type { UpdateAvailable, UpdateScheduleState } from "../ui/src/api/types.ts"; +import { + controlUiSessionPath, + createControlUiMockBootstrapConfig, + createControlUiMockGatewayInitScript, + createControlUiMockSameOriginGatewayScript, + prepareControlUiMockGatewayScenario, + type ControlUiMockGatewayScenario, +} from "../ui/src/test-helpers/control-ui-e2e.ts"; +import { + createControlUiChatHistoryMessage as chatHistoryMessage, + createControlUiMockSessionRow as sessionRow, +} from "../ui/src/test-helpers/control-ui-session-fixtures.ts"; +import { workboardUi } from "../ui/src/test-helpers/control-ui-workboard-fixture.ts"; +import { buildWorkboardMocks } from "../ui/src/test-helpers/control-ui-workboard-fixtures.ts"; +import { installWorkboardBoardMock } from "../ui/src/test-helpers/control-ui-workboard-mocks.ts"; +import { createOfflineDeviceNode } from "../ui/src/test-helpers/devices-fixtures.ts"; +import { + resolveExternalPackageAliasesForVite, + resolveSourcePackageAliasesForVite, + resolveTsconfigPathAliasesForVite, +} from "../ui/vite.config.ts"; +import { approvalMockInitScript } from "./control-ui-mock-approvals.ts"; +import { + buildChatAttachmentHistory, + createChatAttachmentFixturePlugin, +} from "./control-ui-mock-attachments.ts"; +import { + backgroundTasksMockInitScript, + buildBackgroundTasksMock, +} from "./control-ui-mock-background-tasks.ts"; +import { + buildChannelsPairingMock, + buildChannelsStatusMock, + buildChannelWizardMocks, +} from "./control-ui-mock-channels.ts"; +import { buildCronMocks } from "./control-ui-mock-cron.ts"; +import { createStandaloneMockIsolationPlugins } from "./control-ui-mock-isolation.ts"; +import { + buildPluginCatalogMock, + buildPluginDiscoveryCategoriesMock, + buildPluginDiscoveryMock, + buildPluginInspectMock, + pluginLifecycleMockInitScript, +} from "./control-ui-mock-plugins.ts"; +import { createControlUiPreviewInitScript } from "./control-ui-mock-preview.ts"; +import { skillLibraryMockInitScript } from "./control-ui-mock-skill-library.ts"; +import { + buildSkillWorkshopMocks, + skillWorkshopMockInitScript, +} from "./control-ui-mock-skill-workshop.js"; + +type CliOptions = { + allowedHosts: string[]; + fixture?: + | "approval" + | "attachments" + | "avatars" + | "board" + | "code-fences" + | "dashboards" + | "goal" + | "plugins-dense" + | "sidebar-roster" + | "swarm" + | "update-available" + | "update-blocked" + | "update-failed" + | "workboard" + | "workboard-states"; + host: string; + operatorScopes?: string[]; + port: number; +}; + +type SessionListOptions = { + owners?: readonly SessionActorFixture[]; + hasMore: boolean; + nextOffset: number | null; + offset?: number; + totalCount: number; +}; + +type SessionActorFixture = { type: "human" | "agent"; id: string; label: string }; + +const MOCK_ACTOR_PETER: SessionActorFixture = { + type: "human", + id: "profile-peter", + label: "Peter", +}; +const MOCK_ACTOR_MIRA: SessionActorFixture = { + type: "human", + id: "profile-mira", + label: "Mira", +}; +// Rows carry explicit owners the way the gateway projects createdActor fallbacks. +const MOCK_SESSION_OWNERS: readonly SessionActorFixture[] = [MOCK_ACTOR_PETER, MOCK_ACTOR_MIRA]; +const MOCK_AVATAR_PROFILES = [ + { id: "profile-avatar-normal", name: "Mira", fill: "#5d7fc5", label: "Normal profile image" }, + { + id: "profile-avatar-dark", + name: "Rowan", + fill: "#0e1015", + label: "Dark surface-matching profile image", + }, + { + id: "profile-avatar-light", + name: "Sol", + fill: "#faf9f7", + label: "Light surface-matching profile image", + }, +] as const; + +const SESSION_PAGE_SIZE = 50; +const TOTAL_TELEGRAM_SESSIONS = 180; +const ATTENTION_FIXTURE_EXPIRES_AT = Date.parse("2099-01-01T00:00:00.000Z"); +const NARRATION_DEMO_SESSION_KEY = "agent:main:sidebar-narration-demo"; +const NARRATION_DEMO_RUN_ID = "mock-sidebar-narration-run"; +const OBSERVER_DEMO_SESSION_KEY = "agent:main:session-observer-demo"; +const OBSERVER_DEMO_RUN_ID = "mock-session-observer-run"; +const PLAN_DEMO_RUN_ID = "mock-plan-run"; +type UpdateFixture = { + available: UpdateAvailable; + runResponse: unknown; + schedule: UpdateScheduleState; + statusResponse: unknown; +}; + +function buildUpdateFixture(fixture: CliOptions["fixture"], nowMs: number): UpdateFixture | null { + if ( + fixture !== "update-available" && + fixture !== "update-blocked" && + fixture !== "update-failed" + ) { + return null; + } + + if (fixture === "update-available") { + const available: UpdateAvailable = { + currentVersion: "2026.8.1", + latestVersion: "2026.8.2", + channel: "latest", + }; + const schedule: UpdateScheduleState = { + channel: "stable", + autoEnabled: false, + install: { kind: "package" }, + target: { kind: "package", version: available.latestVersion }, + }; + return { + available, + schedule, + statusResponse: { + sentinel: null, + updateAvailable: available, + effectiveChannel: "stable", + schedule, + }, + runResponse: { + ok: true, + result: { + status: "ok", + mode: "global", + before: { version: available.currentVersion }, + after: { version: available.latestVersion }, + steps: [], + durationMs: 12_000, + }, + }, + }; + } + + const currentSha = "83b321ba7d31c04cc6f7a38c87932ec0172b5461"; + const upstreamSha = "ea2ab707d3ab6f9351dcdb2f3c05054097fbfa62"; + const available: UpdateAvailable = { + currentVersion: "2026.8.1", + latestVersion: "2026.8.1", + channel: "dev", + currentSha, + upstreamRef: "origin/main", + upstreamSha, + commitsBehind: 2, + commits: [ + { sha: "f6c71c4", subject: "Keep update status authoritative" }, + { sha: "ea2ab70", subject: "Move update actions into Inbox" }, + ], + }; + const baseSchedule: UpdateScheduleState = { + channel: "dev", + autoEnabled: true, + install: { + kind: "git", + git: { + status: "behind", + currentSha, + commitsBehind: 2, + commitAtMs: nowMs - 2 * 86_400_000, + installedAtMs: nowMs - 7 * 86_400_000, + }, + }, + target: { + kind: "git", + upstreamRef: available.upstreamRef ?? "origin/main", + upstreamSha, + commitsBehind: 2, + }, + }; + + if (fixture === "update-blocked") { + const schedule: UpdateScheduleState = { + ...baseSchedule, + campaign: { + id: "mock-update-waiting-for-idle", + state: "waiting-for-idle", + announcedAtMs: nowMs - 2 * 60_000, + forceAtMs: nowMs + 13 * 60_000, + updatedAtMs: nowMs, + }, + }; + return { + available, + schedule, + statusResponse: { + sentinel: null, + updateAvailable: available, + effectiveChannel: "dev", + schedule, + }, + runResponse: { + ok: true, + result: { + status: "skipped", + mode: "git", + reason: "managed-service-handoff-started", + before: { version: available.currentVersion, sha: currentSha }, + steps: [], + durationMs: 0, + }, + handoff: { status: "started" }, + }, + }; + } + + const schedule: UpdateScheduleState = { + ...baseSchedule, + campaign: { + id: "mock-update-before-failure", + state: "waiting-for-idle", + announcedAtMs: nowMs - 2 * 60_000, + forceAtMs: nowMs + 13 * 60_000, + updatedAtMs: nowMs, + }, + }; + const result: UpdateRunResult = { + status: "error", + mode: "git", + root: "/mock/openclaw", + reason: "build-failed", + before: { version: available.currentVersion, sha: currentSha }, + after: { version: available.latestVersion, sha: upstreamSha }, + steps: [ + { + name: "build", + command: "pnpm build", + cwd: "/mock/openclaw", + durationMs: 8_420, + stdoutTail: "", + stderrTail: "tsc: error TS2345", + exitCode: 1, + }, + ], + durationMs: 11_640, + }; + return { + available, + schedule, + runResponse: { ok: false, result }, + statusResponse: { + sentinel: buildUpdateRestartSentinelPayload({ result, meta: {}, nowMs }), + updateAvailable: available, + effectiveChannel: "dev", + schedule: baseSchedule, + }, + }; +} + +const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const uiRoot = path.join(repoRoot, "ui"); +const boardFixturePath = "/__fixtures/board/"; +const boardFixtureHtml = ` + + + + + + OpenClaw Board Fixture + + + + + +
+ + +`; + +function mockFileHash(value: string): string { + return createHash("sha256").update(value, "utf8").digest("hex"); +} + +function parseArgs(args: string[]): CliOptions { + const options: CliOptions = { allowedHosts: [], host: "127.0.0.1", port: 5187 }; + for (let i = 0; i < args.length; i += 1) { + const arg = expectDefined(args[i], `control UI mock argument at index ${i}`); + if (arg === "--allowed-host") { + const allowedHost = args[++i]?.trim(); + if (allowedHost) { + options.allowedHosts.push(allowedHost); + } + } else if (arg.startsWith("--allowed-host=")) { + const allowedHost = arg.slice("--allowed-host=".length).trim(); + if (allowedHost) { + options.allowedHosts.push(allowedHost); + } + } else if (arg === "--host") { + options.host = args[++i] ?? options.host; + } else if (arg.startsWith("--host=")) { + options.host = arg.slice("--host=".length) || options.host; + } else if (arg === "--fixture") { + options.fixture = parseFixture(args[++i]); + } else if (arg.startsWith("--fixture=")) { + options.fixture = parseFixture(arg.slice("--fixture=".length)); + } else if (arg === "--port") { + options.port = parsePort(args[++i], options.port); + } else if (arg.startsWith("--port=")) { + options.port = parsePort(arg.slice("--port=".length), options.port); + } else if (arg === "--operator-scopes") { + options.operatorScopes = parseOperatorScopes(args[++i]); + } else if (arg.startsWith("--operator-scopes=")) { + options.operatorScopes = parseOperatorScopes(arg.slice("--operator-scopes=".length)); + } + } + return options; +} + +function parseFixture(value: string | undefined): CliOptions["fixture"] { + if (!value) { + return undefined; + } + if ( + value !== "approval" && + value !== "attachments" && + value !== "avatars" && + value !== "board" && + value !== "code-fences" && + value !== "dashboards" && + value !== "goal" && + value !== "plugins-dense" && + value !== "sidebar-roster" && + value !== "swarm" && + value !== "update-available" && + value !== "update-blocked" && + value !== "update-failed" && + value !== "workboard" && + value !== "workboard-states" + ) { + throw new Error(`Unknown Control UI mock fixture: ${value}`); + } + return value; +} + +function parsePort(value: string | undefined, fallback: number): number { + const parsed = Number(value); + return Number.isInteger(parsed) && parsed > 0 && parsed < 65_536 ? parsed : fallback; +} + +function parseOperatorScopes(value: string | undefined): string[] | undefined { + const scopes = (value ?? "") + .split(",") + .map((scope) => scope.trim()) + .filter(Boolean); + return scopes.length > 0 ? scopes : undefined; +} + +function sessionsListResponse(sessions: Array<{ key: string }>, options: SessionListOptions) { + return { + count: sessions.length, + defaults: { + contextTokens: 200_000, + model: "gpt-5-mini", + modelProvider: "openai", + }, + hasMore: options.hasMore, + limitApplied: 50, + nextOffset: options.nextOffset, + ...(options.owners ? { owners: options.owners } : {}), + offset: options.offset ?? 0, + path: "", + // Cases select membership; canonical metadata comes from the scenario's rows. + sessions: sessions.map(({ key }) => ({ key })), + totalCount: options.totalCount, + ts: Date.now(), + }; +} + +function pagedSessionsListResponse( + sessions: Array<{ key: string }>, + offset: number, + owners?: readonly SessionActorFixture[], +) { + const normalizedOffset = Math.max(0, Math.floor(offset)); + const page = sessions.slice(normalizedOffset, normalizedOffset + SESSION_PAGE_SIZE); + const nextOffset = normalizedOffset + SESSION_PAGE_SIZE; + return sessionsListResponse(page, { + owners, + hasMore: nextOffset < sessions.length, + nextOffset: nextOffset < sessions.length ? nextOffset : null, + offset: normalizedOffset, + totalCount: sessions.length, + }); +} + +function buildSessionRows(params: { + baseTime: number; + count: number; + keyPrefix: string; + labelPrefix: string; + model?: string; + modelProvider?: string; +}) { + return Array.from({ length: params.count }, (_value, index) => { + const ordinal = index + 1; + const padded = String(ordinal).padStart(3, "0"); + return sessionRow( + `agent:${params.keyPrefix}-${padded}`, + `${params.labelPrefix} ${padded}`, + params.baseTime - ordinal * 60_000, + { model: params.model, modelProvider: params.modelProvider }, + ); + }); +} + +function buildSessionListCases( + sessions: Array<{ key: string }>, + matchBase: Record = {}, + owners?: readonly SessionActorFixture[], +): Array<{ match: Record; response: unknown }> { + const cases: Array<{ match: Record; response: unknown }> = []; + for (let offset = SESSION_PAGE_SIZE; offset < sessions.length; offset += SESSION_PAGE_SIZE) { + cases.push({ + match: { ...matchBase, offset }, + response: pagedSessionsListResponse(sessions, offset, owners), + }); + } + cases.push({ + match: matchBase, + response: pagedSessionsListResponse(sessions, 0, owners), + }); + return cases; +} + +function buildSearchSessionListCases( + sessions: Array<{ key: string }>, + searchTerms: string[], +): Array<{ match: Record; response: unknown }> { + return searchTerms.flatMap((search) => buildSessionListCases(sessions, { search })); +} + +function buildActivitySessionRows(baseTime: number) { + const owners = { + molty: { type: "agent", id: "profile-molty", label: "Molty" }, + riley: { type: "human", id: "presence-riley", label: "Riley" }, + colin: { type: "human", id: "presence-colin", label: "Colin" }, + patrick: { + type: "human", + id: "presence-patricia", + label: "Patrick", + }, + unresolved: { type: "human", id: "147591189530201337" }, + } as const; + const hour = 60 * 60 * 1000; + const day = 24 * hour; + const fixtures = [ + ["release-check", "Release readiness check", owners.riley, 5 * 60_000], + ["api-notes", "Gateway API notes", owners.molty, 20 * 60_000], + ["design-review", "Activity feed design review", owners.colin, hour], + ["archive-audit", "Archive retention audit", owners.unresolved, 3 * hour], + ["support-handoff", "Support handoff", owners.patrick, 6 * hour], + ["mobile-smoke", "Mobile layout smoke test", owners.riley, 18 * hour], + ["provider-matrix", "Provider matrix cleanup", owners.molty, 30 * hour], + ["docs-pass", "Operator docs pass", owners.colin, 2 * day], + ["queue-review", "Queue behavior review", owners.patrick, 2.5 * day], + ["identity-trace", "Identity trace", owners.unresolved, 3 * day], + ["channel-followup", "Channel delivery follow-up", owners.riley, 4 * day], + ["tooling-refresh", "Tooling refresh", owners.molty, 5 * day], + ["fixture-polish", "Mock fixture polish", owners.colin, 6 * day], + ["weekly-summary", "Weekly activity summary", owners.patrick, 6.5 * day], + ] as const; + const automationKeys = new Set(["release-check", "api-notes", "design-review"]); + return fixtures.map(([key, label, owner, age]) => + sessionRow(`agent:activity:${key}`, label, baseTime - age, { + ...(key === "archive-audit" + ? { + activeRunIds: ["mock-activity-live-run"], + hasActiveRun: true, + observerDigest: { + headline: "Waiting on a fictional mock approval", + health: "waiting-on-user", + revision: 1, + runId: "mock-activity-live-run", + updatedAt: baseTime - age, + }, + status: "running", + } + : {}), + createdActor: owner, + hasAutomation: automationKeys.has(key), + owner: { actor: owner }, + }), + ); +} + +function usageCostTotals(totalTokens: number, totalCost = 0) { + return { + input: Math.round(totalTokens * 0.2), + output: Math.round(totalTokens * 0.1), + cacheRead: Math.round(totalTokens * 0.6), + cacheWrite: Math.round(totalTokens * 0.1), + totalTokens, + totalCost, + inputCost: totalCost, + outputCost: 0, + cacheReadCost: 0, + cacheWriteCost: 0, + missingCostEntries: 0, + }; +} + +// Model Providers settings fixtures: auth state plus live plan/quota/billing +// snapshots so the /settings/model-providers page renders fully in the mock. +function buildSessionDiffMock() { + const appPatch = [ + "diff --git a/src/app.ts b/src/app.ts", + "index 1111111..2222222 100644", + "--- a/src/app.ts", + "+++ b/src/app.ts", + "@@ -12,4 +12,5 @@ export function bootstrap() {", + " const config = readSettings();", + "- const client = createClient(config);", + "+ const client = createClient(config, { retries: 3 });", + '+ client.on("error", reportError);', + " return client;", + "@@ -181,3 +182,3 @@ export function shutdown() {", + " flushQueues();", + '- logger.info("bye");', + '+ logger.info("shutdown complete");', + "", + ].join("\n"); + const readmePatch = [ + "diff --git a/README.md b/README.md", + "new file mode 100644", + "--- /dev/null", + "+++ b/README.md", + "@@ -0,0 +1,3 @@", + "+# Demo", + "+", + "+Mock harness session diff fixture.", + "", + ].join("\n"); + return { + sessionKey: "main", + root: "/tmp/openclaw-mock-checkout", + branch: "feature/session-diff-panel", + baseRef: "main", + files: [ + { + path: "src/app.ts", + status: "modified", + additions: 3, + deletions: 2, + patch: appPatch, + }, + { + path: "README.md", + status: "added", + additions: 3, + deletions: 0, + untracked: true, + patch: readmePatch, + }, + { + path: "assets/logo.png", + status: "modified", + additions: 0, + deletions: 0, + binary: true, + }, + ], + additions: 6, + deletions: 2, + }; +} + +function buildModelProviderMocks(baseTime: number) { + const hour = 60 * 60 * 1000; + const expiry = (remainingMs: number, label: string) => ({ + at: baseTime + remainingMs, + remainingMs, + label, + }); + const costDaily = Array.from({ length: 14 }, (_, index) => { + const date = new Date(baseTime - (13 - index) * 24 * hour); + const iso = `${date.getUTCFullYear()}-${String(date.getUTCMonth() + 1).padStart(2, "0")}-${String(date.getUTCDate()).padStart(2, "0")}`; + const amount = 4 + Math.round(Math.abs(Math.sin(index)) * 900) / 100; + return { + date: iso, + amount, + requests: 120 + index * 7, + inputTokens: 2_400_000 + index * 90_000, + cacheReadTokens: 9_000_000, + cacheWriteTokens: 400_000, + outputTokens: 310_000, + totalTokens: 12_110_000 + index * 90_000, + }; + }); + const anthropicUsage = { + provider: "anthropic", + displayName: "Claude", + plan: "Max 20x", + windows: [ + { label: "5h", usedPercent: 38, resetAt: baseTime + 2.4 * hour }, + { label: "Week", usedPercent: 61, resetAt: baseTime + 68 * hour }, + { label: "Opus", usedPercent: 24, resetAt: baseTime + 68 * hour }, + ], + costHistory: { + unit: "USD", + periodDays: 14, + daily: costDaily, + models: [ + { + name: "claude-sonnet-4-6", + inputTokens: 0, + cacheReadTokens: 0, + cacheWriteTokens: 0, + outputTokens: 0, + totalTokens: 96_000_000, + }, + { + name: "claude-opus-4-8", + inputTokens: 0, + cacheReadTokens: 0, + cacheWriteTokens: 0, + outputTokens: 0, + totalTokens: 31_000_000, + }, + ], + categories: [ + { name: "Sessions", amount: 61.13 }, + { name: "Code Assist", amount: 18.4 }, + ], + }, + }; + const openaiUsage = { + provider: "openai", + displayName: "OpenAI", + plan: "Pro", + windows: [ + { label: "5h", usedPercent: 12, resetAt: baseTime + 3.1 * hour }, + { label: "Week", usedPercent: 44, resetAt: baseTime + 100 * hour }, + ], + billing: [{ type: "balance", label: "Credits", amount: 341, unit: "credits" }], + }; + const openrouterUsage = { + provider: "openrouter", + displayName: "OpenRouter", + windows: [], + billing: [{ type: "balance", amount: 12.34, unit: "USD" }], + }; + const copilotUsage = { + provider: "github-copilot", + displayName: "GitHub Copilot", + plan: "Business", + windows: [{ label: "Premium requests", usedPercent: 71, resetAt: baseTime + 21 * 24 * hour }], + }; + return { + authStatus: { + ts: baseTime, + providers: [ + { + provider: "anthropic", + displayName: "Claude", + status: "ok", + expiry: expiry(11 * 24 * hour, "11d"), + profiles: [ + { + profileId: "anthropic:default", + type: "oauth", + status: "ok", + expiry: expiry(11 * 24 * hour, "11d"), + }, + ], + usage: { + providerId: "anthropic", + plan: anthropicUsage.plan, + windows: anthropicUsage.windows, + }, + }, + { + provider: "openai", + displayName: "OpenAI", + status: "ok", + expiry: expiry(6 * 24 * hour, "6d"), + profiles: [ + { + profileId: "openai:codex", + type: "oauth", + status: "ok", + expiry: expiry(6 * 24 * hour, "6d"), + }, + ], + usage: { + providerId: "openai", + plan: openaiUsage.plan, + windows: openaiUsage.windows, + billing: openaiUsage.billing, + }, + }, + { + provider: "github-copilot", + displayName: "GitHub Copilot", + status: "expiring", + expiry: expiry(26 * 60 * 1000, "26m"), + profiles: [ + { + profileId: "github-copilot:default", + type: "token", + status: "expiring", + expiry: expiry(26 * 60 * 1000, "26m"), + }, + ], + usage: { + providerId: "github-copilot", + plan: copilotUsage.plan, + windows: copilotUsage.windows, + }, + }, + { + provider: "openrouter", + displayName: "OpenRouter", + status: "static", + profiles: [{ profileId: "openrouter:default", type: "api_key", status: "static" }], + }, + { + provider: "google", + displayName: "Gemini", + status: "missing", + profiles: [], + }, + ], + }, + usageStatus: { + updatedAt: baseTime, + providers: [anthropicUsage, openaiUsage, openrouterUsage, copilotUsage], + }, + models: [ + { id: "claude-opus-4-8", name: "Claude Opus 4.8", provider: "anthropic", available: true }, + { + id: "claude-fable-5", + name: "Claude Fable 5", + provider: "anthropic", + available: true, + contextWindow: 1_000_000, + contextWindows: [ + { id: "200k", label: "200K", contextWindow: 200_000 }, + { id: "1m", label: "1M", contextWindow: 1_000_000 }, + ], + contextWindowDefault: "1m", + }, + { + id: "claude-sonnet-4-6", + name: "Claude Sonnet 4.6", + provider: "anthropic", + available: true, + contextWindow: 200_000, + thinkingLevels: [ + { id: "low", label: "Low" }, + { id: "medium", label: "Medium" }, + { id: "high", label: "High" }, + ], + thinkingDefault: "medium", + reasoning: true, + supportsTools: true, + }, + { + id: "gpt-5-mini", + name: "GPT-5 Mini", + provider: "openai", + available: true, + contextWindow: 200_000, + thinkingLevels: [ + { id: "low", label: "Low" }, + { id: "medium", label: "Medium" }, + { id: "high", label: "High" }, + ], + thinkingDefault: "medium", + reasoning: true, + supportsTools: true, + }, + { id: "gpt-5", name: "GPT-5", provider: "openai", available: true }, + { id: "gemini-3-pro", name: "Gemini 3 Pro", provider: "google", available: false }, + { id: "openrouter/auto", name: "OpenRouter Auto", provider: "openrouter", available: true }, + ], + }; +} + +// Deterministic year of daily activity so the settings profile heatmap, +// streaks, and stat strip render with a lively fixture in the mock harness. +function buildProfileUsageMocks(baseTime: number) { + const daily: Array> = []; + let lifetimeTokens = 0; + for (let daysAgo = 364; daysAgo >= 0; daysAgo -= 1) { + const date = new Date(baseTime - daysAgo * 24 * 60 * 60 * 1000); + const iso = `${date.getFullYear()}-${String(date.getMonth() + 1).padStart(2, "0")}-${String(date.getDate()).padStart(2, "0")}`; + const weekendDamper = date.getDay() === 0 || date.getDay() === 6 ? 0.3 : 1; + const quietDay = daysAgo % 19 === 4 ? 0 : 1; + const wave = (Math.sin(daysAgo / 6) + 1.4) * 1_400_000_000; + const spike = daysAgo % 47 === 0 ? 6_000_000_000 : 0; + const tokens = Math.round((wave + spike) * weekendDamper * quietDay); + lifetimeTokens += tokens; + daily.push({ date: iso, ...usageCostTotals(tokens, tokens / 1e9) }); + } + return { + cost: { + updatedAt: baseTime, + days: daily.length, + daily, + totals: usageCostTotals(lifetimeTokens, lifetimeTokens / 1e9), + }, + sessions: { + updatedAt: baseTime, + startDate: daily[0]?.date, + endDate: daily[daily.length - 1]?.date, + sessions: [ + { + key: "agent:openclaw-mock:marathon", + label: "Release night marathon", + usage: { ...usageCostTotals(4_000_000_000), durationMs: (59 * 60 + 4) * 60 * 1000 }, + }, + { + key: "agent:openclaw-mock:daily", + label: "Daily driver", + usage: { ...usageCostTotals(900_000_000), durationMs: 3 * 60 * 60 * 1000 }, + }, + ], + totals: usageCostTotals(lifetimeTokens, lifetimeTokens / 1e9), + aggregates: { + sessionCount: 48_212, + longestSessionDurationMs: (59 * 60 + 4) * 60 * 1000, + messages: { + total: 2_787_815, + user: 1_400_000, + assistant: 1_387_815, + toolCalls: 42_380, + toolResults: 42_380, + errors: 128, + }, + tools: { + totalCalls: 42_380, + uniqueTools: 205, + tools: [ + { name: "exec", count: 6_418 }, + { name: "browser", count: 5_256 }, + { name: "message", count: 4_708 }, + { name: "read", count: 4_489 }, + { name: "sessions_list", count: 3_066 }, + ], + }, + byModel: [ + { + provider: "anthropic", + model: "claude-sonnet-4-6", + count: 9_000, + totals: usageCostTotals(Math.round(lifetimeTokens * 0.7)), + }, + { + provider: "openai", + model: "gpt-5-mini", + count: 4_000, + totals: usageCostTotals(Math.round(lifetimeTokens * 0.3)), + }, + ], + byProvider: [ + { + provider: "anthropic", + count: 9_000, + totals: usageCostTotals(Math.round(lifetimeTokens * 0.7), 184.2), + }, + { + provider: "openai", + count: 4_000, + totals: usageCostTotals(Math.round(lifetimeTokens * 0.3), 96.4), + }, + ], + byAgent: [ + { agentId: "openclaw-mock", totals: usageCostTotals(Math.round(lifetimeTokens * 0.8)) }, + { agentId: "alpha", totals: usageCostTotals(Math.round(lifetimeTokens * 0.2)) }, + ], + byChannel: [ + { channel: "whatsapp", totals: usageCostTotals(Math.round(lifetimeTokens * 0.5)) }, + { channel: "telegram", totals: usageCostTotals(Math.round(lifetimeTokens * 0.3)) }, + { channel: "discord", totals: usageCostTotals(Math.round(lifetimeTokens * 0.2)) }, + ], + daily: [], + }, + }, + }; +} + +/** + * Small but coherent config fixture so the schema-driven settings pages are + * demoable: `config.schema` covers a boolean, an enum, numbers, and strings + * across a few real section keys, and `config.get` returns a matching + * snapshot with the hash `config.set`/`config.apply` are guarded by. + */ +function buildConfigMocks(options: { swarmEnabled?: boolean; workboardEnabled?: boolean } = {}) { + const config = { + logging: { level: "info", consoleTimestamps: true }, + messages: { queueLimit: 5, responsePrefix: "" }, + gateway: { port: 18789, bind: "127.0.0.1", publicOrigin: "https://gateway.example" }, + agents: { defaults: { thinkingDefault: "medium" } }, + commands: { native: "auto", nativeSkills: "auto" }, + models: { mode: "merge" }, + ui: { prefs: { locale: "en" } }, + ...(options.swarmEnabled ? { tools: { swarm: true } } : {}), + ...(options.workboardEnabled ? { plugins: { entries: { workboard: { enabled: true } } } } : {}), + channels: { + whatsapp: { + enabled: true, + allowFrom: ["whatsapp-demo-user"], + dmPolicy: "pairing", + groupPolicy: "allowlist", + selfChatMode: "off", + }, + }, + mcp: { + servers: { + context7: { url: "https://mcp.context7.com/mcp", transport: "streamable-http" }, + github: { + url: "https://api.githubcopilot.com/mcp/", + transport: "streamable-http", + auth: "oauth", + }, + "local-tools": { command: "npx", args: ["some-mcp-server", "--stdio"], enabled: false }, + }, + }, + }; + const schema = { + type: "object", + title: "OpenClaw config", + properties: { + logging: { + type: "object", + title: "Logging", + properties: { + level: { + type: "string", + title: "Log level", + description: "Minimum severity written to the gateway log.", + enum: ["silent", "error", "warn", "info", "debug"], + }, + consoleTimestamps: { + type: "boolean", + title: "Console timestamps", + description: "Prefix console log lines with a timestamp.", + }, + }, + }, + talk: { + type: "object", + title: "Talk", + properties: { + interruptOnSpeech: { + type: "boolean", + title: "Talk Interrupt on Speech", + description: "Stop speaking when the user talks over the assistant.", + }, + realtime: { + type: "object", + title: "Talk Realtime", + properties: { + provider: { + type: "string", + title: "Talk Realtime Provider", + description: "Active realtime voice provider id, such as openai or google.", + }, + model: { + type: "string", + title: "Talk Realtime Model", + description: "Realtime voice model for browser Talk sessions.", + }, + speakerVoice: { + type: "string", + title: "Talk Realtime Speaker Voice", + description: "Built-in realtime voice id.", + }, + }, + }, + }, + }, + messages: { + type: "object", + title: "Messages", + properties: { + queueLimit: { + type: "integer", + title: "Queue limit", + description: "Maximum queued inbound messages per session.", + minimum: 0, + }, + responsePrefix: { + type: "string", + title: "Response prefix", + description: "Optional text prepended to outbound replies.", + }, + }, + }, + commands: { + type: "object", + title: "Commands", + properties: { + native: { + title: "Native Commands", + default: "auto", + anyOf: [{ type: "boolean" }, { type: "string", const: "auto" }], + }, + nativeSkills: { + title: "Native Skill Commands", + default: "auto", + anyOf: [{ type: "boolean" }, { type: "string", const: "auto" }], + }, + }, + }, + gateway: { + type: "object", + title: "Gateway", + properties: { + port: { type: "integer", title: "Port", minimum: 1, maximum: 65535 }, + bind: { type: "string", title: "Bind address" }, + // Zod's .url() emits `format`; keep one such leaf in the fixture so the + // form stays provably editable for plugin URL/email settings. + publicOrigin: { type: "string", title: "Public origin", format: "uri" }, + }, + }, + agents: { + type: "object", + title: "Agents", + properties: { + defaults: { + type: "object", + title: "Defaults", + properties: { + thinkingDefault: { + type: "string", + title: "Default thinking level", + enum: ["off", "low", "medium", "high"], + }, + }, + }, + }, + }, + models: { + type: "object", + title: "Models", + properties: { + mode: { + type: "string", + title: "Catalog mode", + enum: ["merge", "replace"], + }, + }, + }, + // Channel settings are the one schema surface the channels page renders, + // so the fixture keeps both tiers represented. + channels: { + type: "object", + title: "Channels", + properties: { + whatsapp: { + type: "object", + title: "WhatsApp", + properties: { + enabled: { type: "boolean", title: "Enabled" }, + allowFrom: { type: "array", title: "Allow from", items: { type: "string" } }, + dmPolicy: { type: "string", title: "DM policy", enum: ["pairing", "open", "off"] }, + groupPolicy: { + type: "string", + title: "Group policy", + enum: ["allowlist", "open", "off"], + }, + // Channel-specific leaves carry their help from the plugin's own + // uiHints in production; the fixture uses schema descriptions, + // which resolve through the same field-meta fallback. + selfChatMode: { + type: "string", + title: "Self chat mode", + description: "Same-phone setup (bot uses your personal WhatsApp number).", + enum: ["off", "notes"], + }, + configWrites: { type: "boolean", title: "Config writes" }, + streaming: { + type: "object", + title: "Streaming", + properties: { + progress: { + type: "object", + properties: { + maxLines: { type: "integer", title: "Progress max lines" }, + toolProgress: { type: "boolean", title: "Progress tool lines" }, + }, + }, + }, + }, + healthMonitor: { + type: "object", + title: "Health monitor", + properties: { + enabled: { type: "boolean", title: "Enabled" }, + }, + }, + mediaMaxMb: { type: "number", title: "Media max MB" }, + }, + }, + }, + }, + }, + }; + const get = { + path: "~/.openclaw/openclaw.json", + exists: true, + raw: `${JSON.stringify(config, null, 2)}\n`, + hash: "mock-config-hash", + appliedConfigHash: "mock-config-hash", + valid: true, + config, + issues: [], + }; + const writeAck = { ok: true, path: get.path, hash: get.hash, config }; + return { + get, + set: writeAck, + apply: { + ...writeAck, + sentinel: { + persisted: true, + payload: { + kind: "config-apply", + status: "ok", + ts: 0, + message: null, + doctorHint: "openclaw doctor --non-interactive", + stats: { mode: "config.apply", root: get.path, requiresRestart: false }, + }, + }, + }, + schema: { + schema, + // Resolve tiers and shared channel help the way the gateway does so the + // mock reproduces the real split and subtext instead of bare labels. + uiHints: applySharedChannelFieldHelp( + applyResolvedConfigTierHints( + schema, + // Seed with base hints so the mock carries the gateway's labels, + // help, and docsUrl metadata instead of bare tier scaffolding. + applyConfigTierHints(buildBaseHints(), { includePluginOwnedChannels: true }), + ), + ), + version: "mock-config-schema", + generatedAt: new Date(0).toISOString(), + }, + }; +} + +function buildScrollableChatHistory(baseTime: number): unknown[] { + const messages: unknown[] = Array.from({ length: 10 }, (_, index) => { + const timestamp = baseTime + index * 3 * 60_000; + return [ + chatHistoryMessage("user", `Review section ${index + 1}.`, timestamp), + chatHistoryMessage( + "assistant", + `## Section ${index + 1}\n\n- Source reviewed.\n- **Follow-up:** capture the result.`, + timestamp + 35_000, + ), + ]; + }).flat(); + + // Paired tool calls/results exercise the collapsed activity rollup. + const workTurnBase = baseTime + 31 * 60_000; + messages.push( + chatHistoryMessage( + "user", + "Mock work request: refactor the render guard and rerun the suite.", + workTurnBase, + ), + chatHistoryMessage( + "assistant", + "Checking the guard implementation before editing.", + workTurnBase + 5_000, + ), + { + role: "assistant", + content: [ + { + type: "tool_call", + id: "mock-work-read", + name: "read", + args: { path: "ui/src/pages/chat/chat-thread.ts" }, + }, + { + type: "tool_result", + id: "mock-work-read", + name: "read", + text: "Read the transcript grouping boundary and its caller.", + }, + ], + timestamp: workTurnBase + 12_000, + }, + { + role: "assistant", + content: [ + { + type: "tool_call", + id: "mock-work-exec", + name: "exec", + args: { command: "focused transcript tests" }, + }, + { + type: "tool_result", + id: "mock-work-exec", + name: "exec", + text: "Synthetic command completed.", + }, + ], + timestamp: workTurnBase + 95_000, + }, + chatHistoryMessage("assistant", "The sample command completed.", workTurnBase + 172_000), + { + role: "assistant", + content: [ + { + type: "toolCall", + id: "mock-work-yield", + name: "yield", + arguments: { + message: + "Waiting for the two visible implementation owners; resume on progress/completion to coordinate shared review and verification.", + }, + }, + ], + timestamp: workTurnBase + 173_000, + }, + ); + + return messages; +} + +function buildApprovalChatHistory(baseTime: number): unknown[] { + const runId = "mock-production-export-run"; + return [ + chatHistoryMessage( + "user", + "Trace the scroll-anchor regression and prepare a safe production export.", + baseTime, + ), + { + role: "assistant", + content: [ + { + type: "text", + text: "I found the expansion boundary and will inspect the export command before running it.", + }, + { + type: "tool_call", + id: "approval-read", + name: "read", + args: { path: "ui/src/pages/chat/chat-thread.ts" }, + }, + { + type: "tool_result", + id: "approval-read", + name: "read", + text: "The final code block expands after the virtual row is measured.", + }, + ], + timestamp: baseTime + 20_000, + __openclaw: { id: "approval-analysis", runId, seq: 1 }, + }, + { + ...chatHistoryMessage( + "assistant", + "The export requires operator approval before it can leave the mock workstation.", + baseTime + 45_000, + ), + __openclaw: { id: "approval-wait", runId, seq: 2 }, + }, + { + role: "assistant", + content: [ + { type: "text", text: "The preview process exited before the export receipt arrived." }, + ], + stopReason: "error", + errorMessage: "Preview process exited with status 1.", + timestamp: baseTime + 90_000, + __openclaw: { id: "approval-error", runId, seq: 3 }, + }, + ]; +} + +function buildCancelledChatHistory(baseTime: number): unknown[] { + const runId = "mock-cancelled-run"; + return [ + chatHistoryMessage( + "user", + "Reconcile model usage and stop if the provider session expires.", + baseTime, + ), + { + ...chatHistoryMessage( + "assistant", + "I collected the usage window and started comparing the configured default.", + baseTime + 15_000, + ), + __openclaw: { id: "cancelled-partial", runId, seq: 1 }, + }, + { + ...chatHistoryMessage( + "assistant", + "The operator cancelled the run before it changed any defaults.", + baseTime + 35_000, + ), + stopReason: "stop", + openclawAbort: { aborted: true, origin: "operator", runId }, + __openclaw: { id: "cancelled-terminal", runId, seq: 2 }, + }, + ]; +} + +function buildFixtureSummaryHistory(baseTime: number, title: string, details: string): unknown[] { + return [ + chatHistoryMessage("user", title, baseTime), + chatHistoryMessage("assistant", details, baseTime + 35_000), + ]; +} + +function buildAvatarChatHistory(baseTime: number): unknown[] { + return [ + ...MOCK_AVATAR_PROFILES.map(({ id, label, name }, index) => ({ + ...chatHistoryMessage("user", label, baseTime + index * 60_000), + __openclaw: { + senderId: id, + senderIdentity: { type: "profile", id }, + senderName: name, + senderProfileAvatarUrl: `/api/users/${id}/avatar`, + }, + })), + { + ...chatHistoryMessage("user", "Attributed initials", baseTime + 180_000), + __openclaw: { + senderId: "profile-avatar-initials", + senderIdentity: { type: "profile", id: "profile-avatar-initials" }, + senderName: "Alex Morgan", + }, + }, + ]; +} + +function buildCodeFenceChatHistory(baseTime: number): unknown[] { + const proseFence = (language: string, label: string) => { + const lines = Array.from({ length: 16 }, (_, index) => `${label} line ${index + 1}`); + return `\`\`\`${language}\n${lines.join("\n")}\n\`\`\``; + }; + const jsonLines = Array.from({ length: 18 }, (_, index) => ` "item-${index + 1}",`); + jsonLines[jsonLines.length - 1] = ` "item-${jsonLines.length}"`; + return [ + chatHistoryMessage("assistant", proseFence("text", "Plain text"), baseTime), + chatHistoryMessage("assistant", proseFence("md", "Markdown alias"), baseTime + 1_000), + chatHistoryMessage("assistant", proseFence("markdown", "Markdown"), baseTime + 2_000), + chatHistoryMessage( + "assistant", + `\`\`\`json\n[\n${jsonLines.join("\n")}\n]\n\`\`\``, + baseTime + 3_000, + ), + ]; +} + +function searchPrefixes(term: string): string[] { + return Array.from({ length: term.length }, (_value, index) => term.slice(0, index + 1)); +} + +function syntheticAvatar(color: "coral" | "green"): string { + const background = color === "coral" ? "#ffdbcc" : "#d5efd4"; + const fill = color === "coral" ? "#e87355" : "#65a879"; + const artwork = ` + + + + + + + + `; + return `data:image/svg+xml;base64,${Buffer.from(artwork).toString("base64")}`; +} + +async function createChatPickerScenario( + fixture?: CliOptions["fixture"], +): Promise { + const baseTime = Date.parse("2026-05-22T09:00:00.000Z"); + const rosterTime = Date.now(); + const rosterAgents = [ + { + id: "main", + name: "Molty", + theme: "Your everyday coordinator", + emoji: "🦞", + avatar: syntheticAvatar("coral"), + preview: "I am organizing the next steps for the sample project.", + sessionLabels: ["Project next steps", "Weekly review"], + }, + { + id: "forge", + name: "Forge", + theme: "Builds and maintains your projects", + emoji: "", + avatar: "", + preview: "The sample dashboard is ready for your review.", + sessionLabels: ["Sample dashboard", "Build notes"], + }, + { + id: "scout", + name: "Scout", + theme: "Finds answers and connects ideas", + emoji: "🔭", + avatar: "", + preview: "I found three useful approaches for the research outline.", + sessionLabels: ["Research outline", "Source notes"], + }, + { + id: "bloom", + name: "Bloom", + theme: "Turns rough drafts into clear stories", + emoji: "🌱", + avatar: syntheticAvatar("green"), + preview: "The welcome guide now has a shorter opening and clear examples.", + sessionLabels: ["Welcome guide", "Story ideas"], + }, + ] as const; + const pickerInventory = process.env.MOCK_PICKER_INVENTORY === "1"; + const selfProfile: UserProfile = { + id: "presence-riley", + displayName: "Riley", + avatarMime: null, + mergedInto: null, + createdAt: baseTime, + updatedAt: baseTime, + emails: ["riley@example.com"], + githubIdentity: null, + hasAvatar: false, + }; + const devicePairSetupCode = Buffer.from( + JSON.stringify({ + url: "wss://gateway.example.test", + bootstrapToken: "mock-bootstrap-token", + }), + "utf8", + ).toString("base64url"); + const devicePairQrDataUrl = await qrcode.toDataURL(devicePairSetupCode, { + errorCorrectionLevel: "M", + margin: 2, + width: 360, + }); + const whatsappLoginQrDataUrl = await qrcode.toDataURL("mock-whatsapp-login", { + errorCorrectionLevel: "M", + margin: 2, + width: 360, + }); + const workspaceFiles = [ + { + missing: false, + name: "AGENTS.md", + path: "/mock/workspace/AGENTS.md", + size: 2148, + updatedAtMs: baseTime - 120_000, + }, + { + missing: false, + name: "plan.md", + path: "/mock/workspace/plan.md", + size: 912, + updatedAtMs: baseTime - 90_000, + }, + { + missing: false, + name: "notes/context.md", + path: "/mock/workspace/notes/context.md", + size: 1620, + updatedAtMs: baseTime - 30_000, + }, + ]; + const workspaceListCases = ["main", "alpha", "openclaw-mock"].map((agentId) => ({ + match: { agentId }, + response: { + agentId, + files: workspaceFiles, + workspace: "/mock/workspace", + }, + })); + const workspaceFileContentByName = new Map([ + [ + "AGENTS.md", + "# AGENTS.md\n\nMock workspace instructions for the composer rail.\n\n- Keep tool output compact.\n- Prefer right-rail context over modal previews.\n", + ], + [ + "plan.md", + "# Composer polish plan\n\n1. Keep the composer controls calm.\n2. Move session selection into the sidebar.\n3. Keep model, reasoning, and speed choices discoverable without taking over the page.\n", + ], + [ + "notes/context.md", + "# Context notes\n\nThe right rail should feel like workspace context, not a modal pasted beside the chat.\n\n## Current focus\n\n- Markdown previews need readable dark-mode chrome.\n- Empty or unavailable content should show a quiet state instead of an empty card.\n- File previews should load from the same mock scenario as the file list.\n", + ], + ]); + const workspaceFileCases = ["main", "alpha", "openclaw-mock"].flatMap((agentId) => + workspaceFiles.map((file) => ({ + match: { agentId, name: file.name }, + response: { + agentId, + file: { + ...file, + content: workspaceFileContentByName.get(file.name) ?? "", + }, + workspace: "/mock/workspace", + }, + })), + ); + const sessionFiles = [ + { + kind: "modified", + missing: false, + name: "chat.ts", + path: "ui/src/ui/views/chat.ts", + size: 48320, + updatedAtMs: baseTime - 20_000, + }, + { + kind: "modified", + missing: false, + name: "sidebar.css", + path: "ui/src/styles/chat/sidebar.css", + size: 18840, + updatedAtMs: baseTime - 18_000, + }, + { + kind: "read", + missing: false, + name: "artifacts.ts", + path: "src/gateway/server-methods/artifacts.ts", + size: 21876, + updatedAtMs: baseTime - 300_000, + }, + { + kind: "read", + missing: false, + name: "sessions.ts", + path: "packages/gateway-protocol/src/schema/sessions.ts", + size: 16542, + updatedAtMs: baseTime - 420_000, + }, + ]; + const sessionWorkspaceRoot = "/mock/workspace"; + const sessionFileContentByPath = new Map([ + [ + "ui/src/ui/views/chat.ts", + 'function renderSessionWorkspaceRail() {\n return html``;\n}\n', + ], + [ + "ui/src/styles/chat/sidebar.css", + ".chat-workspace-rail__section-title {\n color: var(--muted);\n text-transform: uppercase;\n}\n", + ], + [ + "src/gateway/server-methods/artifacts.ts", + "// Artifact gateway methods collect generated artifacts from session transcripts.\n", + ], + [ + "packages/gateway-protocol/src/schema/sessions.ts", + "export const SessionsFilesListParamsSchema = Type.Object({ sessionKey: NonEmptyString });\n", + ], + [ + "package.json", + '{\n "name": "openclaw",\n "scripts": { "dev:ui:mock": "tsx scripts/control-ui-mock-dev.ts" }\n}\n', + ], + [ + "ui/vite.config.ts", + "export default function controlUiViteConfig() {\n return { server: { strictPort: true } };\n}\n", + ], + [ + "ui/src/e2e/chat-flow.e2e.test.ts", + "it('keeps the session workspace useful while browsing files', async () => {\n await page.getByText('Project files').waitFor();\n});\n", + ], + ]); + const sessionFileCases = [ + { + match: { sessionKey: "agent:main:main" }, + response: { + browser: { + entries: [ + { + kind: "directory", + name: "packages", + path: "packages", + sessionKind: "read", + updatedAtMs: baseTime - 420_000, + }, + { + kind: "directory", + name: "src", + path: "src", + sessionKind: "read", + updatedAtMs: baseTime - 300_000, + }, + { + kind: "directory", + name: "ui", + path: "ui", + sessionKind: "modified", + updatedAtMs: baseTime - 20_000, + }, + { + kind: "file", + name: "package.json", + path: "package.json", + size: 92750, + updatedAtMs: baseTime - 800_000, + }, + ], + path: "", + }, + files: sessionFiles, + root: sessionWorkspaceRoot, + sessionKey: "agent:main:main", + }, + }, + ]; + const sessionFileGetCases = sessionFiles.map((file) => ({ + match: { sessionKey: "agent:main:main", path: file.path }, + response: { + file: { + ...file, + content: sessionFileContentByPath.get(file.path) ?? "", + // Fake CAS token so the file panel offers edit mode against the mock. + hash: mockFileHash(sessionFileContentByPath.get(file.path) ?? ""), + }, + root: sessionWorkspaceRoot, + sessionKey: "agent:main:main", + }, + })); + const sessionFileSetCases = sessionFiles.map((file) => ({ + match: { sessionKey: "agent:main:main", path: file.path }, + response: { + file: { + ...file, + kind: "modified", + workspacePath: file.path, + hash: mockFileHash(`${file.path}:saved`), + updatedAtMs: baseTime, + }, + root: sessionWorkspaceRoot, + sessionKey: "agent:main:main", + }, + })); + const lobsterSvg = ` + + + + + + + + + + openclaw session artifact +`; + const lobsterArtifact = { + id: "artifact-openclaw-lobster", + type: "image", + title: "openclaw-lobster-preview.svg", + mimeType: "image/svg+xml", + sizeBytes: Buffer.byteLength(lobsterSvg, "utf8"), + source: "session-transcript", + download: { mode: "bytes" }, + }; + // Five-zone sidebar fixture: main session (hidden behind the identity card, + // its child promoted to Threads), threads with a running tree, group rows, + // and a worktree row for the Coding zone. + const mainChildRow = sessionRow( + "agent:main:lisbon-trip", + "Lisbon trip planning", + baseTime - 120_000, + { + spawnedBy: "agent:main:main", + unread: true, + }, + ); + const taxChildRow = sessionRow( + "agent:main:subagent:tax-receipts", + "Reading receipts", + baseTime - 30_000, + { + spawnedBy: "agent:main:tax-research", + hasActiveRun: true, + status: "running", + startedAt: baseTime - 200_000, + runtimeMs: 200_000, + }, + ); + const swarmGroupId = "swarm:agent:main:main:mock-turn"; + const swarmChildRows = + fixture === "swarm" + ? [ + sessionRow("agent:main:subagent:swarm-plan", "National polling", baseTime - 9_000, { + parentSessionKey: "agent:main:main", + spawnedBy: "agent:main:main", + swarmGroupId, + swarmPhase: "Research", + swarmPhaseRank: 0, + status: "done", + }), + sessionRow("agent:main:subagent:swarm-work", "Work and labor", baseTime - 8_000, { + hasActiveRun: true, + parentSessionKey: "agent:main:main", + spawnedBy: "agent:main:main", + swarmGroupId, + swarmLog: "Comparing labor, education, and consumer signals.", + swarmPhase: "Research", + swarmPhaseRank: 0, + status: "running", + }), + sessionRow("agent:main:subagent:swarm-health", "Health", baseTime - 7_000, { + hasActiveRun: true, + parentSessionKey: "agent:main:main", + spawnedBy: "agent:main:main", + swarmGroupId, + swarmPhase: "Research", + swarmPhaseRank: 0, + status: "running", + }), + sessionRow("agent:main:subagent:swarm-trust", "Governance and trust", baseTime - 6_000, { + parentSessionKey: "agent:main:main", + spawnedBy: "agent:main:main", + subagentRunState: "active", + swarmGroupId, + swarmPhase: "Research", + swarmPhaseRank: 0, + status: undefined, + }), + sessionRow("agent:main:subagent:swarm-media", "Media signals", baseTime - 5_000, { + parentSessionKey: "agent:main:main", + spawnedBy: "agent:main:main", + swarmGroupId, + swarmPhase: "Research", + swarmPhaseRank: 0, + status: "failed", + }), + ] + : []; + const workboardMocks = buildWorkboardMocks( + Date.now(), + MOCK_ACTOR_PETER, + fixture === "workboard-states", + ); + const activityTime = Date.now(); + const activitySessions = buildActivitySessionRows(activityTime); + const dashboardGallerySessions = + fixture === "dashboards" + ? [ + sessionRow("agent:main:dashboard:release-health", "Release health", baseTime - 3_000, { + boardFace: "dashboard", + createdActor: MOCK_ACTOR_MIRA, + hasActiveRun: true, + status: "running", + }), + sessionRow("agent:main:dashboard:model-spend", "Model spend", baseTime - 8_000, { + boardFace: "dashboard", + createdActor: MOCK_ACTOR_PETER, + }), + sessionRow("agent:main:dashboard:support-radar", "Support radar", baseTime - 18_000, { + boardFace: "dashboard", + createdActor: MOCK_ACTOR_MIRA, + }), + sessionRow("agent:main:dashboard:ci-signal", "CI signal", baseTime - 42_000, { + boardFace: "dashboard", + createdActor: MOCK_ACTOR_PETER, + }), + sessionRow("agent:main:dashboard:community-pulse", "Community pulse", baseTime - 75_000, { + boardFace: "dashboard", + createdActor: MOCK_ACTOR_MIRA, + }), + sessionRow("agent:main:dashboard:gateway-fleet", "Gateway fleet", baseTime - 130_000, { + boardFace: "dashboard", + createdActor: MOCK_ACTOR_PETER, + }), + ] + : []; + const activeGoal = { + schemaVersion: 1 as const, + id: "goal-mobile-parity", + objective: + "Make Goal work on mobile exactly as it does on desktop while preserving action access, progress visibility, timing, token usage, and composer space across narrow viewports. Keep the collapsed state compact enough for active conversations, while making the expanded state comfortable to scan and operate with one hand. Preserve clear hierarchy between the objective, elapsed time, token budget, and available actions without letting long content push the composer out of reach. Ensure the full objective remains readable when it contains detailed constraints, acceptance criteria, rollout notes, and operational context that cannot be reduced to a short summary. Account for long-running sessions whose goals accumulate multiple requirements, edge cases, validation steps, ownership notes, and deployment considerations. The operator should be able to review all of that context without losing access to the Goal controls or forcing the message composer below the visible viewport.", + status: "active" as const, + createdAt: activityTime - 14 * 60_000, + updatedAt: activityTime - 30_000, + tokenStart: 120_000, + tokenStartFresh: true, + tokensUsed: 127_000, + tokenBudget: 300_000, + continuationTurns: 3, + }; + const rosterSessions = rosterAgents.flatMap((agent, index) => [ + sessionRow(`agent:${agent.id}:main`, agent.name, rosterTime - (index + 1) * 720_000, { + agentId: agent.id, + isMain: true, + lastMessagePreview: agent.preview, + }), + ...agent.sessionLabels.map((label, sessionIndex) => { + const working = agent.id === "forge" && sessionIndex === 0; + return sessionRow( + `agent:${agent.id}:sample-${sessionIndex + 1}`, + label, + rosterTime - (index + 1) * 60_000 - sessionIndex * 5_000, + { + agentId: agent.id, + pinned: sessionIndex === 0, + lastMessagePreview: agent.preview, + unread: agent.id === "scout" && sessionIndex === 1, + hasActiveRun: working, + ...(working ? { activeRunIds: ["mock-forge-dashboard"], status: "running" } : {}), + }, + ); + }), + ]); + if (fixture === "sidebar-roster") { + rosterSessions.push( + sessionRow("agent:forge:subagent:layout", "Check sidebar layout", rosterTime - 20_000, { + agentId: "forge", + spawnedBy: "agent:forge:sample-1", + status: "done", + }), + sessionRow("agent:forge:subagent:contrast", "Check nested contrast", rosterTime - 10_000, { + agentId: "forge", + spawnedBy: "agent:forge:subagent:layout", + hasActiveRun: true, + status: "running", + unread: true, + }), + ); + } + const gallerySessions = [ + ...rosterSessions.filter((row) => !row.key.startsWith("agent:main:")), + ...activitySessions, + ...dashboardGallerySessions, + ...(fixture === "workboard" || fixture === "workboard-states" + ? [ + sessionRow(workboardMocks.sessionKey, "Product operations dashboard", baseTime, { + boardFace: "dashboard", + pinned: true, + }), + ...workboardMocks.cardSessions, + ] + : []), + sessionRow("agent:main:main", "Molty", rosterTime - 1_000, { + agentId: "main", + isMain: true, + lastMessagePreview: rosterAgents[0].preview, + activeRunIds: [PLAN_DEMO_RUN_ID], + childSessions: ["agent:main:lisbon-trip", ...swarmChildRows.map((row) => row.key)], + hasActiveRun: true, + ...(fixture === "avatars" ? { kind: "global" as const } : {}), + status: "running", + totalTokens: 170_000, + totalTokensFresh: true, + ...(fixture === "goal" ? { goal: activeGoal } : {}), + }), + ...swarmChildRows, + sessionRow(OBSERVER_DEMO_SESSION_KEY, "Session observer demo", baseTime - 3_000, { + activeRunIds: [OBSERVER_DEMO_RUN_ID], + hasActiveRun: true, + lastReadAt: baseTime + 2_000, + observerDigest: { + headline: "Opening the focused observer tests", + health: "on-track", + revision: 1, + runId: OBSERVER_DEMO_RUN_ID, + updatedAt: baseTime - 2_000, + }, + startedAt: baseTime - 4_000, + status: "running", + }), + sessionRow(NARRATION_DEMO_SESSION_KEY, "Sidebar narration demo", baseTime - 15_000, { + createdActor: MOCK_ACTOR_MIRA, + hasActiveRun: true, + owner: { actor: MOCK_ACTOR_MIRA }, + startedAt: baseTime - 45_000, + status: "running", + }), + sessionRow("agent:main:tax-research", "Tax filing research", baseTime - 60_000, { + hasActiveRun: true, + status: "running", + childSessions: ["agent:main:subagent:tax-receipts"], + pinned: true, + }), + sessionRow("agent:main:cloud-refactor", "Cloud refactor worker", baseTime - 70_000, { + hasActiveRun: true, + status: "running", + startedAt: baseTime - 3_500_000, + execCwd: "/workspace/openclaw", + placement: { + state: "active", + generation: 3, + createdAtMs: baseTime - 3_600_000, + updatedAtMs: baseTime - 20_000, + stateChangedAtMs: baseTime - 3_500_000, + environmentId: "worker:9f2c4e7a81d24b06a5c3f8e1b7d94c1a", + providerId: "machine0", + profileId: "team", + machine: { class: "medium", os: "linux", osLabel: "Linux", cpu: 4, memoryGb: 16 }, + activeOwnerEpoch: 4, + workerBundleHash: "b".repeat(64), + workspaceBaseManifestRef: "sha256:cloud-refactor-base", + remoteWorkspaceDir: "/workspace/openclaw", + diskSpace: { + status: "ok", + availableBytes: 61 * 1024 ** 3, + totalBytes: 100 * 1024 ** 3, + observedAtMs: baseTime - 20_000, + }, + }, + }), + sessionRow( + "agent:main:production-export", + "Investigate transcript scroll-anchor regression when the final code block expands", + baseTime - 75_000, + { + category: "Research", + createdActor: MOCK_ACTOR_MIRA, + execCwd: "/Users/demo/Projects/clawdbot", + owner: { actor: MOCK_ACTOR_MIRA }, + }, + ), + sessionRow("agent:main:model-budget", "Model budget review", baseTime - 80_000, { + category: "Research", + execCwd: "/Users/demo/Projects/openclaw", + owner: { actor: { type: "human", id: "presence-riley", label: "Riley" } }, + status: "failed", + lastRunError: "Model access expired: openai/gpt-5-mini", + }), + // Running rows with participants exercise the paired run trace in the lead slot. + sessionRow("agent:main:release-prep", "Release 2026.9.4 preparation", baseTime - 82_000, { + createdActor: MOCK_ACTOR_PETER, + execCwd: "/Users/demo/Work/openclaw", + hasActiveRun: true, + owner: { actor: MOCK_ACTOR_PETER }, + participantCount: 1, + participants: [{ identity: { type: "profile", id: "profile-mira" }, label: "Mira" }], + status: "running", + }), + sessionRow("agent:main:release-notes", "Release notes review", baseTime - 83_000, { + createdActor: MOCK_ACTOR_MIRA, + execCwd: "/Users/demo/Work/openclaw", + hasActiveRun: true, + owner: { actor: MOCK_ACTOR_MIRA }, + participantCount: 2, + participants: [ + { identity: { type: "profile", id: "profile-riley" }, label: "Riley" }, + { identity: { type: "profile", id: "profile-sam" }, label: "Sam" }, + ], + status: "running", + }), + sessionRow("agent:main:work-openclaw", "OpenClaw work checkout", baseTime - 85_000, { + createdActor: MOCK_ACTOR_PETER, + execCwd: "/Users/demo/Work/openclaw", + lastReadAt: baseTime - 120_000, + owner: { actor: MOCK_ACTOR_PETER }, + participantCount: 4, + participants: [ + { identity: { type: "profile", id: "profile-mira" }, label: "Mira" }, + { identity: { type: "profile", id: "profile-riley" }, label: "Riley" }, + { identity: { type: "profile", id: "profile-sam" }, label: "Sam" }, + { identity: { type: "profile", id: "profile-lee" }, label: "Lee" }, + ], + observerDigest: { + headline: "Done: fixed the flaky retry-window test", + health: "done", + revision: 1, + runId: "mock-idle-final-run", + updatedAt: baseTime - 40_000, + }, + unread: true, + }), + mainChildRow, + sessionRow("agent:main:home-server", "Home server migration", baseTime - 240_000, { + execCwd: "/Users/demo/Projects", + execNode: "a1b2c3d4e5f60718293a4b5c6d7e8f90a1b2c3d4e5f60718293a4b5c6d7e8f90", + hasAutomation: true, + pinned: true, + }), + sessionRow("agent:main:whatsapp:group:family", "Family", baseTime - 90_000, { + kind: "group", + channel: "whatsapp", + unread: true, + }), + sessionRow("agent:main:discord:channel:openclaw-dev", "#openclaw-dev", baseTime - 300_000, { + kind: "group", + channel: "discord", + }), + sessionRow("agent:main:sidebar-zones", "sidebar zones", baseTime - 150_000, { + worktree: { + id: "wt-sidebar-zones", + branch: "claude/sidebar-agent-zones", + repoRoot: "~/Projects/openclaw", + }, + }), + // Second repo plus a spawned worktree checkout so the sidebar's + // Project grouping shows several sections and the worktree fold. + sessionRow("agent:main:clawdbot-vite", "Vite upgrade spike", baseTime - 160_000, { + worktree: { + id: "wt-clawdbot-vite", + branch: "openclaw/vite-upgrade", + repoRoot: "~/Projects/clawdbot", + }, + }), + sessionRow("agent:main:project-grouping", "Sidebar project grouping", baseTime - 170_000, { + spawnedCwd: "~/Projects/openclaw/.claude/worktrees/groups-c7c338", + }), + ...buildSessionRows({ + baseTime: baseTime - 400_000, + count: 3, + keyPrefix: "main:history", + labelPrefix: "Long running session", + }), + ]; + const sessions = fixture === "sidebar-roster" ? rosterSessions : gallerySessions; + const archivedSessions = [ + sessionRow("agent:main:archived-launch-notes", "Archived launch notes", baseTime - 86_400_000, { + archived: true, + archivedBy: MOCK_ACTOR_MIRA, + createdActor: MOCK_ACTOR_PETER, + totalTokens: 42_000, + }), + sessionRow( + "agent:main:discord:channel:archived-lounge", + "#archived-lounge", + baseTime - 172_800_000, + { + archived: true, + channel: "discord", + kind: "group", + totalTokens: 18_000, + }, + ), + ]; + const telegramSessions = buildSessionRows({ + baseTime: baseTime - 30_000, + count: TOTAL_TELEGRAM_SESSIONS, + keyPrefix: "telegram", + labelPrefix: "Telegram investigation", + }); + const claudeSessions = buildSessionRows({ + baseTime: baseTime - 45_000, + count: 75, + keyPrefix: "model-claude", + labelPrefix: "Model search result", + model: "claude-sonnet-4-6", + modelProvider: "anthropic", + }); + // Profile fixtures track the real clock so streaks and the trailing-year + // heatmap stay filled no matter when the mock harness runs. + const profileUsage = buildProfileUsageMocks(Date.now()); + const modelProviders = buildModelProviderMocks(Date.now()); + const skillWorkshop = buildSkillWorkshopMocks(Date.now()); + const richAttention = fixture === "approval"; + const cronMocks = buildCronMocks(Date.now(), { + richAttention, + ...(fixture === "sidebar-roster" ? { secondAgentId: "forge" } : {}), + }); + const updateFixtureNow = Date.now(); + const updateFixture = buildUpdateFixture(fixture, updateFixtureNow); + const updateSchedule = updateFixture?.schedule ?? null; + const heldUpdateSchedule: UpdateScheduleState | null = updateSchedule?.campaign + ? { + ...updateSchedule, + campaign: { + ...updateSchedule.campaign, + holdUntilMs: updateFixtureNow + 60 * 60_000, + updatedAtMs: updateFixtureNow, + }, + } + : null; + const modelAuthStatus = richAttention + ? { + ...modelProviders.authStatus, + providers: modelProviders.authStatus.providers.map((provider) => + provider.provider === "google" + ? { + ...provider, + displayName: "Google Gemini", + status: "expired" as const, + profiles: [ + { + profileId: "shared engineering", + type: "oauth" as const, + status: "expired" as const, + expiry: { + at: updateFixtureNow - 12 * 60_000, + remainingMs: -12 * 60_000, + label: "12m ago", + }, + }, + ], + } + : provider, + ), + } + : modelProviders.authStatus; + const channelWizard = buildChannelWizardMocks(); + const configMocks = buildConfigMocks({ + swarmEnabled: fixture === "swarm", + workboardEnabled: fixture === "workboard" || fixture === "workboard-states", + }); + const fixtureSessionKey = + fixture === "approval" + ? "agent:main:production-export" + : fixture === "dashboards" + ? "agent:main:dashboard:release-health" + : fixture === "update-available" + ? "agent:main:home-server" + : fixture === "update-blocked" + ? "agent:main:model-budget" + : fixture === "update-failed" + ? "agent:main:cloud-refactor" + : fixture === "workboard" || fixture === "workboard-states" + ? workboardMocks.sessionKey + : "agent:main:main"; + const summaryHistory = buildFixtureSummaryHistory( + baseTime, + "Inspect this session.", + "The current state is available in the session controls.", + ); + const fixtureHistories: Partial, unknown[]>> = { + approval: buildApprovalChatHistory(baseTime), + attachments: buildChatAttachmentHistory(baseTime), + avatars: buildAvatarChatHistory(baseTime), + "code-fences": buildCodeFenceChatHistory(baseTime), + "sidebar-roster": [ + chatHistoryMessage("user", "Help me organize the sample project.", rosterTime - 120_000), + chatHistoryMessage( + "assistant", + "The project outline is ready. Forge is building the sample dashboard, Scout is reviewing sources, and Bloom is drafting the welcome guide.", + rosterTime - 60_000, + ), + ], + "update-blocked": buildCancelledChatHistory(baseTime), + }; + const historyMessages = fixture + ? (fixtureHistories[fixture] ?? summaryHistory) + : buildScrollableChatHistory(baseTime); + const planInFlightRun = { + runId: PLAN_DEMO_RUN_ID, + text: "", + plan: { + explanation: "Keep the Control UI change focused", + steps: [ + { step: "Inspect the transcript renderer", status: "completed" }, + { step: "Confirm the plan event contract", status: "completed" }, + { step: "Remove the duplicate card summary", status: "in_progress" }, + { step: "Run focused UI tests", status: "pending" }, + { step: "Capture browser proof", status: "pending" }, + ], + }, + }; + const backgroundTasks = buildBackgroundTasksMock(baseTime); + const custodianHistory = { + turns: [ + { + role: "user", + text: "Can you check whether this system is ready?", + at: baseTime - 18 * 60_000, + }, + { + role: "assistant", + text: "Everything important is connected. I’ll keep watching for changes.", + at: baseTime - 17 * 60_000, + }, + { + role: "user", + text: "Please remember that I prefer concise updates.", + at: baseTime - 16 * 60_000, + }, + ], + } satisfies SystemAgentChatHistoryResult; + const custodianChanges = { + entries: [ + { + id: "mock-system-agent-model", + at: baseTime - 6 * 60_000, + kind: "operation", + source: "system-agent", + summary: "Updated the default model for the main agent", + changedPaths: ["agents.defaults.model"], + }, + { + id: "mock-plugin-install", + at: baseTime - 42 * 60_000, + kind: "config-write", + source: "plugin-install", + summary: "Enabled the Telegram plugin", + changedPaths: ["plugins.entries.telegram.enabled"], + }, + { + id: "mock-doctor-repair", + at: baseTime - 2 * 60 * 60_000, + kind: "config-write", + source: "doctor", + summary: "Repaired a stale channel account reference", + changedPaths: ["channels.whatsapp.defaultAccount"], + }, + ], + } satisfies SystemChangesListResult; + const scenario: ControlUiMockGatewayScenario = { + assistantAgentId: "main", + assistantName: "Molty", + defaultAgentId: "main", + gatewayBootId: "mock-gateway-boot-1", + serverBuildId: "mock", + updateSchedule, + updateAvailable: updateFixture?.available ?? null, + // Advertised Gateway methods gate session actions (see + // ui/src/lib/session-method-access.ts). Omitting the mutation methods left + // every session context-menu row disabled, so the harness could not show + // the menu operators actually see. browser.request/terminal.open likewise + // gate the chat header's panel toggles, which stayed invisible here. + featureMethods: [ + "browser.request", + "chat.abort", + "chat.history", + "chat.send", + "config.patch", + "config.schema", + "cron.add", + "cron.list", + "cron.remove", + "cron.run", + "cron.runs", + "cron.status", + "cron.update", + "chat.metadata", + "chat.startup", + "question.list", + "openclaw.changes.list", + "openclaw.chat", + "openclaw.chat.history", + "progressCard.get", + "plugins.inspect", + "plugins.install", + "plugins.list", + "plugins.search", + "plugins.setEnabled", + "plugins.uninstall", + "sessions.delete", + "sessions.diff", + "sessions.files.set", + "sessions.fork", + "sessions.groups.delete", + "sessions.groups.list", + "sessions.groups.put", + "sessions.groups.rename", + "sessions.patch", + "sessions.patchMany", + "sessions.search", + "skills.workshop.read", + "skills.proposals.apply", + "skills.proposals.evaluate", + "skills.proposals.historyScan", + "skills.proposals.historyStatus", + "skills.proposals.inspect", + "skills.proposals.list", + "skills.proposals.reject", + "skills.proposals.requestRevision", + "skills.library.activate", + "skills.library.import", + "skills.library.list", + "skills.library.mutate", + "skills.library.read", + "skills.library.save", + "skills.library.upload", + "tasks.cancel", + "tasks.get", + "tasks.list", + "sessions.catalog.list", + "sessions.catalog.read", + "sessions.create", + "system.info", + "desktop.observe", + "environments.list", + "terminal.open", + ...(updateFixture ? ["update.hold", "update.run", "update.status"] : []), + ...(fixture === "workboard" || fixture === "workboard-states" + ? [ + "board.get", + "cron.get", + "workboard.boards.list", + "workboard.boards.upsert", + "workboard.cards.list", + "workboard.cards.move", + "workboard.cards.create", + "workboard.cards.update", + "workboard.cards.comment", + "workboard.cards.archive", + "workboard.cards.delete", + "workboard.cards.stats", + ] + : []), + ], + ...(fixture === "workboard" || fixture === "workboard-states" ? workboardUi : {}), + controlUiWidgetKinds: [ + { pluginId: "session", kind: "session:progress", label: "Session progress" }, + ...(fixture === "workboard" || fixture === "workboard-states" + ? [ + { pluginId: "workboard", kind: "workboard:board", label: "Workboard board" }, + { pluginId: "workboard", kind: "workboard:card", label: "Workboard card" }, + { pluginId: "workboard", kind: "workboard:mini", label: "Workboard summary" }, + ] + : []), + ], + // Terminal has a second gate beyond the advertised method (see + // ui/src/lib/terminal-availability.ts). + terminalEnabled: true, + // The mock rows span several owners; advertise the multi-identity policy + // so people-aware UI (People sort, Person grouping) is exercisable here. + hasMultipleSessionSharingIdentities: true, + historyMessages, + sessionGroups: ["Research"], + sessionTranscripts: { + ...backgroundTasks.sessionTranscripts, + "agent:main:main": { + messages: + fixtureSessionKey === "agent:main:main" + ? historyMessages + : buildScrollableChatHistory(baseTime), + ...(fixture === undefined || fixture === "goal" ? { inFlightRun: planInFlightRun } : {}), + }, + "agent:main:production-export": { messages: buildApprovalChatHistory(baseTime) }, + "agent:main:tax-research": { messages: summaryHistory }, + "agent:main:model-budget": { messages: buildCancelledChatHistory(baseTime) }, + "agent:main:archived-launch-notes": { + messages: buildFixtureSummaryHistory( + baseTime, + "Save the launch decision for the archive.", + "Launch notes archived: rollout completed, the rollback window closed, and no follow-up action remains.", + ), + }, + [OBSERVER_DEMO_SESSION_KEY]: { + messages: buildFixtureSummaryHistory( + baseTime, + "Rerun the focused observer test and stream the result.", + "I reproduced the assertion and am checking the narrowed event path now.", + ), + inFlightRun: { + runId: OBSERVER_DEMO_RUN_ID, + text: "Rerunning the focused test and comparing the latest event sequence…", + startedAt: baseTime - 4_000, + events: [], + }, + }, + "agent:main:dashboard:release-health": { + messages: summaryHistory, + }, + "agent:main:home-server": { messages: summaryHistory }, + "agent:main:cloud-refactor": { messages: summaryHistory }, + [workboardMocks.sessionKey]: { messages: summaryHistory }, + ...(fixture === "workboard" || fixture === "workboard-states" + ? workboardMocks.cardSessionHistories + : {}), + }, + // Lights up the footer facepile and who's-online roster; the email-only + // entry keeps the roster's no-display-name row exercised. + presenceUsers: [ + ...(fixture === "avatars" + ? MOCK_AVATAR_PROFILES.map(({ id, name }) => ({ + id, + name, + avatarUrl: `/api/users/${id}/avatar`, + })) + : []), + { + self: true, + id: selfProfile.id, + name: selfProfile.displayName ?? undefined, + email: selfProfile.emails[0], + avatarUrl: `/api/users/${selfProfile.id}/avatar`, + }, + { + id: "presence-colin", + name: "Colin", + email: "colin@example.com", + onlineSince: activityTime - 47 * 60_000, + lastActivityAt: activityTime - 2 * 60_000, + deviceFamily: "Mac", + platform: "macOS", + timeZone: "America/Los_Angeles", + watchedSessions: ["agent:activity:design-review", "agent:main:main"], + }, + { + id: "presence-colin", + name: "Colin", + email: "colin@example.com", + onlineSince: activityTime - 47 * 60_000, + lastActivityAt: activityTime - 2 * 60_000, + deviceFamily: "Mac", + platform: "macOS", + timeZone: "America/Los_Angeles", + watchedSessions: ["agent:activity:design-review"], + }, + { + id: "presence-patricia", + name: "Patrick", + email: "patrick.erichsen@example.com", + onlineSince: activityTime - 12 * 60_000, + lastActivityAt: activityTime - 30_000, + deviceFamily: "iPhone", + platform: "iOS", + timeZone: "Europe/Stockholm", + watchedSessions: ["agent:activity:support-handoff"], + }, + ], + methodResponses: { + ...cronMocks, + "agents.list": { + agents: rosterAgents.map(({ id, name, theme, emoji, avatar }) => ({ + id, + name, + identity: { name, theme, emoji, avatar }, + model: { primary: "example/model-small" }, + })), + defaultId: "main", + mainKey: "main", + scope: "per-sender", + }, + "agent.identity.get": { + cases: rosterAgents.map(({ id, name, theme, emoji, avatar }) => ({ + match: { agentId: id }, + response: { agentId: id, name, theme, emoji, avatar, avatarStatus: "data" }, + })), + }, + "progressCard.get": { card: null }, + "users.self": { profile: selfProfile }, + // Talk settings page pickers: realtime catalog with the model/voice + // suggestion lists the gateway emits for provider entries. + "talk.catalog": { + modes: ["realtime", "stt-tts", "transcription"], + transports: ["webrtc", "provider-websocket", "gateway-relay", "managed-room"], + brains: ["agent-consult", "direct-tools", "none"], + speech: { providers: [] }, + transcription: { providers: [] }, + realtime: { + ready: true, + activeProvider: "openai", + providers: [ + { + id: "openai", + label: "OpenAI Realtime Voice", + configured: true, + defaultModel: "gpt-realtime-2.1", + transports: ["webrtc", "gateway-relay"], + models: ["gpt-realtime-2.1", "gpt-realtime-2.1-mini", "gpt-realtime-2"], + voices: [ + "alloy", + "ash", + "ballad", + "cedar", + "coral", + "echo", + "marin", + "sage", + "shimmer", + "verse", + ], + modes: ["realtime"], + brains: ["agent-consult"], + supportsBrowserSession: true, + }, + { + id: "xai", + label: "xAI Grok Voice", + configured: false, + defaultModel: "grok-voice-latest", + transports: ["gateway-relay"], + voices: ["eve", "ara", "rex", "sal", "leo"], + modes: ["realtime"], + brains: ["agent-consult"], + supportsBrowserSession: false, + }, + ], + }, + }, + // Coding session catalogs so the sidebar's catalog sections (header + // right-click menu, hide/restore preference) are exercised in the mock. + // Ids must match registered plugin catalogs (`claude`, `codex`) or the + // sidebar cannot resolve bundled brand marks. + "sessions.catalog.list": { + catalogs: [ + { + id: "codex", + label: "Codex", + capabilities: { continueSession: true, archive: false, startTerminal: true }, + hosts: [ + { + hostId: "gateway", + label: "This Mac", + kind: "gateway", + connected: true, + sessions: [ + { + threadId: "codex-thread-1", + name: "Release checklist sweep", + cwd: "/Users/demo/projects/openclaw", + status: "idle", + updatedAt: baseTime - 10 * 60_000, + archived: false, + canContinue: true, + canArchive: false, + }, + { + threadId: "codex-thread-2", + name: "Sidebar context-menu proof", + cwd: "/Users/demo/projects/openclaw", + status: "idle", + updatedAt: baseTime - 45 * 60_000, + archived: false, + canContinue: true, + canArchive: false, + }, + ], + }, + ], + }, + { + id: "claude", + label: "Claude Code", + capabilities: { continueSession: true, archive: false }, + hosts: [ + { + hostId: "gateway", + label: "This Mac", + kind: "gateway", + connected: true, + sessions: [ + { + threadId: "claude-thread-1", + name: "Docs refresh", + cwd: "/Users/demo/projects/peekaboo", + status: "idle", + updatedAt: baseTime - 30 * 60_000, + archived: false, + canContinue: true, + canArchive: false, + }, + ], + }, + ], + }, + ], + }, + "sessions.catalog.read": { + cases: [ + { + match: { catalogId: "codex", hostId: "gateway", threadId: "codex-thread-1" }, + response: { + hostId: "gateway", + threadId: "codex-thread-1", + items: [ + { + id: "release-checklist-answer", + type: "agentMessage", + text: "The release checklist is complete and ready for review.", + }, + { + id: "release-checklist-request", + type: "userMessage", + text: "Please sweep the release checklist for anything we missed.", + }, + ], + }, + }, + { + match: { catalogId: "codex", hostId: "gateway", threadId: "codex-thread-2" }, + response: { + hostId: "gateway", + threadId: "codex-thread-2", + items: [ + { + id: "sidebar-context-menu-answer", + type: "agentMessage", + text: "The sidebar context menu behaves as expected.", + }, + ], + }, + }, + { + match: { + catalogId: "claude", + hostId: "gateway", + threadId: "claude-thread-1", + }, + response: { + hostId: "gateway", + threadId: "claude-thread-1", + items: [ + { + id: "docs-refresh-answer", + type: "agentMessage", + text: "The documentation refresh is ready for review.", + }, + ], + }, + }, + ], + }, + "system.info": { + machineName: "Mock-Workstation", + hostname: "mock-workstation.invalid", + platform: "darwin", + release: "25.0.0", + arch: "arm64", + osLabel: "macOS 26.5", + nodeVersion: "24.15.0", + pid: 4242, + uptimeMs: (11 * 24 + 4) * 3_600_000, + loadAverage: [3.2, 2.8, 2.4], + cpuCount: 16, + memoryTotalBytes: 68_719_476_736, + memoryFreeBytes: 34_359_738_368, + diskTotalBytes: 1_000_000_000_000, + diskAvailableBytes: 640_000_000_000, + diskPath: "/Users/demo/.openclaw", + defaultAgentUtilityModel: { + status: "auto", + model: "anthropic/claude-haiku-4-5", + }, + }, + "fs.listDir": { + cases: [ + { + match: { path: "/Users/demo/Projects/openclaw" }, + response: { + path: "/Users/demo/Projects/openclaw", + parent: "/Users/demo/Projects", + home: "/Users/demo", + entries: [ + { name: "ui", path: "/Users/demo/Projects/openclaw/ui" }, + { name: "src", path: "/Users/demo/Projects/openclaw/src" }, + { name: "docs", path: "/Users/demo/Projects/openclaw/docs" }, + { name: "packages", path: "/Users/demo/Projects/openclaw/packages" }, + ], + }, + }, + { + match: { path: "/Users/demo/Projects" }, + response: { + path: "/Users/demo/Projects", + parent: "/Users/demo", + home: "/Users/demo", + entries: [ + { name: "openclaw", path: "/Users/demo/Projects/openclaw" }, + { name: "clawdbot", path: "/Users/demo/Projects/clawdbot" }, + { name: "sweetistics", path: "/Users/demo/Projects/sweetistics" }, + { name: "Peekaboo", path: "/Users/demo/Projects/Peekaboo" }, + ], + }, + }, + { + match: {}, + response: { + path: "/Users/demo", + parent: "/Users", + home: "/Users/demo", + entries: [ + { name: "Projects", path: "/Users/demo/Projects" }, + { name: "Downloads", path: "/Users/demo/Downloads" }, + { name: ".config", path: "/Users/demo/.config", hidden: true }, + ], + }, + }, + ], + }, + "worktrees.branches": { + cases: [ + { + match: { repoRoot: "/Users/demo/Projects/openclaw" }, + response: { + repoRoot: "/Users/demo/Projects/openclaw", + branches: [ + { kind: "local", name: "main" }, + { kind: "local", name: "steipete/place-picker" }, + ], + repositoryStatus: "git", + defaultBranch: "main", + headBranch: "main", + }, + }, + { + match: { repoRoot: "/Users/demo/Projects/clawdbot" }, + response: { + repoRoot: "/Users/demo/Projects/clawdbot", + branches: [ + { kind: "local", name: "main" }, + { kind: "local", name: "steipete/storage-selector-design" }, + ], + repositoryStatus: "git", + defaultBranch: "main", + headBranch: "main", + }, + }, + ], + }, + "environments.list": { + environments: [ + { + id: "gateway", + type: "gateway", + label: "Gateway host", + status: "available", + desktop: true, + }, + { + id: "node:a1b2c3d4e5f60718293a4b5c6d7e8f90a1b2c3d4e5f60718293a4b5c6d7e8f90", + type: "node", + label: "Mac Studio", + status: "available", + desktop: true, + ...(pickerInventory + ? { + platform: "darwin", + sessionHost: true, + workerSlots: { total: 4, available: 3 }, + } + : {}), + }, + ...(pickerInventory + ? [ + { + id: "node:mock-macbook-offline", + type: "node", + label: "MacBook Pro", + platform: "darwin", + status: "unavailable", + sessionHost: true, + lastConnectedAtMs: baseTime - 5 * 86_400_000, + lastDisconnectedAtMs: baseTime - (4 * 24 + 13) * 3_600_000, + }, + ] + : []), + ], + profiles: [ + { + id: "aws", + providerId: "aws", + ...(pickerInventory ? { executionModes: ["worker-turn", "remote-exec"] } : {}), + }, + ...(pickerInventory + ? [ + { + id: "crabbox", + providerId: "crabbox", + executionModes: ["worker-turn", "remote-exec"], + }, + { + id: "test-cloud", + providerId: "test-cloud", + executionModes: ["worker-turn", "remote-exec"], + }, + ] + : []), + ], + }, + // config.set/config.apply are served statefully by the mock gateway + // (raw persists, hash advances) because config.get ships a raw fixture. + "config.get": configMocks.get, + "config.set": configMocks.set, + "config.apply": configMocks.apply, + "config.schema": configMocks.schema, + "openclaw.chat.history": custodianHistory, + "openclaw.changes.list": custodianChanges, + // The sidebar recovers pending questions through question.list after the + // hello handshake, so this remains visible after a mock-page refresh. + "question.list": { + questions: [ + { + id: "mock_tax_question", + agentId: "main", + sessionKey: "agent:main:tax-research", + questions: [ + { + id: "filing_status", + header: "Tax filing", + question: "Should I submit the draft return?", + options: [ + { label: "Submit", description: "File the prepared return." }, + { label: "Review", description: "Keep the draft open for review." }, + ], + }, + ], + createdAtMs: baseTime - 60_000, + expiresAtMs: ATTENTION_FIXTURE_EXPIRES_AT, + status: "pending", + }, + ], + }, + "plugin.approval.list": [], + "openclaw.approval.list": [], + "plugin.approval.resolve": { ok: true }, + "sessions.patch": { ok: true }, + "sessions.diff": buildSessionDiffMock(), + // The worktrees page assumes the gateway contract shape; without this + // fixture the mock's {} fallback surfaces as a TypeError banner. + "worktrees.list": { + worktrees: [ + { + id: "wt-mock-1", + name: "fix-session-icons", + repoFingerprint: "a1b2c3d4e5f60718", + repoRoot: "/Users/demo/Projects/openclaw", + path: "/Users/demo/Projects/openclaw/.openclaw/worktrees/fix-session-icons", + branch: "openclaw/fix-session-icons", + baseRef: "origin/main", + ownerKind: "session", + createdAt: baseTime - 3 * 86_400_000, + lastActiveAt: baseTime - 2 * 3_600_000, + }, + { + id: "wt-mock-2", + name: "dashboard-polish", + repoFingerprint: "a1b2c3d4e5f60718", + repoRoot: "/Users/demo/Projects/openclaw", + path: "/Users/demo/Projects/openclaw/.openclaw/worktrees/dashboard-polish", + branch: "openclaw/dashboard-polish", + baseRef: "origin/main", + ownerKind: "manual", + createdAt: baseTime - 9 * 86_400_000, + lastActiveAt: baseTime - 26 * 3_600_000, + }, + ], + }, + "plugins.list": buildPluginCatalogMock({ + installedCopies: fixture === "plugins-dense" ? 10 : 1, + }), + "plugins.catalog.browse": buildPluginDiscoveryMock(), + "plugins.catalog.categories": buildPluginDiscoveryCategoriesMock(), + "plugins.inspect": buildPluginInspectMock({ + installedCopies: fixture === "plugins-dense" ? 10 : 1, + }), + "skills.status": { + workspaceDir: "/Users/demo/Projects/openclaw", + managedSkillsDir: "/Users/demo/.openclaw/skills", + skills: [], + }, + "channels.status": buildChannelsStatusMock(baseTime), + "channels.pairing.list": buildChannelsPairingMock(baseTime), + "channels.pairing.approve": { + cases: [ + { + match: { requestId: "pairing-req-1" }, + response: { + requestId: "pairing-req-1", + senderId: "telegram-demo-user", + notification: "sent", + commandOwnerBootstrap: "not-requested", + }, + }, + { + response: { + requestId: "pairing-req-2", + senderId: "whatsapp-demo-user", + notification: "unsupported", + commandOwnerBootstrap: "not-requested", + }, + }, + ], + }, + "channels.pairing.dismiss": { + cases: [ + { + match: { requestId: "pairing-req-1" }, + response: { requestId: "pairing-req-1", senderId: "telegram-demo-user" }, + }, + { response: { requestId: "pairing-req-2", senderId: "whatsapp-demo-user" } }, + ], + }, + "web.login.start": { + message: "Scan the QR code with WhatsApp to link this device.", + qrDataUrl: whatsappLoginQrDataUrl, + }, + "web.login.wait": { message: "Linked.", connected: true }, + "wizard.start": channelWizard.start, + "wizard.next": channelWizard.next, + "wizard.cancel": { status: "cancelled" }, + "skills.proposals.requestRevision": skillWorkshop.requestRevision, + "usage.cost": profileUsage.cost, + "sessions.usage": profileUsage.sessions, + "models.authStatus": modelAuthStatus, + "update.hold": heldUpdateSchedule + ? { ok: true, schedule: heldUpdateSchedule } + : { ok: false }, + "update.run": updateFixture?.runResponse ?? {}, + "update.status": updateFixture?.statusResponse ?? {}, + "usage.status": modelProviders.usageStatus, + "device.pair.list": { + paired: [ + { + deviceId: "a1b2c3d4e5f60718293a4b5c6d7e8f90a1b2c3d4e5f60718293a4b5c6d7e8f90", + displayName: "Mac Studio", + platform: "darwin", + clientId: "node-host", + clientMode: "node", + roles: ["operator", "node"], + scopes: ["operator.admin", "operator.read", "operator.write"], + approvedVia: "trusted-cidr", + approvedAtMs: baseTime - 3_600_000, + lastSeenAtMs: baseTime - 60_000, + tokens: [ + { role: "node", scopes: [], createdAtMs: baseTime - 3_600_000 }, + { + role: "operator", + scopes: ["operator.admin", "operator.read", "operator.write"], + createdAtMs: baseTime - 3_600_000, + }, + ], + }, + { + deviceId: "0f1e2d3c4b5a69788796a5b4c3d2e1f00f1e2d3c4b5a69788796a5b4c3d2e1f0", + displayName: "Mac mini", + platform: "darwin", + clientId: "node-host", + clientMode: "node", + roles: ["node"], + approvedVia: "trusted-cidr", + approvedAtMs: baseTime - 86_400_000, + lastSeenAtMs: baseTime - 82_800_000, + tokens: [{ role: "node", scopes: [], createdAtMs: baseTime - 86_400_000 }], + }, + { + deviceId: "9988776655443322119988776655443322119988776655443322119988776655", + clientId: "cli", + clientMode: "cli", + platform: "darwin", + roles: ["operator"], + scopes: ["operator.admin", "operator.read", "operator.write"], + approvedVia: "silent", + approvedAtMs: baseTime - 7_200_000, + lastSeenAtMs: baseTime - 7_100_000, + tokens: [ + { + role: "operator", + scopes: ["operator.admin", "operator.read", "operator.write"], + createdAtMs: baseTime - 7_200_000, + }, + ], + }, + { + deviceId: "11223344556677889900aabbccddeeff11223344556677889900aabbccddeeff", + displayName: "iPhone", + platform: "iOS 26.4", + clientId: "openclaw-ios", + clientMode: "ui", + roles: ["operator", "node"], + scopes: ["operator.approvals", "operator.read", "operator.write"], + approvedVia: "bootstrap", + approvedAtMs: baseTime - 172_800_000, + lastSeenAtMs: baseTime - 3_600_000, + tokens: [ + { role: "node", scopes: [], createdAtMs: baseTime - 172_800_000 }, + { + role: "operator", + scopes: ["operator.approvals", "operator.read", "operator.write"], + createdAtMs: baseTime - 172_800_000, + }, + ], + }, + ], + pending: [ + { + requestId: "mock-pending-request", + deviceId: "feedfacecafebeef0123456789abcdeffeedfacecafebeef0123456789abcdef", + displayName: "MacBook Pro", + role: "operator", + roles: ["operator"], + scopes: ["operator.read", "operator.write"], + remoteIp: "192.168.1.20", + ts: baseTime - 30_000, + }, + ], + }, + "device.pair.setupCode": { + auth: "token", + gatewayUrl: "wss://gateway.example.test", + qrDataUrl: devicePairQrDataUrl, + setupCode: devicePairSetupCode, + urlSource: "mock", + }, + "node.list": { + nodes: [ + createOfflineDeviceNode(), + { + nodeId: "a1b2c3d4e5f60718293a4b5c6d7e8f90a1b2c3d4e5f60718293a4b5c6d7e8f90", + displayName: "Mac Studio", + platform: "darwin", + deviceFamily: "Mac", + modelIdentifier: "Mac15,14", + remoteIp: "192.168.1.11", + version: "2026.6.11", + connected: true, + paired: true, + approvalState: "approved", + connectedAtMs: baseTime - 60_000, + hostStats: { + cpuCount: 24, + loadAverage: [3.2, 2.8, 2.4], + memoryTotalBytes: 192 * 1024 ** 3, + memoryFreeBytes: 41 * 1024 ** 3, + diskTotalBytes: 2 * 1024 ** 4, + diskAvailableBytes: 1.2 * 1024 ** 4, + updatedAtMs: baseTime, + }, + caps: [ + "browser", + "canvas", + "screen", + "computer", + "file", + "system", + "mcp", + "local-inference", + "claude-sessions", + "codex-cli-sessions", + "pi-sessions", + ], + commands: [ + "desktop.stream", + "screen.snapshot", + "system.execApprovals.get", + "system.execApprovals.set", + "system.notify", + "system.run", + "system.which", + ], + }, + { + nodeId: "0f1e2d3c4b5a69788796a5b4c3d2e1f00f1e2d3c4b5a69788796a5b4c3d2e1f0", + displayName: "Mac mini", + platform: "darwin", + deviceFamily: "Mac", + modelIdentifier: "Mac16,11", + remoteIp: "192.168.1.12", + version: "2026.6.10", + connected: true, + paired: true, + approvalState: "approved", + lastSeenAtMs: baseTime - 82_800_000, + hostStats: { + cpuCount: 12, + loadAverage: [15.4, 13.8, 11.2], + memoryTotalBytes: 32 * 1024 ** 3, + memoryFreeBytes: 2 * 1024 ** 3, + diskTotalBytes: 1024 ** 4, + diskAvailableBytes: 64 * 1024 ** 3, + updatedAtMs: baseTime, + }, + caps: ["browser", "computer", "file", "system", "codex-cli-sessions"], + commands: ["desktop.stream", "screen.snapshot", "system.run"], + }, + { + nodeId: "11223344556677889900aabbccddeeff11223344556677889900aabbccddeeff", + displayName: "iPhone", + platform: "iOS 26.4", + deviceFamily: "iPhone", + modelIdentifier: "iPhone17,2", + remoteIp: "192.168.1.30", + version: "2026.6.11", + connected: true, + paired: true, + approvalState: "approved", + lastSeenAtMs: baseTime - 3_600_000, + caps: ["camera", "canvas", "contacts", "device", "location"], + commands: [ + "camera.list", + "contacts.search", + "device.info", + "location.get", + "system.run", + ], + }, + ], + }, + "system-presence": [ + { + host: "gateway-mock.local", + ip: "192.168.1.10", + version: "2026.6.11", + platform: "macos 26.5.2", + deviceFamily: "Mac", + modelIdentifier: "Mac14,12", + lastInputSeconds: 42, + mode: "gateway", + reason: "self", + instanceId: "mock-gateway-instance", + text: "Gateway: gateway-mock.local (192.168.1.10) · app 2026.6.11 · mode gateway · reason self", + ts: baseTime, + }, + { + host: "Mac Studio", + ip: "192.168.1.11", + version: "2026.6.11", + platform: "macos 26.5.2", + deviceFamily: "Mac", + modelIdentifier: "Mac15,14", + lastInputSeconds: 177, + mode: "node", + reason: "periodic", + deviceId: "a1b2c3d4e5f60718293a4b5c6d7e8f90a1b2c3d4e5f60718293a4b5c6d7e8f90", + instanceId: "a1b2c3d4e5f60718293a4b5c6d7e8f90a1b2c3d4e5f60718293a4b5c6d7e8f90", + roles: ["node"], + text: "Node: Mac Studio (192.168.1.11) · app 2026.6.11 · last input 177s ago · mode node · reason periodic", + ts: baseTime - 30_000, + }, + { + host: "openclaw-control-ui", + version: "2026.6.11", + platform: "macos 26.5.2", + mode: "webchat", + reason: "connect", + roles: ["operator"], + instanceId: "mock-unpaired-webchat", + text: "Node: openclaw-control-ui · mode webchat", + ts: baseTime - 10_000, + }, + ], + "agents.files.get": { + cases: workspaceFileCases, + }, + "agents.files.list": { + cases: workspaceListCases, + }, + "sessions.files.get": { + cases: sessionFileGetCases, + }, + "sessions.files.set": { + cases: sessionFileSetCases, + }, + "sessions.files.list": { + cases: [ + { + match: { sessionKey: "agent:main:main", path: "ui" }, + response: { + browser: { + entries: [ + { + kind: "directory", + name: "src", + path: "ui/src", + sessionKind: "modified", + updatedAtMs: baseTime - 20_000, + }, + { + kind: "file", + name: "vite.config.ts", + path: "ui/vite.config.ts", + size: 9860, + updatedAtMs: baseTime - 900_000, + }, + ], + parentPath: "", + path: "ui", + }, + files: sessionFiles, + root: sessionWorkspaceRoot, + sessionKey: "agent:main:main", + }, + }, + { + match: { sessionKey: "agent:main:main", search: "chat" }, + response: { + browser: { + entries: [ + { + kind: "file", + name: "chat.ts", + path: "ui/src/ui/views/chat.ts", + sessionKind: "modified", + size: 48320, + updatedAtMs: baseTime - 20_000, + }, + { + kind: "file", + name: "chat-flow.e2e.test.ts", + path: "ui/src/e2e/chat-flow.e2e.test.ts", + size: 24950, + updatedAtMs: baseTime - 25_000, + }, + ], + path: "", + search: "chat", + }, + files: sessionFiles, + root: sessionWorkspaceRoot, + sessionKey: "agent:main:main", + }, + }, + ...sessionFileCases, + ], + }, + "artifacts.list": { + cases: [ + { + match: { sessionKey: "agent:main:main" }, + response: { artifacts: [lobsterArtifact] }, + }, + ], + }, + "artifacts.download": { + cases: [ + { + match: { sessionKey: "agent:main:main", artifactId: lobsterArtifact.id }, + response: { + artifact: lobsterArtifact, + data: Buffer.from(lobsterSvg, "utf8").toString("base64"), + encoding: "base64", + }, + }, + ], + }, + "sessions.companion.ask": { + cases: [ + { + match: { sessionKey: OBSERVER_DEMO_SESSION_KEY }, + response: { + answer: "It is rerunning the focused test to check whether the latest fix is stable.", + ts: baseTime + 2_000, + }, + }, + ], + }, + // Saturated-main fixture so the debug page and overlay render queued and + // group-budget states, not just idle lanes. + "diagnostics.lanes": { + ts: baseTime, + lanes: [ + { + lane: "cron", + queuedCount: 0, + activeCount: 1, + maxConcurrent: 4, + draining: false, + generation: 1, + }, + { + lane: "cron-nested", + queuedCount: 0, + activeCount: 1, + maxConcurrent: 4, + draining: false, + generation: 1, + group: "cron-hooks", + groupActive: 2, + groupBudget: 4, + }, + { + lane: "hook-dispatch", + queuedCount: 2, + activeCount: 1, + maxConcurrent: 4, + draining: false, + generation: 1, + group: "cron-hooks", + groupActive: 2, + groupBudget: 4, + reservedForLane: 1, + blockedBy: "group-budget", + }, + { + lane: "main", + queuedCount: 3, + activeCount: 16, + maxConcurrent: 16, + draining: false, + generation: 7, + blockedBy: "lane", + }, + { + lane: "nested", + queuedCount: 0, + activeCount: 0, + maxConcurrent: 1, + draining: false, + generation: 1, + }, + { + lane: "subagent", + queuedCount: 5, + activeCount: 8, + maxConcurrent: 8, + draining: false, + generation: 4, + blockedBy: "lane", + }, + ], + dynamic: { + laneCount: 23, + activeCount: 9, + queuedCount: 4, + queuedLaneCount: 3, + }, + }, + status: { + eventLoop: { utilization: 0.42, cpuCoreRatio: 0.24, delayP99Ms: 12, delayMaxMs: 87 }, + processMemory: { + rssBytes: 432 * 1_048_576, + heapUsedBytes: 210 * 1_048_576, + heapTotalBytes: 280 * 1_048_576, + }, + uptimeMs: 5_412_000, + }, + "last-heartbeat": { ts: baseTime }, + "sessions.list": { + cases: [ + // Child fetches must precede the catch-all page case (subset match). + { + match: { spawnedBy: "agent:main:main" }, + response: pagedSessionsListResponse([mainChildRow, ...swarmChildRows], 0), + }, + { + match: { spawnedBy: "agent:main:tax-research" }, + response: pagedSessionsListResponse([taxChildRow], 0), + }, + ...buildSearchSessionListCases(telegramSessions, searchPrefixes("telegram")), + ...buildSearchSessionListCases(claudeSessions, [ + ...searchPrefixes("claude"), + ...searchPrefixes("claude-sonnet-4-6"), + ...searchPrefixes("anthropic"), + ]), + ...buildSessionListCases( + fixture === "sidebar-roster" ? sessions : [...sessions, ...archivedSessions], + {}, + MOCK_SESSION_OWNERS, + ), + ], + }, + "sessions.search": { results: [] }, + ...(fixture === "workboard" || fixture === "workboard-states" + ? workboardMocks.methodResponses + : {}), + }, + models: modelProviders.models, + repeatingSessionEvents: { + intervalMs: 3_000, + events: [ + { + event: "session.observer", + payload: { + headline: "Reading the failing test and its board caller", + health: "on-track", + revision: 2, + runId: OBSERVER_DEMO_RUN_ID, + sessionKey: OBSERVER_DEMO_SESSION_KEY, + updatedAt: baseTime + 1_000, + }, + }, + { + event: "session.observer", + payload: { + assessment: + "The first fix was incomplete, so the agent is narrowing the assertion path.", + headline: "Third run of the same vitest file - two assertions still failing", + health: "grinding", + planProgress: { completed: 2, total: 4 }, + revision: 3, + runId: OBSERVER_DEMO_RUN_ID, + sessionKey: OBSERVER_DEMO_SESSION_KEY, + updatedAt: baseTime + 4_000, + }, + }, + { + event: "session.observer", + payload: { + assessment: "Repeated identical failures suggest the current approach needs a reset.", + headline: "Same failure five runs in a row - it may be circling", + health: "stuck", + planProgress: { completed: 2, total: 4 }, + revision: 4, + runId: OBSERVER_DEMO_RUN_ID, + sessionKey: OBSERVER_DEMO_SESSION_KEY, + updatedAt: baseTime + 7_000, + }, + }, + { + event: "agent", + payload: { + // replace: the demo replays the same snapshot each cycle; without + // it the controller's cumulative-length dedupe drops the repeats. + data: { replace: true, text: "Rebasing onto main and rerunning the sidebar suite." }, + runId: NARRATION_DEMO_RUN_ID, + sessionKey: NARRATION_DEMO_SESSION_KEY, + stream: "assistant", + }, + }, + { + event: "session.tool", + payload: { + data: { name: "exec" }, + runId: NARRATION_DEMO_RUN_ID, + sessionKey: NARRATION_DEMO_SESSION_KEY, + stream: "tool", + }, + }, + ], + }, + sessionArchiveFiltering: true, + sessions: [ + ...sessions, + ...backgroundTasks.sessions, + ...archivedSessions, + ...telegramSessions, + ...claudeSessions, + taxChildRow, + ], + sessionKey: fixtureSessionKey, + workspace: "/Users/demo/Projects/openclaw", + workspaceGit: true, + }; + if (fixture === "sidebar-roster") { + const teamTasks = backgroundTasks.tasks.slice(0, 2).map((task, index) => { + const agent = expectDefined(rosterAgents[index], "team task agent"); + return Object.assign({}, task, { + agentId: agent.id, + title: agent.sessionLabels[0], + sessionKey: `agent:${agent.id}:main`, + ownerKey: `agent:${agent.id}:main`, + childSessionKey: `agent:${agent.id}:sample-1`, + }); + }); + scenario.methodResponses = { + ...scenario.methodResponses, + "sessions.catalog.list": { catalogs: [] }, + "tasks.list": { + cases: [ + ...rosterAgents.map(({ id }) => ({ + match: { agentId: id }, + response: { tasks: teamTasks.filter((task) => task.agentId === id) }, + })), + { response: { tasks: teamTasks } }, + ], + }, + "tasks.get": { + cases: teamTasks.map((task) => ({ + match: { taskId: task.id }, + response: { task }, + })), + }, + "cron.list": { + cases: [ + ...rosterAgents.flatMap(({ id }) => + cronMocks["cron.list"].cases.map((entry) => { + const jobs = entry.response.jobs.filter((job) => job.agentId === id); + return { + match: { ...entry.match, agentId: id }, + response: { ...entry.response, jobs, total: jobs.length }, + }; + }), + ), + ...cronMocks["cron.list"].cases, + ], + }, + }; + scenario.sessions = rosterSessions; + scenario.repeatingSessionEvents = { events: [] }; + scenario.sessionGroups = []; + } + return scenario; +} + +function escapeScriptContent(script: string): string { + return script.replaceAll(" { + const prepared = await prepareControlUiMockGatewayScenario( + fixture === "workboard-states" ? { ...scenario, nativePlugins: [] } : scenario, + ); + if (fixture === "workboard-states") { + const { assets, catalog } = await buildWorkboardStatesAssets(); + for (const [url, asset] of assets) { + prepared.assets.set(url, asset); + } + prepared.scenario = { + ...prepared.scenario, + featureMethods: [ + ...new Set([ + ...(prepared.scenario.featureMethods ?? []), + "plugins.controlUi.list", + "plugins.controlUi.report", + ]), + ], + methodResponses: { + ...prepared.scenario.methodResponses, + "plugins.controlUi.list": catalog, + "plugins.controlUi.report": { ok: true }, + }, + }; + } + const initScript = escapeScriptContent(createControlUiMockGatewayInitScript(prepared.scenario)); + const sameOriginGatewayScript = escapeScriptContent(createControlUiMockSameOriginGatewayScript()); + const newAgentWelcome = await buildNewAgentWelcome({ + engine: { noteAssistantMessage: () => {} }, + }); + const statefulInitScript = escapeScriptContent( + createControlUiPreviewInitScript(newAgentWelcome) + + skillLibraryMockInitScript(prepared.scenario.models) + + pluginLifecycleMockInitScript() + + skillWorkshopMockInitScript(Date.now()) + + (fixture === "sidebar-roster" ? "" : backgroundTasksMockInitScript(Date.now())) + + approvalMockInitScript(fixture === "approval") + + (fixture === "workboard" || fixture === "workboard-states" + ? `(() => { const __name = (target) => target; (${installWorkboardBoardMock.toString()})(${JSON.stringify(buildWorkboardMocks(Date.now(), MOCK_ACTOR_PETER, fixture === "workboard-states"))}); })();` + : ""), + ); + const bootstrapBody = JSON.stringify(createControlUiMockBootstrapConfig(prepared.scenario)); + const pluginIconIds = new Set( + buildPluginCatalogMock() + .plugins.filter((plugin) => plugin.hasIcon) + .map((plugin) => plugin.id), + ); + const attachmentThemeToggle = + fixture === "attachments" || fixture === "avatars" + ? ` + +` + : ""; + return { + configureServer(server) { + if (fixture === "avatars") { + const avatarFills = new Map( + MOCK_AVATAR_PROFILES.map(({ fill, id }) => [`/api/users/${id}/avatar`, fill]), + ); + server.middlewares.use((req, res, next) => { + const pathname = new URL(req.url ?? "/", "http://openclaw.invalid").pathname; + const fill = avatarFills.get(pathname); + if (!fill) { + next(); + return; + } + res.statusCode = 200; + res.setHeader("content-type", "image/svg+xml"); + res.end( + ``, + ); + }); + } + server.middlewares.use((req, res, next) => { + const prefix = "/__openclaw__/plugin-icon/"; + const pathname = new URL(req.url ?? "/", "http://openclaw.invalid").pathname; + if (!pathname.startsWith(prefix)) { + next(); + return; + } + const pluginId = decodeURIComponent(pathname.slice(prefix.length)); + if (!pluginIconIds.has(pluginId)) { + next(); + return; + } + const icon = path.join(repoRoot, "extensions", pluginId, "assets", "icon.png"); + if (!fs.existsSync(icon)) { + next(); + return; + } + res.statusCode = 200; + res.setHeader("content-type", "image/png"); + res.end(fs.readFileSync(icon)); + }); + server.middlewares.use((req, res, next) => { + const pathname = req.url?.split("?", 1)[0]; + if (!pathname?.startsWith(controlUiPluginAssetRoot())) { + next(); + return; + } + const asset = prepared.assets.get(pathname); + res.statusCode = asset ? 200 : 404; + if (asset) { + res.setHeader("content-type", asset.contentType); + } + res.end(asset?.body); + }); + server.middlewares.use(CONTROL_UI_BOOTSTRAP_CONFIG_PATH, (_req, res) => { + res.statusCode = 200; + res.setHeader("content-type", "application/json"); + res.end(bootstrapBody); + }); + }, + // ui/vite.config.ts registers a placeholder bootstrap-config middleware and + // config-file plugins load first, so without "pre" its stub answers every + // request and the scenario's bootstrap fields never reach the app. + enforce: "pre", + name: "openclaw-control-ui-mock-gateway", + transformIndexHtml(html) { + const rosterPreferenceScript = ``; + return html.replace( + "", + `${attachmentThemeToggle} \n \n${rosterPreferenceScript}\n `, + ); + }, + }; +} + +async function buildWorkboardStatesAssets() { + const { build } = await import("esbuild"); + const rootDir = path.join(repoRoot, "extensions/workboard"); + const entry = path.join(rootDir, "browser/index.ts"); + let transformed = false; + const result = await build({ + absWorkingDir: rootDir, + entryPoints: { index: entry }, + outdir: path.join(rootDir, "dist/control-ui/fixture"), + platform: "browser", + target: "es2022", + format: "esm", + bundle: true, + write: false, + minify: true, + legalComments: "none", + sourcemap: false, + tsconfigRaw: { + compilerOptions: { experimentalDecorators: true, useDefineForClassFields: false }, + }, + alias: buildPluginLoaderAliasMap(entry, process.argv[1], import.meta.url, "src"), + plugins: [ + { + name: "workboard-state-projections", + setup(builder) { + builder.onLoad( + { filter: /[/\\]pages[/\\]workboard[/\\]view-card\.ts$/ }, + async ({ path: sourcePath }) => { + const source = await fs.promises.readFile(sourcePath, "utf8"); + const lifecycleImport = + /\bgetWorkboardLifecycle,(?=[\s\S]*?from "\.\.\/\.\.\/lib\/workboard\/index\.ts")/g; + const alertImport = + /\bgetCardAlerts,(?=[\s\S]*?from "\.\.\/\.\.\/lib\/workboard\/card-alerts\.ts")/g; + if ( + [...source.matchAll(lifecycleImport)].length !== 1 || + [...source.matchAll(alertImport)].length !== 1 + ) { + throw new Error( + "Workboard state fixture import seam changed; update its exact import transform.", + ); + } + transformed = true; + return { + loader: "ts", + resolveDir: path.dirname(sourcePath), + contents: + source.replace(lifecycleImport, "").replace(alertImport, "") + + `\nimport { getFixtureLifecycle as getWorkboardLifecycle, getFixtureAlerts as getCardAlerts } from ${JSON.stringify(fileURLToPath(new URL("./control-ui-workboard-state-projections.ts", import.meta.url)))};\n`, + }; + }, + ); + }, + }, + ], + }); + if (!transformed) { + throw new Error("Workboard state fixture did not reach the real card renderer."); + } + const revision = createHash("sha256"); + for (const file of result.outputFiles) { + revision.update(file.contents); + } + const hash = revision.digest("hex"); + const prefix = `${controlUiPluginAssetPrefix("workboard")}${hash}/`; + const assets = new Map( + result.outputFiles.map((file) => [ + `${prefix}${path.basename(file.path)}`, + { + body: Buffer.from(file.contents), + contentType: file.path.endsWith(".css") ? "text/css" : "text/javascript", + }, + ]), + ); + return { + assets, + catalog: { + revision: hash, + diagnostics: [], + plugins: [ + { + pluginId: "workboard", + name: "workboard", + revision: hash, + entryUrl: `${prefix}index.js`, + styles: [`${prefix}index.css`], + }, + ], + }, + }; +} + +function createBoardFixturePlugin(): Plugin { + return { + name: "openclaw-control-ui-board-fixture", + configureServer(server) { + server.middlewares.use(boardFixturePath, (_req, res, next) => { + void server + .transformIndexHtml(boardFixturePath, boardFixtureHtml) + .then((html) => { + res.statusCode = 200; + res.setHeader("content-type", "text/html; charset=utf-8"); + res.end(html); + }) + .catch((error: unknown) => { + next(error as Error); + }); + }); + }, + }; +} + +function hostForUrl(boundAddress: string, requestedHost: string): string { + const host = boundAddress === "0.0.0.0" || boundAddress === "::" ? requestedHost : boundAddress; + const reachableHost = host === "0.0.0.0" || host === "::" ? "127.0.0.1" : host; + return reachableHost.includes(":") ? `[${reachableHost}]` : reachableHost; +} + +function resolveServerUrl( + server: ViteDevServer, + requestedHost: string, + pathname = "/chat", +): string { + const address = server.httpServer?.address(); + if (!address || typeof address === "string") { + throw new Error("Control UI mock server did not expose a TCP port"); + } + return `http://${hostForUrl(address.address, requestedHost)}:${address.port}${pathname}`; +} + +async function waitForShutdown(): Promise { + await new Promise((resolve) => { + process.once("SIGINT", resolve); + process.once("SIGTERM", resolve); + }); +} + +const options = parseArgs(process.argv.slice(2)); +const scenario = await createChatPickerScenario(options.fixture); +if (options.operatorScopes) { + scenario.operatorScopes = options.operatorScopes; +} +// Vite replaces its deps cache when fixture plugins or mode change. Concurrent +// mocks need separate owners so one startup cannot invalidate another's modules. +const cacheRoot = path.join(repoRoot, ".artifacts", "control-ui-mock-vite"); +await mkdir(cacheRoot, { recursive: true }); +const cacheDir = await mkdtemp(path.join(cacheRoot, "server-")); +// Vite's SIGTERM handler calls process.exit() after closing, bypassing finally. +// The process owns this cache, so its exit hook is the single cleanup path. +process.once("exit", () => rmSync(cacheDir, { recursive: true, force: true })); +let server: ViteDevServer | undefined; +try { + server = await createServer({ + base: "/", + cacheDir, + clearScreen: false, + configFile: path.join(uiRoot, "vite.config.ts"), + define: { + "globalThis.OPENCLAW_CONTROL_UI_BUILD_INFO": JSON.stringify({ + version: "2026.7.10", + commit: "0123456789abcdef0123456789abcdef01234567", + commitAt: "2026-07-10T11:22:33.000Z", + builtAt: "2026-07-10T12:34:56.000Z", + branch: null, + dirty: null, + release: false, + buildId: scenario.serverBuildId, + }), + }, + logLevel: "error", + optimizeDeps: { + ...(options.fixture === "board" + ? { entries: [path.join(uiRoot, "src", "test-helpers", "board-fixture.ts")] } + : {}), + include: ["diff", "lit/directives/repeat.js"], + }, + plugins: [ + ...createStandaloneMockIsolationPlugins(), + await createMockGatewayPlugin(scenario, options.fixture), + createBoardFixturePlugin(), + ...(options.fixture === "attachments" ? [createChatAttachmentFixturePlugin()] : []), + ], + publicDir: path.join(uiRoot, "public"), + resolve: { + alias: [ + ...resolveExternalPackageAliasesForVite(), + ...resolveSourcePackageAliasesForVite(), + ...resolveTsconfigPathAliasesForVite(), + ], + }, + root: uiRoot, + server: { + allowedHosts: options.allowedHosts, + host: options.host, + port: options.port, + strictPort: true, + }, + }); + + await server.listen(); + console.log( + `[control-ui-mock] ${resolveServerUrl(server, options.host, controlUiSessionPath(scenario.sessionKey ?? "agent:main:main"))}`, + ); + console.log( + `[control-ui-mock] board fixture: ${resolveServerUrl(server, options.host, boardFixturePath)}`, + ); + await waitForShutdown(); +} finally { + await server?.close(); +} diff --git a/scripts/control-ui-mock-skill-workshop.ts b/scripts/control-ui-mock-skill-workshop.ts new file mode 100644 index 0000000000000000000000000000000000000000..cc3985eedd620187ab8d3e7ec483571d90f97e09 --- /dev/null +++ b/scripts/control-ui-mock-skill-workshop.ts @@ -0,0 +1,293 @@ +import type { + SkillsProposalEvaluateResult, + SkillsProposalInspectResult, + SkillsProposalRecordResult, + SkillsProposalsListResult, +} from "../packages/gateway-protocol/src/schema/agents-models-skills.js"; +import type { ControlUiMockGateway } from "../ui/src/test-helpers/control-ui-e2e.ts"; + +export function buildSkillWorkshopMocks(baseTime: number) { + const hour = 60 * 60 * 1000; + const day = 24 * hour; + const proposals = [ + { + id: "prop-release-tweets", + kind: "update", + status: "pending", + title: "Tighten release tweet drafting", + description: "Capture the changelog-to-tweet flow the agent keeps re-deriving.", + skillName: "release-tweets", + skillKey: "release-tweets", + createdAt: new Date(baseTime - 2 * hour).toISOString(), + updatedAt: new Date(baseTime - hour).toISOString(), + scanState: "clean", + }, + { + id: "prop-crawler-etiquette", + kind: "create", + status: "pending", + title: "Add crawler etiquette skill", + description: "Rate limits and robots.txt handling learned during the docs sweep.", + skillName: "crawler-etiquette", + skillKey: "crawler-etiquette", + createdAt: new Date(baseTime - 3 * day).toISOString(), + updatedAt: new Date(baseTime - 2 * day).toISOString(), + scanState: "clean", + }, + { + id: "prop-changelog-style", + kind: "update", + status: "applied", + title: "Changelog bullet style", + description: "One bullet per entry, no hard wraps.", + skillName: "changelog-style", + skillKey: "changelog-style", + createdAt: new Date(baseTime - 6 * day).toISOString(), + updatedAt: new Date(baseTime - 5 * day).toISOString(), + scanState: "clean", + }, + ] satisfies SkillsProposalsListResult["proposals"]; + const revisionHash = "b".repeat(64); + const recordFor = (proposal: (typeof proposals)[number]): SkillsProposalRecordResult => ({ + schema: "openclaw.skill-workshop.proposal.v1", + id: proposal.id, + kind: proposal.kind, + status: proposal.status, + title: proposal.title, + description: proposal.description, + createdAt: proposal.createdAt, + updatedAt: proposal.updatedAt, + createdBy: "skill-workshop", + proposedVersion: "2", + draftFile: "PROPOSAL.md", + draftHash: "a".repeat(64), + target: { + source: "openclaw-workshop", + skillName: proposal.skillName, + skillKey: proposal.skillKey, + skillDir: `.agents/skills/${proposal.skillKey}`, + skillFile: `.agents/skills/${proposal.skillKey}/SKILL.md`, + }, + scan: { + state: proposal.scanState, + scannedAt: new Date(baseTime - hour).toISOString(), + critical: 0, + warn: 0, + info: 0, + findings: [], + }, + }); + const evaluation: SkillsProposalEvaluateResult["evaluation"] = { + id: "evaluation-control-ui-mock", + proposedVersion: "2", + revisionHash, + trigger: "manual", + startedAt: new Date(baseTime - 20_000).toISOString(), + completedAt: new Date(baseTime - 18_000).toISOString(), + outcomes: [ + { + pluginId: "fixture-quality", + pluginVersion: "1.0.0", + evaluatorId: "readability", + status: "completed", + result: { + summary: "The workflow is bounded and includes a recovery step.", + decision: "pass", + decisionReason: "No blocking findings in the sanitized fixture.", + }, + }, + ], + }; + return { + list: { + schema: "openclaw.skill-workshop.proposals-manifest.v1", + updatedAt: new Date(baseTime - hour).toISOString(), + proposals, + installedSkills: proposals + .filter((proposal) => proposal.status === "applied") + .map((proposal) => ({ + name: proposal.skillName, + skillKey: proposal.skillKey, + description: proposal.description, + })), + }, + inspect: { + cases: proposals.map((proposal) => ({ + match: { proposalId: proposal.id }, + response: { + record: { + ...recordFor(proposal), + ...(proposal.id === "prop-release-tweets" ? { evaluation } : {}), + }, + revisionHash, + content: [ + `# ${proposal.title}`, + "", + proposal.description, + "", + "## Steps", + "1. Gather the source material.", + "2. Apply the documented workflow.", + ].join("\n"), + supportFiles: [], + }, + })), + }, + evaluation, + requestRevision: { runId: "skill-workshop-revision-mock", status: "started" }, + }; +} + +/** Each agent's proposal records own both mutation replies and subsequent reads. */ +function installSkillWorkshopMock(seed: ReturnType): void { + const gateway = (window as Window & { openclawControlUiE2eGateway?: ControlUiMockGateway }) + .openclawControlUiE2eGateway; + if (!gateway) { + return; + } + const scopes = new Map< + string, + { + list: SkillsProposalsListResult; + details: Map; + } + >(); + for (const method of [ + "list", + "inspect", + "evaluate", + "apply", + "reject", + "historyStatus", + "historyScan", + "read", + ]) { + gateway.setRequestHandler( + method === "read" ? "skills.workshop.read" : `skills.proposals.${method}`, + ({ params: input, respond }) => { + if (method === "historyStatus" || method === "historyScan") { + respond({ + __mockError: { + code: "INVALID_REQUEST", + message: + "Historical batch scans are retired. Start a learning session from Workshop to review past conversations.", + }, + }); + return; + } + const params = (input ?? {}) as { + agentId?: string; + proposalId?: string; + expectedRevisionHash?: string; + name?: string; + }; + const agentId = params.agentId ?? "main"; + let scope = scopes.get(agentId); + if (!scope) { + scope = { + list: { + ...structuredClone(seed.list), + schema: "openclaw.skill-workshop.proposals-manifest.v1", + }, + details: new Map( + seed.inspect.cases.map((entry) => [ + entry.match.proposalId, + structuredClone(entry.response), + ]), + ), + }; + scopes.set(agentId, scope); + } + if (method === "list") { + respond(scope.list); + return; + } + if (method === "read") { + const installed = scope.list.installedSkills.find((skill) => skill.name === params.name); + const detail = + installed && + Array.from(scope.details.values()).find( + (candidate) => + candidate.record.status === "applied" && + candidate.record.target.skillKey === installed.skillKey, + ); + respond( + installed && detail + ? { ...installed, content: detail.content } + : { + __mockError: { + code: "INVALID_REQUEST", + message: "Mock Workshop skill not found.", + }, + }, + ); + return; + } + const detail = params.proposalId ? scope.details.get(params.proposalId) : undefined; + const reject = (message: string) => + respond({ __mockError: { code: "INVALID_REQUEST", message } }); + if (!detail) { + reject("Unknown mock proposal; refresh the Workshop."); + return; + } + if (method === "inspect") { + respond(detail); + return; + } + if (detail.record.status !== "pending") { + reject("Only pending proposals can be evaluated, applied, or rejected."); + return; + } + if (params.expectedRevisionHash && params.expectedRevisionHash !== detail.revisionHash) { + reject("The proposal revision changed; refresh the proposal before retrying."); + return; + } + const now = new Date().toISOString(); + detail.record.updatedAt = now; + if (method === "evaluate") { + const evaluation = structuredClone(seed.evaluation); + evaluation.startedAt = now; + evaluation.completedAt = now; + detail.record.evaluation = evaluation; + } else if (method === "apply") { + detail.record.status = "applied"; + detail.record.appliedAt = now; + const skillKey = detail.record.target.skillKey; + const installed = { + name: detail.record.kind === "create" ? skillKey : detail.record.target.skillName, + skillKey, + description: detail.record.description, + }; + scope.list.installedSkills = [ + ...scope.list.installedSkills.filter((skill) => skill.skillKey !== skillKey), + installed, + ]; + } else { + detail.record.status = "rejected"; + detail.record.rejectedAt = now; + } + const entry = scope.list.proposals.find((proposal) => proposal.id === detail.record.id); + if (entry) { + entry.status = detail.record.status; + entry.updatedAt = now; + entry.revisionHash = detail.revisionHash; + } + scope.list.updatedAt = now; + if (method === "evaluate") { + respond({ record: detail.record, evaluation: detail.record.evaluation }); + } else if (method === "apply") { + respond({ + record: detail.record, + targetSkillFile: `.agents/skills/${detail.record.target.skillKey}/SKILL.md`, + }); + } else { + respond(detail.record); + } + }, + ); + } +} + +export function skillWorkshopMockInitScript(baseTime: number): string { + return `(() => { const __name = (target) => target; (${installSkillWorkshopMock.toString()})(${JSON.stringify(buildSkillWorkshopMocks(baseTime))}); })();`; +} diff --git a/scripts/copy-bundled-plugin-metadata.mts b/scripts/copy-bundled-plugin-metadata.mts new file mode 100644 index 0000000000000000000000000000000000000000..c48b08ed2f217b6b9b67e148aa84293cdc774ca1 --- /dev/null +++ b/scripts/copy-bundled-plugin-metadata.mts @@ -0,0 +1,384 @@ +// Copies bundled plugin metadata into generated runtime locations. +import fs from "node:fs"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { + isPluginActivityToolName, + MAX_PLUGIN_ACTIVITY_TOOL_ICONS, + PLUGIN_ACTIVITY_ICON_PATH, + PLUGIN_TOOL_ACTIVITY_ICON_DIR, + PORTABLE_PLUGIN_ICON_PATH, +} from "../src/plugins/portable-icon-paths.ts"; +import { + collectSourceCheckoutPluginBuildEntries, + mapPluginCatalogEntries, +} from "./lib/bundled-plugin-build-entries.mjs"; +import { linkSourcePluginDependencies } from "./lib/bundled-plugin-dependency-links.mjs"; +import { assertRealOutputRoot } from "./lib/output-root-guard.mjs"; +import { + mergeGeneratedChannelConfigs, + readGeneratedBundledChannelConfigs, + resolvePluginRuntimeChannelMetadata, +} from "./lib/plugin-npm-package-manifest.mts"; +import { isRecord } from "./lib/record-shared.mjs"; +import { + removeFileIfExists, + removePathIfExists, + writeTextFileIfChanged, +} from "./runtime-postbuild-shared.mjs"; + +const GENERATED_BUNDLED_SKILLS_DIR = "bundled-skills"; +const TRANSIENT_COPY_ERROR_CODES = new Set(["EEXIST", "ENOENT", "ENOTEMPTY", "EBUSY"]); +const COPY_RETRY_DELAYS_MS = [10, 25, 50]; + +type CopyMetadataParams = { cwd?: string; repoRoot?: string; env?: NodeJS.ProcessEnv }; +type SkillPathParams = { + distPluginDir: string; + manifest: Record; + pluginDir: string; + repoRoot: string; +}; + +function rewritePackageExtensions(entries: unknown, extension: string): string[] | undefined { + if (!Array.isArray(entries)) { + return undefined; + } + + return entries + .map((entry) => rewritePackageEntry(entry, extension)) + .filter((entry) => entry !== undefined); +} + +function rewritePackageEntry(entry: unknown, extension: string): string | undefined { + if (typeof entry !== "string" || entry.trim().length === 0) { + return undefined; + } + const normalized = entry.replace(/^\.\//, ""); + const rewritten = normalized.replace(/\.[^.]+$/u, extension); + return `./${rewritten}`; +} + +function ensurePathInsideRoot(rootDir: string, rawPath: string): string { + const resolved = path.resolve(rootDir, rawPath); + const relative = path.relative(rootDir, resolved); + if ( + relative === "" || + relative === "." || + (!relative.startsWith(`..${path.sep}`) && relative !== ".." && !path.isAbsolute(relative)) + ) { + return resolved; + } + throw new Error(`path escapes plugin root: ${rawPath}`); +} + +function normalizeManifestRelativePath(rawPath: string): string { + return rawPath.replaceAll("\\", "/").replace(/^\.\//u, ""); +} + +function resolveDeclaredSkillSourcePath(params: { + pluginDir: string; + rawPath: string; + repoRoot: string; +}): string { + const normalized = normalizeManifestRelativePath(params.rawPath); + const pluginLocalPath = ensurePathInsideRoot(params.pluginDir, normalized); + if (fs.existsSync(pluginLocalPath)) { + return pluginLocalPath; + } + if (!/^node_modules(?:\/|$)/u.test(normalized)) { + return pluginLocalPath; + } + return ensurePathInsideRoot(params.repoRoot, normalized); +} + +function resolveBundledSkillTarget(rawPath: string) { + const normalized = normalizeManifestRelativePath(rawPath); + if (/^node_modules(?:\/|$)/u.test(normalized)) { + // Bundled dist/plugin roots must not publish nested node_modules trees. Relocate + // dependency-backed skill assets into a dist-owned directory and rewrite the manifest. + const trimmed = normalized.replace(/^node_modules\/?/u, ""); + if (!trimmed) { + throw new Error(`node_modules skill path must point to a package: ${rawPath}`); + } + const bundledRelativePath = `${GENERATED_BUNDLED_SKILLS_DIR}/${trimmed}`; + return { + manifestPath: `./${bundledRelativePath}`, + outputPath: bundledRelativePath, + }; + } + return { + manifestPath: rawPath, + outputPath: normalized, + }; +} + +function isTransientCopyError(error: unknown): boolean { + return ( + isRecord(error) && typeof error.code === "string" && TRANSIENT_COPY_ERROR_CODES.has(error.code) + ); +} + +function sleepSync(ms: number): void { + if (!Number.isFinite(ms) || ms <= 0) { + return; + } + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); +} + +function copySkillPathWithRetry(params: { + copyOptions: fs.CopySyncOptions; + sourcePath: string; + targetPath: string; +}): void { + const maxAttempts = COPY_RETRY_DELAYS_MS.length + 1; + for (let attempt = 0; attempt < maxAttempts; attempt += 1) { + try { + removePathIfExists(params.targetPath); + fs.mkdirSync(path.dirname(params.targetPath), { recursive: true }); + fs.cpSync(params.sourcePath, params.targetPath, params.copyOptions); + return; + } catch (error) { + if (!isTransientCopyError(error) || attempt === maxAttempts - 1) { + throw error; + } + sleepSync(COPY_RETRY_DELAYS_MS[attempt] ?? 0); + } + } +} + +function copyDeclaredPluginSkillPaths(params: SkillPathParams): string[] { + const skills = Array.isArray(params.manifest.skills) ? params.manifest.skills : []; + const pluginId = + typeof params.manifest.id === "string" ? params.manifest.id : path.basename(params.pluginDir); + const copiedSkills: string[] = []; + for (const raw of skills) { + if (typeof raw !== "string" || raw.trim().length === 0) { + continue; + } + const sourcePath = resolveDeclaredSkillSourcePath({ + rawPath: raw, + pluginDir: params.pluginDir, + repoRoot: params.repoRoot, + }); + const target = resolveBundledSkillTarget(raw); + if (!fs.existsSync(sourcePath)) { + // Some Docker/lightweight builds intentionally omit optional plugin-local + // dependencies. Only advertise skill paths that were actually bundled. + console.warn( + `[bundled-plugin-metadata] skipping missing skill path ${sourcePath} (plugin ${pluginId})`, + ); + continue; + } + const targetPath = ensurePathInsideRoot(params.distPluginDir, target.outputPath); + const shouldExcludeNestedNodeModules = /^node_modules(?:\/|$)/u.test( + normalizeManifestRelativePath(raw), + ); + if (shouldExcludeNestedNodeModules) { + removePathIfExists( + ensurePathInsideRoot(params.distPluginDir, normalizeManifestRelativePath(raw)), + ); + } + copySkillPathWithRetry({ + sourcePath, + targetPath, + copyOptions: { + dereference: true, + force: true, + recursive: true, + filter: (candidatePath: string) => { + if (!shouldExcludeNestedNodeModules || candidatePath === sourcePath) { + return true; + } + const relativeCandidate = path.relative(sourcePath, candidatePath).replaceAll("\\", "/"); + return !relativeCandidate.split("/").includes("node_modules"); + }, + }, + }); + copiedSkills.push(target.manifestPath); + } + return copiedSkills; +} + +function copyPresentationAsset( + pluginDir: string, + distPluginDir: string, + relativePath: string, +): void { + const source = path.join(pluginDir, relativePath); + const target = path.join(distPluginDir, relativePath); + let sourceIsFile = false; + try { + sourceIsFile = fs.lstatSync(source).isFile(); + } catch { + // Missing or unreadable presentation assets must not invalidate the plugin package. + } + if (!sourceIsFile) { + removePathIfExists(target); + return; + } + removePathIfExists(target); + fs.mkdirSync(path.dirname(target), { recursive: true }); + fs.copyFileSync(source, target); +} + +function copyPluginIcons(pluginDir: string, distPluginDir: string): void { + copyPresentationAsset(pluginDir, distPluginDir, PORTABLE_PLUGIN_ICON_PATH); + copyPresentationAsset(pluginDir, distPluginDir, PLUGIN_ACTIVITY_ICON_PATH); + const sourceDir = path.join(pluginDir, PLUGIN_TOOL_ACTIVITY_ICON_DIR); + removePathIfExists(path.join(distPluginDir, PLUGIN_TOOL_ACTIVITY_ICON_DIR)); + let entries: fs.Dirent[]; + try { + if (!fs.lstatSync(sourceDir).isDirectory()) { + return; + } + entries = fs.readdirSync(sourceDir, { withFileTypes: true }); + } catch { + return; + } + if (entries.length > MAX_PLUGIN_ACTIVITY_TOOL_ICONS) { + return; + } + const toolIcons = entries + .filter( + (entry) => + entry.isFile() && + entry.name.endsWith(".svg") && + isPluginActivityToolName(entry.name.slice(0, -4)), + ) + .map((entry) => entry.name) + .toSorted(); + for (const fileName of toolIcons) { + copyPresentationAsset( + pluginDir, + distPluginDir, + path.join(PLUGIN_TOOL_ACTIVITY_ICON_DIR, fileName), + ); + } +} + +/** + * Copies bundled plugin metadata and package extension files. + */ +export function copyBundledPluginMetadata(params: CopyMetadataParams = {}): void { + const repoRoot = params.cwd ?? params.repoRoot ?? process.cwd(); + const env = params.env ?? process.env; + const extensionsRoot = path.join(repoRoot, "extensions"); + const distExtensionsRoot = path.join(repoRoot, "dist", "extensions"); + if (!fs.existsSync(extensionsRoot)) { + return; + } + // Fail closed before any dist/extensions removal: a symlinked dist root + // would redirect recursive deletes into the link target. + assertRealOutputRoot(path.join(repoRoot, "dist")); + + const buildEntries = new Map( + collectSourceCheckoutPluginBuildEntries({ cwd: repoRoot, env }).map((entry) => [ + entry.id, + entry, + ]), + ); + const generatedChannelConfigsByPlugin = readGeneratedBundledChannelConfigs(repoRoot); + const sourcePluginDirs = new Set(); + for (const dirent of fs.readdirSync(extensionsRoot, { withFileTypes: true })) { + if (!dirent.isDirectory()) { + continue; + } + + const pluginDir = path.join(extensionsRoot, dirent.name); + const manifestPath = path.join(pluginDir, "openclaw.plugin.json"); + const distPluginDir = path.join(distExtensionsRoot, dirent.name); + const packageJsonPath = path.join(pluginDir, "package.json"); + const parsedPackageJson: unknown = fs.existsSync(packageJsonPath) + ? JSON.parse(fs.readFileSync(packageJsonPath, "utf8")) + : undefined; + const packageJson = isRecord(parsedPackageJson) ? parsedPackageJson : undefined; + const buildEntry = buildEntries.get(dirent.name); + if (!buildEntry) { + removePathIfExists(distPluginDir); + continue; + } + const distNodeModules = path.join(distPluginDir, "node_modules"); + // Remove only dist-owned entries, including an old directory link itself, + // before skill cleanup or an isolated/unified profile transition. + fs.rmSync(distNodeModules, { recursive: true, force: true }); + + sourcePluginDirs.add(dirent.name); + + const distManifestPath = path.join(distPluginDir, "openclaw.plugin.json"); + const distPackageJsonPath = path.join(distPluginDir, "package.json"); + + if (fs.existsSync(manifestPath)) { + const manifest: unknown = JSON.parse(fs.readFileSync(manifestPath, "utf8")); + if (!isRecord(manifest)) { + throw new Error(`invalid plugin manifest: ${manifestPath}`); + } + const pluginId = typeof manifest.id === "string" ? manifest.id : undefined; + const mergedManifest = mergeGeneratedChannelConfigs( + mapPluginCatalogEntries(manifest, (entry: string) => + rewritePackageEntry(entry, buildEntry.runtimeExtension), + ), + pluginId ? generatedChannelConfigsByPlugin.get(pluginId) : undefined, + ); + // Generated skill assets live under a dedicated dist-owned directory. + removePathIfExists(path.join(distPluginDir, GENERATED_BUNDLED_SKILLS_DIR)); + const copiedSkills = copyDeclaredPluginSkillPaths({ + manifest: mergedManifest, + pluginDir, + distPluginDir, + repoRoot, + }); + const bundledManifest = Array.isArray(mergedManifest.skills) + ? { ...mergedManifest, skills: copiedSkills } + : mergedManifest; + writeTextFileIfChanged(distManifestPath, `${JSON.stringify(bundledManifest, null, 2)}\n`); + copyPluginIcons(pluginDir, distPluginDir); + } else { + removeFileIfExists(distManifestPath); + removeFileIfExists(path.join(distPluginDir, PORTABLE_PLUGIN_ICON_PATH)); + removePathIfExists(path.join(distPluginDir, PLUGIN_ACTIVITY_ICON_PATH)); + removePathIfExists(path.join(distPluginDir, PLUGIN_TOOL_ACTIVITY_ICON_DIR)); + } + + if (!fs.existsSync(packageJsonPath)) { + removeFileIfExists(distPackageJsonPath); + continue; + } + if (packageJson && isRecord(packageJson.openclaw)) { + const extension = buildEntry.runtimeExtension; + const channel = resolvePluginRuntimeChannelMetadata(packageJson.openclaw.channel, { + pluginDir: dirent.name, + runtimeBuildOutputs: rewritePackageExtensions(buildEntry.sourceEntries, extension) ?? [], + runtimeRoot: ".", + }); + packageJson.openclaw = { + ...packageJson.openclaw, + ...(channel ? { channel } : {}), + extensions: rewritePackageExtensions(packageJson.openclaw.extensions, extension), + ...(typeof packageJson.openclaw.setupEntry === "string" + ? { setupEntry: rewritePackageEntry(packageJson.openclaw.setupEntry, extension) } + : {}), + }; + } + + writeTextFileIfChanged(distPackageJsonPath, `${JSON.stringify(packageJson, null, 2)}\n`); + if (buildEntry.isolated) { + linkSourcePluginDependencies(pluginDir, distNodeModules); + } + } + + if (!fs.existsSync(distExtensionsRoot)) { + return; + } + + for (const dirent of fs.readdirSync(distExtensionsRoot, { withFileTypes: true })) { + if (!dirent.isDirectory() || sourcePluginDirs.has(dirent.name)) { + continue; + } + const distPluginDir = path.join(distExtensionsRoot, dirent.name); + removePathIfExists(distPluginDir); + } +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { + copyBundledPluginMetadata(); +} diff --git a/scripts/crabbox-setup.mts b/scripts/crabbox-setup.mts new file mode 100644 index 0000000000000000000000000000000000000000..3a6a2799aa92f0650f2fe7b718cad2436b9b1c0c --- /dev/null +++ b/scripts/crabbox-setup.mts @@ -0,0 +1,19 @@ +import { appendFile } from "node:fs/promises"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { + ensureManagedCrabboxBinary, + resolveCrabboxBinary, +} from "../extensions/crabbox/cli-runtime-api.js"; +import { resolvePathEnvKey } from "./windows-cmd-helpers.mjs"; + +const cli = await ensureManagedCrabboxBinary({ + binary: resolveCrabboxBinary({ + openclawRoot: resolve(dirname(fileURLToPath(import.meta.url)), ".."), + pathEnv: process.env[resolvePathEnvKey(process.env)], + }), +}); +if (process.env.GITHUB_PATH) { + await appendFile(process.env.GITHUB_PATH, `${dirname(cli.binary)}\n`); +} +console.log(JSON.stringify(cli)); diff --git a/scripts/crabbox-source-receiver.mts b/scripts/crabbox-source-receiver.mts new file mode 100644 index 0000000000000000000000000000000000000000..da52c5e3ec5e9ef61124b427540d0d1afe3a3b84 --- /dev/null +++ b/scripts/crabbox-source-receiver.mts @@ -0,0 +1,293 @@ +import type { CrabboxSourceCapsule } from "./crabbox-source-capsule.mts"; + +// This program travels in the locally constructed command, independently of the +// uploaded files. Neither receiver Git metadata nor the capsule can choose its identity. +const receiver = String.raw` +const fs = require("node:fs"); +const path = require("node:path"); +const { createHash } = require("node:crypto"); +const { isUtf8 } = require("node:buffer"); +const { spawnSync } = require("node:child_process"); +const expected = JSON.parse(process.argv[1]); +const syncRoot = process.cwd(); +const cwd = process.argv[2] ?? syncRoot; +let temporary; +function fail(message) { throw new Error(message); } +function stat(file) { + try { return fs.lstatSync(file); } catch (error) { + if (error.code === "ENOENT" || error.code === "ENOTDIR") return null; + throw error; + } +} +function safePath(file) { + if (!file || file.includes("\\") || file.split("/").some(part => + !part || part === "." || part === ".." || part.toLowerCase() === ".git")) fail("unsafe source path"); + return file; +} +function hashFile(file, algorithm, blob = false) { + const fd = fs.openSync(file, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW); + try { + const info = fs.fstatSync(fd); + if (!info.isFile()) fail("source entry is not a regular file"); + const hash = createHash(algorithm); + if (blob) hash.update("blob " + info.size + "\0"); + const buffer = Buffer.alloc(65536); + let size; + while ((size = fs.readSync(fd, buffer)) > 0) hash.update(buffer.subarray(0, size)); + return hash.digest("hex"); + } finally { fs.closeSync(fd); } +} +try { + if (process.argv[2] && (cwd === syncRoot || cwd.startsWith(syncRoot + path.sep) || syncRoot.startsWith(cwd + path.sep))) + fail("Testbox execution and sync workspaces overlap; stop this lease and warm a fresh one"); + const capsule = path.join(syncRoot, ".openclaw-crabbox-changed-gate.bundle"); + if (!stat(capsule)?.isFile() || hashFile(capsule, "sha256") !== expected.digest) + fail("missing or mismatched source capsule; rerun from the local candidate"); + // Native cleanup owns only syncRoot. Source application and the payload share + // the prepared workspace, so ignored runtime never enters the native delete walk. + process.chdir(cwd); + temporary = fs.mkdtempSync(path.join(cwd, ".openclaw-source-")); + const bundle = path.join(temporary, "source.bundle"); + fs.copyFileSync(capsule, bundle); + if (hashFile(bundle, "sha256") !== expected.digest) fail("source capsule changed during import"); + const gitDir = path.join(temporary, "git"); + const env = { ...process.env, GIT_DIR: gitDir, GIT_WORK_TREE: cwd, + GIT_INDEX_FILE: path.join(gitDir, "index"), GIT_OPTIONAL_LOCKS: "0" }; + delete env.GIT_COMMON_DIR; + delete env.GIT_OBJECT_DIRECTORY; + delete env.GIT_ALTERNATE_OBJECT_DIRECTORIES; + delete env.GIT_SHALLOW_FILE; + function git(args, options = {}) { + const { encoding, ...spawnOptions } = options; + const result = spawnSync("git", ["-c", "core.hooksPath=/dev/null", "-c", "core.fsmonitor=false", ...args], + { cwd, env, maxBuffer: 64 * 1024 * 1024, ...spawnOptions }); + if (result.status !== 0) fail("source Git operation failed: " + args[0]); + if (encoding === "buffer") return result.stdout; + if (result.stdout === null) return ""; + if (!isUtf8(result.stdout)) fail("unsupported non-UTF-8 Git metadata"); + return result.stdout.toString("utf8"); + } + git(["init", "-q"]); + git(["remote", "add", "origin", "https://github.com/openclaw/openclaw.git"]); + git(["fetch", "-q", "--depth=2", "origin", expected.baseSha + ":refs/remotes/origin/main"]); + if (git(["rev-parse", "refs/remotes/origin/main"]).trim() !== expected.baseSha) + fail("source base mismatch"); + git(["fetch", "-q", bundle, "refs/openclaw/source-capsule:refs/heads/openclaw-source"]); + for (const [ref, value] of [ + ["refs/heads/openclaw-source", expected.carrier], + ["refs/heads/openclaw-source^{tree}", expected.tree], + ["refs/heads/openclaw-source^", expected.baseSha], + ]) if (git(["rev-parse", ref]).trim() !== value) fail("source capsule identity mismatch"); + function entries(tree, directory = gitDir) { + const output = git(["ls-tree", "-r", "-z", tree], { encoding: "buffer", + env: { ...env, GIT_DIR: directory, GIT_INDEX_FILE: path.join(directory, "index") } }); + if (!isUtf8(output)) fail("unsupported non-UTF-8 source paths"); + return output.subarray(0, -1).toString("utf8") + .split("\0").filter(Boolean).map(row => { + const match = /^(100644|100755|120000) blob ([a-f0-9]{40})\t([\s\S]+)$/.exec(row); + if (!match) fail("unsupported source tree entry"); + return { mode: match[1], oid: match[2], file: safePath(match[3]) }; + }); + } + // The prepared workspace can contain newer workflow source. Its committed tree + // owns only cleanup candidates; the capsule alone selects the executed source. + const previous = new Map((process.argv[2] ? entries("HEAD", path.join(cwd, ".git")) : []) + .map(entry => [entry.file, entry])); + const files = entries(expected.tree); + const metadata = JSON.parse(git(["show", "-s", "--format=%B", expected.carrier])); + if (!Array.isArray(metadata.deleted) || metadata.deleted.some(file => typeof file !== "string")) + fail("invalid source deletion inventory"); + const deleted = new Set(metadata.deleted.map(safePath)); + const selected = new Set(files.map(entry => entry.file)); + const directories = new Set(); + for (const { file } of files) { + let parent = path.posix.dirname(file); + while (parent !== ".") { directories.add(parent); parent = path.posix.dirname(parent); } + } + // Never follow an old directory symlink while removing or creating source entries. + function reachable(file) { + let parent = path.posix.dirname(file); + while (parent !== ".") { + if (!stat(parent)?.isDirectory()) return false; + parent = path.posix.dirname(parent); + } + return true; + } + function remove(file) { + safePath(file); + if (reachable(file)) fs.rmSync(file, { recursive: true, force: true }); + } + // The producer owns privacy-filtered deletions, including ignored entries lost + // from old indexes. A directory with unknown contents is not ours to erase. + for (const file of [...deleted].sort((a, b) => b.split("/").length - a.split("/").length)) { + if (selected.has(file) || directories.has(file)) fail("conflicting source deletion"); + if (reachable(file)) { + if (stat(file)?.isDirectory()) fs.rmdirSync(file); + else fs.rmSync(file, { force: true }); + } + } + for (const directory of [...directories].sort((a, b) => a.split("/").length - b.split("/").length)) { + if (!stat(directory)?.isDirectory()) { + remove(directory); + fs.mkdirSync(directory); + } + } + // Batch raw blobs once; checkout/archive would apply attributes or export rules. + const blobPath = path.join(temporary, "blobs"); + const output = fs.openSync(blobPath, "wx"); + try { git(["cat-file", "--batch"], { + input: files.map(entry => entry.oid + "\n").join(""), stdio: ["pipe", output, "pipe"] + }); } finally { fs.closeSync(output); } + const input = fs.openSync(blobPath, "r"); + const buffer = Buffer.alloc(65536); + let cursor = 0; + let end = 0; + function take(count) { + if (cursor === end) { end = fs.readSync(input, buffer); cursor = 0; } + if (end === 0) fail("truncated source blobs"); + const chunk = buffer.subarray(cursor, Math.min(end, cursor + count)); + cursor += chunk.length; + return chunk; + } + try { + for (const entry of files) { + let header = ""; + for (;;) { + const byte = take(1)[0]; + if (byte === 10) break; + header += String.fromCharCode(byte); + if (header.length > 100) fail("invalid source blob header"); + } + const match = /^([a-f0-9]{40}) blob (\d+)$/.exec(header); + if (!match || match[1] !== entry.oid) fail("source blob identity mismatch"); + let remaining = Number(match[2]); + remove(entry.file); + const fd = entry.mode === "120000" ? null : fs.openSync(entry.file, "wx", 0o600); + const target = []; + try { + while (remaining > 0) { + const bytes = take(remaining); + remaining -= bytes.length; + if (fd === null) target.push(Buffer.from(bytes)); + else fs.writeFileSync(fd, bytes); + } + } finally { if (fd !== null) fs.closeSync(fd); } + if (take(1)[0] !== 10) fail("invalid source blob terminator"); + if (fd === null) fs.symlinkSync(Buffer.concat(target), entry.file); + else fs.chmodSync(entry.file, entry.mode === "100755" ? 0o755 : 0o644); + } + } finally { fs.closeSync(input); } + git(["read-tree", expected.tree]); + fs.rmSync(capsule); + for (;;) { + const extras = git(["ls-files", "--others", "--exclude-standard", "-z"]).split("\0") + .filter(file => file && !file.startsWith(path.basename(temporary) + "/")); + if (extras.length === 0) break; + // Settle ancestor ignore rules first: removing a negated rule can protect + // descendant rules already listed, while removing an exclusion can reveal more. + const ignore = extras.filter(file => path.posix.basename(file) === ".gitignore") + .sort((a, b) => a.split("/").length - b.split("/").length)[0]; + for (const file of ignore ? [ignore] : extras) { + const entry = previous.get(file); + if (!entry) fail("unexpected source entry: " + file); + // Preserve untracked, ignored, and modified runtime state. Only unchanged + // committed extras may be retired after the new source's ignore rules apply. + verify(entry); + fs.unlinkSync(file); + deleted.add(file); + } + // Each pass removes committed files. Retiring their .gitignore rules can + // expose more entries, so verification ends only with an empty inventory. + } + for (const file of deleted) { + if (reachable(file) && stat(file)) fail("source deletion mismatch: " + file); + } + // Verify filesystem bytes, kind, and executable bit independently of either index. + function verify({ file, mode, oid }) { + if (!reachable(file)) fail("source parent mismatch: " + file); + const info = stat(file); + let actual; + if (mode === "120000") { + if (!info?.isSymbolicLink()) fail("source kind mismatch: " + file); + const target = fs.readlinkSync(file, { encoding: "buffer" }); + actual = createHash("sha1").update("blob " + target.length + "\0").update(target).digest("hex"); + } else { + if (!info?.isFile() || Boolean(info.mode & 0o100) !== (mode === "100755")) + fail("source mode mismatch: " + file); + actual = hashFile(file, "sha1", true); + } + if (actual !== oid) fail("source bytes mismatch: " + file); + } + for (const entry of files) verify(entry); + if (expected.alias) git(["update-ref", expected.alias, expected.baseSha]); + git(["symbolic-ref", "HEAD", "refs/heads/openclaw-source"]); + const sourceIndex = git(["ls-files", "--stage", "-v", "-z"], { encoding: "buffer" }); + fs.rmSync(".git", { recursive: true, force: true }); + fs.renameSync(gitDir, path.join(cwd, ".git")); + env.GIT_DIR = path.join(cwd, ".git"); + env.GIT_INDEX_FILE = path.join(env.GIT_DIR, "index"); + const sourceGit = stat(env.GIT_DIR); + // The payload consumes Git identity as well as bytes. Lifecycle hooks may refresh + // index caches or hooksPath, but cannot change source membership or comparison refs. + function verifySource() { + const currentGit = stat(env.GIT_DIR); + if (!currentGit?.isDirectory() || currentGit.dev !== sourceGit.dev || currentGit.ino !== sourceGit.ino || + !stat(env.GIT_INDEX_FILE)?.isFile() || stat(path.join(env.GIT_DIR, "commondir"))) + fail("source Git owner mismatch"); + for (const entry of files) verify(entry); + for (const file of deleted) { + if (reachable(file) && stat(file)) fail("source deletion mismatch: " + file); + } + if (!git(["ls-files", "--stage", "-v", "-z"], { encoding: "buffer" }).equals(sourceIndex)) + fail("source index mismatch"); + if (git(["symbolic-ref", "HEAD"]).trim() !== "refs/heads/openclaw-source") + fail("source HEAD mismatch"); + for (const [ref, value] of [ + ["HEAD", expected.carrier], ["refs/remotes/origin/main", expected.baseSha], + ...(expected.alias ? [[expected.alias, expected.baseSha]] : []), + ]) if (git(["rev-parse", ref]).trim() !== value) fail("source comparison ref mismatch: " + ref); + const extras = git(["ls-files", "--others", "--exclude-standard", "-z"]) + .split("\0").filter(file => file && !file.startsWith(path.basename(temporary) + "/")); + if (extras.length) fail("unexpected source entry: " + extras[0]); + } + verifySource(); + if (selected.has("pnpm-lock.yaml")) { + const installer = ".github/actions/setup-node-env/install-dependencies.sh"; + if (!selected.has(installer) || !stat(installer)?.isFile()) + fail("selected source lacks a regular dependency install owner"); + // Hydration belongs to workflow source. Reconcile through the selected + // source's install owner before any caller payload can run. + const installEnv = { ...env, CI: "true", GITHUB_WORKSPACE: cwd, + NODE_BIN: path.dirname(process.execPath), FROZEN_LOCKFILE: "true", + DEPENDENCY_CACHE: "false", DEPENDENCY_CACHE_HIT: "false" }; + for (const key of ["GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE"]) delete installEnv[key]; + process.stderr.write("[crabbox] reconciling selected-source dependencies\n"); + const install = spawnSync("bash", [installer], { cwd, env: installEnv, stdio: ["inherit", 2, 2] }); + if (install.status !== 0) fail("selected-source frozen install failed; payload was not run"); + verifySource(); + } + process.stderr.write("[crabbox] verified source=" + expected.sourceSha + " tree=" + expected.tree + " carrier=" + expected.carrier + "\n"); +} catch (error) { + process.stderr.write("[crabbox] source verification failed: " + error.message + "\n"); + process.exitCode = 2; +} finally { + if (temporary) fs.rmSync(temporary, { recursive: true, force: true }); +} +`; + +export function remoteSourceBootstrap( + capsule: CrabboxSourceCapsule, + alias: string, + testboxWorkspace: boolean, +) { + const quote = (value: string) => `'${value.replaceAll("'", "'\\''")}'`; + const { sourceSha, baseSha, tree, carrier, digest } = capsule; + const command = `node -e ${quote(receiver)} ${quote(JSON.stringify({ sourceSha, baseSha, tree, carrier, digest, alias }))}`; + if (!testboxWorkspace) { + return command; + } + return [ + 'openclaw_source_root="$(cd ./.git/crabbox-artifact-root && pwd -P)" || { echo "[crabbox] missing prepared Testbox execution workspace; stop this lease and warm a fresh one" >&2; exit 2; };', + `${command} "$openclaw_source_root" && cd "$openclaw_source_root"`, + ].join(" "); +} diff --git a/scripts/crabbox-untrusted-bootstrap.sh b/scripts/crabbox-untrusted-bootstrap.sh new file mode 100644 index 0000000000000000000000000000000000000000..08d22edeb88ccae08c92709f7b2034bd5e12c90c --- /dev/null +++ b/scripts/crabbox-untrusted-bootstrap.sh @@ -0,0 +1,223 @@ +#!/usr/bin/env bash +set -euo pipefail + +node_version="24.19.0" +pnpm_spec="pnpm@12.3.4+sha512.961aa41fb077da3a04a441d9f8e15ebc0c96da8ef710b2eb67bf9ee7cb0610eabd48f1fd85f51cffe73846785fa0f87c56a3a872a1d893f8446741b5cce45457" +# Trusted main's historical pin is needed to validate older contributor heads. +historical_pnpm_spec="pnpm@12.1.0+sha512.d9b8276d97f6ec86e49815877f91ee9f63cee61f2063b304e43b6dab8fa07ce8a9afd46d2facd39f921e6a9d06b3c75a81349c7b888c2d22886bae0229901037" + +if [[ $# -lt 2 ]]; then + echo "usage: $0 [args...]" >&2 + exit 2 +fi +expected_head_sha="$1" +shift +unset NODE_OPTIONS NODE_PATH COREPACK_NPM_REGISTRY COREPACK_INTEGRITY_KEYS COREPACK_ENV_FILE +export COREPACK_ENV_FILE=0 + +imds_token="$( + /usr/bin/curl -fsS --connect-timeout 2 --max-time 5 -X PUT \ + -H "X-aws-ec2-metadata-token-ttl-seconds: 60" \ + http://169.254.169.254/latest/api/token +)" +iam_status="$( + /usr/bin/curl -sS --connect-timeout 2 --max-time 5 -o /dev/null -w "%{http_code}" \ + -H "X-aws-ec2-metadata-token: ${imds_token}" \ + http://169.254.169.254/latest/meta-data/iam/security-credentials/ +)" +if [[ "$iam_status" != "404" ]]; then + echo "refusing untrusted bootstrap: IAM credentials endpoint returned ${iam_status}" >&2 + exit 1 +fi + +actual_head_sha="$(/usr/bin/git rev-parse HEAD)" +if [[ "$actual_head_sha" != "$expected_head_sha" ]]; then + echo "refusing untrusted run: expected HEAD ${expected_head_sha}, got ${actual_head_sha}" >&2 + exit 1 +fi + +case "$(/usr/bin/uname -m)" in + x86_64) + node_arch="x64" + node_sha256="14b342e71204f811bde6153be8e04b62aef63c236fef92b55f9c83154b409647" + ;; + aarch64 | arm64) + node_arch="arm64" + node_sha256="01443c1e1a29e531ccad5a46fefa6df490d2189c49f7955904aecdbb0fe86fdc" + ;; + *) + echo "unsupported architecture: $(/usr/bin/uname -m)" >&2 + exit 2 + ;; +esac + +archive="node-v${node_version}-linux-${node_arch}.tar.xz" +base_url="https://nodejs.org/dist/v${node_version}" +install_root="/opt/openclaw-untrusted-node-v${node_version}-${node_arch}" +corepack_home="/opt/openclaw-untrusted-corepack" +archive_root="/opt/crabbox/toolchain-archives" +tmp_dir="$(/usr/bin/mktemp -d)" +run_home="" +# Invoked by the EXIT trap. +# shellcheck disable=SC2329 +cleanup() { + /bin/rm -rf -- "$tmp_dir" + if [[ -n "$run_home" ]]; then + /bin/rm -rf -- "$run_home" + fi +} +trap cleanup EXIT + +verify_archive() { + local name="$1" checksum="$2" algorithm="$3" + printf '%s %s\n' "$checksum" "$tmp_dir/$name" | "/usr/bin/sha${algorithm}sum" --check --status +} + +copy_verified_archive() { + local name="$1" checksum="$2" algorithm="$3" + # Never verify a shared path and then extract it: authenticate the private copy. + [[ -f "$archive_root/$name" ]] && + /bin/cp -- "$archive_root/$name" "$tmp_dir/$name" && + verify_archive "$name" "$checksum" "$algorithm" +} + +if ! copy_verified_archive "$archive" "$node_sha256" 256; then + /usr/bin/curl -fsSL --connect-timeout 10 --max-time 300 --retry 2 \ + -o "$tmp_dir/$archive" "$base_url/$archive" + if ! verify_archive "$archive" "$node_sha256" 256; then + echo "refusing untrusted bootstrap: downloaded $archive differs from the trusted digest" >&2 + exit 1 + fi +fi + +sudo /usr/bin/env -i PATH=/usr/bin:/bin /bin/bash -s -- \ + "$install_root" "$tmp_dir/$archive" <<'INSTALL_NODE' +set -euo pipefail +umask 022 +install_root="$1" +archive_path="$2" +/bin/rm -rf -- "$install_root" +/usr/bin/mkdir -p "$install_root" +/usr/bin/tar -xJf "$archive_path" -C "$install_root" --strip-components=1 +INSTALL_NODE + +candidate_package_json="$PWD/package.json" +pnpm_spec="$( + cd "$install_root" + "$install_root/bin/node" - "$candidate_package_json" "$pnpm_spec" "$historical_pnpm_spec" <<'PACKAGE_MANAGER' +const fs = require("node:fs"); +const [file, current, historical] = process.argv.slice(2); +let pkg; +try { + pkg = JSON.parse(fs.readFileSync(file, "utf8")); +} catch { + console.error("refusing untrusted run: invalid package.json"); + process.exit(1); +} +if (pkg?.packageManager !== current && pkg?.packageManager !== historical) { + console.error("refusing untrusted run: packageManager pin differs from trusted main or approved history"); + process.exit(1); +} +process.stdout.write(pkg.packageManager); +PACKAGE_MANAGER +)" + +pnpm_version="${pnpm_spec#pnpm@}" +pnpm_version="${pnpm_version%%+*}" +pnpm_native_sha512="" +case "$pnpm_version:$node_arch" in + 12.3.4:x64) + pnpm_native_sha512="d99a8e9523e47f05f5879711f853e259ff3e17eda1653ff74ef8542b9b22807ab06900888aaf11ec21b186774ab3adc9b5c2e2d9ad50a68fb05ff128c9f8f225" + ;; + 12.3.4:arm64) + pnpm_native_sha512="b7bd40540ecb46a88a4f2679c4c61a65cda7e437dda4c6dfa2466e8883971c138cd371029c5d2de226306810ea26056394a6143b0685fdb4506a318d038709e3" + ;; +esac +pnpm_archive="pnpm-${pnpm_version}.tgz" +pnpm_native_archive="exe.linux-${node_arch}-${pnpm_version}.tgz" +pnpm_seed="" +if [[ -n "$pnpm_native_sha512" ]] && + copy_verified_archive "$pnpm_archive" "${pnpm_spec##*+sha512.}" 512 && + copy_verified_archive "$pnpm_native_archive" "$pnpm_native_sha512" 512; then + pnpm_seed="$tmp_dir" +fi + +# Only public toolchain artifacts need shared access; keep caller state private. +sudo /usr/bin/env -i PATH=/usr/bin:/bin /bin/bash -s -- \ + "$install_root" "$corepack_home" "$pnpm_spec" "$pnpm_seed" "$node_arch" <<'INSTALL' +set -euo pipefail +umask 022 +install_root="$1" +corepack_home="$2" +pnpm_spec="$3" +pnpm_seed="$4" +node_arch="$5" +/bin/rm -rf -- "$corepack_home" +/usr/bin/mkdir -p "$corepack_home" +if [[ -n "$pnpm_seed" ]]; then + pnpm_version="${pnpm_spec#pnpm@}" + pnpm_version="${pnpm_version%%+*}" + pnpm_root="$corepack_home/v1/pnpm/$pnpm_version" + /usr/bin/mkdir -p "$pnpm_root" + /usr/bin/tar -xzf "$pnpm_seed/pnpm-${pnpm_version}.tgz" -C "$pnpm_root" --strip-components=1 + native_root="$pnpm_root/node_modules/@pnpm/exe.linux-${node_arch}" + /usr/bin/mkdir -p "$native_root" + /usr/bin/tar -xzf "$pnpm_seed/exe.linux-${node_arch}-${pnpm_version}.tgz" \ + -C "$native_root" --strip-components=1 + # Corepack 0.35's v1 cache format, generated only from authenticated archives. + "$install_root/bin/node" - "$pnpm_root/.corepack" "$pnpm_spec" <<'METADATA' +const fs = require("node:fs"); +const [file, spec] = process.argv.slice(2); +fs.writeFileSync(file, JSON.stringify({ + locator: { name: "pnpm", reference: spec.slice("pnpm@".length) }, + bin: { pnpm: "./bin/pnpm.mjs", pnpx: "./bin/pnpx.mjs" }, + hash: spec.slice(spec.indexOf("+") + 1), +})); +METADATA + export COREPACK_ENABLE_NETWORK=0 +fi +# Corepack must select and warm the approved pin outside the untrusted checkout. +cd "$install_root" +/usr/bin/env \ + COREPACK_ENABLE_DOWNLOAD_PROMPT=0 \ + COREPACK_HOME="$corepack_home" \ + PATH="$install_root/bin:/usr/bin:/bin" \ + "$install_root/bin/corepack" enable --install-directory "$install_root/bin" +/usr/bin/env \ + COREPACK_ENABLE_DOWNLOAD_PROMPT=0 \ + COREPACK_HOME="$corepack_home" \ + PATH="$install_root/bin:/usr/bin:/bin" \ + "$install_root/bin/corepack" prepare "$pnpm_spec" --activate +/usr/bin/env \ + COREPACK_ENABLE_DOWNLOAD_PROMPT=0 \ + COREPACK_HOME="$corepack_home" \ + PATH="$install_root/bin:/usr/bin:/bin" \ + "$install_root/bin/corepack" "$pnpm_spec" --version +INSTALL + +for tool in node npm npx corepack pnpm pnpx; do + if [[ -e "$install_root/bin/$tool" ]]; then + sudo /usr/bin/ln -sfn "$install_root/bin/$tool" "/usr/local/bin/$tool" + fi +done + +/usr/local/bin/node --version + +actual_package_manager="$( + /usr/local/bin/node -e \ + 'const fs = require("node:fs"); const pkg = JSON.parse(fs.readFileSync("package.json", "utf8")); process.stdout.write(pkg.packageManager || "")' +)" +if [[ "$actual_package_manager" != "$pnpm_spec" ]]; then + echo "refusing untrusted run: packageManager pin changed during bootstrap" >&2 + exit 1 +fi + +run_home="$(/usr/bin/mktemp -d)" +export HOME="$run_home" +export COREPACK_HOME="$corepack_home" +export COREPACK_ENABLE_DOWNLOAD_PROMPT=0 +/usr/local/bin/pnpm --version +/usr/local/bin/pnpm install --frozen-lockfile +command_status=0 +"$@" || command_status=$? +exit "$command_status" diff --git a/scripts/crabbox-wrapper-providers.mts b/scripts/crabbox-wrapper-providers.mts new file mode 100644 index 0000000000000000000000000000000000000000..80ddfe8da967fee1ad8a21ade4b52e96be611939 --- /dev/null +++ b/scripts/crabbox-wrapper-providers.mts @@ -0,0 +1,87 @@ +const providerAliases = new Map([ + ["blacksmith", "blacksmith-testbox"], + ["cf", "cloudflare"], + ["container", "local-container"], + ["docker", "local-container"], + ["exe", "exe-dev"], + ["exedev", "exe-dev"], + ["google", "gcp"], + ["google-cloud", "gcp"], + ["local-docker", "local-container"], + ["namespace", "namespace-devbox"], + ["namespace-devboxes", "namespace-devbox"], + ["rail", "railway"], + ["railwayapp", "railway"], + ["run-pod", "runpod"], + ["runpodio", "runpod"], + ["sem", "semaphore"], + ["static", "ssh"], + ["static-ssh", "ssh"], + ["tensorlake-sbx", "tensorlake"], + ["tl", "tensorlake"], +]); + +export function canonicalProviderName(provider: string) { + return providerAliases.get(provider) ?? provider; +} + +function addProviderNames(names: Set, text: string) { + for (const name of text + .replace(/\s+\(defaults?\b.*$/u, "") + .split(/\s*(?:,|\||\bor\b)\s*/u) + .map((s) => s.trim()) + .filter(Boolean)) { + if (/^[a-z0-9][a-z0-9-]*$/u.test(name)) { + names.add(name); + } + } +} + +function providerListContinuation(line: string, previousText: string) { + const match = line.match( + /^\s*((?:or\s+)?[a-z0-9][a-z0-9-]*(?:\s*(?:,|\||\bor\b)\s*(?:or\s+)?[a-z0-9][a-z0-9-]*)*\s*(?:,|\|)?)(?:\s+\(defaults?\b.*)?\s*$/u, + ); + if (!match) { + return ""; + } + if (/[,|]\s*$/u.test(previousText) || /[,|]|\bor\b|\(defaults?\b/u.test(line)) { + return match[1] ?? ""; + } + return ""; +} + +export function parseProvidersFromHelp(text: string) { + const names = new Set(); + const lines = text.split(/\r?\n/u); + for (let index = 0; index < lines.length; index += 1) { + const line = lines[index] ?? ""; + const providerMatch = line.match(/provider:\s*([a-z0-9][a-z0-9, -]*)(?:\s*\(defaults?\b|$)/u); + if (providerMatch) { + let providerText = providerMatch[1] ?? ""; + while (!/\(defaults?\b/u.test(lines[index] ?? "") && index + 1 < lines.length) { + const continuation = providerListContinuation(lines[index + 1] ?? "", providerText); + if (!continuation) { + break; + } + index += 1; + providerText = `${providerText} ${continuation}`; + } + addProviderNames(names, providerText); + continue; + } + + const flagMatch = line.match( + /^\s+-{1,2}provider(?:[=\s]+)([a-z0-9][a-z0-9|, -]*)(?:\s{2,}|\s+\(|$)/u, + ); + const flagProviders = flagMatch?.[1] ?? ""; + if (/[,|]|\bor\b/u.test(flagProviders)) { + addProviderNames(names, flagProviders); + } + } + return [...names]; +} + +export function isProviderAdvertised(provider: string, advertisedProviders: readonly string[]) { + const canonicalProvider = canonicalProviderName(provider); + return advertisedProviders.includes(provider) || advertisedProviders.includes(canonicalProvider); +} diff --git a/scripts/crabbox-wrapper.mjs b/scripts/crabbox-wrapper.mjs new file mode 100644 index 0000000000000000000000000000000000000000..9ebc9117043dd708dc0873ad56cc9adf233f66b6 --- /dev/null +++ b/scripts/crabbox-wrapper.mjs @@ -0,0 +1,4 @@ +#!/usr/bin/env node +import { runTsxCliShim } from "./lib/tsx-cli-shim.mjs"; + +await runTsxCliShim(import.meta.url, { implementation: "./crabbox-wrapper.mts" }); diff --git a/scripts/dated-todo-scan.mts b/scripts/dated-todo-scan.mts new file mode 100644 index 0000000000000000000000000000000000000000..176984dfc8c8564935cc25423923b6ff99c79884 --- /dev/null +++ b/scripts/dated-todo-scan.mts @@ -0,0 +1,415 @@ +#!/usr/bin/env node +import { spawnSync } from "node:child_process"; +import { mkdirSync, readFileSync, statSync, writeFileSync } from "node:fs"; +import { basename, dirname, extname, resolve } from "node:path"; + +const DEFAULT_OUTPUT = ".artifacts/dated-todo-candidates.json"; +const MAX_FILE_BYTES = 2 * 1024 * 1024; +const MAX_CANDIDATES = 5_000; +const TEXT_EXTENSIONS = new Set([ + ".bash", + ".cjs", + ".css", + ".cts", + ".go", + ".html", + ".java", + ".js", + ".json", + ".jsx", + ".kt", + ".kts", + ".md", + ".mdx", + ".mjs", + ".mts", + ".php", + ".py", + ".rb", + ".rs", + ".scss", + ".sh", + ".swift", + ".toml", + ".ts", + ".tsx", + ".xml", + ".yaml", + ".yml", + ".zsh", +]); +const EXCLUDED_SEGMENTS = new Set([ + ".artifacts", + ".generated", + ".git", + ".i18n", + ".next", + "__fixtures__", + "build", + "coverage", + "dist", + "dist-runtime", + "fixtures", + "generated", + "i18n", + "locales", + "node_modules", + "test-fixtures", + "translations", + "vendor", +]); +const EXCLUDED_BASENAMES = new Set([ + "bun.lock", + "bun.lockb", + "CHANGELOG.md", + "package-lock.json", + "pnpm-lock.yaml", + "yarn.lock", +]); +const TODO_PATTERN = + /\b(?:TODO|FIXME|HACK|removeAfter|remove\s+after|delete\s+after|until|deadline|expires?|expiry|expiration|deprecated|deprecation|window|re-?enable|temporary)\b/iu; +const DATE_PATTERN = + /(?:\b20\d{2}-\d{2}-\d{2}(?=\b|T)|\b(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:t(?:ember)?)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\s+(?:\d{1,2}(?:st|nd|rd|th)?(?:,\s*|\s+))?20\d{2}\b)/iu; +const ISO_DATE_PREFILTER = String.raw`20[0-9]{2}-[0-9]{2}-[0-9]{2}`; +const MONTH_DATE_PREFILTER = String.raw`(Jan(uary)?|Feb(ruary)?|Mar(ch)?|Apr(il)?|May|Jun(e)?|Jul(y)?|Aug(ust)?|Sep(t(ember)?)?|Oct(ober)?|Nov(ember)?|Dec(ember)?)\s+([0-9]{1,2}(st|nd|rd|th)?(,\s*|\s+))?20[0-9]{2}`; +const GIT_MONTH_DATE_PREFILTER = MONTH_DATE_PREFILTER.replaceAll(String.raw`\s`, "[[:space:]]"); + +type ScanOptions = { root: string; output: string; compatReport?: string }; +type LoadedFile = { file: string; lines: string[] }; +type Candidate = { + file: string; + line: number; + text: string; + source: "scan" | "compat-registry"; +}; +type CompatRecord = { + code?: unknown; + removeAfter?: unknown; + replacement?: unknown; + status?: unknown; +}; +type CompatReport = { compat?: { records?: CompatRecord[] } }; + +function parseArgs(argv: string[]): ScanOptions { + const options: ScanOptions = { + root: process.cwd(), + output: DEFAULT_OUTPUT, + compatReport: undefined, + }; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === "--help" || arg === "-h") { + process.stdout.write( + "Usage: node --import tsx scripts/dated-todo-scan.mts [--root ] [--output ] [--compat-report ]\n", + ); + process.exit(0); + } + const value = argv[index + 1]; + if (!value || value.startsWith("--")) { + throw new Error(`Missing value for ${arg}`); + } + if (arg === "--root") { + options.root = value; + } else if (arg === "--output") { + options.output = value; + } else if (arg === "--compat-report") { + options.compatReport = value; + } else { + throw new Error(`Unknown argument: ${arg}`); + } + index += 1; + } + options.root = resolve(options.root); + options.output = resolve(options.root, options.output); + if (options.compatReport) { + options.compatReport = resolve(options.root, options.compatReport); + } + return options; +} + +function run(command: string, args: string[], cwd: string, maxBuffer = 32 * 1024 * 1024): string { + const result = spawnSync(command, args, { + cwd, + encoding: "utf8", + maxBuffer, + stdio: ["ignore", "pipe", "pipe"], + }); + if (result.error) { + throw new Error(`Failed to run ${command}: ${result.error.message}`); + } + if (result.status !== 0) { + throw new Error( + `${command} ${args.join(" ")} failed (${result.status ?? "unknown"}): ${result.stderr.trim()}`, + ); + } + return result.stdout; +} + +function runGitGrep(root: string, paths: string[]): string { + const result = spawnSync( + "git", + [ + "grep", + "-l", + "-I", + "-i", + "-E", + "-e", + ISO_DATE_PREFILTER, + "-e", + GIT_MONTH_DATE_PREFILTER, + "--", + ...paths, + ], + { + cwd: root, + encoding: "utf8", + maxBuffer: 32 * 1024 * 1024, + stdio: ["ignore", "pipe", "pipe"], + }, + ); + if (result.error) { + throw new Error(`Failed to run git grep: ${result.error.message}`); + } + if (result.status !== 0 && result.status !== 1) { + throw new Error(`git grep failed (${result.status ?? "unknown"}): ${result.stderr.trim()}`); + } + return result.stdout; +} + +function runDatePrefilter(root: string, paths: string[]): string { + const result = spawnSync( + "rg", + [ + "-l", + "-i", + "--hidden", + "--no-ignore", + "--no-messages", + "-e", + ISO_DATE_PREFILTER, + "-e", + MONTH_DATE_PREFILTER, + "--glob", + "!.git/**", + "--glob", + "!.i18n/**", + "--glob", + "!node_modules/**", + "--glob", + "!dist/**", + "--glob", + "!dist-runtime/**", + ...[...EXCLUDED_SEGMENTS].flatMap((segment) => ["--glob", `!**/${segment}/**`]), + ...paths, + ], + { + cwd: root, + encoding: "utf8", + maxBuffer: 32 * 1024 * 1024, + stdio: ["ignore", "pipe", "pipe"], + }, + ); + if (result.error && "code" in result.error && result.error.code === "ENOENT") { + return runGitGrep(root, paths); + } + if (result.error) { + throw new Error(`Failed to run rg: ${result.error.message}`); + } + if (result.status !== 0 && result.status !== 1) { + throw new Error(`rg failed (${result.status ?? "unknown"}): ${result.stderr.trim()}`); + } + return result.stdout; +} + +function isScannablePath(file: string): boolean { + const normalized = file.replaceAll("\\", "/"); + const parts = normalized.split("/"); + const name = basename(normalized); + return ( + TEXT_EXTENSIONS.has(extname(name).toLowerCase()) && + !EXCLUDED_BASENAMES.has(name) && + !parts.some((part) => EXCLUDED_SEGMENTS.has(part)) && + !/(?:^|[.-])generated(?:[.-]|$)/iu.test(name) && + !/(?:^|[.-])api-baseline(?:[.-]|$)/iu.test(name) + ); +} + +function compactText(lines: string[]): string { + return [...new Set(lines.map((line) => line.trim()).filter(Boolean))] + .join(" | ") + .replaceAll(/\s+/gu, " ") + .slice(0, 500); +} + +function loadFiles(root: string, files: string[]): LoadedFile[] { + return files + .filter(Boolean) + .filter(isScannablePath) + .toSorted() + .flatMap((file) => { + const absolutePath = resolve(root, file); + if (statSync(absolutePath).size > MAX_FILE_BYTES) { + return []; + } + return [ + { + file: file.replaceAll("\\", "/"), + lines: readFileSync(absolutePath, "utf8").split(/\r?\n/u), + }, + ]; + }); +} + +function loadPrefilteredFiles(root: string): LoadedFile[] { + const tracked = new Set( + run("git", ["ls-files", "--cached", "-z"], root) + .split("\0") + .filter((file) => file && isScannablePath(file)), + ); + const files = runDatePrefilter(root, ["."]) + .split("\n") + .map((file) => file.replace(/^\.\//u, "")) + .filter((file) => tracked.has(file)); + return loadFiles(root, files); +} + +function loadCompatFiles(root: string): LoadedFile[] { + const output = run("git", ["ls-files", "--cached", "-z", "--", "src/plugins/compat"], root); + return loadFiles(root, output.split("\0")); +} + +function collectScanCandidates(files: LoadedFile[]): Candidate[] { + const candidates: Candidate[] = []; + for (const { file, lines } of files) { + for (let index = 0; index < lines.length; index += 1) { + if (!TODO_PATTERN.test(lines[index] ?? "")) { + continue; + } + const nearbyDates: string[] = []; + for ( + let nearby = Math.max(0, index - 1); + nearby <= Math.min(lines.length - 1, index + 1); + nearby += 1 + ) { + if (DATE_PATTERN.test(lines[nearby] ?? "")) { + nearbyDates.push(lines[nearby] ?? ""); + } + } + if (nearbyDates.length === 0) { + continue; + } + candidates.push({ + file, + line: index + 1, + text: compactText([lines[index] ?? "", ...nearbyDates]), + source: "scan", + }); + } + } + return candidates; +} + +function readCompatReport(options: ScanOptions): CompatReport { + if (options.compatReport) { + return JSON.parse(readFileSync(options.compatReport, "utf8")) as CompatReport; + } + const reportScript = resolve(options.root, "scripts/plugin-boundary-report.ts"); + const output = run( + process.execPath, + ["--import", "tsx", reportScript, "--json"], + options.root, + 16 * 1024 * 1024, + ); + return JSON.parse(output) as CompatReport; +} + +function findCompatLocation(code: string, files: LoadedFile[]): { file: string; line: number } { + const escapedCode = code.replaceAll(/[.*+?^${}()|[\]\\]/gu, "\\$&"); + const codeField = new RegExp(`\\bcode\\s*:\\s*["']${escapedCode}["']`, "u"); + for (const { file, lines } of files) { + const index = lines.findIndex((line) => codeField.test(line)); + if (index >= 0) { + return { file, line: index + 1 }; + } + } + return { file: "src/plugins/compat/registry.ts", line: 1 }; +} + +function collectCompatCandidates(report: CompatReport, files: LoadedFile[]): Candidate[] { + const records = report?.compat?.records; + if (!Array.isArray(records)) { + throw new Error("Plugin boundary report is missing compat.records"); + } + const locationFiles = [...files].toSorted((left, right) => { + const leftCompat = left.file.startsWith("src/plugins/compat/") ? 0 : 1; + const rightCompat = right.file.startsWith("src/plugins/compat/") ? 0 : 1; + return leftCompat - rightCompat || left.file.localeCompare(right.file); + }); + return records + .filter( + (record) => + record?.status === "deprecated" && + typeof record.code === "string" && + typeof record.removeAfter === "string", + ) + .map((record) => { + const code = record.code as string; + const removeAfter = record.removeAfter as string; + const location = findCompatLocation(code, locationFiles); + return { + file: location.file, + line: location.line, + text: compactText([ + `${code}: removeAfter ${removeAfter}`, + typeof record.replacement === "string" ? `replacement ${record.replacement}` : "", + ]), + source: "compat-registry", + }; + }); +} + +function sortAndDeduplicate(candidates: Candidate[]): Candidate[] { + const unique = new Map(); + for (const candidate of candidates) { + unique.set( + `${candidate.source}\0${candidate.file}\0${candidate.line}\0${candidate.text}`, + candidate, + ); + } + return [...unique.values()].toSorted( + (left, right) => + left.file.localeCompare(right.file) || + left.line - right.line || + left.source.localeCompare(right.source) || + left.text.localeCompare(right.text), + ); +} + +function main(): void { + const options = parseArgs(process.argv.slice(2)); + const scanCandidates = collectScanCandidates(loadPrefilteredFiles(options.root)); + const compatCandidates = collectCompatCandidates( + readCompatReport(options), + loadCompatFiles(options.root), + ); + const candidates = sortAndDeduplicate([...scanCandidates, ...compatCandidates]); + if (candidates.length > MAX_CANDIDATES) { + throw new Error( + `Dated TODO prefilter produced ${candidates.length} candidates, above the ${MAX_CANDIDATES} safety cap`, + ); + } + mkdirSync(dirname(options.output), { recursive: true }); + writeFileSync(options.output, `${JSON.stringify(candidates, null, 2)}\n`); + process.stdout.write( + `dated-todo-scan: wrote ${candidates.length} candidates (${scanCandidates.length} scan, ${compatCandidates.length} compat-registry) to ${options.output}\n`, + ); +} + +try { + main(); +} catch (error) { + const message = error instanceof Error ? error.message : String(error); + process.stderr.write(`dated-todo-scan: ${message}\n`); + process.exitCode = 1; +} diff --git a/scripts/debug-claude-usage.ts b/scripts/debug-claude-usage.ts new file mode 100644 index 0000000000000000000000000000000000000000..2c4aa320ac3a63475c2b9010c57be5ecb978752b --- /dev/null +++ b/scripts/debug-claude-usage.ts @@ -0,0 +1,539 @@ +// Debug Claude Usage script supports OpenClaw repository automation. +import { execFileSync } from "node:child_process"; +import crypto from "node:crypto"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { expectDefined } from "../packages/normalization-core/src/expect.js"; +import { normalizeOptionalString } from "../packages/normalization-core/src/string-coerce.js"; +import { requireOptionArgument } from "./lib/arg-utils.mts"; +import { readBoundedResponseText } from "./lib/bounded-response.mjs"; +import { + maskIdentifier, + parseStrictIntegerOption, + previewForDevToolLog, + redactHomePath, +} from "./lib/dev-tooling-safety.ts"; + +type Args = { + agentId: string; + help: boolean; + reveal: boolean; + sessionKey?: string; +}; + +type FetchOptions = { + fetchImpl?: typeof fetch; + timeoutMs?: number; +}; + +const DEFAULT_FETCH_TIMEOUT_MS = 30_000; +const FETCH_RESPONSE_MAX_BYTES = 256 * 1024; + +const mask = (value: string) => { + return maskIdentifier( + value, + value.trim().length >= 12 ? 6 : 4, + value.trim().length >= 12 ? 6 : 4, + ); +}; + +const parseArgs = (args = process.argv.slice(2)): Args => { + let agentId = "main"; + let help = false; + let reveal = false; + let sessionKey: string | undefined; + + for (let i = 0; i < args.length; i++) { + const arg = expectDefined(args[i], `Claude usage argument at index ${i}`); + if (arg === "--agent") { + agentId = parseNonBlankArgValue(requireOptionArgument(args, i, "--agent"), "--agent"); + i += 1; + continue; + } + if (arg.startsWith("--agent=")) { + agentId = parseNonBlankArgValue(parseInlineArgValue(arg, "--agent"), "--agent"); + continue; + } + if (arg === "--help" || arg === "-h") { + help = true; + continue; + } + if (arg === "--reveal") { + reveal = true; + continue; + } + if (arg === "--session-key") { + sessionKey = parseNonBlankArgValue( + requireOptionArgument(args, i, "--session-key"), + "--session-key", + ); + i += 1; + continue; + } + if (arg.startsWith("--session-key=")) { + sessionKey = parseNonBlankArgValue( + parseInlineArgValue(arg, "--session-key"), + "--session-key", + ); + continue; + } + throw new Error(`Unknown argument: ${arg}`); + } + + return { agentId, help, reveal, sessionKey }; +}; + +function parseInlineArgValue(arg: string, label: string): string { + const value = arg.slice(`${label}=`.length); + if (!value) { + throw new Error(`${label} requires a value`); + } + return value; +} + +function parseNonBlankArgValue(value: string, label: string): string { + const normalized = normalizeOptionalString(value); + if (!normalized) { + throw new Error(`${label} requires a value`); + } + return normalized; +} + +function printUsage(): void { + console.log(`Usage: node --import tsx scripts/debug-claude-usage.ts [options] + +Options: + --agent OpenClaw agent id to inspect (default: main) + --session-key Claude web session key override + --reveal Print token/session values instead of masked identifiers + --help, -h Show this help message`); +} + +const loadAuthProfiles = (agentId: string) => { + const stateRoot = process.env.OPENCLAW_STATE_DIR?.trim() || path.join(os.homedir(), ".openclaw"); + const authPath = path.join(stateRoot, "agents", agentId, "agent", "auth-profiles.json"); + if (!fs.existsSync(authPath)) { + throw new Error(`Missing: ${authPath}`); + } + const store = JSON.parse(fs.readFileSync(authPath, "utf8")) as { + profiles?: Record; + }; + return { authPath, store }; +}; + +const CLAUDE_COOKIE_HOST_SQL = + "(host_key = 'claude.ai' OR host_key = '.claude.ai' OR host_key LIKE '%.claude.ai')"; +const CLAUDE_FIREFOX_COOKIE_HOST_SQL = + "(host = 'claude.ai' OR host = '.claude.ai' OR host LIKE '%.claude.ai')"; + +const pickAnthropicTokens = (store: { + profiles?: Record; +}): Array<{ profileId: string; token: string }> => { + const profiles = store.profiles ?? {}; + const found: Array<{ profileId: string; token: string }> = []; + for (const [id, cred] of Object.entries(profiles)) { + if (cred?.provider !== "anthropic") { + continue; + } + const token = cred.type === "token" ? cred.token?.trim() : undefined; + if (token) { + found.push({ profileId: id, token }); + } + } + return found; +}; + +const resolveFetchTimeoutMs = (raw = process.env.OPENCLAW_DEBUG_CLAUDE_USAGE_FETCH_TIMEOUT_MS) => { + return parseStrictIntegerOption({ + fallback: DEFAULT_FETCH_TIMEOUT_MS, + label: "OPENCLAW_DEBUG_CLAUDE_USAGE_FETCH_TIMEOUT_MS", + min: 1, + raw, + }); +}; + +const withFetchTimeout = async ( + label: string, + timeoutMs: number, + run: (signal: AbortSignal) => Promise, +): Promise => { + const controller = new AbortController(); + let timeout: ReturnType | undefined; + const timeoutPromise = new Promise((_resolve, reject) => { + timeout = setTimeout(() => { + const error = new Error(`${label} exceeded timeout of ${timeoutMs}ms`); + reject(error); + controller.abort(error); + }, timeoutMs); + }); + try { + return await Promise.race([run(controller.signal), timeoutPromise]); + } finally { + if (timeout) { + clearTimeout(timeout); + } + } +}; + +const fetchText = async ( + label: string, + url: string, + init: RequestInit, + options: FetchOptions = {}, +) => { + const fetchImpl = options.fetchImpl ?? fetch; + const timeoutMs = options.timeoutMs ?? resolveFetchTimeoutMs(); + return await withFetchTimeout(label, timeoutMs, async (signal) => { + const res = await fetchImpl(url, { ...init, signal }); + const text = await readBoundedResponseText(res, label, FETCH_RESPONSE_MAX_BYTES, { signal }); + return { res, text }; + }); +}; + +const fetchAnthropicOAuthUsage = async (token: string, options: FetchOptions = {}) => { + const { res, text } = await fetchText( + "Anthropic OAuth usage request", + "https://api.anthropic.com/api/oauth/usage", + { + headers: { + Authorization: `Bearer ${token}`, + Accept: "application/json", + "anthropic-version": "2023-06-01", + "anthropic-beta": "oauth-2025-04-20", + "User-Agent": "openclaw-debug", + }, + }, + options, + ); + return { status: res.status, contentType: res.headers.get("content-type"), text }; +}; + +const readClaudeCliKeychain = (): { + accessToken: string; + expiresAt?: number; + scopes?: string[]; +} | null => { + if (process.platform !== "darwin") { + return null; + } + try { + const raw = execFileSync( + "security", + ["find-generic-password", "-s", "Claude Code-credentials", "-w"], + { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"], timeout: 5000 }, + ); + const parsed = JSON.parse(raw.trim()) as Record; + const oauth = parsed?.claudeAiOauth as Record | undefined; + if (!oauth || typeof oauth !== "object") { + return null; + } + const accessToken = oauth.accessToken; + if (typeof accessToken !== "string" || !accessToken.trim()) { + return null; + } + const expiresAt = typeof oauth.expiresAt === "number" ? oauth.expiresAt : undefined; + const scopes = Array.isArray(oauth.scopes) + ? oauth.scopes.filter((v): v is string => typeof v === "string") + : undefined; + return { accessToken, expiresAt, scopes }; + } catch { + return null; + } +}; + +const chromeServiceNameForPath = (cookiePath: string): string => { + if (cookiePath.includes("/Arc/")) { + return "Arc Safe Storage"; + } + if (cookiePath.includes("/BraveSoftware/")) { + return "Brave Safe Storage"; + } + if (cookiePath.includes("/Microsoft Edge/")) { + return "Microsoft Edge Safe Storage"; + } + if (cookiePath.includes("/Chromium/")) { + return "Chromium Safe Storage"; + } + return "Chrome Safe Storage"; +}; + +const readKeychainPassword = (service: string): string | null => { + try { + const out = execFileSync("security", ["find-generic-password", "-w", "-s", service], { + encoding: "utf8", + stdio: ["ignore", "pipe", "ignore"], + timeout: 5000, + }); + const pw = out.trim(); + return pw ? pw : null; + } catch { + return null; + } +}; + +const decryptChromeCookieValue = (encrypted: Buffer, service: string): string | null => { + if (encrypted.length < 4) { + return null; + } + const prefix = encrypted.subarray(0, 3).toString("utf8"); + if (prefix !== "v10" && prefix !== "v11") { + return null; + } + + const password = readKeychainPassword(service); + if (!password) { + return null; + } + + const key = crypto.pbkdf2Sync(password, "saltysalt", 1003, 16, "sha1"); + const iv = Buffer.alloc(16, 0x20); + const data = encrypted.subarray(3); + + try { + const decipher = crypto.createDecipheriv("aes-128-cbc", key, iv); + decipher.setAutoPadding(true); + const decrypted = Buffer.concat([decipher.update(data), decipher.final()]); + const text = decrypted.toString("utf8").trim(); + return text ? text : null; + } catch { + return null; + } +}; + +const queryChromeCookieDb = (cookieDb: string): string | null => { + try { + const out = execFileSync( + "sqlite3", + [ + "-readonly", + cookieDb, + ` + SELECT + COALESCE(NULLIF(value,''), hex(encrypted_value)) + FROM cookies + WHERE ${CLAUDE_COOKIE_HOST_SQL} + AND name = 'sessionKey' + LIMIT 1; + `, + ], + { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"], timeout: 5000 }, + ).trim(); + if (!out) { + return null; + } + if (out.startsWith("sk-ant-")) { + return out; + } + const hex = out.replace(/[^0-9A-Fa-f]/g, ""); + if (!hex) { + return null; + } + const buf = Buffer.from(hex, "hex"); + const service = chromeServiceNameForPath(cookieDb); + const decrypted = decryptChromeCookieValue(buf, service); + return decrypted && decrypted.startsWith("sk-ant-") ? decrypted : null; + } catch { + return null; + } +}; + +const queryFirefoxCookieDb = (cookieDb: string): string | null => { + try { + const out = execFileSync( + "sqlite3", + [ + "-readonly", + cookieDb, + ` + SELECT value + FROM moz_cookies + WHERE ${CLAUDE_FIREFOX_COOKIE_HOST_SQL} + AND name = 'sessionKey' + LIMIT 1; + `, + ], + { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"], timeout: 5000 }, + ).trim(); + return out && out.startsWith("sk-ant-") ? out : null; + } catch { + return null; + } +}; + +const browserRootLabel = (root: string): string => path.basename(root) || "browser"; + +const findClaudeSessionKey = (): { sessionKey: string; source: string } | null => { + if (process.platform !== "darwin") { + return null; + } + + const firefoxRoot = path.join( + os.homedir(), + "Library", + "Application Support", + "Firefox", + "Profiles", + ); + if (fs.existsSync(firefoxRoot)) { + for (const entry of fs.readdirSync(firefoxRoot)) { + const db = path.join(firefoxRoot, entry, "cookies.sqlite"); + if (!fs.existsSync(db)) { + continue; + } + const value = queryFirefoxCookieDb(db); + if (value) { + return { sessionKey: value, source: `firefox:${entry}` }; + } + } + } + + const chromeCandidates = [ + path.join(os.homedir(), "Library", "Application Support", "Google", "Chrome"), + path.join(os.homedir(), "Library", "Application Support", "Chromium"), + path.join(os.homedir(), "Library", "Application Support", "Arc"), + path.join(os.homedir(), "Library", "Application Support", "BraveSoftware", "Brave-Browser"), + path.join(os.homedir(), "Library", "Application Support", "Microsoft Edge"), + ]; + + for (const root of chromeCandidates) { + if (!fs.existsSync(root)) { + continue; + } + const profiles = fs + .readdirSync(root) + .filter((name) => name === "Default" || name.startsWith("Profile ")); + for (const profile of profiles) { + const db = path.join(root, profile, "Cookies"); + if (!fs.existsSync(db)) { + continue; + } + const value = queryChromeCookieDb(db); + if (value) { + return { sessionKey: value, source: `chromium:${browserRootLabel(root)}/${profile}` }; + } + } + } + + return null; +}; + +const fetchClaudeWebUsage = async (sessionKey: string, options: FetchOptions = {}) => { + const headers = { + Cookie: `sessionKey=${sessionKey}`, + Accept: "application/json", + "User-Agent": + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15", + }; + const { res: orgRes, text: orgText } = await fetchText( + "Claude organizations request", + "https://claude.ai/api/organizations", + { headers }, + options, + ); + if (!orgRes.ok) { + return { ok: false as const, step: "organizations", status: orgRes.status, body: orgText }; + } + const orgs = JSON.parse(orgText) as Array<{ uuid?: string }>; + const orgId = orgs?.[0]?.uuid; + if (!orgId) { + return { ok: false as const, step: "organizations", status: 200, body: orgText }; + } + + const { res: usageRes, text: usageText } = await fetchText( + "Claude usage request", + `https://claude.ai/api/organizations/${orgId}/usage`, + { headers }, + options, + ); + return usageRes.ok + ? { ok: true as const, orgId, body: usageText } + : { ok: false as const, step: "usage", status: usageRes.status, body: usageText }; +}; + +const main = async (argv = process.argv.slice(2)) => { + const opts = parseArgs(argv); + if (opts.help) { + printUsage(); + return; + } + + const { authPath, store } = loadAuthProfiles(opts.agentId); + console.log(`Auth file: ${redactHomePath(authPath)}`); + + const keychain = readClaudeCliKeychain(); + if (keychain) { + console.log( + `Claude Code CLI keychain: accessToken=${opts.reveal ? keychain.accessToken : mask(keychain.accessToken)} scopes=${keychain.scopes?.join(",") ?? "(unknown)"}`, + ); + const oauth = await fetchAnthropicOAuthUsage(keychain.accessToken); + console.log( + `OAuth usage (keychain): HTTP ${oauth.status} (${oauth.contentType ?? "no content-type"})`, + ); + console.log(previewForDevToolLog(oauth.text, 200)); + } else { + console.log("Claude Code CLI keychain: missing/unreadable"); + } + + const anthropic = pickAnthropicTokens(store); + if (anthropic.length === 0) { + console.log("Auth profiles: no Anthropic token profiles found"); + } else { + for (const entry of anthropic) { + console.log( + `Auth profiles: ${entry.profileId} token=${opts.reveal ? entry.token : mask(entry.token)}`, + ); + const oauth = await fetchAnthropicOAuthUsage(entry.token); + console.log( + `OAuth usage (${entry.profileId}): HTTP ${oauth.status} (${oauth.contentType ?? "no content-type"})`, + ); + console.log(previewForDevToolLog(oauth.text, 200)); + } + } + + const envSessionKey = + process.env.CLAUDE_AI_SESSION_KEY?.trim() || process.env.CLAUDE_WEB_SESSION_KEY?.trim(); + const discoveredSession = opts.sessionKey || envSessionKey ? null : findClaudeSessionKey(); + const sessionKey = opts.sessionKey?.trim() || envSessionKey || discoveredSession?.sessionKey; + const source = opts.sessionKey + ? "--session-key" + : envSessionKey + ? "env" + : (discoveredSession?.source ?? "auto"); + + if (!sessionKey) { + console.log( + "Claude web: no sessionKey found (try --session-key or export CLAUDE_AI_SESSION_KEY)", + ); + return; + } + + console.log( + `Claude web: sessionKey=${opts.reveal ? sessionKey : mask(sessionKey)} (source: ${source})`, + ); + const web = await fetchClaudeWebUsage(sessionKey); + if (!web.ok) { + console.log(`Claude web: ${web.step} HTTP ${web.status}`); + console.log(previewForDevToolLog(web.body, 400)); + return; + } + console.log(`Claude web: org=${web.orgId} OK`); + console.log(previewForDevToolLog(web.body, 400)); +}; + +export const testing = { + CLAUDE_COOKIE_HOST_SQL, + fetchAnthropicOAuthUsage, + parseArgs, + resolveFetchTimeoutMs, +}; + +if (import.meta.url === pathToFileURL(path.resolve(process.argv[1] ?? "")).href) { + await main().catch((error: unknown) => { + console.error( + previewForDevToolLog(error instanceof Error ? error.message : String(error), 800), + ); + process.exitCode = 1; + }); +} diff --git a/scripts/diffs-shiki-curated.ts b/scripts/diffs-shiki-curated.ts new file mode 100644 index 0000000000000000000000000000000000000000..77488a39561f21874c0c70c621ee34c0d7f209f9 --- /dev/null +++ b/scripts/diffs-shiki-curated.ts @@ -0,0 +1,59 @@ +// Diffs Shiki Curated script supports OpenClaw repository automation. +import { + createBundledHighlighter, + createCssVariablesTheme, + createSingletonShorthands, + getTokenStyleObject, + guessEmbeddedLanguages, + normalizeTheme, + stringifyTokenStyle, +} from "@shikijs/core"; +import { + createJavaScriptRegexEngine, + defaultJavaScriptRegexConstructor, +} from "@shikijs/engine-javascript"; +import { createOnigurumaEngine, loadWasm } from "@shikijs/engine-oniguruma"; +import { bundledThemes } from "shiki/themes"; +import { + bundledLanguagesBase, + bundledLanguagesInfo, +} from "../extensions/diffs/src/shiki-curated-languages.js"; +export * from "@shikijs/core"; +export { bundledLanguagesBase, bundledLanguagesInfo }; +export { bundledThemes, bundledThemesInfo } from "shiki/themes"; + +export const bundledLanguagesAlias = Object.fromEntries( + bundledLanguagesInfo.flatMap((language) => + ("aliases" in language ? language.aliases : []).map((alias) => [alias, language.import]), + ), +); +export const bundledLanguages = { + ...bundledLanguagesBase, + ...bundledLanguagesAlias, +}; + +export const createHighlighter = createBundledHighlighter({ + langs: bundledLanguages, + themes: bundledThemes, + engine: () => createOnigurumaEngine(import("shiki/wasm")), +}); + +const shorthands = createSingletonShorthands(createHighlighter, { guessEmbeddedLanguages }); + +export const codeToHtml = shorthands.codeToHtml; +export const codeToHast = shorthands.codeToHast; +export const codeToTokens = shorthands.codeToTokens; +export const codeToTokensBase = shorthands.codeToTokensBase; +export const codeToTokensWithThemes = shorthands.codeToTokensWithThemes; +export const getSingletonHighlighter = shorthands.getSingletonHighlighter; +export const getLastGrammarState = shorthands.getLastGrammarState; +export { + createCssVariablesTheme, + createJavaScriptRegexEngine, + createOnigurumaEngine, + defaultJavaScriptRegexConstructor, + getTokenStyleObject, + loadWasm, + normalizeTheme, + stringifyTokenStyle, +}; diff --git a/scripts/docker-channel-promote.mjs b/scripts/docker-channel-promote.mjs new file mode 100644 index 0000000000000000000000000000000000000000..0de6661d2a566f669882963ed595d4358ebd5184 --- /dev/null +++ b/scripts/docker-channel-promote.mjs @@ -0,0 +1,302 @@ +#!/usr/bin/env node + +import { execFileSync } from "node:child_process"; +import process from "node:process"; +import { parseArgs } from "node:util"; +import { isDirectRunUrl } from "./lib/direct-run.mjs"; +import { isMissingManifestError } from "./lib/docker-manifest-error.mjs"; +import { + parseDockerImageConfigVersion, + resolveDockerReleasePolicy, +} from "./lib/docker-release-policy.mjs"; +import { compareReleaseVersions } from "./lib/release-version.mjs"; +import { parsePlatform, verifyDockerAttestations } from "./verify-docker-attestations.mjs"; + +const DOCKER_TIMEOUT_MS = 120_000; +const REQUIRED_PLATFORMS = Object.freeze([ + parsePlatform("linux/amd64"), + parsePlatform("linux/arm64"), +]); +const VARIANTS = Object.freeze([ + { aliasKey: "default", suffix: "" }, + { aliasKey: "slim", suffix: "-slim" }, + { aliasKey: "browser", suffix: "-browser" }, +]); + +/** @typedef {{ imageTagSuffix?: string; images: string[]; includeBrowser?: boolean; version: string }} DockerPromotionParams */ +/** + * @typedef {object} DockerExecOptions + * @property {"utf8"} encoding + * @property {"SIGKILL"} killSignal + * @property {number} maxBuffer + * @property {["ignore", "pipe", "pipe"]} stdio + * @property {number} timeout + */ +/** @typedef {(command: string, args: string[], options: DockerExecOptions) => string} DockerExec */ +/** + * @typedef {object} DockerAttestationParams + * @property {DockerExec} execFileSyncImpl + * @property {string[]} imageRefs + * @property {(message: string) => void} log + * @property {Array<{ architecture: string; os: string; variant?: string }>} requiredPlatforms + */ +/** + * @typedef {object} DockerPromotionOptions + * @property {boolean} [allowRollback] + * @property {DockerExec} [execFileSyncImpl] + * @property {(message: string) => void} [log] + * @property {(params: DockerAttestationParams) => void} [verifyAttestationsImpl] + */ + +/** + * Build the version-specific source to moving-alias promotion plan. + * + * @param {DockerPromotionParams} params + */ +export function createDockerChannelPromotionPlan({ + version, + imageTagSuffix = "", + images, + includeBrowser = true, +}) { + if (images.length === 0) { + throw new Error("At least one --image is required."); + } + if (imageTagSuffix !== "" && !/^-r[0-9]{8}$/u.test(imageTagSuffix)) { + throw new Error(`Invalid Docker image tag suffix "${imageTagSuffix}".`); + } + const policy = resolveDockerReleasePolicy(version); + const promotions = []; + for (const image of images) { + for (const { aliasKey, suffix } of VARIANTS) { + const aliases = policy.movingAliases[aliasKey]; + if (aliases.length === 0 || (!includeBrowser && aliasKey === "browser")) { + continue; + } + promotions.push({ + image, + sourceRef: `${image}:${version}${imageTagSuffix}${suffix}`, + targetRefs: aliases.map((alias) => `${image}:${alias}`), + }); + } + } + if (promotions.length === 0) { + throw new Error(`Docker ${policy.channel} releases have no moving aliases to promote.`); + } + return { channel: policy.channel, promotions, version: policy.version }; +} + +function runDocker(args, execFileSyncImpl) { + return execFileSyncImpl("docker", args, { + encoding: "utf8", + killSignal: "SIGKILL", + maxBuffer: 20 * 1024 * 1024, + stdio: ["ignore", "pipe", "pipe"], + timeout: DOCKER_TIMEOUT_MS, + }); +} + +function inspectManifestDigest(imageRef, execFileSyncImpl) { + const raw = runDocker( + ["buildx", "imagetools", "inspect", imageRef, "--format", "{{json .Manifest}}"], + execFileSyncImpl, + ); + let digest; + try { + digest = JSON.parse(raw).digest; + } catch (error) { + throw new Error(`Could not parse the manifest for ${imageRef}.`, { cause: error }); + } + if (typeof digest !== "string" || !/^sha256:[a-f0-9]{64}$/.test(digest)) { + throw new Error(`The manifest for ${imageRef} did not contain a valid sha256 digest.`); + } + return digest; +} + +function formatPlatform(platform) { + const suffix = platform.variant ? `/${platform.variant}` : ""; + return `${platform.os}/${platform.architecture}${suffix}`; +} + +function inspectImageVersion(imageRef, execFileSyncImpl, { allowMissing = false } = {}) { + const versions = new Map(); + for (const [index, platform] of REQUIRED_PLATFORMS.entries()) { + const platformName = formatPlatform(platform); + let raw; + try { + // In formatted multi-platform inspection, Buildx keys .Image by os/arch. + // Read every promoted platform rather than trusting one config label. + raw = runDocker( + [ + "buildx", + "imagetools", + "inspect", + imageRef, + "--format", + `{{json (index .Image "${platformName}")}}`, + ], + execFileSyncImpl, + ); + } catch (error) { + if (allowMissing && index === 0 && isMissingManifestError(error, imageRef)) { + return null; + } + throw error; + } + + const version = parseDockerImageConfigVersion(raw, imageRef, platformName); + versions.set(platformName, version); + } + const uniqueVersions = new Set(versions.values()); + if (uniqueVersions.size !== 1) { + const details = [...versions].map(([platform, version]) => `${platform}=${version}`).join(", "); + throw new Error(`${imageRef} has inconsistent platform versions: ${details}.`); + } + return uniqueVersions.values().next().value; +} + +function verifySourceVersions(resolved, version, execFileSyncImpl) { + for (const promotion of resolved) { + const sourceVersion = inspectImageVersion(promotion.sourceDigestRef, execFileSyncImpl); + if (sourceVersion !== version) { + throw new Error( + `${promotion.sourceDigestRef} reports version ${sourceVersion}, expected ${version}.`, + ); + } + } +} + +function preventChannelRollback(resolved, version, execFileSyncImpl) { + for (const promotion of resolved) { + for (const targetRef of promotion.targetRefs) { + const currentVersion = inspectImageVersion(targetRef, execFileSyncImpl, { + allowMissing: true, + }); + if (currentVersion === null) { + continue; + } + const comparison = compareReleaseVersions(version, currentVersion); + if (comparison === null) { + throw new Error( + `Cannot compare candidate version ${version} with ${targetRef} version ${currentVersion}.`, + ); + } + if (comparison < 0) { + throw new Error( + `Refusing to move ${targetRef} backward from ${currentVersion} to ${version}. ` + + "An approved repair may rerun with --allow-rollback.", + ); + } + } + } +} + +/** + * Promote every planned alias and verify the registry result. + * + * @param {DockerPromotionParams} params + * @param {DockerPromotionOptions} [options] + */ +export function promoteDockerChannel(params, options = {}) { + const execFileSyncImpl = options.execFileSyncImpl ?? execFileSync; + const log = options.log ?? console.log; + const verifyAttestationsImpl = options.verifyAttestationsImpl ?? verifyDockerAttestations; + const plan = createDockerChannelPromotionPlan(params); + + // Resolve every version-specific source before the first alias write. A missing + // release variant must not leave the channel partially promoted. + const resolved = plan.promotions.map((promotion) => { + const sourceDigest = inspectManifestDigest(promotion.sourceRef, execFileSyncImpl); + return { + ...promotion, + sourceDigest, + sourceDigestRef: `${promotion.image}@${sourceDigest}`, + }; + }); + + // Attestation checks and writes share these digest refs so a concurrent tag + // rewrite cannot swap the content between verification and promotion. + verifyAttestationsImpl({ + imageRefs: resolved.map((promotion) => promotion.sourceDigestRef), + requiredPlatforms: REQUIRED_PLATFORMS, + execFileSyncImpl, + log, + }); + verifySourceVersions(resolved, plan.version, execFileSyncImpl); + if (!options.allowRollback) { + preventChannelRollback(resolved, plan.version, execFileSyncImpl); + } + + for (const promotion of resolved) { + const targetArgs = promotion.targetRefs.flatMap((targetRef) => ["--tag", targetRef]); + runDocker( + [ + "buildx", + "imagetools", + "create", + "--prefer-index=false", + ...targetArgs, + promotion.sourceDigestRef, + ], + execFileSyncImpl, + ); + for (const targetRef of promotion.targetRefs) { + const targetDigest = inspectManifestDigest(targetRef, execFileSyncImpl); + if (targetDigest !== promotion.sourceDigest) { + throw new Error( + `${targetRef} resolved to ${targetDigest}, expected ${promotion.sourceDigest}.`, + ); + } + log(`Verified ${targetRef} -> ${promotion.sourceDigest}.`); + } + } + return plan; +} + +function printHelp() { + console.log( + "Usage: node scripts/docker-channel-promote.mjs --version YYYY.M.P --image REGISTRY/IMAGE [--image REGISTRY/IMAGE] [--image-tag-suffix -rYYYYMMDD] [--allow-rollback]", + ); +} + +function main() { + const { values } = parseArgs({ + args: process.argv.slice(2), + options: { + "allow-rollback": { type: "boolean" }, + help: { type: "boolean", short: "h" }, + image: { type: "string", multiple: true }, + "image-tag-suffix": { type: "string", default: "" }, + version: { type: "string" }, + }, + strict: true, + }); + if (values.help) { + printHelp(); + return; + } + const version = values.version?.trim(); + if (!version) { + throw new Error("--version is required."); + } + const images = (values.image ?? []).map((image) => image.trim()); + if (images.length === 0 || images.some((image) => image.length === 0)) { + throw new Error("At least one non-empty --image is required."); + } + const plan = promoteDockerChannel( + { version, imageTagSuffix: values["image-tag-suffix"], images }, + { allowRollback: values["allow-rollback"] }, + ); + console.log(`Promoted Docker ${plan.channel} aliases for ${plan.version}.`); +} + +if (isDirectRunUrl(process.argv[1], import.meta.url)) { + try { + main(); + } catch (error) { + console.error( + `docker-channel-promote: ${error instanceof Error ? error.message : String(error)}`, + ); + process.exitCode = 1; + } +} diff --git a/scripts/docker-e2e-timings.mts b/scripts/docker-e2e-timings.mts new file mode 100644 index 0000000000000000000000000000000000000000..39c5324be312c0ecf9dc6efa1894569298fe6220 --- /dev/null +++ b/scripts/docker-e2e-timings.mts @@ -0,0 +1,173 @@ +#!/usr/bin/env node +// Summarizes Docker E2E timing artifacts. +// Accepts scheduler summary.json or lane-timings.json so agents can see the +// slowest lanes and phase critical path before deciding what to rerun. +import { isRecord } from "@openclaw/normalization-core/record-coerce"; +import { readDockerE2eJsonArtifact } from "./lib/docker-e2e-json-artifacts.mts"; +import { parsePositiveInt } from "./lib/numeric-options.mjs"; + +function usage() { + return "Usage: node --import tsx scripts/docker-e2e-timings.mts [--limit N]"; +} + +function parseArgs(argv: string[]) { + const options = { file: "", help: false, limit: 12 }; + const readLimit = (raw: string | undefined): number => { + if (!raw || raw.startsWith("-")) { + throw new Error("--limit requires a value"); + } + return parsePositiveInt(raw, "--limit"); + }; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]!; + if (arg === "--help" || arg === "-h") { + options.help = true; + } else if (arg === "--limit") { + options.limit = readLimit(argv[(index += 1)]); + } else if (arg?.startsWith("--limit=")) { + options.limit = readLimit(arg.slice("--limit=".length)); + } else if (arg?.startsWith("-")) { + throw new Error(`unknown argument: ${arg}\n${usage()}`); + } else if (!options.file) { + options.file = arg; + } else { + throw new Error(`unknown argument: ${arg}\n${usage()}`); + } + } + if (options.help) { + return options; + } + if (!options.file) { + throw new Error(usage()); + } + return options; +} + +function readJson(file: string) { + const value = readDockerE2eJsonArtifact(file); + return isRecord(value) ? value : {}; +} + +function seconds(value: unknown): number { + return typeof value === "number" && Number.isFinite(value) ? value : 0; +} + +function scalarText(value: unknown, fallback = ""): string { + if (typeof value === "string") { + return value; + } + if (typeof value === "number" || typeof value === "boolean") { + return String(value); + } + return fallback; +} + +function durationBetween(startedAt: unknown, finishedAt: unknown): number { + if (typeof startedAt !== "string" || typeof finishedAt !== "string") { + return 0; + } + const started = Date.parse(startedAt); + const finished = Date.parse(finishedAt); + if (!Number.isFinite(started) || !Number.isFinite(finished) || finished < started) { + return 0; + } + return Math.round((finished - started) / 1000); +} + +function summarizeSummary(summary: Record, limit: number): void { + const lanes = (Array.isArray(summary.lanes) ? summary.lanes : []) + .filter(isRecord) + .map((lane) => ({ + imageKind: scalarText(lane.imageKind), + name: typeof lane.name === "string" ? lane.name : "", + seconds: seconds(lane.elapsedSeconds), + status: lane.status === 0 ? "pass" : `fail ${scalarText(lane.status, "unknown")}`, + timedOut: lane.timedOut === true, + })) + .filter((lane) => lane.name) + .toSorted((left, right) => right.seconds - left.seconds || left.name.localeCompare(right.name)); + const phases = (Array.isArray(summary.phases) ? summary.phases : []) + .filter(isRecord) + .map((phase) => ({ + name: typeof phase.name === "string" ? phase.name : "", + seconds: seconds(phase.elapsedSeconds), + status: scalarText(phase.status), + })) + .filter((phase) => phase.name); + const wallSeconds = durationBetween(summary.startedAt, summary.finishedAt); + const totalLaneSeconds = lanes.reduce((total, lane) => total + lane.seconds, 0); + const criticalPathSeconds = + phases.reduce((total, phase) => total + phase.seconds, 0) || + wallSeconds || + lanes[0]?.seconds || + 0; + + console.log(`Status: ${scalarText(summary.status, "unknown")}`); + if (wallSeconds > 0) { + console.log(`Wall seconds: ${wallSeconds}`); + } + console.log(`Lane seconds total: ${totalLaneSeconds}`); + console.log(`Approx critical path seconds: ${criticalPathSeconds}`); + if (wallSeconds > 0 && totalLaneSeconds > 0) { + console.log(`Approx parallelism: ${(totalLaneSeconds / wallSeconds).toFixed(1)}x`); + } + if (phases.length > 0) { + console.log(""); + console.log("Phases:"); + for (const phase of phases.toSorted((left, right) => right.seconds - left.seconds)) { + console.log(`- ${phase.name}: ${phase.seconds}s ${phase.status}`); + } + } + console.log(""); + console.log(`Slowest lanes (top ${Math.min(limit, lanes.length)}):`); + for (const lane of lanes.slice(0, limit)) { + console.log( + `- ${lane.name}: ${lane.seconds}s ${lane.status}${lane.timedOut ? " timeout" : ""}${ + lane.imageKind ? ` image=${lane.imageKind}` : "" + }`, + ); + } +} + +function summarizeTimingStore(store: Record, limit: number): void { + const laneStore = isRecord(store.lanes) ? store.lanes : {}; + const lanes = Object.entries(laneStore) + .filter((entry): entry is [string, Record] => isRecord(entry[1])) + .map(([name, lane]) => ({ + name, + seconds: seconds(lane.durationSeconds), + status: lane.status === 0 ? "pass" : `fail ${scalarText(lane.status, "unknown")}`, + updatedAt: scalarText(lane.updatedAt), + })) + .toSorted((left, right) => right.seconds - left.seconds || left.name.localeCompare(right.name)); + console.log(`Updated: ${scalarText(store.updatedAt, "unknown")}`); + console.log(`Known lanes: ${lanes.length}`); + console.log(""); + console.log(`Slowest lanes (top ${Math.min(limit, lanes.length)}):`); + for (const lane of lanes.slice(0, limit)) { + console.log(`- ${lane.name}: ${lane.seconds}s ${lane.status} ${lane.updatedAt}`.trim()); + } +} + +function main(): void { + const options = parseArgs(process.argv.slice(2)); + if (options.help) { + console.log(usage()); + return; + } + const payload = readJson(options.file); + if (Array.isArray(payload.lanes)) { + summarizeSummary(payload, options.limit); + } else if (isRecord(payload.lanes)) { + summarizeTimingStore(payload, options.limit); + } else { + throw new Error(`Unsupported Docker E2E timing artifact: ${options.file}`); + } +} + +try { + main(); +} catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); +} diff --git a/scripts/docker-e2e.mts b/scripts/docker-e2e.mts new file mode 100644 index 0000000000000000000000000000000000000000..8dab7e44e43874601c2de89e46d8d764b706641b --- /dev/null +++ b/scripts/docker-e2e.mts @@ -0,0 +1,170 @@ +// Docker E2E CI helper. +// Converts scheduler JSON into GitHub Actions outputs and compact markdown +// summaries so the workflow does not duplicate Docker E2E planning logic. +import { asOptionalRecord } from "@openclaw/normalization-core/record-coerce"; +import { readDockerE2eJsonArtifact } from "./lib/docker-e2e-json-artifacts.mts"; + +function recordOrEmpty(value: unknown) { + return asOptionalRecord(value) ?? {}; +} + +function recordArray(value: unknown) { + return Array.isArray(value) ? value.map(recordOrEmpty) : []; +} + +function usage() { + return [ + "Usage:", + " node scripts/docker-e2e.mjs github-outputs ", + " node scripts/docker-e2e.mjs summary ", + " node scripts/docker-e2e.mjs failed-reruns <summary.json>", + ].join("\n"); +} + +function boolOutput(value: unknown) { + return value ? "1" : "0"; +} + +function githubOutputs(value: unknown) { + const plan = recordOrEmpty(value); + const needs = recordOrEmpty(plan.needs); + const credentials = Array.isArray(plan.credentials) + ? plan.credentials.filter( + (credential: unknown): credential is string => typeof credential === "string", + ) + : []; + const requiredPackages: unknown[] = Array.isArray(plan.requiredPrepublishPluginPackages) + ? plan.requiredPrepublishPluginPackages + : []; + return [ + `credentials=${credentials.join(",")}`, + `needs_bare_image=${boolOutput(needs.bareImage)}`, + `needs_e2e_image=${boolOutput(needs.e2eImage)}`, + `needs_functional_image=${boolOutput(needs.functionalImage)}`, + `needs_live_image=${boolOutput(needs.liveImage)}`, + `needs_package=${boolOutput(needs.package)}`, + `needs_prepublish_plugin_registry=${boolOutput(needs.prepublishPluginRegistry)}`, + `required_prepublish_plugin_packages=${JSON.stringify(requiredPackages)}`, + ]; +} + +function markdownCell(value: unknown) { + const text = + typeof value === "string" || typeof value === "number" || typeof value === "boolean" + ? String(value) + : ""; + return text.replaceAll("|", "\\|"); +} + +function inlineCode(value: unknown) { + const text = + typeof value === "string" || typeof value === "number" || typeof value === "boolean" + ? String(value) + : ""; + return `\`${text.replaceAll("`", "\\`")}\``; +} + +function formatSeconds(value: unknown) { + const seconds = Number(value); + if (!Number.isFinite(seconds) || seconds < 0) { + return ""; + } + const rounded = Math.round(seconds); + const minutes = Math.floor(rounded / 60); + const rest = rounded % 60; + return minutes > 0 ? `${minutes}m ${rest}s` : `${rest}s`; +} + +function summaryMarkdown(value: unknown, title: string) { + const summary = recordOrEmpty(value); + const lanes = recordArray(summary.lanes); + const slowest = lanes + .filter((lane) => Number.isFinite(Number(lane.elapsedSeconds))) + .toSorted((a, b) => Number(b.elapsedSeconds) - Number(a.elapsedSeconds)) + .slice(0, 8); + const lines = [ + `### ${title}`, + "", + `Status: ${inlineCode(summary.status)}`, + "", + "| Lane | Status | Seconds | Timed out | Rerun |", + "| --- | ---: | ---: | --- | --- |", + ]; + for (const lane of lanes) { + const status = lane.status === 0 ? "pass" : `fail ${markdownCell(lane.status)}`; + lines.push( + `| ${inlineCode(lane.name)} | ${status} | ${markdownCell(lane.elapsedSeconds)} | ${lane.timedOut ? "yes" : "no"} | ${inlineCode(lane.rerunCommand)} |`, + ); + } + + if (slowest.length > 0) { + lines.push("", "| Slowest lane | Duration | Status |", "| --- | ---: | --- |"); + for (const lane of slowest) { + const status = lane.status === 0 ? "pass" : `fail ${markdownCell(lane.status)}`; + lines.push( + `| ${inlineCode(lane.name)} | ${markdownCell(formatSeconds(lane.elapsedSeconds))} | ${status} |`, + ); + } + } + + const phases = recordArray(summary.phases); + if (phases.length > 0) { + lines.push("", "| Phase | Duration | Status | Image kind |", "| --- | ---: | --- | --- |"); + for (const phase of phases) { + lines.push( + `| ${inlineCode(phase.name)} | ${markdownCell(formatSeconds(phase.elapsedSeconds))} | ${markdownCell(phase.status)} | ${markdownCell(phase.imageKind)} |`, + ); + } + } + const failedReruns = failedRerunCommands(summary); + if (failedReruns.length > 0) { + lines.push("", "Failed lane reruns:", ""); + for (const command of failedReruns) { + lines.push(`- ${inlineCode(command)}`); + } + } + return lines.join("\n"); +} + +function failedRerunCommands(value: unknown) { + const summary = recordOrEmpty(value); + const lanes = recordArray(summary.lanes); + return lanes.flatMap((lane) => + lane.status !== 0 && typeof lane.rerunCommand === "string" && lane.rerunCommand + ? [lane.rerunCommand] + : [], + ); +} + +function main(argv: string[] = process.argv.slice(2)) { + const [command, file, ...args] = argv; + if (command === "--help" || command === "-h") { + process.stdout.write(`${usage()}\n`); + return 0; + } + if (!command || !file) { + throw new Error(usage()); + } + + if (command === "github-outputs") { + process.stdout.write(`${githubOutputs(readDockerE2eJsonArtifact(file)).join("\n")}\n`); + } else if (command === "summary") { + const title = args.join(" ").trim(); + if (!title) { + throw new Error(usage()); + } + process.stdout.write(`${summaryMarkdown(readDockerE2eJsonArtifact(file), title)}\n`); + } else if (command === "failed-reruns") { + process.stdout.write(`${failedRerunCommands(readDockerE2eJsonArtifact(file)).join("\n")}\n`); + } else { + throw new Error(`unknown command: ${command}\n${usage()}`); + } + return 0; +} + +try { + process.exitCode = main(); +} catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; +} diff --git a/scripts/docs-link-audit.mts b/scripts/docs-link-audit.mts new file mode 100644 index 0000000000000000000000000000000000000000..0742fa9e39d53c926b3106c3020a397e5e8dfb31 --- /dev/null +++ b/scripts/docs-link-audit.mts @@ -0,0 +1,712 @@ +#!/usr/bin/env node + +// Audits docs links against the shared publishing parser and route contract. +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { createProcessor } from "@mdx-js/mdx"; +import { isRecord } from "@openclaw/normalization-core/record-coerce"; +import { err, ok, type Result } from "@openclaw/normalization-core/result"; +import MarkdownIt from "markdown-it"; +import type { Nodes } from "mdast"; +import { + CLAWHUB_REPO_ENV, + resolveClawHubRepoPath, + syncClawHubDocsTree, +} from "./docs-sync-publish.mjs"; +import { parseDocsDocument, resolveDocsFragment } from "./lib/docs-markdown.mjs"; +import { + addRoute, + collectMirroredDocsRoutes, + collectNavPageEntries, + normalizeRoute, +} from "./lib/docs-published-routes.mts"; +import { resolveRedirects } from "./lib/docs-redirects.mjs"; + +const ROOT = process.cwd(); +const DOCS_DIR = path.join(ROOT, "docs"); +const DOCS_JSON_PATH = path.join(DOCS_DIR, "docs.json"); +const ROOT_MARKDOWN_FILES = ["README.md", "CONTRIBUTING.md", "SECURITY.md"]; +const MDX_PROCESSOR = createProcessor({ format: "mdx" }); +const MARKDOWN_PARSER = new MarkdownIt({ html: false }); +const HTML_MARKDOWN_PARSER = new MarkdownIt({ html: true }); +type MarkdownToken = ReturnType<typeof MARKDOWN_PARSER.parse>[number]; +const VERBATIM_MDX_ELEMENTS = new Set(["code", "pre", "script", "style", "textarea"]); +if (!fs.existsSync(DOCS_DIR) || !fs.statSync(DOCS_DIR).isDirectory()) { + console.error("docs:check-links: missing docs directory; run from repo root."); + process.exit(1); +} + +if (!fs.existsSync(DOCS_JSON_PATH)) { + console.error("docs:check-links: missing docs/docs.json."); + process.exit(1); +} + +function walk(dir: string): string[] { + const out: string[] = []; + const entries = fs.readdirSync(dir, { withFileTypes: true }); + for (const entry of entries) { + if (entry.name.startsWith(".")) { + continue; + } + const full = path.join(dir, entry.name); + if (entry.isDirectory()) { + out.push(...walk(full)); + } else if (entry.isFile()) { + out.push(full); + } + } + return out; +} + +function escapeHtmlAttribute(value: string) { + return value + .replaceAll("&", "&") + .replaceAll('"', """) + .replaceAll("<", "<") + .replaceAll(">", ">"); +} + +function collectMarkdownCodeLines(tokens: MarkdownToken[]): Set<number> { + const lines = new Set<number>(); + for (const token of tokens) { + if ((token.type === "fence" || token.type === "code_block") && token.map) { + for (let line = token.map[0]; line < token.map[1]; line++) { + lines.add(line); + } + } + } + return lines; +} + +/** + * Projects parsed Markdown links onto their source lines. MDX parsing owns the + * normal path; markdown-it is a tolerant fallback for legacy malformed pages. + */ +function projectExternalLinkMarkdown(raw: string) { + // Lychee also receives every original source file for HTML attributes and bare URLs. + // This line-stable companion contains only parsed Markdown links hidden by MDX blocks. + const projected: string[] = raw.split("\n").map((line) => (line.endsWith("\r") ? "\r" : "")); + let projectedLinks = 0; + const appendLink = (line: number | undefined, url: string | null | undefined) => { + if ( + line === undefined || + !Number.isInteger(line) || + line < 1 || + line > projected.length || + !url + ) { + return; + } + const index = line - 1; + const projectedLine = projected[index]; + if (projectedLine === undefined) { + return; + } + const suffix = projectedLine.endsWith("\r") ? "\r" : ""; + const existing = suffix ? projectedLine.slice(0, -1) : projectedLine; + const separator = existing ? " " : ""; + projected[index] = + `${existing}${separator}<a href="${escapeHtmlAttribute(url)}">link</a>${suffix}`; + projectedLinks += 1; + }; + + try { + const tree = MDX_PROCESSOR.parse(raw); + const definitions = new Map<string, string>(); + const collectDefinitions = (node: Nodes): void => { + if (node.type === "definition" && !definitions.has(node.identifier)) { + definitions.set(node.identifier, node.url); + } + if ("children" in node) { + for (const child of node.children) { + collectDefinitions(child); + } + } + }; + collectDefinitions(tree); + + const collectLinks = (node: Nodes, verbatim = false): void => { + const nextVerbatim = + verbatim || + ((node.type === "mdxJsxFlowElement" || node.type === "mdxJsxTextElement") && + node.name !== null && + VERBATIM_MDX_ELEMENTS.has(node.name)); + if (!nextVerbatim) { + if (node.type === "link" || node.type === "image") { + appendLink(node.position?.start.line, node.url); + } else if (node.type === "linkReference" || node.type === "imageReference") { + appendLink(node.position?.start.line, definitions.get(node.identifier)); + } + } + if ("children" in node) { + for (const child of node.children) { + collectLinks(child, nextVerbatim); + } + } + }; + collectLinks(tree); + } catch { + const rawLines = raw.split("\n"); + const inlineVerbatimLinks = new Map<string, number>(); + const transparentEnv: Record<string, unknown> = {}; + const transparentTokens = MARKDOWN_PARSER.parse(raw, transparentEnv); + const fallbackCodeLines = collectMarkdownCodeLines(transparentTokens); + const childUrl = (child: MarkdownToken): string | undefined => { + const value = + child.type === "link_open" + ? child.attrGet("href") + : child.type === "image" + ? child.attrGet("src") + : undefined; + return typeof value === "string" ? value : undefined; + }; + const sourceLineForUrl = (token: MarkdownToken, url: string): number => { + if (!token.map) { + return 0; + } + const escapedUrl = url.replaceAll("&", "&"); + for (let line = token.map[0]; line < token.map[1]; line += 1) { + if (rawLines[line]?.includes(url) || rawLines[line]?.includes(escapedUrl)) { + return line; + } + for (const inlineToken of MARKDOWN_PARSER.parseInline( + rawLines[line] ?? "", + transparentEnv, + )) { + if ((inlineToken.children ?? []).some((child) => childUrl(child) === url)) { + return line; + } + } + } + return token.map[0]; + }; + const inlineVerbatimStack: string[] = []; + for (const [line, rawLine] of rawLines.entries()) { + if (inlineVerbatimStack.length === 0 && fallbackCodeLines.has(line)) { + continue; + } + for (const token of HTML_MARKDOWN_PARSER.parseInline(rawLine, transparentEnv)) { + for (const child of token.children ?? []) { + if (child.type === "html_inline") { + const closingTag = child.content.match(/^<\/([a-z][A-Za-z0-9.:_-]*)[\t ]*>$/u)?.[1]; + if (closingTag) { + const openingIndex = inlineVerbatimStack.lastIndexOf(closingTag); + if (openingIndex >= 0) { + inlineVerbatimStack.length = openingIndex; + } + continue; + } + const openingTag = child.content.match( + /^<([a-z][A-Za-z0-9.:_-]*)(?:[\t ][^<>]*?)?(\/?)>$/u, + ); + const openingName = openingTag?.[1]; + if (openingName && !openingTag[2] && VERBATIM_MDX_ELEMENTS.has(openingName)) { + inlineVerbatimStack.push(openingName); + } + continue; + } + const url = childUrl(child); + if (inlineVerbatimStack.length > 0 && url) { + const key = `${line}\0${url}`; + inlineVerbatimLinks.set(key, (inlineVerbatimLinks.get(key) ?? 0) + 1); + } + } + } + } + for (const token of transparentTokens) { + if (token.type !== "inline" || !token.map) { + continue; + } + for (const child of token.children ?? []) { + const url = childUrl(child); + if (!url) { + continue; + } + const sourceLine = sourceLineForUrl(token, url); + const key = `${sourceLine}\0${url}`; + const hiddenOccurrences = inlineVerbatimLinks.get(key) ?? 0; + if (hiddenOccurrences > 0) { + inlineVerbatimLinks.set(key, hiddenOccurrences - 1); + continue; + } + appendLink(sourceLine + 1, url); + } + } + } + + return { text: projected.join("\n"), projectedLinks }; +} + +/** + * Writes a parallel docs tree that exposes Markdown nested in HTML/MDX blocks. + * Original inputs still cover tag attributes; projected inputs cover children. + */ +export function prepareExternalLinkAuditTree(repoRoot: string, outputDir: string) { + const root = path.resolve(repoRoot); + const docsRoot = path.join(root, "docs"); + const outputRoot = path.resolve(outputDir); + if (fs.existsSync(outputRoot)) { + throw new Error(`external-link audit input already exists: ${outputRoot}`); + } + const outputFromDocs = path.relative(docsRoot, outputRoot); + if ( + outputFromDocs === "" || + (!outputFromDocs.startsWith(`..${path.sep}`) && + outputFromDocs !== ".." && + !path.isAbsolute(outputFromDocs)) + ) { + throw new Error("external-link audit output must be outside docs"); + } + + const sourcePaths = [ + ...walk(docsRoot).filter((filePath) => /\.mdx?$/iu.test(filePath)), + ...ROOT_MARKDOWN_FILES.map((filename) => path.join(root, filename)), + ]; + let projectedLinks = 0; + for (const sourcePath of sourcePaths) { + const targetPath = path.join(outputRoot, path.relative(root, sourcePath)); + const raw = fs.readFileSync(sourcePath, "utf8"); + const projected = projectExternalLinkMarkdown(raw); + projectedLinks += projected.projectedLinks; + fs.mkdirSync(path.dirname(targetPath), { recursive: true }); + fs.writeFileSync(targetPath, projected.text, "utf8"); + } + + return { files: sourcePaths.length, projectedLinks }; +} + +function normalizeSlashes(p: string) { + return p.replace(/\\/g, "/"); +} + +function isLocalizedDocPath(p: string) { + return /^\/?[a-z]{2}(?:-[A-Za-z]{2,8})+\//.test(p); +} + +function isGeneratedTranslatedDoc(relPath: string) { + return isLocalizedDocPath(relPath); +} + +function createRedirectMap(docsConfig: Record<string, unknown>): Map<string, string> { + const redirects = new Map<string, string>(); + const redirectEntries = Array.isArray(docsConfig.redirects) ? docsConfig.redirects : []; + for (const item of redirectEntries) { + if (!isRecord(item)) { + continue; + } + const source = normalizeRoute(typeof item.source === "string" ? item.source : ""); + const destination = normalizeRoute( + typeof item.destination === "string" ? item.destination : "", + ); + redirects.set(source, destination); + } + return redirects; +} + +function buildAuditIndex( + docsDir = DOCS_DIR, + options: { allowExternalClawHubRoutes?: boolean } = {}, +) { + const docsJsonPath = path.join(docsDir, "docs.json"); + const parsedConfig: unknown = JSON.parse(fs.readFileSync(docsJsonPath, "utf8")); + if (!isRecord(parsedConfig)) { + throw new Error(`${docsJsonPath} must contain an object`); + } + const docsConfig = parsedConfig; + const redirects = createRedirectMap(docsConfig); + const allFiles = walk(docsDir); + const relAllFiles = new Set(allFiles.map((abs) => normalizeSlashes(path.relative(docsDir, abs)))); + const markdownFiles = allFiles.filter((abs) => { + if (!/\.(md|mdx)$/i.test(abs)) { + return false; + } + const rel = normalizeSlashes(path.relative(docsDir, abs)); + return !isGeneratedTranslatedDoc(rel); + }); + const routes = new Set<string>(); + + for (const abs of markdownFiles) { + const rel = normalizeSlashes(path.relative(docsDir, abs)); + const slug = rel.replace(/\.(md|mdx)$/i, ""); + addRoute(routes, slug); + } + + // Without a ClawHub checkout the mirrored tree is absent, so its pages cannot be + // parsed. The navigation still declares the published routes, so keep them + // resolvable and remember which ones are only backed by that declaration. + const mirroredRoutes = new Set<string>(); + if (options.allowExternalClawHubRoutes === true) { + for (const route of collectMirroredDocsRoutes(docsConfig.navigation)) { + routes.add(route); + mirroredRoutes.add(route); + } + } + + return { + docsDir, + docsConfig, + redirects, + allFiles, + relAllFiles, + markdownFiles, + routes, + mirroredRoutes, + }; +} + +export function resolveRoute( + route: string, + { redirects, routes }: { redirects: Map<string, string>; routes: Set<string> }, +) { + let current = normalizeRoute(route); + if (current === "/") { + return { ok: true, terminal: "/" }; + } + + const seen = new Set([current]); + while (redirects.has(current)) { + current = normalizeRoute(redirects.get(current) ?? ""); + if (seen.has(current)) { + return { ok: false, terminal: current, loop: true }; + } + seen.add(current); + } + return { ok: routes.has(current), terminal: current }; +} + +/** Prepares a docs directory, mirroring ClawHub docs when available. */ +export function prepareMirroredDocsDir( + sourceDir = DOCS_DIR, + options: { + resolveClawHubRepoPathImpl?: ( + value?: string, + options?: { required?: boolean }, + ) => string | undefined; + syncClawHubDocsTreeImpl?: ( + targetDocsDir: string, + options?: { repoPath?: string; required?: boolean }, + ) => unknown; + } = {}, +) { + const sourceRoot = path.resolve(sourceDir); + if (sourceRoot !== path.resolve(DOCS_DIR)) { + return { dir: sourceRoot, mirroredClawHub: false, cleanup: () => {} }; + } + + const resolveClawHubRepoPathImpl = options.resolveClawHubRepoPathImpl ?? resolveClawHubRepoPath; + const syncClawHubDocsTreeImpl = options.syncClawHubDocsTreeImpl ?? syncClawHubDocsTree; + const clawhubRepo = resolveClawHubRepoPathImpl("", { required: false }); + if (!clawhubRepo) { + return { dir: sourceRoot, mirroredClawHub: false, cleanup: () => {} }; + } + + const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-docs-link-audit-")); + const tempDir = path.join(tempRoot, "docs"); + try { + fs.cpSync(sourceRoot, tempDir, { recursive: true }); + syncClawHubDocsTreeImpl(tempDir, { repoPath: clawhubRepo, required: false }); + return { + dir: tempDir, + mirroredClawHub: true, + cleanup: () => fs.rmSync(tempRoot, { recursive: true, force: true }), + }; + } catch (error) { + fs.rmSync(tempRoot, { recursive: true, force: true }); + throw error; + } +} + +function parseAuditUrl( + href: string, + base = "https://docs.openclaw.ai", +): Result<{ hostname: string; pathname: string; hash: string }, string> { + try { + const url = new URL(href, base); + return ok({ + hostname: url.hostname, + pathname: + url.hostname === "docs.openclaw.ai" ? decodeURIComponent(url.pathname) : url.pathname, + hash: url.hash, + }); + } catch (error) { + return err(error instanceof URIError ? "malformed URL path" : "malformed URL"); + } +} + +/** + * Explains that a fragment into a ClawHub-mirrored route could not be checked + * because the upstream source is absent from this checkout. + */ +function mirroredFragmentReason(terminal: string, hash: string) { + return `fragment unverified without the ClawHub source checkout (terminal: ${terminal}${hash}); set ${CLAWHUB_REPO_ENV}`; +} + +/** + * Audits local docs links against route, file, and redirect indexes. + */ +export function auditDocsLinks( + options: { docsDir?: string; allowExternalClawHubRoutes?: boolean; anchors?: boolean } = {}, +) { + const docsDir = options.docsDir ?? DOCS_DIR; + const index = buildAuditIndex(docsDir, { + allowExternalClawHubRoutes: options.allowExternalClawHubRoutes === true, + }); + const broken: Array<{ file: string; line: number; link: string; reason: string }> = []; + let checked = 0; + let unverifiedMirroredFragments = 0; + + // The publisher writes physical/index routes; frontmatter permalinks do not + // create pages. Only emitted pages and configured redirects can prove fragments. + const pages = new Map<string, ReturnType<typeof parseDocsDocument>>(); + const pageRoute = (rel: string) => + normalizeRoute(rel.replace(/\.mdx?$/i, "").replace(/(?:^|\/)index$/, "")); + for (const abs of index.markdownFiles) { + const rel = normalizeSlashes(path.relative(index.docsDir, abs)); + pages.set( + pageRoute(rel), + parseDocsDocument(fs.readFileSync(abs, "utf8"), undefined, { + sourceFile: abs, + root: path.dirname(index.docsDir), + pageRoute: pageRoute(rel), + mapLink: (href: string, line: number | undefined) => ({ href, line: line ?? 0 }), + }), + ); + } + const collisions = [...pages].flatMap(([route, document]) => + document.collisions.map((collision) => ({ route, ...collision })), + ); + if (options.anchors) { + for (const collision of collisions) { + if (collision.reason === "duplicate authored/canonical ID") { + broken.push({ + file: collision.route, + line: 0, + link: `#${collision.id}`, + reason: collision.reason, + }); + } + } + } + const redirectTargets = new Map<string, ReturnType<typeof parseAuditUrl>>(); + if (options.anchors) { + const records = resolveRedirects({ + redirects: index.docsConfig.redirects ?? [], + pages: [...pages.keys(), ...index.mirroredRoutes].map((route) => ({ + route, + markdownRoute: `${route}.md`, + })), + localeCodes: ["en"], + prefixes: [], + publicPath: (route: string) => route, + onError: (source: string, error: Error) => + broken.push({ file: "docs.json", line: 0, link: source, reason: error.message }), + }); + for (const record of records) { + const destinationResult = parseAuditUrl(record.destination); + redirectTargets.set(record.source, destinationResult); + if (!destinationResult.ok) { + broken.push({ + file: "docs.json", + line: 0, + link: record.source, + reason: destinationResult.error, + }); + continue; + } + const destination = destinationResult.value; + if (destination.hostname !== "docs.openclaw.ai") { + continue; + } + const page = pages.get(destination.pathname); + if (page && destination.hash && !resolveDocsFragment(destination.hash, new Set(page.ids))) { + broken.push({ + file: "docs.json", + line: 0, + link: record.source, + reason: `redirect fragment not found: ${record.destination}`, + }); + continue; + } + // A redirect into a ClawHub-mirrored route now resolves off the navigation + // declaration, which proves the route but not the fragment. Say so instead + // of letting the destination fragment go silently unchecked. + if (!page && destination.hash) { + const terminal = normalizeRoute(destination.pathname); + if (index.mirroredRoutes.has(terminal)) { + unverifiedMirroredFragments++; + broken.push({ + file: "docs.json", + line: 0, + link: record.source, + reason: mirroredFragmentReason(terminal, destination.hash), + }); + } + } + } + } + for (const abs of index.markdownFiles) { + const rel = normalizeSlashes(path.relative(index.docsDir, abs)); + const document = pages.get(pageRoute(rel))!; + for (const { href: raw, line } of document.links) { + const local = !/^(?:[a-z][a-z0-9+.-]*:|\/\/)/i.test(raw); + if (!options.anchors && (!local || raw.startsWith("#"))) { + continue; + } + const urlResult = parseAuditUrl(raw, `https://docs.openclaw.ai${pageRoute(rel)}`); + if (urlResult.ok && urlResult.value.hostname !== "docs.openclaw.ai") { + continue; + } + checked++; + if (!urlResult.ok) { + broken.push({ file: rel, line, link: raw, reason: urlResult.error }); + continue; + } + const url = urlResult.value; + const route = pageRoute(url.pathname).replace(/^\/en(?:\/|$)/, "/"); + const destinationResult = + options.anchors && !pages.has(route) && redirectTargets.has(route) + ? redirectTargets.get(route)! + : urlResult; + if (!destinationResult.ok) { + broken.push({ file: rel, line, link: raw, reason: destinationResult.error }); + continue; + } + const destination = destinationResult.value; + if (destination.hostname !== "docs.openclaw.ai") { + continue; + } + const terminal = pageRoute(destination.pathname).replace(/^\/en(?:\/|$)/, "/"); + const page = pages.get(terminal); + const resolved = resolveRoute(route, { redirects: index.redirects, routes: index.routes }); + // A route the navigation mirrors from ClawHub is published even though no + // Markdown page exists locally. Its existence is proven by the declaration; + // only its fragments need the real source checkout. + const mirroredOnly = !page && index.mirroredRoutes.has(terminal); + if (mirroredOnly) { + // Plain mode does not inspect fragments at all, so a declared mirrored + // route resolves there whether or not the link carries one. Only anchor + // mode reports the fragment it cannot verify without the source. + if (options.anchors && destination.hash) { + unverifiedMirroredFragments++; + broken.push({ + file: rel, + line, + link: raw, + reason: mirroredFragmentReason(terminal, destination.hash), + }); + } + continue; + } + if ( + !page && + (options.anchors || !resolved.ok) && + !index.relAllFiles.has(url.pathname.slice(1)) + ) { + broken.push({ + file: rel, + line, + link: raw, + reason: `route/file not found (terminal: ${resolved.terminal})`, + }); + } else if ( + options.anchors && + destination.hash && + page && + (/\.mdx?$/.test(destination.pathname) || + !resolveDocsFragment(destination.hash, new Set(page.ids))) + ) { + broken.push({ + file: rel, + line, + link: raw, + reason: /\.mdx?$/.test(destination.pathname) + ? "fragment requires an HTML page, not raw Markdown" + : `fragment not found (terminal: ${terminal}${destination.hash})`, + }); + } + } + } + + for (const page of collectNavPageEntries(index.docsConfig.navigation || [])) { + if (isGeneratedTranslatedDoc(page)) { + continue; + } + checked++; + const route = normalizeRoute(page); + const resolvedRoute = resolveRoute(route, { + redirects: index.redirects, + routes: index.routes, + }); + if (resolvedRoute.ok) { + continue; + } + + broken.push({ + file: "docs.json", + line: 0, + link: page, + reason: `navigation page not published (terminal: ${resolvedRoute.terminal})`, + }); + } + + return { checked, broken, collisions, unverifiedMirroredFragments }; +} + +/** Runs the docs link audit CLI. */ +function runDocsLinkAuditCli() { + const args = process.argv.slice(2); + if (args[0] === "--prepare-external-links") { + if (args.length !== 2 || !args[1]) { + console.error("usage: docs-link-audit.mjs --prepare-external-links <output-dir>"); + return 1; + } + const result = prepareExternalLinkAuditTree(ROOT, path.resolve(ROOT, args[1])); + console.log(`prepared_external_link_files=${result.files}`); + console.log(`projected_markdown_links=${result.projectedLinks}`); + return 0; + } + + const mirroredDocsDir = prepareMirroredDocsDir(DOCS_DIR); + try { + const { checked, broken, collisions, unverifiedMirroredFragments } = auditDocsLinks({ + docsDir: mirroredDocsDir.dir, + allowExternalClawHubRoutes: !mirroredDocsDir.mirroredClawHub, + anchors: args.includes("--anchors"), + }); + console.log(`checked_internal_links=${checked}`); + console.log(`broken_links=${broken.length}`); + if (args.includes("--anchors")) { + console.log( + `omitted_compatibility_aliases=${collisions.filter((item) => item.reason === "compatibility alias collision").length}`, + ); + console.log(`mirrored_clawhub_docs=${mirroredDocsDir.mirroredClawHub ? "yes" : "no"}`); + if (!mirroredDocsDir.mirroredClawHub) { + console.error( + `docs:check-links: ClawHub docs are authored in openclaw/clawhub and are absent from this checkout. ` + + `/clawhub/** routes were accepted from docs.json navigation and their fragments were not checked` + + `${unverifiedMirroredFragments > 0 ? ` (${unverifiedMirroredFragments} reported unverified)` : ""}. ` + + `Set ${CLAWHUB_REPO_ENV} to a ClawHub checkout to verify them.`, + ); + } + } + + for (const item of broken) { + console.log(`${item.file}:${item.line || "unknown"} :: ${item.link} :: ${item.reason}`); + } + + return broken.length > 0 ? 1 : 0; + } finally { + mirroredDocsDir.cleanup(); + } +} + +function isCliEntry() { + const cliArg = process.argv[1]; + return cliArg ? import.meta.url === pathToFileURL(cliArg).href : false; +} + +if (isCliEntry()) { + process.exit(runDocsLinkAuditCli()); +} diff --git a/scripts/docs-list.d.ts b/scripts/docs-list.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..9ab34dd84879afcd6090f4c8c01053a709802364 --- /dev/null +++ b/scripts/docs-list.d.ts @@ -0,0 +1,4 @@ +export function renderDocsHeadingMap( + docsDir?: string, + options?: { relativePath?: (base: string, fullPath: string) => string }, +): string; diff --git a/scripts/docs-spellcheck.sh b/scripts/docs-spellcheck.sh new file mode 100644 index 0000000000000000000000000000000000000000..ed5551d4cc996084c09727363a9dc5a21858a095 --- /dev/null +++ b/scripts/docs-spellcheck.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Bash 5.3+ can deadlock writing heredoc pipes on macOS before the reader starts. +if [[ ${OSTYPE:-} == darwin* && $BASH != /bin/bash ]] && ((BASH_VERSINFO[0] > 5 || (BASH_VERSINFO[0] == 5 && BASH_VERSINFO[1] >= 3))); then + exec /bin/bash "$0" "$@" +fi +set -euo pipefail + +mode="${1:-}" +write_flag=() +if [[ "$mode" == "--write" ]]; then + write_flag=(-w) +fi + +args=( + README.md + docs + --skip=*.png,*.jpg,*.jpeg,*.gif,*.svg + -D + - + -D + scripts/codespell-dictionary.txt + -I + scripts/codespell-ignore.txt + ${write_flag[@]+"${write_flag[@]}"} +) + +if command -v codespell >/dev/null 2>&1; then + codespell "${args[@]}" + exit 0 +fi + +if command -v python3 >/dev/null 2>&1; then + python3 -m pip install --user --disable-pip-version-check --break-system-packages codespell >/dev/null 2>&1 || \ + python3 -m pip install --user --disable-pip-version-check codespell >/dev/null 2>&1 + + user_bin="$(python3 - <<'PY' +import site +print(f"{site.USER_BASE}/bin") +PY +)" + if [[ -x "${user_bin}/codespell" ]]; then + "${user_bin}/codespell" "${args[@]}" + exit 0 + fi +fi + +echo "codespell unavailable: install codespell or python3" >&2 +exit 1 diff --git a/scripts/docs-sync-publish.mjs b/scripts/docs-sync-publish.mjs new file mode 100644 index 0000000000000000000000000000000000000000..b076be0745787f9abdb329031200fda92309b024 --- /dev/null +++ b/scripts/docs-sync-publish.mjs @@ -0,0 +1,974 @@ +#!/usr/bin/env node + +// Syncs source docs into the generated publish tree. +import { execFileSync } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { isDeepStrictEqual } from "node:util"; +import { renderDocsHeadingMap } from "./docs-list.js"; +import { requireOptionArgument } from "./lib/arg-utils.runtime.mjs"; +import { repairMintlifyAccordionIndentation } from "./lib/mintlify-accordion.mjs"; +import { resolveRepoRoot } from "./lib/repo-root.mjs"; + +const ROOT = resolveRepoRoot(import.meta.url); +const SOURCE_DOCS_DIR = path.join(ROOT, "docs"); +const SOURCE_CONFIG_PATH = path.join(SOURCE_DOCS_DIR, "docs.json"); +const SLUGIFY_PACKAGE = "@sindresorhus/slugify"; +const INTERNAL_DOCS_DIRS = ["internal"]; +const DEFAULT_CLAWHUB_SOURCE_REPO = "openclaw/clawhub"; +const CLAWHUB_DOCS_TARGET_DIR = "clawhub"; +export const CLAWHUB_REPO_ENV = "OPENCLAW_DOCS_SYNC_CLAWHUB_REPO"; +const DEFAULT_CLAWHUB_REPO_CANDIDATES = [ + path.resolve(ROOT, "..", "clawhub-docs-clawhub"), + path.resolve(ROOT, "..", "clawhub"), +]; +const SYNC_SUPPORT_FILES = [ + // File URLs declare the copied runtime closure without executing modules: + // source sync runs before parser dependencies are installed. + { + source: new URL("./lib/docs-markdown.mjs", import.meta.url), + target: path.join(".openclaw-sync", "lib", "docs-markdown.mjs"), + }, + { + source: new URL("./lib/docs-redirects.mjs", import.meta.url), + target: path.join(".openclaw-sync", "lib", "docs-redirects.mjs"), + }, + { + source: path.join(ROOT, "scripts", "check-docs-mdx.mjs"), + target: path.join(".openclaw-sync", "check-docs-mdx.mjs"), + }, + { + source: path.join(ROOT, "scripts", "check-docs-mdx.mts"), + target: path.join(".openclaw-sync", "check-docs-mdx.mts"), + }, + { + source: path.join(ROOT, "scripts", "lib", "arg-utils.runtime.mjs"), + target: path.join(".openclaw-sync", "lib", "arg-utils.runtime.mjs"), + }, + { + source: path.join(ROOT, "scripts", "lib", "tsx-cli-shim.mjs"), + target: path.join(".openclaw-sync", "lib", "tsx-cli-shim.mjs"), + }, + { + source: path.join(ROOT, "scripts", "lib", "local-check-runtime.mts"), + target: path.join(".openclaw-sync", "lib", "local-check-runtime.mts"), + }, + { + source: path.join(ROOT, "scripts", "tsx.mjs"), + target: path.join(".openclaw-sync", "tsx.mjs"), + }, + { + source: path.join(ROOT, "scripts", "lib", "mintlify-accordion.mjs"), + target: path.join(".openclaw-sync", "lib", "mintlify-accordion.mjs"), + }, + { + source: path.join(ROOT, ".github", "codex", "prompts", "docs-mdx-repair.md"), + target: path.join(".openclaw-sync", "docs-mdx-repair.md"), + }, +]; +const GENERATED_LOCALES = [ + { + language: "zh-Hans", + dir: "zh-CN", + navFile: "zh-Hans-navigation.json", + tmFile: "zh-CN.tm.jsonl", + navMode: "overlay", + }, + { + language: "zh-Hant", + dir: "zh-TW", + navFile: "zh-Hant-navigation.json", + tmFile: "zh-TW.tm.jsonl", + navMode: "clone-en", + }, + { + language: "ja", + dir: "ja-JP", + navFile: "ja-navigation.json", + tmFile: "ja-JP.tm.jsonl", + navMode: "clone-en", + }, + { + language: "es", + dir: "es", + navFile: "es-navigation.json", + tmFile: "es.tm.jsonl", + navMode: "clone-en", + }, + { + language: "pt-BR", + dir: "pt-BR", + navFile: "pt-BR-navigation.json", + tmFile: "pt-BR.tm.jsonl", + navMode: "clone-en", + }, + { + language: "ko", + dir: "ko", + navFile: "ko-navigation.json", + tmFile: "ko.tm.jsonl", + navMode: "clone-en", + }, + { + language: "de", + dir: "de", + navFile: "de-navigation.json", + tmFile: "de.tm.jsonl", + navMode: "clone-en", + }, + { + language: "fr", + dir: "fr", + navFile: "fr-navigation.json", + tmFile: "fr.tm.jsonl", + navMode: "clone-en", + }, + { + language: "hi", + dir: "hi", + navFile: "hi-navigation.json", + tmFile: "hi.tm.jsonl", + navMode: "clone-en", + }, + { + language: "ar", + dir: "ar", + navFile: "ar-navigation.json", + tmFile: "ar.tm.jsonl", + navMode: "clone-en", + }, + { + language: "it", + dir: "it", + navFile: "it-navigation.json", + tmFile: "it.tm.jsonl", + navMode: "clone-en", + }, + { + language: "vi", + dir: "vi", + navFile: "vi-navigation.json", + tmFile: "vi.tm.jsonl", + navMode: "clone-en", + }, + { + language: "nl", + dir: "nl", + navFile: "nl-navigation.json", + tmFile: "nl.tm.jsonl", + navMode: "clone-en", + }, + { + language: "fa", + dir: "fa", + navFile: "fa-navigation.json", + tmFile: "fa.tm.jsonl", + navMode: "clone-en", + // Mintlify rejected `fa` in navigation.languages when this override was + // added (2026-04-29). The `fa/` docs tree is still generated and published; + // only the navigation entry is withheld. Keep generated docs and + // translation memory so the locale stays available once the docs host + // accepts it. Re-test against the host before removing this flag. + navigation: false, + }, + { + language: "tr", + dir: "tr", + navFile: "tr-navigation.json", + tmFile: "tr.tm.jsonl", + navMode: "clone-en", + }, + { + language: "uk", + dir: "uk", + navFile: "uk-navigation.json", + tmFile: "uk.tm.jsonl", + navMode: "clone-en", + }, + { + language: "id", + dir: "id", + navFile: "id-navigation.json", + tmFile: "id.tm.jsonl", + navMode: "clone-en", + }, + { + language: "pl", + dir: "pl", + navFile: "pl-navigation.json", + tmFile: "pl.tm.jsonl", + navMode: "clone-en", + }, + { + language: "th", + dir: "th", + navFile: "th-navigation.json", + tmFile: "th.tm.jsonl", + navMode: "clone-en", + // Mintlify rejected `th` in navigation.languages when this override was + // added (2026-04-29). The `th/` docs tree is still generated and published; + // only the navigation entry is withheld. Keep generated docs and + // translation memory so the locale stays available once the docs host + // accepts it. Re-test against the host before removing this flag. + navigation: false, + }, + { + language: "ru", + dir: "ru", + navFile: "ru-navigation.json", + tmFile: "ru.tm.jsonl", + navMode: "clone-en", + }, +]; + +export function parseArgs(argv) { + const args = { + target: "", + sourceRepo: "", + sourceSha: "", + clawhubRepo: process.env[CLAWHUB_REPO_ENV] || "", + clawhubSourceRepo: + process.env.OPENCLAW_DOCS_SYNC_CLAWHUB_SOURCE_REPO || DEFAULT_CLAWHUB_SOURCE_REPO, + clawhubSourceSha: process.env.OPENCLAW_DOCS_SYNC_CLAWHUB_SOURCE_SHA || "", + }; + + for (let index = 0; index < argv.length; index += 1) { + const part = argv[index]; + switch (part) { + case "--target": + args.target = requireOptionArgument(argv, index, part); + index += 1; + break; + case "--source-repo": + args.sourceRepo = requireOptionArgument(argv, index, part); + index += 1; + break; + case "--source-sha": + args.sourceSha = requireOptionArgument(argv, index, part); + index += 1; + break; + case "--clawhub-repo": + args.clawhubRepo = requireOptionArgument(argv, index, part); + index += 1; + break; + case "--clawhub-source-repo": + args.clawhubSourceRepo = requireOptionArgument(argv, index, part); + index += 1; + break; + case "--clawhub-source-sha": + args.clawhubSourceSha = requireOptionArgument(argv, index, part); + index += 1; + break; + default: + throw new Error(`unknown arg: ${part}`); + } + } + + if (!args.target) { + throw new Error("missing --target"); + } + + return args; +} + +function run(command, args, options = {}) { + execFileSync(command, args, { + cwd: ROOT, + stdio: "inherit", + ...options, + }); +} + +function ensureDir(dirPath) { + fs.mkdirSync(dirPath, { recursive: true }); +} + +function normalizeSlashes(value) { + return value.replace(/\\/g, "/"); +} + +function walkFiles(entryPath, out = []) { + if (!fs.existsSync(entryPath)) { + return out; + } + + const stat = fs.statSync(entryPath); + if (stat.isFile()) { + out.push(entryPath); + return out; + } + + for (const entry of fs.readdirSync(entryPath, { withFileTypes: true })) { + if (entry.name === "node_modules" || entry.name === ".git") { + continue; + } + walkFiles(path.join(entryPath, entry.name), out); + } + return out; +} + +function walkMarkdownFiles(entryPath, out = []) { + if (!fs.existsSync(entryPath)) { + return out; + } + + const stat = fs.statSync(entryPath); + if (stat.isFile()) { + if (/\.mdx?$/i.test(entryPath)) { + out.push(entryPath); + } + return out; + } + + for (const entry of fs.readdirSync(entryPath, { withFileTypes: true })) { + if (entry.name === "node_modules" || entry.name === ".git") { + continue; + } + walkMarkdownFiles(path.join(entryPath, entry.name), out); + } + return out; +} + +function readJson(filePath) { + return JSON.parse(fs.readFileSync(filePath, "utf8")); +} + +function writeJson(filePath, value) { + ensureDir(path.dirname(filePath)); + fs.writeFileSync(filePath, `${JSON.stringify(value, null, 2)}\n`); +} + +function getGitHeadSha(repoPath) { + try { + return execFileSync("git", ["-C", repoPath, "rev-parse", "HEAD"], { + cwd: ROOT, + encoding: "utf8", + stdio: ["ignore", "pipe", "ignore"], + }).trim(); + } catch { + return ""; + } +} + +/** + * Resolves the local ClawHub repository path used for docs mirroring. + */ +export function resolveClawHubRepoPath(value = "", options = {}) { + const required = options.required !== false; + const candidates = [ + value, + process.env[CLAWHUB_REPO_ENV] || "", + ...DEFAULT_CLAWHUB_REPO_CANDIDATES, + ].filter((candidate) => candidate.trim().length > 0); + + for (const candidate of candidates) { + const repoPath = path.resolve(candidate); + if (fs.existsSync(path.join(repoPath, "docs"))) { + return repoPath; + } + } + + if (required) { + throw new Error(`missing ClawHub docs source; pass --clawhub-repo or set ${CLAWHUB_REPO_ENV}`); + } + return ""; +} + +function prefixLocalePage(entry, localeDir) { + if (typeof entry === "string") { + return `${localeDir}/${entry}`; + } + if (Array.isArray(entry)) { + return entry.map((item) => prefixLocalePage(item, localeDir)); + } + if (!entry || typeof entry !== "object") { + return entry; + } + + const clone = { ...entry }; + if (typeof clone.page === "string") { + clone.page = `${localeDir}/${clone.page}`; + } + if (Array.isArray(clone.pages)) { + clone.pages = clone.pages.map((item) => prefixLocalePage(item, localeDir)); + } + return clone; +} + +function prefixLocaleNavGroup(group, localeDir) { + const clone = { ...group }; + if (Array.isArray(clone.pages)) { + clone.pages = clone.pages.map((entry) => prefixLocalePage(entry, localeDir)); + } + return clone; +} + +function prefixLocaleNavTab(tab, localeDir) { + const clone = { ...tab }; + if (Array.isArray(clone.pages)) { + clone.pages = clone.pages.map((entry) => prefixLocalePage(entry, localeDir)); + } + if (Array.isArray(clone.groups)) { + clone.groups = clone.groups.map((group) => prefixLocaleNavGroup(group, localeDir)); + } + return clone; +} + +function cloneEnglishLanguageNav(englishNav, locale) { + if (!englishNav) { + throw new Error("docs/docs.json is missing navigation.languages.en"); + } + return { + ...englishNav, + language: locale.language, + tabs: Array.isArray(englishNav.tabs) + ? englishNav.tabs + .filter((tab) => tab?.tab !== "ClawHub") + .map((tab) => prefixLocaleNavTab(tab, locale.dir)) + : englishNav.tabs, + }; +} + +function collectNavPages(entry, pages = new Set()) { + if (typeof entry === "string") { + pages.add(entry); + return pages; + } + if (Array.isArray(entry)) { + for (const item of entry) { + collectNavPages(item, pages); + } + return pages; + } + if (!entry || typeof entry !== "object") { + return pages; + } + if (typeof entry.page === "string") { + pages.add(entry.page); + } + collectNavPages(entry.pages, pages); + collectNavPages(entry.groups, pages); + collectNavPages(entry.tabs, pages); + return pages; +} + +function findBestNavMatchIndex(candidates, overlayEntry, excludedIndexes = new Set()) { + const overlayPages = collectNavPages(overlayEntry); + let bestIndex = -1; + let bestScore = 0; + for (const [index, candidate] of candidates.entries()) { + if (excludedIndexes.has(index)) { + continue; + } + const candidatePages = collectNavPages(candidate); + let score = 0; + for (const page of overlayPages) { + if (candidatePages.has(page)) { + score += 1; + } + } + if (score > bestScore) { + bestIndex = index; + bestScore = score; + } + } + return bestIndex; +} + +export function applyLocaleNavLabelOverlay(fullNav, labelOverlay) { + const tabs = Array.isArray(fullNav.tabs) + ? fullNav.tabs.map((tab) => ({ + ...tab, + groups: Array.isArray(tab.groups) ? tab.groups.map((group) => ({ ...group })) : tab.groups, + })) + : fullNav.tabs; + const composed = { ...fullNav, tabs }; + if (!Array.isArray(tabs) || !Array.isArray(labelOverlay?.tabs)) { + return composed; + } + + for (const overlayTab of labelOverlay.tabs) { + const tabIndex = findBestNavMatchIndex(tabs, overlayTab); + if (tabIndex < 0) { + continue; + } + const tab = tabs[tabIndex]; + if (typeof overlayTab.tab === "string") { + tab.tab = overlayTab.tab; + } + if (!Array.isArray(tab.groups) || !Array.isArray(overlayTab.groups)) { + continue; + } + const matchedGroupIndexes = new Set(); + for (const overlayGroup of overlayTab.groups) { + const groupIndex = findBestNavMatchIndex(tab.groups, overlayGroup, matchedGroupIndexes); + if (groupIndex >= 0 && typeof overlayGroup.group === "string") { + tab.groups[groupIndex].group = overlayGroup.group; + matchedGroupIndexes.add(groupIndex); + } + } + } + return composed; +} + +function composeLocaleNav(locale, englishNav) { + const cloned = cloneEnglishLanguageNav(englishNav, locale); + if (!locale.navFile) { + return cloned; + } + const overlayPath = path.join(SOURCE_DOCS_DIR, ".i18n", locale.navFile); + if (!fs.existsSync(overlayPath)) { + console.warn( + `docs-sync-publish: missing navigation overlay ${locale.navFile} for locale ${locale.language}; publishing the English sidebar labels for that locale.`, + ); + return cloned; + } + return applyLocaleNavLabelOverlay(cloned, readJson(overlayPath)); +} + +export function composeDocsConfig() { + const sourceConfig = readJson(SOURCE_CONFIG_PATH); + const languages = sourceConfig?.navigation?.languages; + + if (!Array.isArray(languages)) { + throw new Error("docs/docs.json is missing navigation.languages"); + } + + const englishNav = languages.find((entry) => entry?.language === "en"); + const generatedLanguageSet = new Set( + GENERATED_LOCALES.filter((entry) => entry.navigation !== false).map((entry) => entry.language), + ); + const withoutGenerated = languages.filter((entry) => !generatedLanguageSet.has(entry?.language)); + const enIndex = withoutGenerated.findIndex((entry) => entry?.language === "en"); + const generated = GENERATED_LOCALES.filter((entry) => entry.navigation !== false).map((entry) => + composeLocaleNav(entry, englishNav), + ); + if (enIndex === -1) { + withoutGenerated.push(...generated); + } else { + withoutGenerated.splice(enIndex + 1, 0, ...generated); + } + + return { + ...sourceConfig, + navigation: { + ...sourceConfig.navigation, + languages: withoutGenerated, + }, + }; +} + +export function reportOrphanLocaleDocs(targetDocsDir) { + let orphaned = 0; + for (const locale of GENERATED_LOCALES) { + const localeDir = path.join(targetDocsDir, locale.dir); + if (!fs.existsSync(localeDir)) { + continue; + } + for (const filePath of walkMarkdownFiles(localeDir)) { + const relativePath = path.relative(localeDir, filePath); + // Check the assembled publish tree so externally mirrored docs, such as + // ClawHub pages, count as valid English sources too. + const englishBase = path.join(targetDocsDir, relativePath); + const englishMd = englishBase.replace(/\.mdx?$/i, ".md"); + const englishMdx = englishBase.replace(/\.mdx?$/i, ".mdx"); + if (fs.existsSync(englishMd) || fs.existsSync(englishMdx)) { + continue; + } + orphaned += 1; + } + } + + if (orphaned > 0) { + // Translation artifacts update inbound links and delete their old target + // together. Docs sync must not publish the deletion ahead of that step. + console.log(`Deferred ${orphaned} orphan localized doc(s) to translation finalization.`); + } + return orphaned; +} + +function repairGeneratedLocaleDocs(targetDocsDir) { + let repaired = 0; + for (const locale of GENERATED_LOCALES) { + const localeDir = path.join(targetDocsDir, locale.dir); + for (const filePath of walkMarkdownFiles(localeDir)) { + const raw = fs.readFileSync(filePath, "utf8"); + const repairedRaw = repairMintlifyAccordionIndentation(raw); + if (repairedRaw === raw) { + continue; + } + fs.writeFileSync(filePath, repairedRaw); + repaired += 1; + } + } + + if (repaired > 0) { + console.log(`Repaired Mintlify accordion indentation in ${repaired} generated locale doc(s).`); + } +} + +function pruneInternalDocs(targetDocsDir) { + let pruned = 0; + for (const relativeDir of INTERNAL_DOCS_DIRS) { + const dirPath = path.join(targetDocsDir, relativeDir); + if (!fs.existsSync(dirPath)) { + continue; + } + fs.rmSync(dirPath, { recursive: true, force: true }); + pruned += 1; + } + + if (pruned > 0) { + console.log(`Pruned ${pruned} internal-only docs director${pruned === 1 ? "y" : "ies"}.`); + } +} + +function shouldExcludeClawHubDocsPath(relativePath) { + const normalized = normalizeSlashes(relativePath); + return ( + normalized === "specs" || normalized.startsWith("specs/") || normalized.includes("/specs/") + ); +} + +function toClawHubTargetRelativePath(relativePath) { + const normalized = normalizeSlashes(relativePath); + if (normalized === "README.md") { + return ""; + } + if (normalized === "clawhub.md") { + return "index.md"; + } + return normalized.replace(/\/README\.md$/iu, "/index.md"); +} + +function toClawHubDocsRoute(relativePath) { + const targetRelativePath = toClawHubTargetRelativePath(relativePath); + if (!targetRelativePath) { + return ""; + } + + const normalized = targetRelativePath.replace(/\.mdx?$/iu, ""); + if (normalized === "index") { + return `/${CLAWHUB_DOCS_TARGET_DIR}`; + } + if (normalized.endsWith("/index")) { + return `/${CLAWHUB_DOCS_TARGET_DIR}/${normalized.slice(0, -"/index".length)}`; + } + return `/${CLAWHUB_DOCS_TARGET_DIR}/${normalized}`; +} + +function splitLinkTarget(value) { + const match = /^(\S+)(.*)$/su.exec(value); + return { + target: match?.[1] ?? value, + suffix: match?.[2] ?? "", + }; +} + +function splitTargetParts(value) { + const hashIndex = value.indexOf("#"); + const queryIndex = value.indexOf("?"); + const splitIndexes = [hashIndex, queryIndex].filter((index) => index >= 0); + const splitIndex = splitIndexes.length > 0 ? Math.min(...splitIndexes) : -1; + if (splitIndex === -1) { + return { pathPart: value, rest: "" }; + } + return { + pathPart: value.slice(0, splitIndex), + rest: value.slice(splitIndex), + }; +} + +function rewriteClawHubMarkdownLinkTarget(rawTarget, relativeSourceDir, source) { + const { target, suffix } = splitLinkTarget(rawTarget); + if (/^(?:https?:|mailto:|tel:|data:|#)/iu.test(target) || target.startsWith("/")) { + return rawTarget; + } + + const { pathPart, rest } = splitTargetParts(target); + if (!pathPart) { + return rawTarget; + } + + let normalizedRelative; + if (pathPart.startsWith("docs/")) { + normalizedRelative = normalizeSlashes(pathPart.slice("docs/".length)); + } else if ( + pathPart.startsWith("./") || + pathPart.startsWith("../") || + /\.mdx?$/iu.test(pathPart) + ) { + normalizedRelative = normalizeSlashes(path.normalize(path.join(relativeSourceDir, pathPart))); + } else { + return rawTarget; + } + + if (normalizedRelative.startsWith("../")) { + const sourceRef = source.sha || "main"; + const repoRelative = normalizeSlashes( + path.normalize(path.join("docs", relativeSourceDir, pathPart)), + ).replace(/^(?:\.\.\/)+/u, ""); + return `https://github.com/${source.repository}/blob/${sourceRef}/${repoRelative}${rest}${suffix}`; + } + + if (!/\.mdx?$/iu.test(normalizedRelative)) { + return rawTarget; + } + + const route = toClawHubDocsRoute(normalizedRelative); + return route ? `${route}${rest}${suffix}` : rawTarget; +} + +function rewriteClawHubMarkdownLinks(raw, relativeSourcePath, source) { + const relativeSourceDir = normalizeSlashes(path.dirname(relativeSourcePath)); + const baseDir = relativeSourceDir === "." ? "" : relativeSourceDir; + return raw.replace(/(!?\[[^\]]*\]\()([^)]+)(\))/gu, (_match, prefix, target, suffix) => { + return `${prefix}${rewriteClawHubMarkdownLinkTarget(target, baseDir, source)}${suffix}`; + }); +} + +/** + * Mirrors ClawHub docs into the target docs tree. + */ +export function syncClawHubDocsTree(targetDocsDir, options = {}) { + const repoPath = resolveClawHubRepoPath(options.repoPath || "", { + required: options.required !== false, + }); + if (!repoPath) { + return { + repository: options.sourceRepo || DEFAULT_CLAWHUB_SOURCE_REPO, + sha: options.sourceSha || "", + path: "", + files: 0, + }; + } + + const sourceDocsDir = path.join(repoPath, "docs"); + const targetDir = path.join(targetDocsDir, CLAWHUB_DOCS_TARGET_DIR); + const source = { + repository: options.sourceRepo || DEFAULT_CLAWHUB_SOURCE_REPO, + sha: options.sourceSha || getGitHeadSha(repoPath), + }; + + fs.rmSync(targetDir, { recursive: true, force: true }); + ensureDir(targetDir); + + let copied = 0; + for (const sourcePath of walkFiles(sourceDocsDir)) { + const relativeSourcePath = normalizeSlashes(path.relative(sourceDocsDir, sourcePath)); + if (shouldExcludeClawHubDocsPath(relativeSourcePath)) { + continue; + } + + const targetRelativePath = toClawHubTargetRelativePath(relativeSourcePath); + if (!targetRelativePath) { + continue; + } + const targetPath = path.join(targetDir, targetRelativePath); + ensureDir(path.dirname(targetPath)); + + if (/\.mdx?$/iu.test(sourcePath)) { + const raw = fs.readFileSync(sourcePath, "utf8"); + fs.writeFileSync( + targetPath, + rewriteClawHubMarkdownLinks(raw, relativeSourcePath, source), + "utf8", + ); + } else { + fs.copyFileSync(sourcePath, targetPath); + } + copied += 1; + } + + console.log(`Synced ${copied} ClawHub doc asset(s) from ${repoPath}.`); + return { + ...source, + path: repoPath, + files: copied, + }; +} + +function syncDocsTree(targetRoot, options = {}) { + const targetDocsDir = path.join(targetRoot, "docs"); + ensureDir(targetDocsDir); + + const localeFilters = GENERATED_LOCALES.flatMap((entry) => [ + "--filter", + `P ${entry.dir}/`, + "--filter", + `P .i18n/${entry.tmFile}`, + "--exclude", + `${entry.dir}/`, + "--exclude", + `.i18n/${entry.tmFile}`, + ]); + + run("rsync", [ + "-a", + "--delete", + "--filter", + "P .i18n/README.md", + "--exclude", + ".i18n/README.md", + ...INTERNAL_DOCS_DIRS.flatMap((dir) => ["--exclude", `${dir}/`]), + ...localeFilters, + `${SOURCE_DOCS_DIR}/`, + `${targetDocsDir}/`, + ]); + pruneInternalDocs(targetDocsDir); + writePublishedDocsMap(targetDocsDir); + + for (const locale of GENERATED_LOCALES) { + const sourceTmPath = path.join(SOURCE_DOCS_DIR, ".i18n", locale.tmFile); + const targetTmPath = path.join(targetDocsDir, ".i18n", locale.tmFile); + if (!fs.existsSync(targetTmPath) && fs.existsSync(sourceTmPath)) { + ensureDir(path.dirname(targetTmPath)); + fs.copyFileSync(sourceTmPath, targetTmPath); + } + } + + const clawhubSource = syncClawHubDocsTree(targetDocsDir, { + repoPath: options.clawhubRepo, + sourceRepo: options.clawhubSourceRepo, + sourceSha: options.clawhubSourceSha, + }); + reportOrphanLocaleDocs(targetDocsDir); + repairGeneratedLocaleDocs(targetDocsDir); + writeJson(path.join(targetDocsDir, "docs.json"), composeDocsConfig()); + return { clawhub: clawhubSource }; +} + +/** Writes the public heading map into the publish tree without committing an expanded mirror. */ +export function writePublishedDocsMap(targetDocsDir) { + const outputPath = path.join(targetDocsDir, "docs_map.md"); + fs.writeFileSync(outputPath, renderDocsHeadingMap(SOURCE_DOCS_DIR), "utf8"); + return outputPath; +} + +function writeSyncMetadata(targetRoot, args, sources) { + const metadata = { + repository: args.sourceRepo || "", + sha: args.sourceSha || "", + sources: { + openclaw: { + repository: args.sourceRepo || "", + sha: args.sourceSha || "", + }, + clawhub: { + repository: + sources.clawhub.repository || args.clawhubSourceRepo || DEFAULT_CLAWHUB_SOURCE_REPO, + sha: sources.clawhub.sha || args.clawhubSourceSha || "", + }, + }, + syncedAt: new Date().toISOString(), + }; + writeJson(path.join(targetRoot, ".openclaw-sync", "source.json"), metadata); +} + +function sourceSlugifyVersion() { + const version = readJson(path.join(ROOT, "package.json")).devDependencies[SLUGIFY_PACKAGE]; + if (typeof version !== "string" || !/^\d+\.\d+\.\d+(?:[-+][0-9A-Za-z.-]+)?$/.test(version)) { + throw new Error("docs sync requires an exact source slugify version"); + } + return version; +} + +function readPublishDependencies(targetRoot) { + return { + packageJson: readJson(path.join(targetRoot, "package.json")), + packageLock: readJson(path.join(targetRoot, "package-lock.json")), + }; +} + +function matchesSlugifyVersion({ packageJson, packageLock }, version) { + return ( + packageJson.devDependencies?.[SLUGIFY_PACKAGE] === version && + packageLock.lockfileVersion === 3 && + packageLock.packages?.[""]?.devDependencies?.[SLUGIFY_PACKAGE] === version && + packageLock.packages?.[`node_modules/${SLUGIFY_PACKAGE}`]?.version === version + ); +} + +/** Checks sync output here and against fresh publisher main after the workflow rebases. */ +export function validateDocsSyncDependencies(targetRoot, baseline) { + const current = readPublishDependencies(targetRoot); + const version = sourceSlugifyVersion(); + if (!matchesSlugifyVersion(current, version)) { + throw new Error(`docs sync publisher manifest and lock must both pin slugify ${version}`); + } + // Only this dependency's declaration and resolved row belong to source sync. + // Compare objects, not JSON formatting; never absorb unrelated npm or rebase churn. + const expected = structuredClone(baseline); + expected.packageJson.devDependencies[SLUGIFY_PACKAGE] = version; + expected.packageLock.packages[""].devDependencies[SLUGIFY_PACKAGE] = version; + const slugifyPath = `node_modules/${SLUGIFY_PACKAGE}`; + expected.packageLock.packages[slugifyPath] = current.packageLock.packages[slugifyPath]; + if (!isDeepStrictEqual(current, expected)) { + throw new Error( + "docs sync changed unrelated publisher dependencies; reconcile the lock before retrying", + ); + } + for (const entry of SYNC_SUPPORT_FILES) { + const copied = fs.readFileSync(path.join(targetRoot, entry.target)); + if (!fs.readFileSync(entry.source).equals(copied)) { + throw new Error(`docs sync support file differs from source: ${entry.target}`); + } + } +} + +function syncSupportFiles(targetRoot) { + const baseline = readPublishDependencies(targetRoot); + const version = sourceSlugifyVersion(); + if (!matchesSlugifyVersion(baseline, version)) { + // Generate the lock from the publisher's existing graph, without installing + // packages. Parser, manifest and lock are committed together by the workflow. + run( + "npm", + [ + "install", + "--package-lock-only", + "--ignore-scripts", + "--no-audit", + "--no-fund", + "--save-dev", + "--save-exact", + `${SLUGIFY_PACKAGE}@${version}`, + ], + { cwd: targetRoot, timeout: 120_000 }, + ); + } + for (const entry of SYNC_SUPPORT_FILES) { + const targetPath = path.join(targetRoot, entry.target); + ensureDir(path.dirname(targetPath)); + fs.copyFileSync(entry.source, targetPath); + } + validateDocsSyncDependencies(targetRoot, baseline); +} + +function main() { + const args = parseArgs(process.argv.slice(2)); + const targetRoot = path.resolve(args.target); + + if (!fs.existsSync(targetRoot)) { + throw new Error(`target does not exist: ${targetRoot}`); + } + + const clawhubRepo = resolveClawHubRepoPath(args.clawhubRepo); + const sources = syncDocsTree(targetRoot, { + clawhubRepo, + clawhubSourceRepo: args.clawhubSourceRepo, + clawhubSourceSha: args.clawhubSourceSha, + }); + syncSupportFiles(targetRoot); + writeSyncMetadata(targetRoot, args, sources); +} + +function isCliEntry() { + const cliArg = process.argv[1]; + return cliArg ? import.meta.url === pathToFileURL(cliArg).href : false; +} + +if (isCliEntry()) { + main(); +} diff --git a/scripts/ensure-cli-startup-build.mts b/scripts/ensure-cli-startup-build.mts new file mode 100644 index 0000000000000000000000000000000000000000..7163f1a45666197ca3f68ac21a086a6ab4aee47b --- /dev/null +++ b/scripts/ensure-cli-startup-build.mts @@ -0,0 +1,83 @@ +#!/usr/bin/env node + +// Ensures CLI startup benchmark assets are built before checks. +import { spawnSync } from "node:child_process"; +import { existsSync } from "node:fs"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { readPositiveEnvInt } from "./lib/numeric-options.mjs"; +import { resolveRepoRoot } from "./lib/repo-root.mjs"; +const repoRoot = resolveRepoRoot(import.meta.url); +const entryCandidates = ["dist/entry.js", "dist/entry.mjs"]; +const startupMetadataPath = "dist/cli-startup-metadata.json"; +const DEFAULT_BUILD_TIMEOUT_MS = 10 * 60 * 1000; + +type StartupBuildParams = Partial<{ + env: NodeJS.ProcessEnv; + existsSync: typeof existsSync; + killSignal: NodeJS.Signals; + nodeExecPath: string; + rootDir: string; + spawnSync: (...args: Parameters<typeof spawnSync>) => Partial<ReturnType<typeof spawnSync>>; + stdio: "inherit" | "pipe"; + timeoutMs: number; +}>; + +/** + * Resolves the CLI startup build timeout from environment. + */ +export function resolveCliStartupBuildTimeoutMs(env: NodeJS.ProcessEnv = process.env) { + return readPositiveEnvInt("OPENCLAW_CLI_STARTUP_BUILD_TIMEOUT_MS", env, DEFAULT_BUILD_TIMEOUT_MS); +} + +/** + * Reports whether required CLI startup build outputs exist. + */ +export function hasCliStartupBuild(params: StartupBuildParams = {}) { + const rootDir = params.rootDir ?? repoRoot; + const exists = params.existsSync ?? existsSync; + const hasEntry = entryCandidates.some((relativePath) => exists(path.join(rootDir, relativePath))); + return hasEntry && exists(path.join(rootDir, startupMetadataPath)); +} + +/** + * Builds CLI startup assets when required outputs are missing. + */ +export function ensureCliStartupBuild(params: StartupBuildParams = {}) { + const rootDir = params.rootDir ?? repoRoot; + if (hasCliStartupBuild({ rootDir, existsSync: params.existsSync })) { + return { built: false }; + } + + const nodeExecPath = params.nodeExecPath ?? process.execPath; + const spawn = params.spawnSync ?? spawnSync; + const buildScript = path.join(rootDir, "scripts", "build-all.mts"); + + console.error( + "[cli-startup-build] dist startup entry or metadata missing; running cliStartup build profile", + ); + const result = spawn(nodeExecPath, ["--import", "tsx", buildScript, "cliStartup"], { + cwd: rootDir, + env: params.env ?? process.env, + killSignal: params.killSignal ?? "SIGKILL", + stdio: params.stdio ?? "inherit", + timeout: params.timeoutMs ?? resolveCliStartupBuildTimeoutMs(params.env ?? process.env), + }); + if (result.error) { + throw result.error; + } + const status = result.status ?? (result.signal ? 1 : 0); + if (status !== 0) { + throw new Error(`cliStartup build profile failed with exit code ${status}`); + } + return { built: true }; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + try { + ensureCliStartupBuild(); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); + } +} diff --git a/scripts/ensure-extension-memory-build.mts b/scripts/ensure-extension-memory-build.mts new file mode 100644 index 0000000000000000000000000000000000000000..78b73f3e43b892fb1ee3bf0f53d8bf95ad14ac7a --- /dev/null +++ b/scripts/ensure-extension-memory-build.mts @@ -0,0 +1,148 @@ +#!/usr/bin/env node + +// Ensures memory extension runtime entries are built before checks. +import { spawnSync } from "node:child_process"; +import { readdirSync } from "node:fs"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { resolvePluginRootPublicSurfacePath } from "../src/plugins/public-surface-runtime.js"; +import { + collectBundledPluginBuildEntries, + NON_PACKAGED_BUNDLED_PLUGIN_DIRS, +} from "./lib/bundled-plugin-build-entries.mjs"; +import { readPositiveEnvInt } from "./lib/numeric-options.mjs"; +import { resolveRepoRoot } from "./lib/repo-root.mjs"; +const repoRoot = resolveRepoRoot(import.meta.url); +const DEFAULT_BUILD_TIMEOUT_MS = 10 * 60 * 1000; + +type ExtensionMemoryBuildParams = { + env?: NodeJS.ProcessEnv; + killSignal?: NodeJS.Signals; + nodeExecPath?: string; + requiredExtensionIds?: string[]; + rootDir?: string; + spawnSync?: ( + command: string, + args: string[], + options: Record<string, unknown>, + ) => { error?: Error; signal?: NodeJS.Signals | null; status: number | null }; + stdio?: "inherit" | "pipe"; + timeoutMs?: number; +}; + +/** + * Resolves the extension memory build timeout from environment. + */ +export function resolveExtensionMemoryBuildTimeoutMs(env: NodeJS.ProcessEnv = process.env) { + return readPositiveEnvInt( + "OPENCLAW_EXTENSION_MEMORY_BUILD_TIMEOUT_MS", + env, + DEFAULT_BUILD_TIMEOUT_MS, + ); +} + +function collectExpectedExtensionMemoryEntryIds(rootDir: string, env: NodeJS.ProcessEnv) { + try { + const entries = collectBundledPluginBuildEntries({ cwd: rootDir, env }); + return entries + .filter( + (entry) => + entry.hasManifest && + !NON_PACKAGED_BUNDLED_PLUGIN_DIRS.has(entry.id) && + entry.sourceEntries.length > 0, + ) + .map((entry) => entry.id) + .toSorted((left, right) => left.localeCompare(right)); + } catch { + return null; + } +} + +/** Discovers built index entries for readiness checks and RSS profiling. */ +export function findBuiltExtensionMemoryEntries(rootDir: string = repoRoot) { + const entries = new Map<string, string>(); + // Prefer canonical root output over package-local builds; source-only entries + // returned by the public-surface resolver must never enter a built-RSS run. + for (const extensionsDir of [ + path.join(rootDir, "dist", "extensions"), + path.join(rootDir, "extensions"), + ]) { + let extensionIds: string[]; + try { + extensionIds = readdirSync(extensionsDir); + } catch { + continue; + } + for (const dir of extensionIds) { + if (entries.has(dir)) { + continue; + } + const file = resolvePluginRootPublicSurfacePath({ + pluginRoot: path.join(extensionsDir, dir), + artifactBasename: "index.js", + }); + if (file && /\.[cm]?js$/u.test(file)) { + entries.set(dir, file); + } + } + } + return [...entries] + .map(([dir, file]) => ({ dir, file })) + .toSorted((a, b) => a.dir.localeCompare(b.dir)); +} + +/** Reports whether all required built memory extension entries exist. */ +export function hasBuiltExtensionMemoryEntries(params: ExtensionMemoryBuildParams = {}) { + const rootDir = params.rootDir ?? repoRoot; + const builtIds = new Set(findBuiltExtensionMemoryEntries(rootDir).map((entry) => entry.dir)); + const requiredExtensionIds = + (params.requiredExtensionIds?.length ?? 0) > 0 + ? params.requiredExtensionIds + : collectExpectedExtensionMemoryEntryIds(rootDir, params.env ?? process.env); + if (!requiredExtensionIds || requiredExtensionIds.length === 0) { + return builtIds.size > 0; + } + return requiredExtensionIds.every((id) => builtIds.has(id)); +} + +/** + * Builds memory extension entries when required outputs are missing. + */ +export function ensureExtensionMemoryBuild(params: ExtensionMemoryBuildParams = {}) { + const rootDir = params.rootDir ?? repoRoot; + if (hasBuiltExtensionMemoryEntries(params)) { + return { built: false }; + } + + const nodeExecPath = params.nodeExecPath ?? process.execPath; + const spawn = params.spawnSync ?? spawnSync; + const buildScript = path.join(rootDir, "scripts", "build-all.mts"); + + console.error( + "[extension-memory-build] required built plugin entries missing; running cliStartup build profile", + ); + const result = spawn(nodeExecPath, ["--import", "tsx", buildScript, "cliStartup"], { + cwd: rootDir, + env: params.env ?? process.env, + killSignal: params.killSignal ?? "SIGKILL", + stdio: params.stdio ?? "inherit", + timeout: params.timeoutMs ?? resolveExtensionMemoryBuildTimeoutMs(params.env ?? process.env), + }); + if (result.error) { + throw result.error; + } + const status = result.status ?? (result.signal ? 1 : 0); + if (status !== 0) { + throw new Error(`cliStartup build profile failed with exit code ${status}`); + } + return { built: true }; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + try { + ensureExtensionMemoryBuild(); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); + } +} diff --git a/scripts/firecrawl-compare.ts b/scripts/firecrawl-compare.ts new file mode 100644 index 0000000000000000000000000000000000000000..165b900e44b0ec6ba84cec91f32b2d32bbb0c38b --- /dev/null +++ b/scripts/firecrawl-compare.ts @@ -0,0 +1,158 @@ +// Firecrawl Compare script supports OpenClaw repository automation. +import { pathToFileURL } from "node:url"; +import { truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice"; +import { fetchFirecrawlContent } from "../extensions/firecrawl/api.ts"; +import { formatErrorMessage } from "../src/infra/errors.ts"; +import { extractReadableContent } from "../src/web-fetch/content-extractors.runtime.js"; +import { readBoundedResponseText } from "./lib/bounded-response.mjs"; + +const DEFAULT_URLS = [ + "https://en.wikipedia.org/wiki/Web_scraping", + "https://news.ycombinator.com/", + "https://www.apple.com/iphone/", + "https://www.nytimes.com/", + "https://www.reddit.com/r/javascript/", +]; + +const urls = process.argv.slice(2); +const targets = urls.length > 0 ? urls : DEFAULT_URLS; +const apiKey = process.env.FIRECRAWL_API_KEY; +const baseUrl = process.env.FIRECRAWL_BASE_URL ?? "https://api.firecrawl.dev"; + +const userAgent = + "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_7_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"; +const timeoutMs = 30_000; +const FETCH_HTML_MAX_BYTES = 5 * 1024 * 1024; + +function truncate(value: string, max = 180): string { + if (!value) { + return ""; + } + return value.length > max ? `${truncateUtf16Safe(value, max)}…` : value; +} + +async function fetchHtml( + url: string, + fetchImpl: typeof fetch = fetch, +): Promise<{ + ok: boolean; + status: number; + contentType: string; + finalUrl: string; + body: string; +}> { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + try { + const res = await fetchImpl(url, { + method: "GET", + headers: { Accept: "*/*", "User-Agent": userAgent }, + signal: controller.signal, + }); + const contentType = res.headers.get("content-type") ?? "application/octet-stream"; + const body = await readBoundedResponseText(res, "local HTML fetch", FETCH_HTML_MAX_BYTES, { + signal: controller.signal, + }); + return { + ok: res.ok, + status: res.status, + contentType, + finalUrl: res.url || url, + body, + }; + } finally { + clearTimeout(timer); + } +} + +async function run() { + if (!apiKey) { + console.log("FIRECRAWL_API_KEY not set. Firecrawl comparisons will be skipped."); + } + + for (const url of targets) { + console.log(`\n=== ${url}`); + let localStatus; + let localTitle = ""; + let localText = ""; + let localError: string | undefined; + + try { + const res = await fetchHtml(url); + if (!res.ok) { + localStatus = `http ${res.status}`; + } else if (!res.contentType.includes("text/html")) { + localStatus = `non-html (${res.contentType})`; + } else { + const readable = await extractReadableContent({ + html: res.body, + url: res.finalUrl, + extractMode: "markdown", + }); + if (readable?.text) { + localStatus = "readability"; + localTitle = readable.title ?? ""; + localText = readable.text; + } else { + localStatus = "readability-empty"; + } + } + } catch (error) { + localStatus = "error"; + localError = formatErrorMessage(error); + } + + console.log(`local: ${localStatus} len=${localText.length} title=${truncate(localTitle, 80)}`); + if (localError) { + console.log(`local error: ${localError}`); + } + if (localText) { + console.log(`local sample: ${truncate(localText)}`); + } + + if (apiKey) { + try { + const firecrawl = await fetchFirecrawlContent({ + url, + extractMode: "markdown", + apiKey, + baseUrl, + onlyMainContent: true, + maxAgeMs: 172_800_000, + proxy: "auto", + storeInCache: true, + timeoutSeconds: 60, + }); + console.log( + `firecrawl: ok len=${firecrawl.text.length} title=${truncate( + firecrawl.title ?? "", + 80, + )} status=${firecrawl.status ?? "n/a"}`, + ); + if (firecrawl.warning) { + console.log(`firecrawl warning: ${firecrawl.warning}`); + } + if (firecrawl.text) { + console.log(`firecrawl sample: ${truncate(firecrawl.text)}`); + } + } catch (error) { + const message = formatErrorMessage(error); + console.log(`firecrawl: error ${message}`); + } + } + } + + process.exit(0); +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { + run().catch((error: unknown) => { + console.error(error); + process.exit(1); + }); +} + +export const testing = { + fetchHtml, + truncate, +}; diff --git a/scripts/format-docs.mts b/scripts/format-docs.mts new file mode 100644 index 0000000000000000000000000000000000000000..e826f25de3d3460fd65293d346a73d306edab613 --- /dev/null +++ b/scripts/format-docs.mts @@ -0,0 +1,275 @@ +#!/usr/bin/env node + +// Formats docs Markdown/MDX and repairs Mintlify accordion indentation. +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { resolveRepoToolBinPath } from "./lib/local-check-runtime.mts"; +import { repairMintlifyAccordionIndentation } from "./lib/mintlify-accordion.mjs"; +import { outputTail, spawnOutputText } from "./lib/output-tail.mts"; +import { resolveRepoRoot } from "./lib/repo-root.mjs"; +import { buildCmdExeCommandLine, resolveWindowsCmdExePath } from "./windows-cmd-helpers.mjs"; +const ROOT = resolveRepoRoot(import.meta.url); +const CHECK = process.argv.includes("--check"); +const DOCS_FORMAT_MAX_BUFFER_BYTES = 1024 * 1024 * 16; +const DOCS_FORMAT_MAX_COMMAND_LINE_BYTES = 24 * 1024; +const FAILURE_OUTPUT_TAIL_BYTES = 16 * 1024; + +type CommandResult = { + error?: Error; + signal?: NodeJS.Signals | null; + status: number | null; + stderr?: string | Buffer; + stdout?: string | Buffer; +}; +type FormatDeps = { + existsSync?: (candidate: string) => boolean; + spawnSync?: (command: string, args: string[], options: Record<string, unknown>) => CommandResult; +}; +type OxfmtParams = { + comSpec?: string; + env?: NodeJS.ProcessEnv; + existsSync?: (candidate: string) => boolean; + maxCommandLineBytes?: number; + nodeExecPath?: string; + platform?: NodeJS.Platform; + repoRoot?: string; +}; +type FormatDocsParams = OxfmtParams & { check?: boolean; root?: string }; +type CommandInvocation = { args: string[]; command: string }; + +function commandFailureMessage( + label: string, + result: CommandResult, + invocation: CommandInvocation, +) { + const details: string[] = []; + if (invocation) { + details.push(`command: ${invocation.command}`); + if (invocation.args.length > 0) { + const previewArgs = invocation.args.slice(0, 12).join(" "); + const suffix = invocation.args.length > 12 ? ` ... (${invocation.args.length} args)` : ""; + details.push(`args: ${previewArgs}${suffix}`); + } + } + if (result.error?.message) { + details.push(result.error.message); + } + if (result.status !== null && result.status !== undefined && result.status !== 0) { + details.push(`exit status: ${result.status}`); + } + if (result.signal) { + details.push(`signal: ${result.signal}`); + } + const stderrTail = outputTail(result.stderr, FAILURE_OUTPUT_TAIL_BYTES); + if (stderrTail) { + details.push(`stderr tail:\n${stderrTail}`); + } + const stdoutTail = outputTail(result.stdout, FAILURE_OUTPUT_TAIL_BYTES); + if (stdoutTail) { + details.push(`stdout tail:\n${stdoutTail}`); + } + return `${label} failed${details.length > 0 ? `:\n${details.join("\n")}` : ""}`; +} + +export function docsFiles(root = ROOT, deps: FormatDeps = {}) { + const spawnSyncImpl = deps.spawnSync ?? spawnSync; + const result = spawnSyncImpl("git", ["ls-files", "docs/**/*.md", "docs/**/*.mdx", "README.md"], { + cwd: root, + encoding: "utf8", + maxBuffer: DOCS_FORMAT_MAX_BUFFER_BYTES, + }); + if (result.error || result.status !== 0) { + throw new Error( + commandFailureMessage("git ls-files", result, { + command: "git", + args: ["ls-files", "docs/**/*.md", "docs/**/*.mdx", "README.md"], + }), + ); + } + return spawnOutputText(result.stdout) + .split("\n") + .filter(Boolean) + .filter((relativePath) => (deps.existsSync ?? fs.existsSync)(path.join(root, relativePath))); +} + +function commandLineBytes(args: string[]) { + return args.reduce((total, arg) => total + Buffer.byteLength(arg, "utf8") + 3, 0); +} + +export function chunkFilesForCommand( + files: string[], + prefixArgs: string[], + maxBytes = DOCS_FORMAT_MAX_COMMAND_LINE_BYTES, +) { + const chunks: string[][] = []; + let chunk: string[] = []; + let chunkBytes = commandLineBytes(prefixArgs); + + for (const file of files) { + const fileBytes = Buffer.byteLength(file, "utf8") + 3; + if (chunk.length > 0 && chunkBytes + fileBytes > maxBytes) { + chunks.push(chunk); + chunk = []; + chunkBytes = commandLineBytes(prefixArgs); + } + chunk.push(file); + chunkBytes += fileBytes; + } + + if (chunk.length > 0) { + chunks.push(chunk); + } + + return chunks; +} + +export function resolveOxfmtInvocation(args: string[], params: OxfmtParams = {}) { + const repoRoot = params.repoRoot ?? ROOT; + const platform = params.platform ?? process.platform; + const existsSync = params.existsSync ?? fs.existsSync; + const shimName = platform === "win32" ? "oxfmt.cmd" : "oxfmt"; + const shimPath = resolveRepoToolBinPath(shimName, { cwd: repoRoot, fileExists: existsSync }); + + if (existsSync(shimPath)) { + if (platform === "win32") { + const comSpec = params.comSpec ?? resolveWindowsCmdExePath(params.env ?? process.env); + return { + command: comSpec, + args: ["/d", "/s", "/c", buildCmdExeCommandLine(shimPath, args)], + shell: false, + windowsVerbatimArguments: true, + }; + } + return { + command: shimPath, + args, + shell: false, + }; + } + + return { + command: params.nodeExecPath ?? process.execPath, + args: [path.join(repoRoot, "node_modules", "oxfmt", "bin", "oxfmt"), ...args], + shell: false, + }; +} + +export function runOxfmt(files: string[], params: OxfmtParams = {}, deps: FormatDeps = {}) { + if (files.length === 0) { + return; + } + const repoRoot = params.repoRoot ?? ROOT; + const spawnSyncImpl = deps.spawnSync ?? spawnSync; + const prefixArgs = ["--write", "--threads=1", "--config", path.join(repoRoot, ".oxfmtrc.jsonc")]; + for (const chunk of chunkFilesForCommand( + files, + prefixArgs, + params.maxCommandLineBytes ?? DOCS_FORMAT_MAX_COMMAND_LINE_BYTES, + )) { + const invocation = resolveOxfmtInvocation([...prefixArgs, ...chunk], { + comSpec: params.comSpec, + existsSync: deps.existsSync, + nodeExecPath: params.nodeExecPath, + platform: params.platform, + repoRoot, + }); + const result = spawnSyncImpl(invocation.command, invocation.args, { + cwd: repoRoot, + encoding: "utf8", + maxBuffer: DOCS_FORMAT_MAX_BUFFER_BYTES, + shell: invocation.shell, + windowsVerbatimArguments: invocation.windowsVerbatimArguments, + }); + + if (result.error || result.status !== 0) { + throw new Error(commandFailureMessage("oxfmt", result, invocation)); + } + } +} + +function repairFiles(root: string, files: string[]) { + const changed: string[] = []; + for (const relativePath of files) { + const absolutePath = path.join(root, relativePath); + const raw = fs.readFileSync(absolutePath, "utf8"); + const formatted = repairMintlifyAccordionIndentation(raw); + if (formatted === raw) { + continue; + } + fs.writeFileSync(absolutePath, formatted); + changed.push(relativePath); + } + return changed; +} + +function copyDocsToTemp(root: string, files: string[]) { + const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-docs-format-")); + for (const relativePath of files) { + const source = path.join(root, relativePath); + const target = path.join(tempRoot, relativePath); + fs.mkdirSync(path.dirname(target), { recursive: true }); + fs.copyFileSync(source, target); + } + return tempRoot; +} + +export function formatDocs(params: FormatDocsParams = {}, deps: FormatDeps = {}) { + const root = params.root ?? ROOT; + const check = params.check ?? false; + const changed: string[] = []; + const files = docsFiles(root, deps); + + if (check) { + const tempRoot = copyDocsToTemp(root, files); + try { + runOxfmt( + files.map((relativePath) => path.join(tempRoot, relativePath)), + { ...params, repoRoot: root }, + deps, + ); + repairFiles(tempRoot, files); + for (const relativePath of files) { + const raw = fs.readFileSync(path.join(root, relativePath), "utf8"); + const formatted = fs.readFileSync(path.join(tempRoot, relativePath), "utf8"); + if (formatted !== raw) { + changed.push(relativePath); + } + } + } finally { + fs.rmSync(tempRoot, { recursive: true, force: true }); + } + } else { + runOxfmt(files, { ...params, repoRoot: root }, deps); + changed.push(...repairFiles(root, files)); + } + + return { + changed, + fileCount: files.length, + }; +} + +function main() { + const { changed, fileCount } = formatDocs({ check: CHECK, root: ROOT }); + + if (CHECK && changed.length > 0) { + console.error(`Format issues found in ${changed.length} docs file(s):`); + for (const relativePath of changed) { + console.error(`- ${relativePath}`); + } + process.exit(1); + } + + if (changed.length > 0) { + console.log(`Formatted ${changed.length} docs file(s).`); + } else { + console.log(`Docs formatting clean (${fileCount} files).`); + } +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { + main(); +} diff --git a/scripts/freebsd-service-inspect.mjs b/scripts/freebsd-service-inspect.mjs new file mode 100644 index 0000000000000000000000000000000000000000..eeee90b4cc9edd910ca13b49a3dff4f7ce45d275 --- /dev/null +++ b/scripts/freebsd-service-inspect.mjs @@ -0,0 +1,117 @@ +#!/usr/bin/env node +// Validate the standalone import boundary before loading code as root. +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +class InspectionFailure extends Error { + constructor(reason) { + super(reason); + this.reason = reason; + } +} + +function fail(reason) { + throw new InspectionFailure(reason); +} + +function readStat(filename) { + try { + return fs.lstatSync(filename); + } catch (error) { + if (error.code === "ENOENT") { + return null; + } + throw new InspectionFailure("filesystem-inspection-failed"); + } +} + +function inspectPath(filename) { + // A trusted leaf below a writable directory can be replaced before import. + // Validate the complete path before loading the shared discovery helper. + const parts = filename.split("/").filter(Boolean); + let current = "/"; + let result; + for (const part of ["", ...parts]) { + current = path.join(current, part); + result = readStat(current); + if (!result) { + fail("required-path-missing"); + } + if (result.isSymbolicLink()) { + fail("symbolic-link-needs-owner-inspection"); + } + if (result.uid !== 0 || (result.mode & 0o022) !== 0) { + fail("unsafe-path-ownership"); + } + if (current !== filename && !result.isDirectory()) { + fail("invalid-path-type"); + } + } + return result; +} + +function registerExitCleanup(cleanup) { + const unregister = () => { + process.off("SIGINT", onSigint); + process.off("SIGTERM", onSigterm); + process.off("exit", cleanup); + }; + const onSignal = (signal) => { + unregister(); + cleanup(); + // This standalone entry point owns its exit policy; preserve signal exit + // after reclaiming the query instead of continuing with an unknown result. + process.kill(process.pid, signal); + }; + const onSigint = () => onSignal("SIGINT"); + const onSigterm = () => onSignal("SIGTERM"); + process.once("SIGINT", onSigint); + process.once("SIGTERM", onSigterm); + process.once("exit", cleanup); + return unregister; +} + +try { + if (process.argv.length !== 2) { + fail("no-arguments-accepted"); + } + if (process.platform !== "freebsd") { + fail("freebsd-required"); + } + if (process.geteuid() !== 0) { + fail("root-required"); + } + const discoveryUrl = new URL("./lib/freebsd-service-discovery.mjs", import.meta.url); + // This small bootstrap cannot import the shared path checker before proving + // that the helper and every ancestor are root-owned and not writable by others. + for (const filename of [ + process.execPath, + path.resolve(process.argv[1]), + fileURLToPath(discoveryUrl), + ]) { + if (!inspectPath(filename).isFile()) { + fail("invalid-path-type"); + } + } + const { discoverFreeBsdService } = await import(discoveryUrl.href); + const discovered = await discoverFreeBsdService({ registerExitCleanup }); + if (discovered.status === "unknown") { + fail(discovered.reason); + } + const output = JSON.stringify({ ...discovered, authority: "diagnostic-only" }); + if (Buffer.byteLength(output) > 64 * 1024) { + fail("result-too-large"); + } + process.stdout.write(`${output}\n`); +} catch (error) { + const reason = error instanceof InspectionFailure ? error.reason : "inspection-failed"; + process.stdout.write( + `${JSON.stringify({ schema: 1, service: "openclaw", status: "unknown", reason, authority: "diagnostic-only" })}\n`, + ); + process.stderr.write( + `[freebsd-service-inspect] ${reason}. Inspect the service configuration from its owner's root shell.\n`, + ); + process.stderr.write("[freebsd-service-inspect] FAILED (exit 1)\n"); + process.exitCode = 1; +} diff --git a/scripts/frv-proof-broker.d.mts b/scripts/frv-proof-broker.d.mts new file mode 100644 index 0000000000000000000000000000000000000000..fd686f76362af8288467fcec85776e2dbb2c3f9a --- /dev/null +++ b/scripts/frv-proof-broker.d.mts @@ -0,0 +1,53 @@ +export type BrokerContext = { + actor: string; + correlation: string; + landedSha: string; + prNumber: number; + repository: "openclaw/openclaw"; + runId: number; + workflowSha: string; +}; + +export type GitHubApi = { + request(method: string, path: string, body?: unknown): Promise<unknown>; +}; + +export type FixtureRunExpectation = { + attempt: number; + branch: string; + conclusion: "failure" | "success"; + correlation: string; + headSha: string; + repository: string; + runId?: number; +}; + +export type ProofReceipt = { + actor: string; + correlation: string; + fixtureRunAttempt: 2; + fixtureRunId: number; + landedSha: string; + operation: "noop"; + prNumber: number; + repository: "openclaw/openclaw"; + sourceRef: "refs/heads/main"; + workflowSha: string; +}; + +export function validateBrokerRequest(event: unknown, env: NodeJS.ProcessEnv): BrokerContext; +export function validateFixtureRun( + value: unknown, + expected: FixtureRunExpectation, +): Record<string, unknown>; +export function runProofBroker(options: { + api: GitHubApi; + env: NodeJS.ProcessEnv; + event: unknown; + sleep?: (milliseconds: number) => Promise<void>; +}): Promise<ProofReceipt>; +export function createGitHubApi(options: { + repository: string; + token: string; + fetchImpl?: typeof fetch; +}): GitHubApi; diff --git a/scripts/frv-proof-broker.mjs b/scripts/frv-proof-broker.mjs new file mode 100644 index 0000000000000000000000000000000000000000..5459a9ab0f65e65cc1422260783825a324527294 --- /dev/null +++ b/scripts/frv-proof-broker.mjs @@ -0,0 +1,394 @@ +#!/usr/bin/env node +import { appendFileSync, readFileSync, writeFileSync } from "node:fs"; +import { pathToFileURL } from "node:url"; +import { isRecord } from "./lib/record-shared.mjs"; +import { validateForwardAncestry } from "./pr-lib/crabbox-gate-contract.mjs"; + +const REPOSITORY = "openclaw/openclaw"; +const BROKER_WORKFLOW = ".github/workflows/frv-proof-broker.yml"; +const FIXTURE_WORKFLOW = ".github/workflows/frv-proof-fixture.yml"; +const FIXTURE_WORKFLOW_ID = "frv-proof-fixture.yml"; +const FIXTURE_NAME = "FRV Proof Fixture"; +const FIXTURE_OPERATION = "noop"; +const SHA_PATTERN = /^[0-9a-f]{40}$/u; +const POLL_INTERVAL_MS = 5_000; +const POLL_LIMIT = 120; + +function requiredString(value, label) { + if (typeof value !== "string" || value.length === 0) { + throw new Error(`${label} is required`); + } + return value; +} + +function requiredPositiveInteger(value, label) { + const number = Number(value); + if (!Number.isSafeInteger(number) || number < 1) { + throw new Error(`${label} must be a positive integer`); + } + return number; +} + +function assertExactKeys(value, expected, label) { + if (!isRecord(value)) { + throw new Error(`${label} must be an object`); + } + const compare = (left, right) => left.localeCompare(right); + const actual = Object.keys(value).toSorted(compare); + const wanted = [...expected].toSorted(compare); + if (JSON.stringify(actual) !== JSON.stringify(wanted)) { + throw new Error(`${label} keys must be exactly: ${wanted.join(", ")}`); + } +} + +function requiredEnv(env, name) { + return requiredString(env[name], name); +} + +export function validateBrokerRequest(event, env) { + if (!isRecord(event)) { + throw new Error("GITHUB_EVENT_PATH must contain an object"); + } + const repository = requiredEnv(env, "GITHUB_REPOSITORY"); + if (repository !== REPOSITORY) { + throw new Error(`FRV proof broker requires repository ${REPOSITORY}`); + } + if (requiredEnv(env, "GITHUB_EVENT_NAME") !== "workflow_dispatch") { + throw new Error("FRV proof broker requires workflow_dispatch"); + } + if (requiredEnv(env, "GITHUB_REF") !== "refs/heads/main") { + throw new Error("FRV proof broker must run from refs/heads/main"); + } + const workflowSha = requiredEnv(env, "GITHUB_WORKFLOW_SHA"); + const eventSha = requiredEnv(env, "GITHUB_SHA"); + if (!SHA_PATTERN.test(workflowSha) || workflowSha !== eventSha) { + throw new Error("FRV proof broker requires one exact trusted workflow SHA"); + } + const expectedWorkflowRef = `${REPOSITORY}/${BROKER_WORKFLOW}@refs/heads/main`; + if (requiredEnv(env, "GITHUB_WORKFLOW_REF") !== expectedWorkflowRef) { + throw new Error(`FRV proof broker requires ${expectedWorkflowRef}`); + } + const actor = requiredEnv(env, "GITHUB_ACTOR"); + if (actor !== requiredEnv(env, "GITHUB_TRIGGERING_ACTOR")) { + throw new Error("FRV proof broker actor must match the triggering actor"); + } + const runId = requiredPositiveInteger(requiredEnv(env, "GITHUB_RUN_ID"), "GITHUB_RUN_ID"); + const runAttempt = requiredPositiveInteger( + requiredEnv(env, "GITHUB_RUN_ATTEMPT"), + "GITHUB_RUN_ATTEMPT", + ); + + assertExactKeys(event.inputs, ["landed_sha", "pr_number"], "workflow inputs"); + const inputs = event.inputs; + const prNumber = requiredPositiveInteger(inputs.pr_number, "pr_number"); + const landedSha = requiredString(inputs.landed_sha, "landed_sha"); + if (!SHA_PATTERN.test(landedSha)) { + throw new Error("landed_sha must be exactly 40 lowercase hex characters"); + } + const correlation = `frv-proof-${runId}-${runAttempt}`; + + return { + actor, + correlation, + landedSha, + prNumber, + repository: REPOSITORY, + runId, + workflowSha, + }; +} + +function record(value, label) { + if (!isRecord(value)) { + throw new Error(`${label} must be an object`); + } + return value; +} + +function nestedRecord(value, key, label) { + return record(record(value, label)[key], `${label}.${key}`); +} + +function validateActorPermission(value, actor) { + const permission = requiredString(record(value, "actor permission").permission, "permission"); + if (!new Set(["admin", "maintain", "write"]).has(permission)) { + throw new Error(`actor ${actor} lacks repository write permission`); + } +} + +function validatePullRequest(value, context) { + const pull = record(value, "pull request"); + if ( + pull.number !== context.prNumber || + pull.state !== "closed" || + pull.merged !== true || + typeof pull.merged_at !== "string" || + pull.merged_at.length === 0 + ) { + throw new Error("proof target must be the requested merged pull request"); + } + if (pull.merge_commit_sha !== context.landedSha) { + throw new Error("pull request merge commit does not match the requested landed SHA"); + } + if (nestedRecord(pull, "base", "pull request").ref !== "main") { + throw new Error("pull request base must be main"); + } + if (nestedRecord(pull, "base", "pull request").repo?.full_name !== context.repository) { + throw new Error("pull request base repository does not match"); + } +} + +function validateLandedAncestry(value, context) { + validateForwardAncestry( + value, + { baseSha: context.landedSha, headSha: context.workflowSha }, + "landed controller ancestry", + ); +} + +function validateFixtureWorkflow(value) { + const workflow = record(value, "fixture workflow"); + if ( + workflow.name !== FIXTURE_NAME || + workflow.path !== FIXTURE_WORKFLOW || + workflow.state !== "active" + ) { + throw new Error("fixture workflow identity does not match the fixed active workflow"); + } + requiredPositiveInteger(workflow.id, "fixture workflow id"); +} + +function validateMainRef(value, workflowSha) { + const ref = record(value, "main ref"); + if (ref.ref !== "refs/heads/main") { + throw new Error("main ref identity does not match"); + } + const mainSha = requiredString(nestedRecord(ref, "object", "main ref").sha, "main ref SHA"); + if (!SHA_PATTERN.test(mainSha) || mainSha !== workflowSha) { + throw new Error("trusted main moved before fixture dispatch"); + } +} + +export function validateFixtureRun(value, expected) { + const run = record(value, "fixture run"); + if (nestedRecord(run, "repository", "fixture run").full_name !== expected.repository) { + throw new Error("fixture run repository does not match"); + } + if (run.path !== FIXTURE_WORKFLOW) { + throw new Error("fixture run workflow does not match"); + } + if (run.head_sha !== expected.headSha || run.head_branch !== expected.branch) { + throw new Error("fixture run source does not match the trusted main workflow SHA"); + } + if (run.event !== "workflow_dispatch") { + throw new Error("fixture run event does not match"); + } + const expectedTitle = `${FIXTURE_NAME} [${FIXTURE_OPERATION}] ${expected.correlation}`; + if (run.display_title !== expectedTitle) { + throw new Error("fixture run operation or correlation does not match"); + } + const runId = requiredPositiveInteger(run.id, "fixture run id"); + if (expected.runId !== undefined && runId !== expected.runId) { + throw new Error("fixture run id changed"); + } + if (requiredPositiveInteger(run.run_attempt, "fixture run attempt") !== expected.attempt) { + throw new Error("fixture run attempt does not match"); + } + if (run.status !== "completed" || run.conclusion !== expected.conclusion) { + throw new Error(`fixture run must complete with ${expected.conclusion}`); + } + return run; +} + +function fixtureRunIdentityMatches(value, context) { + if (!isRecord(value)) { + return false; + } + return ( + value.head_branch === "main" && + value.event === "workflow_dispatch" && + value.display_title === `${FIXTURE_NAME} [${FIXTURE_OPERATION}] ${context.correlation}` + ); +} + +async function waitForInitialRun(api, context, sleep) { + const branch = encodeURIComponent("main"); + for (let poll = 0; poll < POLL_LIMIT; poll += 1) { + const response = record( + await api.request( + "GET", + `/actions/workflows/${FIXTURE_WORKFLOW_ID}/runs?event=workflow_dispatch&branch=${branch}&per_page=20`, + ), + "fixture workflow runs", + ); + const runs = response.workflow_runs; + if (!Array.isArray(runs)) { + throw new Error("fixture workflow runs response is invalid"); + } + const candidate = runs.find((run) => fixtureRunIdentityMatches(run, context)); + if (candidate) { + const run = record(candidate, "fixture run"); + if (run.status !== "completed") { + await sleep(POLL_INTERVAL_MS); + continue; + } + return validateFixtureRun(run, { + attempt: 1, + branch: "main", + conclusion: "failure", + correlation: context.correlation, + headSha: context.workflowSha, + repository: context.repository, + }); + } + await sleep(POLL_INTERVAL_MS); + } + throw new Error("timed out waiting for the broker-owned fixture run"); +} + +async function waitForRerun(api, context, fixtureRunId, sleep) { + for (let poll = 0; poll < POLL_LIMIT; poll += 1) { + const run = record(await api.request("GET", `/actions/runs/${fixtureRunId}`), "fixture run"); + const attempt = requiredPositiveInteger(run.run_attempt, "fixture run attempt"); + if (attempt < 2 || run.status !== "completed") { + await sleep(POLL_INTERVAL_MS); + continue; + } + return validateFixtureRun(run, { + attempt: 2, + branch: "main", + conclusion: "success", + correlation: context.correlation, + headSha: context.workflowSha, + repository: context.repository, + runId: fixtureRunId, + }); + } + throw new Error("timed out waiting for the failed-job rerun"); +} + +async function validateMutationAuthority(api, context) { + validateActorPermission( + await api.request("GET", `/collaborators/${encodeURIComponent(context.actor)}/permission`), + context.actor, + ); + validatePullRequest(await api.request("GET", `/pulls/${context.prNumber}`), context); + validateLandedAncestry( + await api.request("GET", `/compare/${context.landedSha}...${context.workflowSha}`), + context, + ); +} + +async function validateFixturePrerequisite(api) { + validateFixtureWorkflow(await api.request("GET", `/actions/workflows/${FIXTURE_WORKFLOW_ID}`)); +} + +async function validateTrustedMain(api, context) { + validateMainRef(await api.request("GET", "/git/ref/heads/main"), context.workflowSha); +} + +export async function runProofBroker({ api, env, event, sleep = setTimeoutPromise }) { + const context = validateBrokerRequest(event, env); + await validateFixturePrerequisite(api); + await validateMutationAuthority(api, context); + await validateTrustedMain(api, context); + + await api.request("POST", `/actions/workflows/${FIXTURE_WORKFLOW_ID}/dispatches`, { + inputs: { + correlation: context.correlation, + operation: FIXTURE_OPERATION, + }, + ref: "main", + }); + const initialRun = await waitForInitialRun(api, context, sleep); + const fixtureRunId = requiredPositiveInteger(initialRun.id, "fixture run id"); + await validateMutationAuthority(api, context); + await api.request("POST", `/actions/runs/${fixtureRunId}/rerun-failed-jobs`); + await waitForRerun(api, context, fixtureRunId, sleep); + return { + actor: context.actor, + correlation: context.correlation, + fixtureRunAttempt: 2, + fixtureRunId, + landedSha: context.landedSha, + operation: FIXTURE_OPERATION, + prNumber: context.prNumber, + repository: context.repository, + sourceRef: "refs/heads/main", + workflowSha: context.workflowSha, + }; +} + +function setTimeoutPromise(milliseconds) { + return new Promise((resolve) => { + setTimeout(resolve, milliseconds); + }); +} + +export function createGitHubApi({ repository, token, fetchImpl = fetch }) { + if (repository !== REPOSITORY) { + throw new Error(`FRV proof broker requires repository ${REPOSITORY}`); + } + const baseUrl = `https://api.github.com/repos/${repository}`; + return { + async request(method, path, body) { + const response = await fetchImpl(`${baseUrl}${path}`, { + body: body === undefined ? undefined : JSON.stringify(body), + headers: { + Accept: "application/vnd.github+json", + Authorization: `Bearer ${token}`, + "Content-Type": "application/json", + "X-GitHub-Api-Version": "2022-11-28", + }, + method, + signal: AbortSignal.timeout(30_000), + }); + if (!response.ok) { + const detail = (await response.text()).slice(0, 500); + throw new Error(`GitHub API ${method} ${path} failed (${response.status}): ${detail}`); + } + if (response.status === 204) { + return null; + } + return response.json(); + }, + }; +} + +async function main() { + const eventPath = requiredEnv(process.env, "GITHUB_EVENT_PATH"); + const event = JSON.parse(readFileSync(eventPath, "utf8")); + const token = requiredEnv(process.env, "GH_TOKEN"); + const api = createGitHubApi({ repository: REPOSITORY, token }); + const receipt = await runProofBroker({ api, env: process.env, event }); + const receiptPath = requiredEnv(process.env, "FRV_PROOF_RECEIPT"); + writeFileSync(receiptPath, `${JSON.stringify(receipt, null, 2)}\n`, { mode: 0o600 }); + if (process.env.GITHUB_OUTPUT) { + appendFileSync(process.env.GITHUB_OUTPUT, `receipt_path=${receiptPath}\n`); + } + if (process.env.GITHUB_STEP_SUMMARY) { + appendFileSync( + process.env.GITHUB_STEP_SUMMARY, + [ + "## FRV failed-job rerun proof", + "", + `- Pull request: #${receipt.prNumber}`, + `- Landed controller: \`${receipt.landedSha}\``, + `- Trusted broker SHA: \`${receipt.workflowSha}\``, + `- Fixture run: \`${receipt.fixtureRunId}\`, attempt \`2\``, + "- Fixed operation: `noop`", + "- Fixture source: trusted `main` at the broker workflow SHA", + "", + ].join("\n"), + ); + } +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { + try { + await main(); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); + } +} diff --git a/scripts/full-release-candidate-contract.mjs b/scripts/full-release-candidate-contract.mjs new file mode 100644 index 0000000000000000000000000000000000000000..01bdb7c2ed949ac106c551a27decd6668b392dac --- /dev/null +++ b/scripts/full-release-candidate-contract.mjs @@ -0,0 +1,707 @@ +#!/usr/bin/env node +import { createHash } from "node:crypto"; +import { readFileSync, writeFileSync } from "node:fs"; +import { pathToFileURL } from "node:url"; +import { canonicalAsciiJson, compareAscii } from "./lib/canonical-json.mjs"; +import { isRecord } from "./lib/record-shared.mjs"; +import { + normalizeUpgradeSurvivorBaselineSpec, + parseUpgradeSurvivorBaselineSpecs, + parseUpgradeSurvivorScenarios, +} from "./lib/upgrade-survivor-policy.mjs"; + +const FULL_RELEASE_CANDIDATE_REQUEST_SCHEMA = "openclaw.full-release-candidate-request/v2"; +const FULL_RELEASE_CANDIDATE_MANIFEST_SCHEMA = "openclaw.full-release-candidate/v2"; +const FULL_RELEASE_CANDIDATE_BINDING_SCHEMA = "openclaw.full-release-candidate-binding/v2"; +const FULL_RELEASE_CANDIDATE_ARTIFACT_PREFIX = "full-release-candidate-v2-"; + +const MANIFEST_MAX_BYTES = 32 * 1024; +const BINDING_MAX_BYTES = 40 * 1024; +const SHA_PATTERN = /^[a-f0-9]{40}$/u; +const SHA256_PATTERN = /^[a-f0-9]{64}$/u; +const POSITIVE_DECIMAL_PATTERN = /^[1-9][0-9]*$/u; +const ISO_TIMESTAMP_PATTERN = + /^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(?:\.[0-9]+)?Z$/u; +const ASCII_PATTERN = /^[\x20-\x7e]+$/u; +const PACKAGE_NAME_PATTERN = /^(?:@[a-z0-9][a-z0-9._-]*\/)?[a-z0-9][a-z0-9._-]*$/u; +const REPOSITORY_PATTERN = /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u; +const WORKFLOW_PATH_PATTERN = /^\.github\/workflows\/[A-Za-z0-9_.-]+\.ya?ml$/u; +const RELEASE_PROFILES = new Set(["minimum", "beta", "stable", "full"]); +const SHARED_IMAGE_POLICIES = new Set(["existing-only", "no-push-artifact"]); +function fail(message) { + throw new Error(message); +} + +function exactKeys(value, keys, label) { + if (!isRecord(value)) { + fail(`${label} must be an object`); + } + const actual = Object.keys(value).toSorted(compareAscii); + const expected = [...keys].toSorted(compareAscii); + if (actual.length !== expected.length || actual.some((key, index) => key !== expected[index])) { + fail(`${label} keys must be exactly: ${expected.join(", ")}`); + } +} + +function ascii(value, label) { + if (typeof value !== "string" || !ASCII_PATTERN.test(value)) { + fail(`${label} must be a non-empty printable ASCII string`); + } + return value; +} + +function sha(value, label) { + const normalized = ascii(value, label); + if (!SHA_PATTERN.test(normalized)) { + fail(`${label} must be a lowercase 40-character commit SHA`); + } + return normalized; +} + +function sha256(value, label) { + if (typeof value !== "string" || !SHA256_PATTERN.test(value)) { + fail(`${label} must be 64 lowercase hexadecimal characters`); + } + return value; +} + +function repository(value, label) { + const normalized = ascii(value, label); + if (!REPOSITORY_PATTERN.test(normalized)) { + fail(`${label} must be an owner/repository slug`); + } + return normalized; +} + +function workflowPath(value, label) { + const normalized = ascii(value, label); + if (!WORKFLOW_PATH_PATTERN.test(normalized)) { + fail(`${label} must be a .github/workflows YAML path`); + } + return normalized; +} + +function positiveDecimal(value, label) { + const normalized = String(value ?? ""); + if (!POSITIVE_DECIMAL_PATTERN.test(normalized)) { + fail(`${label} must be a positive decimal string`); + } + return normalized; +} + +function boolean(value, label) { + if (typeof value !== "boolean") { + fail(`${label} must be boolean`); + } + return value; +} + +function sortedUniquePackages(value, label) { + if (!Array.isArray(value)) { + fail(`${label} must be an array`); + } + const result = value.map((entry, index) => { + const packageName = ascii(entry, `${label}[${index}]`); + if (!PACKAGE_NAME_PATTERN.test(packageName)) { + fail(`${label}[${index}] must be a package name`); + } + return packageName; + }); + if ( + new Set(result).size !== result.length || + result.some((entry, index) => index > 0 && compareAscii(result[index - 1], entry) >= 0) + ) { + fail(`${label} must contain unique package names in ascending ASCII order`); + } + return result; +} + +function stringArray(value, label) { + if (!Array.isArray(value)) { + fail(`${label} must be an array`); + } + return value.map((entry, index) => ascii(entry, `${label}[${index}]`)); +} + +export function buildFullReleaseCandidateRequest(input) { + if (!isRecord(input)) { + fail("full release candidate request input must be an object"); + } + const explicitBaselines = parseUpgradeSurvivorBaselineSpecs( + typeof input.upgradeSurvivorBaselines === "string" ? input.upgradeSurvivorBaselines : undefined, + ); + const defaultBaseline = normalizeUpgradeSurvivorBaselineSpec( + typeof input.upgradeSurvivorBaseline === "string" ? input.upgradeSurvivorBaseline : undefined, + ); + const effectiveBaselines = + explicitBaselines.length > 0 ? explicitBaselines : defaultBaseline ? [defaultBaseline] : []; + const effectiveScenarios = parseUpgradeSurvivorScenarios( + typeof input.upgradeSurvivorScenarios === "string" ? input.upgradeSurvivorScenarios : undefined, + ); + return validateFullReleaseCandidateRequest({ + schema: FULL_RELEASE_CANDIDATE_REQUEST_SCHEMA, + repository: input.repository, + targetSha: input.targetSha, + toolingSha: input.toolingSha, + releaseProfile: input.releaseProfile, + releaseSoak: input.releaseSoak, + upgradeSurvivorBaselines: effectiveBaselines.toSorted(compareAscii), + upgradeSurvivorScenarios: effectiveScenarios.toSorted(compareAscii), + allowFrozenTargetScenarioOmissions: input.allowFrozenTargetScenarioOmissions, + allowUnreleasedChangelog: input.allowUnreleasedChangelog, + packagePublished: input.packagePublished, + sharedImagePolicy: input.sharedImagePolicy, + contractVersions: { + package: 1, + prepublishPluginRegistry: 1, + sharedImage: 1, + }, + }); +} + +export function validateFullReleaseCandidateRequest(value) { + exactKeys( + value, + [ + "allowFrozenTargetScenarioOmissions", + "allowUnreleasedChangelog", + "contractVersions", + "packagePublished", + "releaseProfile", + "releaseSoak", + "repository", + "schema", + "sharedImagePolicy", + "targetSha", + "toolingSha", + "upgradeSurvivorBaselines", + "upgradeSurvivorScenarios", + ], + "full release candidate request", + ); + exactKeys( + value.contractVersions, + ["package", "prepublishPluginRegistry", "sharedImage"], + "full release candidate request contractVersions", + ); + if ( + value.contractVersions.package !== 1 || + value.contractVersions.prepublishPluginRegistry !== 1 || + value.contractVersions.sharedImage !== 1 + ) { + fail("full release candidate request contract versions are invalid"); + } + const releaseProfile = ascii( + value.releaseProfile, + "full release candidate request releaseProfile", + ); + if (!RELEASE_PROFILES.has(releaseProfile)) { + fail("full release candidate request releaseProfile is invalid"); + } + const sharedImagePolicy = ascii( + value.sharedImagePolicy, + "full release candidate request sharedImagePolicy", + ); + if (!SHARED_IMAGE_POLICIES.has(sharedImagePolicy)) { + fail("full release candidate request sharedImagePolicy is invalid"); + } + const baselines = stringArray( + value.upgradeSurvivorBaselines, + "full release candidate request upgradeSurvivorBaselines", + ); + if (new Set(baselines).size !== baselines.length) { + fail("full release candidate request upgradeSurvivorBaselines must be unique"); + } + if ( + baselines.some((entry, index) => index > 0 && compareAscii(baselines[index - 1], entry) >= 0) + ) { + fail( + "full release candidate request upgradeSurvivorBaselines must be in ascending ASCII order", + ); + } + for (const baseline of baselines) { + if (normalizeUpgradeSurvivorBaselineSpec(baseline) !== baseline) { + fail("full release candidate request upgradeSurvivorBaselines are not normalized"); + } + } + const scenarios = stringArray( + value.upgradeSurvivorScenarios, + "full release candidate request upgradeSurvivorScenarios", + ); + if ( + new Set(scenarios).size !== scenarios.length || + scenarios.some((entry, index) => index > 0 && compareAscii(scenarios[index - 1], entry) >= 0) || + JSON.stringify(parseUpgradeSurvivorScenarios(scenarios.join(" "))) !== JSON.stringify(scenarios) + ) { + fail("full release candidate request upgradeSurvivorScenarios are not normalized"); + } + if (value.schema !== FULL_RELEASE_CANDIDATE_REQUEST_SCHEMA) { + fail("full release candidate request schema is invalid"); + } + return { + schema: value.schema, + repository: repository(value.repository, "full release candidate request repository"), + targetSha: sha(value.targetSha, "full release candidate request targetSha"), + toolingSha: sha(value.toolingSha, "full release candidate request toolingSha"), + releaseProfile, + releaseSoak: boolean(value.releaseSoak, "full release candidate request releaseSoak"), + packagePublished: boolean( + value.packagePublished, + "full release candidate request packagePublished", + ), + upgradeSurvivorBaselines: baselines, + upgradeSurvivorScenarios: scenarios, + allowFrozenTargetScenarioOmissions: boolean( + value.allowFrozenTargetScenarioOmissions, + "full release candidate request allowFrozenTargetScenarioOmissions", + ), + allowUnreleasedChangelog: boolean( + value.allowUnreleasedChangelog, + "full release candidate request allowUnreleasedChangelog", + ), + sharedImagePolicy, + contractVersions: { + package: 1, + prepublishPluginRegistry: 1, + sharedImage: 1, + }, + }; +} + +export function canonicalFullReleaseCandidateRequestJson(value) { + return canonicalAsciiJson(validateFullReleaseCandidateRequest(value)); +} + +export function candidateRequestSha256(value) { + return createHash("sha256").update(canonicalFullReleaseCandidateRequestJson(value)).digest("hex"); +} + +export function fullReleaseCandidateArtifactName(requestSha256) { + return `${FULL_RELEASE_CANDIDATE_ARTIFACT_PREFIX}${sha256( + requestSha256, + "full release candidate requestSha256", + )}`; +} + +function artifactIdentity(value, label) { + exactKeys(value, ["digest", "expiresAt", "id", "name", "runAttempt", "runId"], label); + const expiresAt = ascii(value.expiresAt, `${label} expiresAt`); + if (!ISO_TIMESTAMP_PATTERN.test(expiresAt) || Number.isNaN(Date.parse(expiresAt))) { + fail(`${label} expiresAt must be an ISO-8601 UTC timestamp`); + } + return { + digest: sha256(value.digest, `${label} digest`), + expiresAt, + id: positiveDecimal(value.id, `${label} id`), + name: ascii(value.name, `${label} name`), + runAttempt: positiveDecimal(value.runAttempt, `${label} runAttempt`), + runId: positiveDecimal(value.runId, `${label} runId`), + }; +} + +function validatePackage(value, request) { + exactKeys( + value, + ["artifact", "fileName", "packageSha256", "sourceSha", "version"], + "full release candidate package", + ); + const sourceSha = sha(value.sourceSha, "full release candidate package sourceSha"); + if (sourceSha !== request.targetSha) { + fail("full release candidate package sourceSha does not match the request targetSha"); + } + return { + artifact: artifactIdentity(value.artifact, "full release candidate package artifact"), + fileName: ascii(value.fileName, "full release candidate package fileName"), + packageSha256: sha256(value.packageSha256, "full release candidate package packageSha256"), + sourceSha, + version: ascii(value.version, "full release candidate package version"), + }; +} + +function validateRegistry(value, request, requiredPackages) { + exactKeys( + value, + ["artifact", "manifestSha256", "sourceSha"], + "full release candidate prepublishPluginRegistry", + ); + const sourceSha = sha( + value.sourceSha, + "full release candidate prepublishPluginRegistry sourceSha", + ); + if (sourceSha !== request.targetSha || requiredPackages.length === 0) { + fail("full release candidate prerelease registry does not match the request"); + } + return { + artifact: artifactIdentity( + value.artifact, + "full release candidate prepublishPluginRegistry artifact", + ), + manifestSha256: sha256( + value.manifestSha256, + "full release candidate prepublishPluginRegistry manifestSha256", + ), + sourceSha, + }; +} + +function validateSharedImage(value, packageSha256) { + exactKeys( + value, + ["archiveSha256", "artifact", "packageSha256"], + "full release candidate sharedImage", + ); + const imagePackageSha256 = sha256( + value.packageSha256, + "full release candidate sharedImage packageSha256", + ); + if (imagePackageSha256 !== packageSha256) { + fail("full release candidate shared image packageSha256 does not match the package"); + } + return { + archiveSha256: sha256(value.archiveSha256, "full release candidate sharedImage archiveSha256"), + artifact: artifactIdentity(value.artifact, "full release candidate sharedImage artifact"), + packageSha256: imagePackageSha256, + }; +} + +function assertProducedInSameAttempt(manifest) { + const expectedRunId = manifest.producer.runId; + const expectedRunAttempt = manifest.producer.runAttempt; + if ( + manifest.publisher.runId !== expectedRunId || + manifest.publisher.runAttempt !== expectedRunAttempt + ) { + fail("full release candidate publisher was not bound to the declared producer attempt"); + } + for (const [label, artifact] of [ + ["package", manifest.package.artifact], + ["prepublish plugin registry", manifest.prepublishPluginRegistry.artifact], + ["shared image", manifest.sharedImage.artifact], + ]) { + if (artifact.runId !== expectedRunId || artifact.runAttempt !== expectedRunAttempt) { + fail(`full release candidate ${label} artifact was not produced by the declared attempt`); + } + } +} + +function validateCandidateJobIdentity(value, label, request) { + exactKeys( + value, + ["jobId", "jobName", "repository", "runAttempt", "runId", "workflowPath", "workflowSha"], + label, + ); + const identity = { + jobId: positiveDecimal(value.jobId, `${label} jobId`), + jobName: ascii(value.jobName, `${label} jobName`), + repository: repository(value.repository, `${label} repository`), + runAttempt: positiveDecimal(value.runAttempt, `${label} runAttempt`), + runId: positiveDecimal(value.runId, `${label} runId`), + workflowPath: workflowPath(value.workflowPath, `${label} workflowPath`), + workflowSha: sha(value.workflowSha, `${label} workflowSha`), + }; + if (identity.repository !== request.repository || identity.workflowSha !== request.toolingSha) { + fail(`${label} does not match the request`); + } + return identity; +} + +function validateFullReleaseCandidateManifest(value) { + exactKeys( + value, + [ + "package", + "preparation", + "prepublishPluginRegistry", + "producer", + "publisher", + "request", + "requestSha256", + "schema", + "sharedImage", + ], + "full release candidate manifest", + ); + if (value.schema !== FULL_RELEASE_CANDIDATE_MANIFEST_SCHEMA) { + fail("full release candidate manifest schema is invalid"); + } + const request = validateFullReleaseCandidateRequest(value.request); + const requestSha256 = sha256(value.requestSha256, "full release candidate requestSha256"); + if (requestSha256 !== candidateRequestSha256(request)) { + fail("full release candidate requestSha256 does not match the request"); + } + const producer = validateCandidateJobIdentity( + value.producer, + "full release candidate producer", + request, + ); + const publisher = validateCandidateJobIdentity( + value.publisher, + "full release candidate publisher", + request, + ); + exactKeys( + value.preparation, + ["planSha256", "requiredPrepublishPluginPackages"], + "full release candidate preparation", + ); + const preparation = { + planSha256: sha256( + value.preparation.planSha256, + "full release candidate preparation planSha256", + ), + requiredPrepublishPluginPackages: sortedUniquePackages( + value.preparation.requiredPrepublishPluginPackages, + "full release candidate preparation requiredPrepublishPluginPackages", + ), + }; + const packageValue = validatePackage(value.package, request); + const manifest = { + schema: value.schema, + request, + requestSha256, + producer, + publisher, + preparation, + package: packageValue, + prepublishPluginRegistry: validateRegistry( + value.prepublishPluginRegistry, + request, + preparation.requiredPrepublishPluginPackages, + ), + sharedImage: validateSharedImage(value.sharedImage, packageValue.packageSha256), + }; + assertProducedInSameAttempt(manifest); + return manifest; +} + +function buildFullReleaseCandidateManifest(input) { + if (!isRecord(input)) { + fail("full release candidate manifest input must be an object"); + } + return validateFullReleaseCandidateManifest({ + schema: FULL_RELEASE_CANDIDATE_MANIFEST_SCHEMA, + ...input, + }); +} + +function canonicalFullReleaseCandidateManifestJson(value) { + const json = canonicalAsciiJson(validateFullReleaseCandidateManifest(value)); + if (Buffer.byteLength(json) > MANIFEST_MAX_BYTES) { + fail(`full release candidate manifest exceeds ${MANIFEST_MAX_BYTES} bytes`); + } + return json; +} + +function fullReleaseCandidateManifestSha256(value) { + return createHash("sha256") + .update(canonicalFullReleaseCandidateManifestJson(value)) + .digest("hex"); +} + +export function buildFullReleaseCandidateBinding({ artifact, manifest }) { + const validatedManifest = validateFullReleaseCandidateManifest(manifest); + const evidenceArtifact = artifactIdentity(artifact, "full release candidate evidence artifact"); + const expectedName = fullReleaseCandidateArtifactName(validatedManifest.requestSha256); + if ( + evidenceArtifact.name !== expectedName || + evidenceArtifact.runId !== validatedManifest.publisher.runId || + evidenceArtifact.runAttempt !== validatedManifest.publisher.runAttempt + ) { + fail("full release candidate evidence artifact does not match its manifest"); + } + return validateFullReleaseCandidateBinding({ + schema: FULL_RELEASE_CANDIDATE_BINDING_SCHEMA, + request: validatedManifest.request, + requestSha256: validatedManifest.requestSha256, + producer: validatedManifest.producer, + publisher: validatedManifest.publisher, + evidenceArtifact, + manifestSha256: fullReleaseCandidateManifestSha256(validatedManifest), + preparation: validatedManifest.preparation, + package: validatedManifest.package, + prepublishPluginRegistry: validatedManifest.prepublishPluginRegistry, + sharedImage: validatedManifest.sharedImage, + }); +} + +export function validateFullReleaseCandidateBinding(value) { + exactKeys( + value, + [ + "evidenceArtifact", + "manifestSha256", + "package", + "preparation", + "prepublishPluginRegistry", + "producer", + "publisher", + "request", + "requestSha256", + "schema", + "sharedImage", + ], + "full release candidate binding", + ); + if (value.schema !== FULL_RELEASE_CANDIDATE_BINDING_SCHEMA) { + fail("full release candidate binding schema is invalid"); + } + const request = validateFullReleaseCandidateRequest(value.request); + const requestSha256 = sha256(value.requestSha256, "full release candidate binding requestSha256"); + if (requestSha256 !== candidateRequestSha256(request)) { + fail("full release candidate binding requestSha256 does not match the request"); + } + const producer = validateCandidateJobIdentity( + value.producer, + "full release candidate binding producer", + request, + ); + const publisher = validateCandidateJobIdentity( + value.publisher, + "full release candidate binding publisher", + request, + ); + const evidenceArtifact = artifactIdentity( + value.evidenceArtifact, + "full release candidate binding evidenceArtifact", + ); + if ( + evidenceArtifact.name !== fullReleaseCandidateArtifactName(requestSha256) || + evidenceArtifact.runId !== publisher.runId || + evidenceArtifact.runAttempt !== publisher.runAttempt + ) { + fail("full release candidate binding evidence artifact is invalid"); + } + exactKeys( + value.preparation, + ["planSha256", "requiredPrepublishPluginPackages"], + "full release candidate binding preparation", + ); + const preparation = { + planSha256: sha256( + value.preparation.planSha256, + "full release candidate binding preparation planSha256", + ), + requiredPrepublishPluginPackages: sortedUniquePackages( + value.preparation.requiredPrepublishPluginPackages, + "full release candidate binding preparation requiredPrepublishPluginPackages", + ), + }; + const packageValue = validatePackage(value.package, request); + const prepublishPluginRegistry = validateRegistry( + value.prepublishPluginRegistry, + request, + preparation.requiredPrepublishPluginPackages, + ); + const sharedImage = validateSharedImage(value.sharedImage, packageValue.packageSha256); + const manifestSha256 = sha256( + value.manifestSha256, + "full release candidate binding manifestSha256", + ); + const reconstructedManifest = { + schema: FULL_RELEASE_CANDIDATE_MANIFEST_SCHEMA, + request, + requestSha256, + producer, + publisher, + preparation, + package: packageValue, + prepublishPluginRegistry, + sharedImage, + }; + if (manifestSha256 !== fullReleaseCandidateManifestSha256(reconstructedManifest)) { + fail("full release candidate binding manifestSha256 does not match its manifest fields"); + } + const binding = { + schema: value.schema, + request, + requestSha256, + producer, + publisher, + evidenceArtifact, + manifestSha256, + preparation, + package: packageValue, + prepublishPluginRegistry, + sharedImage, + }; + assertProducedInSameAttempt(binding); + if (Buffer.byteLength(canonicalAsciiJson(binding)) > BINDING_MAX_BYTES) { + fail(`full release candidate binding exceeds ${BINDING_MAX_BYTES} bytes`); + } + return binding; +} + +function option(args, name) { + const index = args.indexOf(name); + if (index < 0 || !args[index + 1]) { + fail(`missing ${name}`); + } + return args[index + 1]; +} + +function readJson(path, label) { + let value; + try { + value = JSON.parse(readFileSync(path, "utf8")); + } catch (error) { + fail(`${label} is invalid JSON: ${error instanceof Error ? error.message : String(error)}`); + } + return value; +} + +function runCli() { + const [command, ...args] = process.argv.slice(2); + if (command === "request") { + const request = buildFullReleaseCandidateRequest( + readJson(option(args, "--input"), "request input"), + ); + const requestJson = canonicalFullReleaseCandidateRequestJson(request); + writeFileSync(option(args, "--output"), requestJson); + process.stdout.write( + `${JSON.stringify({ + requestJson: requestJson.slice(0, -1), + requestSha256: candidateRequestSha256(request), + })}\n`, + ); + return; + } + if (command === "manifest") { + const manifest = buildFullReleaseCandidateManifest( + readJson(option(args, "--input"), "manifest input"), + ); + writeFileSync(option(args, "--output"), canonicalFullReleaseCandidateManifestJson(manifest)); + process.stdout.write( + `${JSON.stringify({ + manifestSha256: fullReleaseCandidateManifestSha256(manifest), + requestSha256: manifest.requestSha256, + })}\n`, + ); + return; + } + if (command === "binding") { + const manifest = readJson(option(args, "--manifest"), "candidate manifest"); + const binding = buildFullReleaseCandidateBinding({ + manifest, + artifact: { + name: option(args, "--artifact-name"), + id: option(args, "--artifact-id"), + digest: option(args, "--artifact-digest"), + expiresAt: option(args, "--artifact-expires-at"), + runId: option(args, "--artifact-run-id"), + runAttempt: option(args, "--artifact-run-attempt"), + }, + }); + process.stdout.write(`${JSON.stringify(binding)}\n`); + return; + } + fail("usage: full-release-candidate-contract.mjs <request|manifest|binding> ..."); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + try { + runCli(); + } catch (error) { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + process.exit(1); + } +} diff --git a/scripts/full-release-candidate-reuse.mjs b/scripts/full-release-candidate-reuse.mjs new file mode 100644 index 0000000000000000000000000000000000000000..1e303f53585443376542bbebe0c83db05dbc2d90 --- /dev/null +++ b/scripts/full-release-candidate-reuse.mjs @@ -0,0 +1,405 @@ +#!/usr/bin/env node +import { spawnSync } from "node:child_process"; +import { appendFileSync, readFileSync } from "node:fs"; +import { pathToFileURL } from "node:url"; +import { + canonicalFullReleaseCandidateRequestJson, + candidateRequestSha256, + fullReleaseCandidateArtifactName, + validateFullReleaseCandidateRequest, +} from "./full-release-candidate-contract.mjs"; +import { classifyReleaseGhTransportError } from "./full-release-validation-policy.mjs"; +import { downloadExactActionsArtifactArchive } from "./lib/actions-artifact-archive.mjs"; +import { + CandidateConstituentUnavailableError, + CandidateDiscoveryBudgetError, + CandidateEvaluationLimitError, + candidateArtifactJsonFromBinding, + loadSelectedFullReleaseCandidate, + resolveCandidateBinding, + selectTrustedFullReleaseCandidate, + validateCandidateBinding, + verifySealedFullReleaseCandidate, +} from "./lib/full-release-candidate-reuse.mjs"; +import { isRecord } from "./lib/record-shared.mjs"; + +const MAX_ARTIFACT_PAGES = 10; +const CANDIDATE_DISCOVERY_BUDGET_MS = 8 * 60 * 1000; +const GH_TIMEOUT_MS = 60_000; +const CANDIDATE_GH_TIMEOUT_MS = 20_000; +const CANDIDATE_GH_RETRY_ATTEMPTS = 2; +const GH_RETRY_BASE_DELAY_MS = 1_000; + +function fail(message) { + throw new Error(message); +} + +function requireDiscoveryBudget(deadlineMs) { + if (deadlineMs !== undefined && Date.now() >= deadlineMs) { + throw new CandidateDiscoveryBudgetError("candidate discovery exceeded its time budget"); + } +} + +function requestContract(input) { + const request = validateFullReleaseCandidateRequest(input); + const requestJson = canonicalFullReleaseCandidateRequestJson(request); + return { + request, + requestJson: requestJson.trimEnd(), + requestSha256: candidateRequestSha256(request), + }; +} + +function runGhJson( + repository, + path, + label, + { attempts = 3, deadlineMs, paginate = false, timeoutMs = GH_TIMEOUT_MS } = {}, +) { + let lastError = new Error(`${label} failed`); + for (let attempt = 1; attempt <= attempts; attempt += 1) { + requireDiscoveryBudget(deadlineMs); + const remainingMs = deadlineMs === undefined ? timeoutMs : deadlineMs - Date.now(); + const args = ["api"]; + if (paginate) { + args.push("--paginate", "--slurp"); + } + args.push(`repos/${repository}/${path}`); + const result = spawnSync("gh", args, { + encoding: "utf8", + killSignal: "SIGKILL", + maxBuffer: 2 * 1024 * 1024, + timeout: Math.max(1, Math.min(timeoutMs, remainingMs)), + }); + if (result.error) { + lastError = result.error; + } else if (result.status !== 0) { + lastError = new Error( + `${label} failed: ${result.stderr.trim() || `exit ${result.status ?? "unknown"}`}`, + ); + } else { + try { + return JSON.parse(result.stdout); + } catch (error) { + fail( + `${label} returned invalid JSON: ${error instanceof Error ? error.message : String(error)}`, + ); + } + } + if (attempt === attempts || classifyReleaseGhTransportError(lastError) !== "transient") { + throw lastError; + } + requireDiscoveryBudget(deadlineMs); + const retryDelayMs = + deadlineMs === undefined + ? GH_RETRY_BASE_DELAY_MS * attempt + : Math.min(GH_RETRY_BASE_DELAY_MS * attempt, Math.max(0, deadlineMs - Date.now())); + Atomics.wait( + new Int32Array(new SharedArrayBuffer(Int32Array.BYTES_PER_ELEMENT)), + 0, + 0, + retryDelayMs, + ); + } + throw lastError; +} + +function readCandidateWorkflowJobs(repository, runId, runAttempt, options) { + return readWorkflowJobPages( + repository, + `actions/runs/${runId}/attempts/${runAttempt}/jobs?per_page=100`, + options, + ); +} + +function readCandidateWorkflowHistory(repository, runId, options) { + return readWorkflowJobPages( + repository, + `actions/runs/${runId}/jobs?filter=all&per_page=100`, + options, + ); +} + +function readWorkflowJobPages(repository, path, options) { + const pages = runGhJson(repository, path, "full release candidate workflow jobs", { + ...options, + paginate: true, + }); + if ( + !Array.isArray(pages) || + pages.length === 0 || + pages.some((page) => !isRecord(page) || !Array.isArray(page.jobs)) + ) { + fail("full release candidate workflow job pages are invalid"); + } + const jobs = pages.flatMap((page) => page.jobs); + return { jobs, total_count: pages[0].total_count }; +} + +function readRepositoryArtifacts(repository, requestSha256, options) { + const artifacts = []; + const name = encodeURIComponent(fullReleaseCandidateArtifactName(requestSha256)); + for (let page = 1; page <= MAX_ARTIFACT_PAGES; page += 1) { + const response = runGhJson( + repository, + `actions/artifacts?name=${name}&per_page=100&page=${page}`, + "full release candidate artifact listing", + options, + ); + if (!Array.isArray(response.artifacts)) { + fail("full release candidate artifact listing is invalid"); + } + artifacts.push(...response.artifacts); + if (response.artifacts.length < 100) { + return artifacts; + } + } + return null; +} + +function readJson(path, label) { + try { + return JSON.parse(readFileSync(path, "utf8")); + } catch (error) { + return fail( + `${label} is invalid JSON: ${error instanceof Error ? error.message : String(error)}`, + ); + } +} + +function option(args, name) { + const index = args.indexOf(name); + if (index < 0 || !args[index + 1]) { + fail(`missing ${name}`); + } + return args[index + 1]; +} + +function optionalOption(args, name) { + const index = args.indexOf(name); + return index < 0 ? undefined : option(args, name); +} + +function output(name, value) { + const line = `${name}=${value}\n`; + if (process.env.GITHUB_OUTPUT) { + appendFileSync(process.env.GITHUB_OUTPUT, line); + } else { + process.stdout.write(line); + } +} + +async function discover(args) { + const contract = requestContract( + readJson(option(args, "--request-input"), "candidate request input"), + ); + const expectedRequestSha256 = optionalOption(args, "--expected-request-sha256"); + const expectedPackageSha256 = optionalOption(args, "--expected-package-sha256"); + if (expectedPackageSha256 !== undefined && !/^[a-f0-9]{64}$/u.test(expectedPackageSha256)) { + fail("expected package digest must be a lowercase SHA-256"); + } + if (expectedRequestSha256 !== undefined && expectedRequestSha256 !== contract.requestSha256) { + fail("full release candidate request digest does not match the expected request digest"); + } + const token = process.env.GH_TOKEN; + if (!token) { + fail("GH_TOKEN is required"); + } + output("request_json", contract.requestJson); + output("request_sha256", contract.requestSha256); + let selected; + const deadlineMs = Date.now() + CANDIDATE_DISCOVERY_BUDGET_MS; + const ghOptions = { + attempts: CANDIDATE_GH_RETRY_ATTEMPTS, + deadlineMs, + timeoutMs: CANDIDATE_GH_TIMEOUT_MS, + }; + try { + const artifacts = readRepositoryArtifacts( + contract.request.repository, + contract.requestSha256, + ghOptions, + ); + if (artifacts === null) { + output("state", "unavailable"); + output("reused", "false"); + output("reuse_reason", "candidate artifact inventory exceeded the bounded scan"); + return; + } + selected = await selectTrustedFullReleaseCandidate({ + artifacts, + deadlineMs, + request: contract.request, + readWorkflowRun: async (runId) => + runGhJson( + contract.request.repository, + `actions/runs/${runId}`, + "full release candidate workflow run", + ghOptions, + ), + readWorkflowJobs: async (runId) => + readCandidateWorkflowHistory(contract.request.repository, runId, ghOptions), + }); + } catch (error) { + if ( + error instanceof CandidateDiscoveryBudgetError || + error instanceof CandidateEvaluationLimitError + ) { + output("state", "unavailable"); + output("reused", "false"); + output("reuse_reason", error.message); + return; + } + if (classifyReleaseGhTransportError(error) !== "transient") { + throw error; + } + output("state", "unavailable"); + output("reused", "false"); + output("reuse_reason", "candidate discovery unavailable after bounded retries"); + return; + } + if (!selected) { + output("state", "miss"); + output("reused", "false"); + output("reuse_reason", "no trusted exact candidate artifact"); + return; + } + let binding; + try { + binding = await loadSelectedFullReleaseCandidate({ + deadlineMs, + downloadArchive: (params) => + downloadExactActionsArtifactArchive({ + ...params, + deadlineMs, + retryAttempts: CANDIDATE_GH_RETRY_ATTEMPTS, + timeoutMs: CANDIDATE_GH_TIMEOUT_MS, + }), + readArtifact: async (artifactId) => + runGhJson( + contract.request.repository, + `actions/artifacts/${artifactId}`, + "full release candidate constituent artifact", + ghOptions, + ), + readRunAttempt: async (runId, runAttempt) => + runGhJson( + contract.request.repository, + `actions/runs/${runId}/attempts/${runAttempt}`, + "full release candidate workflow attempt", + ghOptions, + ), + readWorkflowJobs: async (runId, runAttempt) => + readCandidateWorkflowJobs(contract.request.repository, runId, runAttempt, ghOptions), + request: contract.request, + selected, + token, + }); + } catch (error) { + if ( + !(error instanceof CandidateConstituentUnavailableError) && + !(error instanceof CandidateDiscoveryBudgetError) + ) { + throw error; + } + output("state", "unavailable"); + output("reused", "false"); + output("reuse_reason", error.message); + return; + } + if ( + expectedPackageSha256 !== undefined && + binding.package.packageSha256 !== expectedPackageSha256 + ) { + output("state", "miss"); + output("reused", "false"); + output("reuse_reason", "candidate package differs from the prepared publication bytes"); + return; + } + output("state", "hit"); + output("reused", "true"); + output("reuse_reason", "trusted exact candidate artifact"); + output("binding_json", JSON.stringify(binding)); + output("candidate_artifact_json", candidateArtifactJsonFromBinding(binding)); +} + +function resolveBinding(args) { + const input = readJson(option(args, "--input"), "candidate binding input"); + const binding = resolveCandidateBinding(input); + process.stdout.write( + `${JSON.stringify({ + binding, + candidateArtifactJson: binding ? candidateArtifactJsonFromBinding(binding) : "", + })}\n`, + ); +} + +async function verify(args) { + const plan = readJson(option(args, "--plan"), "release execution plan"); + if (plan.candidate === null) { + return; + } + const token = process.env.GH_TOKEN; + if (!token) { + fail("GH_TOKEN is required"); + } + const binding = validateCandidateBinding(plan.candidate); + const ghOptions = { + attempts: CANDIDATE_GH_RETRY_ATTEMPTS, + timeoutMs: CANDIDATE_GH_TIMEOUT_MS, + }; + await verifySealedFullReleaseCandidate({ + binding, + consumerRunAttempt: option(args, "--consumer-run-attempt"), + consumerRunId: option(args, "--consumer-run-id"), + readArtifact: async (artifactId) => + runGhJson( + binding.request.repository, + `actions/artifacts/${artifactId}`, + "sealed full release candidate artifact", + ghOptions, + ), + readRunAttempt: async (runId, runAttempt) => + runGhJson( + binding.request.repository, + `actions/runs/${runId}/attempts/${runAttempt}`, + "sealed full release candidate workflow attempt", + ghOptions, + ), + readWorkflowJobs: async (runId, runAttempt) => + readCandidateWorkflowJobs(binding.request.repository, runId, runAttempt, ghOptions), + downloadArchive: (params) => + downloadExactActionsArtifactArchive({ + ...params, + retryAttempts: CANDIDATE_GH_RETRY_ATTEMPTS, + timeoutMs: CANDIDATE_GH_TIMEOUT_MS, + }), + token, + }); +} + +async function main() { + const [command, ...args] = process.argv.slice(2); + if (command === "discover") { + await discover(args); + return; + } + if (command === "resolve") { + resolveBinding(args); + return; + } + if (command === "verify") { + await verify(args); + return; + } + fail("usage: full-release-candidate-reuse.mjs <discover|resolve|verify> ..."); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + try { + await main(); + } catch (error) { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + process.exit(1); + } +} diff --git a/scripts/full-release-validation-at-sha.mjs b/scripts/full-release-validation-at-sha.mjs new file mode 100644 index 0000000000000000000000000000000000000000..645715a6d6403c61a18a3ab1a445445cf8fc6898 --- /dev/null +++ b/scripts/full-release-validation-at-sha.mjs @@ -0,0 +1,6 @@ +#!/usr/bin/env node +import { runTsxCliShim } from "./lib/tsx-cli-shim.mjs"; + +await runTsxCliShim(import.meta.url, { + implementation: "./full-release-validation-at-sha.mts", +}); diff --git a/scripts/full-release-validation-policy.d.mts b/scripts/full-release-validation-policy.d.mts new file mode 100644 index 0000000000000000000000000000000000000000..d9ebf071b4dda43e9778fa1fac5cd1cb6872b091 --- /dev/null +++ b/scripts/full-release-validation-policy.d.mts @@ -0,0 +1,158 @@ +export const MAX_RELEASE_ARTIFACT_BYTES: number; +export const SPLIT_CHANGELOG_EVIDENCE_REUSE_POLICY: "split-changelog-release-v1"; +export function isSplitChangelogEvidenceDelta(paths: unknown, version: unknown): boolean; +export function classifyReleaseChangelogEvidenceComparison( + comparison: unknown, + identity: { baseSha: string; version?: unknown }, +): { changedPaths: string[]; policy: string }; +export function serializeReleaseArtifact(payload: unknown): string; +export function buildReleaseValidationManifest(input: { + plan: ReleaseRecord; + drain?: ReleaseRecord; + context: ReleaseRecord; +}): ReleaseRecord; +export function assertReleasePublicationKnownBudget( + plan: ReleaseRecord, + context: ReleaseRecord, +): void; +export function normalizeReleaseCoveragePolicy( + input: ReleaseRecord, +): "npm-beta-v1" | "npm-stable-v1" | undefined; +export function validateReleaseCoveragePolicyBinding( + plan: ReleaseRecord | undefined, + validationInputs?: ReleaseRecord, +): void; +export function normalizeReleaseTelegramWaiver(input: ReleaseRecord): string; +export function validateReleaseTelegramWaiverBinding( + plan: ReleaseRecord | undefined, + validationInputs?: ReleaseRecord, +): void; + +export interface ReleaseRecord { + [key: string]: unknown; +} +export interface ReleaseChild extends ReleaseRecord { + key: string; + runAttempt: number | null; + runId: string; +} +export interface ReleaseExecutionPlan extends ReleaseRecord { + sourceAdmissionContract?: "1"; + sourceAdmission?: import("./full-release-publication-contract.mjs").PublicationSourceFact | null; + publicationAdmissionContract?: "1"; + publicationAdmission?: + | import("./full-release-publication-contract.mjs").PublicationAdmission + | null; + children: ReleaseChild[]; + evidenceReuse: ReleaseRecord; + gates: ReleaseRecord[]; +} +export interface ReleaseStateArtifact extends ReleaseRecord { + blockers: ReleaseRecord[]; + children: Record<string, ReleaseRecord>; + errors: ReleaseRecord[]; + parentRunAttempt: number; + sourceParentRunAttempt: number; + state: string; +} +export interface ReleaseChildSpec { + dispatchName: string; + displayName: string; + key: string; + parentJobName: string; + rerunGroups: string[]; + suffix: string; + workflow: string; +} +export type ReleaseGhTransportErrorClass = "ambiguous" | "hard" | "transient"; +export function classifyReleaseGhTransportError(error: unknown): ReleaseGhTransportErrorClass; +export function isReleaseGhArtifactMissingError(error: unknown): boolean; +export function releaseChildSpec(key: string): ReleaseChildSpec; +export function validateReleaseChildRunProvenance( + run: ReleaseRecord, + expected?: ReleaseRecord, +): { + dispatchActor: string; + effectiveRunAttempt: number; + repository: string; + triggeringActor: string; +}; +export function validateReleaseChildDispatchBinding(input: ReleaseRecord): void; +export function buildReleaseExecutionPlan(input: ReleaseRecord): { + children: ReleaseChild[]; + gates: ReleaseRecord[]; +}; +export function buildReleaseExecutionPlanArtifact(input: ReleaseRecord): ReleaseExecutionPlan; +export function validateReleaseExecutionPlanArtifact( + payload: unknown, + expected?: Record<string, unknown>, +): ReleaseExecutionPlan; +export function releaseExecutionPlanSha256(plan: ReleaseRecord): string; +export function releaseCompositeJobsSha256(value: ReleaseRecord): string; +export function compareReleaseJobsByName(left: { name: string }, right: { name: string }): number; +export function composeReleaseAttemptJobs( + attempts: Array<{ jobs: ReleaseRecord[]; runAttempt: number }>, + expected: { effectiveRunAttempt: number; plannedRunAttempt: number }, +): { + effectiveRunAttempt: number; + jobs: ReleaseRecord[]; + plannedRunAttempt: number; + sha256: string; +}; +export function composeReleaseChildAttemptEvidence(input: { + attempts: Array<{ jobs: ReleaseRecord[]; runAttempt: number }>; + expected: ReleaseRecord; + run: ReleaseRecord; +}): ReleaseRecord; + +export function terminalPolicyPass( + child: ReleaseRecord, + releaseProfile: string, + workflowRef: string, +): boolean; + +export function classifyReleaseSnapshot(input: ReleaseRecord): ReleaseStateArtifact; +export function releasePlanGateFailures(gates: ReleaseRecord[]): ReleaseRecord[]; +export function buildReleaseStateArtifact(input: ReleaseRecord): ReleaseStateArtifact; +export function validateReleaseStateArtifact( + payload: unknown, + expected?: Record<string, unknown>, + expectedMode?: string, +): ReleaseStateArtifact; +export function verifyReleaseStateArtifacts( + executionPlanPayload: unknown, + decisionPayload: unknown, + drainPayload: unknown, + expected?: Record<string, unknown>, +): { + decision: ReleaseStateArtifact; + drain: ReleaseStateArtifact; + executionPlan: ReleaseExecutionPlan; + sourceAttempts: { + decision: number; + drain: number; + executionPlan: number; + }; +}; +export function selectReleaseStateArtifacts( + executionPlanPayload: unknown, + decisionCandidates: Array<{ name: string; payload: unknown }>, + drainCandidates: Array<{ name: string; payload: unknown }>, + expected?: Record<string, unknown>, +): { + decision: ReleaseStateArtifact; + drain: ReleaseStateArtifact; + executionPlan: ReleaseExecutionPlan; + sourceAttempts: { + decision: number; + drain: number; + executionPlan: number; + }; +}; +export function formatReleaseStateOutcome(payload: ReleaseRecord): string; +export function releaseStateChildEvidence(child: ReleaseRecord): ReleaseRecord; +export function affectedActiveRunIds( + children: ReleaseRecord[], + blockers: ReleaseRecord[], + cancelledRunIds?: Set<string>, +): string[]; diff --git a/scripts/full-release-validation-state.mjs b/scripts/full-release-validation-state.mjs new file mode 100644 index 0000000000000000000000000000000000000000..9665f96fdff3028a14d38ce275b0b7278e4a9875 --- /dev/null +++ b/scripts/full-release-validation-state.mjs @@ -0,0 +1,1571 @@ +#!/usr/bin/env node +import { execFile } from "node:child_process"; +import { + appendFileSync, + existsSync, + mkdirSync, + readFileSync, + readdirSync, + statSync, + writeFileSync, +} from "node:fs"; +import { dirname, join } from "node:path"; +import process from "node:process"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; +import { validateFullReleaseCandidateRequest } from "./full-release-candidate-contract.mjs"; +import { + createPublicationAdmission, + publicationObservationJson, + publicationSourceReuseIdentity, + validatePublicationAdmissionBinding, + validatePublicationSourceBinding, +} from "./full-release-publication-contract.mjs"; +import { + assertReleasePublicationKnownBudget, + affectedActiveRunIds, + buildReleaseExecutionPlan, + buildReleaseExecutionPlanArtifact, + buildReleaseStateArtifact, + buildReleaseValidationManifest, + classifyReleaseGhTransportError, + classifyReleaseSnapshot, + composeReleaseChildAttemptEvidence, + formatReleaseStateOutcome, + releasePlanGateFailures, + MAX_RELEASE_ARTIFACT_BYTES, + serializeReleaseArtifact, + selectReleaseStateArtifacts, + validateReleaseChildRunProvenance, + validateReleaseCoveragePolicyBinding, + validateReleaseExecutionPlanArtifact, + validateReleaseTelegramWaiverBinding, + verifyReleaseStateArtifacts, +} from "./full-release-validation-policy.mjs"; +import { sortJsonValueKeys } from "./lib/canonical-json.mjs"; + +export * from "./full-release-validation-policy.mjs"; + +const execFileAsync = promisify(execFile); +const RELEASE_SUMMARY_PATH = + process.env.OPENCLAW_RELEASE_CI_SUMMARY_VALIDATOR ?? + fileURLToPath(new URL("./release-ci-summary.mjs", import.meta.url)); +const API_ERROR_PATTERN = + /HTTP [45][0-9][0-9]|API|Bad credentials|rate limit|network|connection|timeout|ETIMEDOUT|ECONNRESET|EAI_AGAIN/u; +const DEFAULT_POLL_INTERVAL_MS = 60_000; +const DEFAULT_HEARTBEAT_INTERVAL_MS = 5 * 60_000; +const GH_TIMEOUT_MS = 60_000; +const TRANSPORT_UNCERTAINTY_MS = 15 * 60_000; +let ghRetryDeadline; + +function stringValue(value, fallback = "") { + return typeof value === "string" ? value : fallback; +} + +function requiredString(value, label) { + const normalized = stringValue(value).trim(); + if (!normalized) { + throw new Error(`${label} is required`); + } + return normalized; +} + +function positiveInteger(value, label) { + const normalized = Number(value); + if (!Number.isSafeInteger(normalized) || normalized < 1) { + throw new Error(`${label} must be a positive integer`); + } + return normalized; +} + +async function abortableSleep(milliseconds, signal) { + let abortError; + await new Promise((resolve) => { + const timer = setTimeout(resolve, milliseconds); + const abort = () => { + clearTimeout(timer); + abortError = signal?.reason instanceof Error ? signal.reason : new Error("operation aborted"); + resolve(); + }; + signal?.addEventListener("abort", abort, { once: true }); + }); + if (abortError instanceof Error) { + throw new Error(abortError.message, { cause: abortError }); + } +} + +const deadlineDelayMs = (delay, deadline, now) => + Number.isFinite(deadline) ? Math.max(0, Math.min(delay, deadline - now)) : delay; + +export function releaseGhRetryDelayMs(attempt, deadlineMonotonicMs, nowMonotonicMs) { + return deadlineDelayMs(Math.min(attempt * 10_000, 60_000), deadlineMonotonicMs, nowMonotonicMs); +} + +async function runGh(args, options = {}) { + const attempts = options.attempts ?? 6; + let lastError; + for (let attempt = 1; attempt <= attempts; attempt += 1) { + try { + const result = await execFileAsync("gh", args, { + encoding: "utf8", + env: process.env, + killSignal: "SIGKILL", + maxBuffer: 64 * 1024 * 1024, + signal: options.signal, + timeout: GH_TIMEOUT_MS, + }); + return result.stdout; + } catch (error) { + lastError = error; + if (options.signal?.aborted) { + throw options.signal.reason ?? error; + } + if (attempt === attempts || classifyReleaseGhTransportError(error) !== "transient") { + throw error; + } + const delay = releaseGhRetryDelayMs(attempt, ghRetryDeadline, performance.now()); + if (delay === 0) { + throw error; + } + await abortableSleep(delay, options.signal); + } + } + throw lastError; +} + +async function githubJson(path, signal) { + return JSON.parse( + await runGh(["api", `repos/${process.env.GITHUB_REPOSITORY}/${path}`], { signal }), + ); +} + +async function githubAttemptJobs(runId, runAttempt, signal) { + return ( + await runGh( + [ + "api", + "--paginate", + `repos/${process.env.GITHUB_REPOSITORY}/actions/runs/${runId}/attempts/${runAttempt}/jobs?per_page=100`, + "--jq", + ".jobs[] | @json", + ], + { signal }, + ) + ) + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line)); +} + +function issue(kind, child, message, extra = {}) { + return { + child: child.key, + kind, + message, + runId: stringValue(child.runId), + url: stringValue(child.url), + ...extra, + }; +} + +export function validateChildBinding(child, run, composite) { + const errors = []; + let provenance = {}; + try { + provenance = validateReleaseChildRunProvenance(run, { + ...child, + plannedRunAttempt: child.runAttempt, + repository: process.env.GITHUB_REPOSITORY, + }); + } catch (error) { + errors.push( + issue("provenance_mismatch", child, error instanceof Error ? error.message : String(error)), + ); + } + return { + ...child, + ...provenance, + compositeJobsSha256: stringValue(composite.sha256), + conclusion: stringValue(run.conclusion), + createdAt: stringValue(run.created_at), + errors, + jobs: composite.jobs, + observedRunAttempts: composite.observedRunAttempts, + plannedRunAttempt: Number(child.runAttempt), + runAttempt: Number(run.run_attempt), + runId: String(run.id), + status: stringValue(run.status), + updatedAt: stringValue(run.updated_at), + url: stringValue(run.html_url, child.url), + workflowRef: stringValue(run.head_branch), + workflowSha: stringValue(run.head_sha), + }; +} + +export async function readChild(child, previous, signal, options = {}) { + if (!child.selected) { + return { ...child, errors: [], jobs: [], status: "skipped" }; + } + if (!child.runId || !child.runAttempt) { + const error = issue("dispatch_missing", child, `${child.key} omitted its exact run identity`); + return { ...child, errors: [error], jobs: [], status: "missing" }; + } + try { + const run = options.readRun + ? await options.readRun(child.runId, signal) + : await githubJson(`actions/runs/${child.runId}`, signal); + const currentAttempt = positiveInteger(run.run_attempt, `${child.key} run attempt`); + const plannedAttempt = positiveInteger(child.runAttempt, `${child.key} planned run attempt`); + if (currentAttempt < plannedAttempt) { + return validateChildBinding(child, run, { + jobs: [], + observedRunAttempts: [], + sha256: "", + }); + } + const attempts = await Promise.all( + Array.from({ length: currentAttempt - plannedAttempt + 1 }, async (_, index) => { + const runAttempt = plannedAttempt + index; + return { + jobs: options.readAttemptJobs + ? await options.readAttemptJobs(child.runId, runAttempt, signal) + : await githubAttemptJobs(child.runId, runAttempt, signal), + runAttempt, + }; + }), + ); + if (run.status !== "completed" && attempts.at(-1)?.jobs.length === 0) { + if (attempts.slice(0, -1).some((attempt) => attempt.jobs.length === 0)) { + throw new Error(`${child.key} child attempt evidence is gapped`); + } + const partial = validateChildBinding(child, run, { + jobs: [], + observedRunAttempts: [], + sha256: "", + }); + return (previous?.compositeJobsSha256 || previous?.transportFailure) && + partial.errors.length === 0 + ? { + ...previous, + conclusion: stringValue(run.conclusion), + status: stringValue(run.status), + } + : partial; + } + const evidence = composeReleaseChildAttemptEvidence({ + attempts, + expected: { + ...child, + plannedRunAttempt: plannedAttempt, + repository: process.env.GITHUB_REPOSITORY, + }, + run, + }); + return validateChildBinding(child, run, { + jobs: evidence.jobs, + observedRunAttempts: evidence.observedRunAttempts, + sha256: evidence.compositeJobsSha256, + }); + } catch (error) { + const degraded = classifyReleaseGhTransportError(error) === "transient"; + const provenanceMismatch = + error instanceof Error && error.message.startsWith("release child provenance changed:"); + const readError = issue( + provenanceMismatch ? "provenance_mismatch" : "api_error", + child, + `${child.key} GitHub read failed: ${error instanceof Error ? error.message : String(error)}`, + ); + ghRetryDeadline ??= degraded ? performance.now() + TRANSPORT_UNCERTAINTY_MS : undefined; + return { + ...child, + ...previous, + errors: degraded + ? (previous?.errors ?? []).filter((entry) => entry.kind === "provenance_mismatch") + : [ + ...(previous?.errors ?? []).filter((entry) => entry.kind === "provenance_mismatch"), + readError, + ], + status: degraded ? "transport_uncertain" : stringValue(previous?.status, "unknown"), + transportFailure: degraded ? { errorClass: "transient" } : undefined, + }; + } +} + +export function updateReleaseTransportEpisode(previous, children, options = {}) { + const monotonicNow = options.monotonicNow ?? performance.now(); + const wallNow = options.wallNow ?? Date.now(); + const uncertain = children.filter((child) => child.transportFailure?.errorClass === "transient"); + const affected = uncertain + .map((child) => ({ + child: child.key, + compositeJobsSha256: stringValue(child.compositeJobsSha256), + errorClass: "transient", + lastValidAt: stringValue(child.updatedAt), + runAttempt: child.runAttempt, + runId: String(child.runId), + })) + .toSorted((left, right) => left.child.localeCompare(right.child, "en")); + if (affected.length === 0) { + return { status: "certain" }; + } + const deadline = options.deadline ?? ghRetryDeadline ?? monotonicNow + TRANSPORT_UNCERTAINTY_MS; + const wallStart = wallNow + deadline - monotonicNow - TRANSPORT_UNCERTAINTY_MS; + const episode = previous?.deadlineMonotonicMs + ? previous + : { + deadlineAt: new Date(wallStart + TRANSPORT_UNCERTAINTY_MS).toISOString(), + deadlineMonotonicMs: deadline, + startedAt: new Date(wallStart).toISOString(), + }; + return { + ...episode, + affected, + error: + monotonicNow >= episode.deadlineMonotonicMs + ? issue( + "transport_deadline_exceeded", + { key: "<collector>" }, + `GitHub transport remained uncertain; affected ${affected.map((child) => `${child.child}:${child.runId}:${child.runAttempt}`).join(",")}`, + ) + : undefined, + status: monotonicNow >= episode.deadlineMonotonicMs ? "expired" : "uncertain", + }; +} + +export function parsePlanInputs(value) { + const parsed = JSON.parse(requiredString(value, "plan inputs JSON")); + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + throw new Error("plan inputs JSON must be an object"); + } + return parsed; +} + +export function hydrateReusedPlan(plan, evidence) { + const byRole = new Map((evidence.children ?? []).map((child) => [child.role, child])); + return plan.map((child) => { + if (!child.selected) { + return child; + } + const reused = byRole.get(child.key); + if (!reused) { + return child; + } + return { + ...child, + displayTitle: reused.displayTitle, + result: "success", + // Reuse keeps the dispatch origin, so a human rerun still composes earlier jobs. + // Verified manifests predating childEvidence only carry the effective attempt. + runAttempt: + evidence.manifest.childEvidence === undefined + ? reused.runAttempt + : evidence.manifest.childEvidence[child.key].plannedRunAttempt, + runId: reused.runId, + url: reused.url, + workflowRef: reused.headBranch, + workflowSha: reused.workflowSha, + }; + }); +} + +function changedPathsValue(value) { + if (Array.isArray(value)) { + return value; + } + try { + const parsed = JSON.parse(stringValue(value, "[]")); + return Array.isArray(parsed) ? parsed : []; + } catch { + return []; + } +} + +async function validateReuse(executionPlan, signal) { + const { children: plan, evidenceReuse, trustedWorkflow } = executionPlan; + if (!evidenceReuse.requested) { + return { blockers: [], children: plan, errors: [] }; + } + try { + const args = [ + RELEASE_SUMMARY_PATH, + "--validate-run", + requiredString(evidenceReuse.selectedRunId, "evidence selected run ID"), + "--repo", + requiredString(process.env.GITHUB_REPOSITORY, "GitHub repository"), + "--trusted-workflow-ref", + requiredString(trustedWorkflow?.ref, "trusted workflow ref"), + "--trusted-workflow-full-ref", + requiredString(trustedWorkflow?.fullRef, "trusted workflow full ref"), + "--trusted-workflow-sha", + requiredString(trustedWorkflow?.sha, "trusted workflow SHA"), + "--verifier-source-sha", + requiredString(executionPlan.workflowSha, "workflow SHA"), + "--verifier-source-file", + RELEASE_SUMMARY_PATH, + "--expected-target-sha", + requiredString(process.env.TARGET_SHA, "target SHA"), + "--expected-evidence-policy", + requiredString(evidenceReuse.policy, "evidence policy"), + "--expected-evidence-sha", + requiredString(evidenceReuse.evidenceSha, "evidence SHA"), + "--expected-root-run-id", + requiredString(evidenceReuse.rootRunId, "evidence root run ID"), + "--expected-selected-run-id", + requiredString(evidenceReuse.selectedRunId, "evidence selected run ID"), + "--expected-changed-paths-json", + JSON.stringify(changedPathsValue(evidenceReuse.changedPaths)), + "--json", + ]; + const result = await execFileAsync(process.execPath, args, { + encoding: "utf8", + env: process.env, + killSignal: "SIGKILL", + maxBuffer: 64 * 1024 * 1024, + signal, + timeout: GH_TIMEOUT_MS * 6, + }); + const evidence = JSON.parse(result.stdout); + if ( + evidence.releaseProfile !== process.env.RELEASE_PROFILE || + evidence.rerunGroup !== process.env.RERUN_GROUP || + !evidence.manifest || + typeof evidence.manifest !== "object" || + Array.isArray(evidence.manifest) + ) { + throw new Error("reused release evidence no longer matches the requested validation"); + } + validateReleaseTelegramWaiverBinding(executionPlan, evidence.manifest.validationInputs); + validateReleaseCoveragePolicyBinding(executionPlan, evidence.manifest.validationInputs); + const source = validatePublicationSourceBinding(evidence.manifest, { + sourceAdmissionContract: executionPlan.sourceAdmissionContract, + }); + if ( + JSON.stringify(publicationSourceReuseIdentity(source)) !== + JSON.stringify(publicationSourceReuseIdentity(executionPlan.sourceAdmission)) + ) { + throw new Error("reused source admission differs from the requested publication source"); + } + return { + blockers: [], + children: hydrateReusedPlan(plan, evidence), + errors: [], + sourceManifest: sortJsonValueKeys(evidence.manifest), + }; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + const entry = { + child: "<evidence>", + kind: API_ERROR_PATTERN.test(message) ? "api_error" : "reused_evidence_invalid", + message, + runId: stringValue(evidenceReuse.selectedRunId), + url: stringValue(evidenceReuse.runUrl), + }; + return API_ERROR_PATTERN.test(message) + ? { blockers: [], children: plan, errors: [entry] } + : { blockers: [entry], children: plan, errors: [] }; + } +} + +function writeArtifact(path, payload) { + const json = serializeReleaseArtifact(payload); + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, json); +} + +function writeResult(path, payload) { + writeArtifact(path, payload); + if (process.env.GITHUB_OUTPUT) { + appendFileSync(process.env.GITHUB_OUTPUT, `state=${payload.state}\n`); + for (const [key, child] of Object.entries(payload.children ?? {})) { + appendFileSync(process.env.GITHUB_OUTPUT, `${key}_conclusion=${child.conclusion ?? ""}\n`); + } + } +} + +function writeExecutionPlan(path, payload) { + writeArtifact(path, payload); + if (process.env.GITHUB_OUTPUT) { + appendFileSync(process.env.GITHUB_OUTPUT, `sha256=${payload.sha256}\n`); + appendFileSync( + process.env.GITHUB_OUTPUT, + `source_parent_attempt=${payload.parentRunAttempt}\n`, + ); + } +} + +function appendSummary(mode, payload) { + if (!process.env.GITHUB_STEP_SUMMARY) { + return; + } + appendFileSync( + process.env.GITHUB_STEP_SUMMARY, + `## ${mode === "decision" ? "Release Decision" : "Diagnostic Drain"}\n\n${formatReleaseStateOutcome(payload)}\n`, + ); +} + +export function formatReleaseStateHeartbeat(mode, decision) { + return `${mode} heartbeat: state=${decision.state} active=${decision.activeRunIds.length} blockers=${decision.blockers.length} errors=${decision.errors.length}`; +} + +async function cancelAffectedChildren(children, blockers, cancelledRunIds, signal) { + const errors = []; + for (const runId of affectedActiveRunIds(children, blockers, cancelledRunIds)) { + const child = children.find((entry) => String(entry.runId) === runId); + try { + await runGh(["run", "cancel", runId], { attempts: 1, signal }); + cancelledRunIds.add(runId); + } catch (error) { + errors.push( + issue( + "api_error", + child ?? { key: "<child>", runId }, + `exact child cancellation failed: ${ + error instanceof Error ? error.message : String(error) + }`, + ), + ); + } + } + return errors; +} + +function readArtifact(path, label) { + try { + if (statSync(path).size > MAX_RELEASE_ARTIFACT_BYTES) { + throw new Error("release artifact exceeds the size limit"); + } + return JSON.parse(readFileSync(path, "utf8")); + } catch (error) { + throw new Error( + `${label} artifact is unreadable: ${error instanceof Error ? error.message : String(error)}`, + { cause: error }, + ); + } +} + +function verifyMode() { + const expected = { + publicationAdmissionContract: + process.env.FULL_RELEASE_PUBLICATION_ADMISSION_CONTRACT || undefined, + sourceAdmissionContract: process.env.FULL_RELEASE_SOURCE_ADMISSION_CONTRACT || undefined, + maxParentRunAttempt: positiveInteger(process.env.GITHUB_RUN_ATTEMPT, "parent run attempt"), + parentRunId: requiredString(process.env.GITHUB_RUN_ID, "parent run ID"), + repository: requiredString(process.env.GITHUB_REPOSITORY, "GitHub repository"), + releaseProfile: requiredString(process.env.RELEASE_PROFILE, "release profile"), + rerunGroup: requiredString(process.env.RERUN_GROUP, "rerun group"), + targetSha: requiredString(process.env.TARGET_SHA, "target SHA"), + workflowRef: requiredString(process.env.GITHUB_REF_NAME, "workflow ref"), + workflowSha: requiredString(process.env.GITHUB_SHA, "workflow SHA"), + }; + const verified = verifyReleaseStateArtifacts( + readArtifact( + requiredString(process.env.RELEASE_EXECUTION_PLAN_PATH, "execution plan path"), + "execution plan", + ), + readArtifact(requiredString(process.env.RELEASE_DECISION_PATH, "decision path"), "decision"), + readArtifact(requiredString(process.env.DIAGNOSTIC_DRAIN_PATH, "drain path"), "drain"), + expected, + ); + appendSummary("decision", verified.decision); + appendSummary("drain", verified.drain); +} + +function planExpected() { + return { + publicationAdmissionContract: + process.env.FULL_RELEASE_PUBLICATION_ADMISSION_CONTRACT || undefined, + sourceAdmissionContract: process.env.FULL_RELEASE_SOURCE_ADMISSION_CONTRACT || undefined, + targetContextRef: process.env.TARGET_CONTEXT_REF || undefined, + coveragePolicy: process.env.COVERAGE_POLICY || undefined, + telegramWaiver: process.env.TELEGRAM_WAIVER ?? "", + ...(process.env.TARGET_VERSION ? { targetVersion: process.env.TARGET_VERSION } : {}), + parentRunId: requiredString(process.env.GITHUB_RUN_ID, "parent run ID"), + repository: requiredString(process.env.GITHUB_REPOSITORY, "GitHub repository"), + releaseProfile: requiredString(process.env.RELEASE_PROFILE, "release profile"), + rerunGroup: requiredString(process.env.RERUN_GROUP, "rerun group"), + targetSha: stringValue(process.env.TARGET_SHA), + workflowSha: requiredString(process.env.GITHUB_SHA, "workflow SHA"), + workflowRef: requiredString(process.env.GITHUB_REF_NAME, "workflow ref"), + }; +} + +function manifestContextFromEnvironment(source) { + const env = process.env; + const coverage = source?.coverage ?? {}; + const inputs = {}; + for (const [key, variable, sourceKey] of [ + ["provider", "PROVIDER", "provider"], + ["mode", "MODE", "mode"], + ["liveSuiteFilter", "LIVE_SUITE_FILTER", "live_suite_filter"], + ["crossOsSuiteFilter", "CROSS_OS_SUITE_FILTER", "cross_os_suite_filter"], + ["releasePackageSpec", "RELEASE_PACKAGE_SPEC", "release_package_spec"], + [ + "packageAcceptancePackageSpec", + "PACKAGE_ACCEPTANCE_PACKAGE_SPEC", + "package_acceptance_package_spec", + ], + ["codexPluginSpec", "CODEX_PLUGIN_SPEC", "codex_plugin_spec"], + ["npmTelegramPackageSpec", "NPM_TELEGRAM_PACKAGE_SPEC", "npm_telegram_package_spec"], + ["npmTelegramProviderMode", "NPM_TELEGRAM_PROVIDER_MODE", "npm_telegram_provider_mode"], + ["npmTelegramScenario", "NPM_TELEGRAM_SCENARIO", "npm_telegram_scenario"], + ["skipPackageTelegramE2e", "SKIP_PACKAGE_TELEGRAM_E2E", "skip_package_telegram_e2e"], + ["allowUnreleasedChangelog", "ALLOW_UNRELEASED_CHANGELOG", "allow_unreleased_changelog"], + [ + "pluginPrereleaseNodeExcludePatternsJson", + "PLUGIN_PRERELEASE_NODE_EXCLUDE_PATTERNS_JSON", + "plugin_prerelease_node_exclude_patterns_json", + ], + ]) { + inputs[key] = env[variable] ?? coverage[sourceKey] ?? ""; + } + inputs.targetContextRef = env.TARGET_CONTEXT_REF ?? source?.targetContextRef ?? ""; + inputs.targetVersion = env.TARGET_VERSION ?? source?.projection?.version ?? ""; + const waiver = env.TELEGRAM_WAIVER ?? coverage.telegram_waiver ?? ""; + if (waiver) { + inputs.telegramWaiver = waiver; + } + return { + runId: env.GITHUB_RUN_ID, + runAttempt: env.GITHUB_RUN_ATTEMPT, + workflowRef: env.GITHUB_REF_NAME, + workflowSha: env.GITHUB_SHA, + workflowFullRef: env.GITHUB_REF, + workflowRefType: env.GITHUB_REF_TYPE, + targetRef: env.TARGET_REF ?? source?.targetContextRef ?? "", + releaseProfile: env.RELEASE_PROFILE ?? coverage.release_profile ?? "", + rerunGroup: env.RERUN_GROUP ?? coverage.rerun_group ?? "", + runReleaseSoak: env.RUN_RELEASE_SOAK ?? coverage.run_release_soak ?? "", + validationInputs: inputs, + publicationArtifacts: { + npmPreflight: JSON.parse(env.QUALIFIED_NPM_BUNDLE_JSON || "null"), + docker: env.PREPARED_DOCKER_MANIFEST_SHA256 + ? { + preparedRunId: env.PREPARED_DOCKER_RUN_ID, + preparedRunAttempt: env.PREPARED_DOCKER_RUN_ATTEMPT, + preparedArtifactName: env.PREPARED_DOCKER_ARTIFACT_NAME, + preparedManifestSha256: env.PREPARED_DOCKER_MANIFEST_SHA256, + } + : null, + }, + }; +} + +function publicationKnownBudget(record, evidenceReuse) { + serializeReleaseArtifact(record); + const source = record.sourceAdmission; + const context = manifestContextFromEnvironment(source); + const candidateRequest = candidateRequestFromEnvironment(); + const inputs = { + childPhaseVersion: 3, + parentRunId: source.runId, + parentRunAttempt: source.runAttempt, + workflowRef: context.workflowRef, + workflowSha: source.workflow.sha, + releaseProfile: context.releaseProfile, + rerunGroup: context.rerunGroup, + targetVersion: context.validationInputs.targetVersion, + runReleaseSoak: context.runReleaseSoak, + coveragePolicy: source.coverage.coverage_policy || undefined, + telegramWaiver: source.coverage.telegram_waiver, + releasePackageSpec: source.coverage.release_package_spec, + npmTelegramPackageSpec: source.coverage.npm_telegram_package_spec, + liveSuiteFilter: source.coverage.live_suite_filter, + candidateRequired: process.env.PUBLICATION_CANDIDATE_REQUIRED === "true", + resolveTargetResult: "success", + evidenceReuse: evidenceReuse?.requested === true, + }; + const built = buildReleaseExecutionPlan(inputs); + const plan = buildReleaseExecutionPlanArtifact({ + ...record, + ...inputs, + evidenceReuse, + attemptEvidenceVersion: 3, + children: built.children, + gates: built.gates, + trustedWorkflow: { + fullRef: source.tooling.ref, + ref: source.tooling.ref.replace(/^refs\/(?:heads|tags)\//u, ""), + sha: source.tooling.sha, + }, + expected: { + ...inputs, + repository: source.repository, + targetSha: source.candidateSha, + candidateRequest, + }, + }); + assertReleasePublicationKnownBudget(plan, context); +} + +async function publicationReuseMode() { + const directory = requiredString(process.env.RUNNER_TEMP, "runner temporary directory"); + const record = readArtifact( + join(directory, "full-release-publication-admission/publication-admission.json"), + "publication admission", + ); + validatePublicationAdmissionBinding(record, { + publicationAdmissionContract: "1", + parentRunId: process.env.GITHUB_RUN_ID, + sourceParentRunAttempt: 1, + workflowSha: process.env.GITHUB_SHA, + workflowRef: process.env.GITHUB_REF_NAME, + }); + let reuse; + let outputs; + if (positiveInteger(process.env.GITHUB_RUN_ATTEMPT, "parent attempt") > 1) { + const { restoreOriginalPublicationAdmission } = await import("./release-ci-summary.mjs"); + const restored = await restoreOriginalPublicationAdmission({ + request: { ...record.sourceAdmission, runAttempt: Number(process.env.GITHUB_RUN_ATTEMPT) }, + }); + reuse = restored.plan.evidenceReuse; + outputs = reuse.requested + ? { + reuse: "true", + evidence_run_id: reuse.selectedRunId, + evidence_root_run_id: reuse.rootRunId, + evidence_run_url: reuse.runUrl, + evidence_sha: reuse.evidenceSha, + evidence_policy: reuse.policy, + changed_paths: JSON.stringify(reuse.changedPaths), + changed_path_count: String(reuse.changedPaths.length), + } + : { reuse: "false", reuse_reason: "original admission did not reuse evidence" }; + } else { + const path = join(directory, "reusable-evidence.outputs"); + if (statSync(path).size > MAX_RELEASE_ARTIFACT_BYTES) { + throw new Error("publication reuse output exceeds the enclosing artifact budget"); + } + outputs = {}; + for (const line of readFileSync(path, "utf8").trimEnd().split("\n")) { + const separator = line.indexOf("="); + const key = line.slice(0, separator); + if ( + separator < 1 || + Object.hasOwn(outputs, key) || + ![ + "reuse", + "reuse_reason", + "evidence_run_id", + "evidence_root_run_id", + "evidence_run_url", + "evidence_sha", + "evidence_policy", + "changed_paths", + "changed_path_count", + "evidence_manifest", + ].includes(key) + ) { + throw new Error("invalid publication reuse output"); + } + outputs[key] = line.slice(separator + 1); + } + if (!["true", "false"].includes(outputs.reuse)) { + throw new Error("publication reuse outcome is missing"); + } + if (outputs.reuse === "true") { + reuse = { + requested: true, + selectedRunId: outputs.evidence_run_id, + rootRunId: outputs.evidence_root_run_id, + runUrl: outputs.evidence_run_url, + evidenceSha: outputs.evidence_sha, + policy: outputs.evidence_policy, + changedPaths: JSON.parse(outputs.changed_paths), + sourceManifest: JSON.parse(outputs.evidence_manifest), + }; + validatePublicationAdmissionBinding( + reuse.sourceManifest, + record.sourceAdmission.validationPurpose === "publish" + ? { publicationAdmissionContract: "1" } + : {}, + ); + } + delete outputs.evidence_manifest; + } + publicationKnownBudget(record, reuse); + // Bulk root evidence remains in the private file/retained plan, never job outputs. + const lines = []; + for (const [key, value] of Object.entries(outputs)) { + if (typeof value !== "string" || /[\r\n]/u.test(value)) { + throw new Error("publication reuse output is not bounded single-line metadata"); + } + lines.push(`${key}=${value}\n`); + } + const output = lines.join(""); + if (Buffer.byteLength(output, "utf16le") > MAX_RELEASE_ARTIFACT_BYTES) { + throw new Error("publication reuse job outputs exceed the UTF-16 size limit"); + } + appendFileSync(process.env.GITHUB_OUTPUT, output); +} + +async function publicationMode(mode) { + const directory = requiredString(process.env.RUNNER_TEMP, "runner temporary directory"); + const sourcePath = join(directory, "publication-source-admission.json"); + const admissionPath = join(directory, "publication-admission.json"); + if (mode === "restore-publication") { + if (positiveInteger(process.env.GITHUB_RUN_ATTEMPT, "parent attempt") <= 1) { + throw new Error("publication restoration requires a later parent attempt"); + } + const request = readArtifact( + join(directory, "publication-source-request.json"), + "publication request", + ); + const { restoreOriginalPublicationAdmission } = await import("./release-ci-summary.mjs"); + const restored = await restoreOriginalPublicationAdmission({ request }); + writeArtifact(sourcePath, restored.source); + writeArtifact(admissionPath, { + sourceAdmissionContract: "1", + sourceAdmission: restored.source, + publicationAdmissionContract: "1", + publicationAdmission: restored.admission, + }); + return; + } + if (positiveInteger(process.env.GITHUB_RUN_ATTEMPT, "parent attempt") !== 1) { + // Only the preceding authenticated restore creates this file on later attempts. + const restored = readArtifact(admissionPath, "restored publication admission"); + validatePublicationAdmissionBinding(restored, { + publicationAdmissionContract: "1", + parentRunId: process.env.GITHUB_RUN_ID, + sourceParentRunAttempt: 1, + workflowSha: process.env.GITHUB_SHA, + workflowRef: process.env.GITHUB_REF_NAME, + }); + publicationKnownBudget(restored); + return; + } + const source = readArtifact(sourcePath, "publication source"); + let admission = null; + if (source.validationPurpose === "publish") { + if (process.env.PUBLICATION_UPLOAD_OUTCOME !== "success") { + throw new Error("publication observation upload did not succeed"); + } + const path = join(directory, "publication-observations.json"); + const observations = readArtifact(path, "publication observations"); + if (readFileSync(path, "utf8") !== publicationObservationJson(observations)) { + throw new Error("publication observation upload bytes are not canonical"); + } + const { createReleaseEvidenceClient, validatePublicationObservationArtifactIdentity } = + await import("./release-ci-summary.mjs"); + const uploaded = { + id: requiredString(process.env.PUBLICATION_ARTIFACT_ID, "publication artifact ID"), + digest: requiredString( + process.env.PUBLICATION_ARTIFACT_DIGEST, + "publication artifact digest", + ), + }; + if (!/^[a-f0-9]{64}$/u.test(uploaded.digest)) { + throw new Error("invalid publication upload action digest"); + } + uploaded.digest = `sha256:${uploaded.digest}`; + const metadata = createReleaseEvidenceClient(source.repository).getArtifact(uploaded.id); + const descriptor = validatePublicationObservationArtifactIdentity(metadata, source, uploaded); + admission = createPublicationAdmission( + source, + observations, + descriptor, + new Date().toISOString(), + ); + } + const record = { + sourceAdmissionContract: "1", + sourceAdmission: source, + publicationAdmissionContract: "1", + publicationAdmission: admission, + }; + validatePublicationAdmissionBinding(record, { publicationAdmissionContract: "1" }); + publicationKnownBudget(record); + writeArtifact(admissionPath, record); +} + +function writeManifestMode() { + const plan = readArtifact(process.env.RELEASE_EXECUTION_PLAN_PATH, "execution plan"); + const drain = readArtifact(process.env.DIAGNOSTIC_DRAIN_PATH, "diagnostic drain"); + const manifest = buildReleaseValidationManifest({ + plan, + drain, + context: manifestContextFromEnvironment(plan.sourceAdmission), + }); + writeArtifact( + join( + requiredString(process.env.RUNNER_TEMP, "runner temporary directory"), + "full-release-validation/full-release-validation-manifest.json", + ), + manifest, + ); +} + +function candidateRequestFromInputs(planInputs) { + return validateFullReleaseCandidateRequest(planInputs.candidateRequestInput); +} + +function candidateRequestFromEnvironment() { + return validateFullReleaseCandidateRequest( + JSON.parse(requiredString(process.env.CANDIDATE_REQUEST_JSON, "candidate request JSON")), + ); +} + +function evidenceReuseFromInputs(planInputs, sourceManifest = {}) { + return { + changedPaths: changedPathsValue(planInputs.evidenceChangedPaths), + evidenceSha: stringValue(planInputs.evidenceSha), + policy: stringValue(planInputs.evidencePolicy), + requested: planInputs.evidenceReuse === true || planInputs.evidenceReuse === "true", + rootRunId: stringValue(planInputs.evidenceRootRunId), + runUrl: stringValue(planInputs.evidenceRunUrl), + selectedRunId: stringValue(planInputs.evidenceRunId), + sourceManifest, + }; +} + +function trustedWorkflowFromInputs(planInputs) { + return planInputs.trustedWorkflow; +} + +function candidateFromInputs(planInputs, gates) { + const candidate = planInputs.candidateEvidence ?? null; + const bindingRequired = gates.some( + (gate) => + ["Acquire full release candidate", "Prepare shared release candidate"].includes(gate.name) && + gate.required, + ); + if (!bindingRequired) { + if (candidate !== null) { + throw new Error("release candidate evidence exists when candidate binding is not required"); + } + return null; + } + if ( + stringValue(planInputs.candidateAcquisitionResult ?? planInputs.candidateBindingResult) === + "success" + ) { + if (!candidate || typeof candidate !== "object" || Array.isArray(candidate)) { + throw new Error("successful release candidate binding omitted producer evidence"); + } + return candidate; + } + if (candidate !== null) { + throw new Error("release candidate evidence exists without successful binding"); + } + return null; +} + +async function planMode() { + const outputPath = requiredString( + process.env.FULL_RELEASE_EXECUTION_PLAN_PATH, + "execution plan output path", + ); + const expected = planExpected(); + const currentAttempt = positiveInteger(process.env.GITHUB_RUN_ATTEMPT, "parent run attempt"); + + if (process.env.FULL_RELEASE_RESTORE_PLAN === "true") { + const restoredPayload = readArtifact(outputPath, "execution plan"); + const restored = validateReleaseExecutionPlanArtifact(restoredPayload, { + ...expected, + ...(restoredPayload.attemptEvidenceVersion !== undefined + ? { candidateRequest: candidateRequestFromEnvironment() } + : {}), + sourceParentRunAttempt: 1, + }); + if (expected.publicationAdmissionContract) { + const source = JSON.parse( + requiredString(process.env.SOURCE_ADMISSION_JSON, "restored source admission"), + ); + const { restoreOriginalPublicationAdmission } = await import("./release-ci-summary.mjs"); + const original = await restoreOriginalPublicationAdmission({ + request: { ...source, runAttempt: currentAttempt }, + }); + if ( + JSON.stringify(sortJsonValueKeys(original.plan)) !== + JSON.stringify(sortJsonValueKeys(restored)) + ) { + throw new Error("cached publication plan differs from its authenticated original"); + } + } + writeExecutionPlan(outputPath, restored); + return; + } + if (currentAttempt !== 1) { + throw new Error("collector retry omitted the immutable attempt-one execution plan"); + } + + const planInputs = { + ...parsePlanInputs(process.env.FULL_RELEASE_PLAN_INPUTS_JSON), + releaseProfile: expected.releaseProfile, + }; + if (expected.publicationAdmissionContract) { + const publication = + planInputs.resolveTargetResult === "success" + ? readArtifact( + requiredString(process.env.PUBLICATION_ADMISSION_PATH, "publication admission path"), + "publication admission", + ) + : { + publicationAdmissionContract: expected.publicationAdmissionContract, + publicationAdmission: null, + }; + Object.assign(planInputs, publication); + } + const attemptEvidenceVersion = Number(planInputs.childPhaseVersion) === 3 ? 3 : 2; + const built = buildReleaseExecutionPlan(planInputs); + const candidate = candidateFromInputs(planInputs, built.gates); + const candidateRequest = candidateRequestFromInputs(planInputs); + const abortController = new AbortController(); + let finished = false; + let plan = buildReleaseExecutionPlanArtifact({ + attemptEvidenceVersion, + sourceAdmissionContract: planInputs.sourceAdmissionContract, + sourceAdmission: planInputs.sourceAdmission, + publicationAdmissionContract: planInputs.publicationAdmissionContract, + publicationAdmission: planInputs.publicationAdmission, + candidate, + coveragePolicy: planInputs.coveragePolicy, + children: built.children, + evidenceReuse: evidenceReuseFromInputs(planInputs), + expected: { ...expected, candidateRequest, parentRunAttempt: currentAttempt }, + gates: built.gates, + releaseProfile: expected.releaseProfile, + rerunGroup: expected.rerunGroup, + telegramWaiver: planInputs.telegramWaiver, + targetVersion: planInputs.targetVersion, + trustedWorkflow: trustedWorkflowFromInputs(planInputs), + }); + const stop = () => { + if (finished) { + return; + } + abortController.abort(new Error("execution plan collection cancelled")); + plan = buildReleaseExecutionPlanArtifact({ + attemptEvidenceVersion, + sourceAdmissionContract: plan.sourceAdmissionContract, + sourceAdmission: plan.sourceAdmission, + publicationAdmissionContract: plan.publicationAdmissionContract, + publicationAdmission: plan.publicationAdmission, + blockers: plan.blockers, + candidate: plan.candidate, + coveragePolicy: plan.coveragePolicy, + children: plan.children, + errors: [ + ...plan.errors, + { + child: "<collector>", + kind: "collector_cancelled", + message: "execution plan collector received a termination signal", + }, + ], + evidenceReuse: plan.evidenceReuse, + expected: { + ...expected, + candidateRequest: plan.candidateRequest, + parentRunAttempt: currentAttempt, + }, + gates: plan.gates, + releaseProfile: expected.releaseProfile, + rerunGroup: expected.rerunGroup, + telegramWaiver: plan.telegramWaiver, + targetVersion: plan.targetVersion, + trustedWorkflow: plan.trustedWorkflow, + }); + writeExecutionPlan(outputPath, plan); + finished = true; + process.exit(1); + }; + process.once("SIGINT", stop); + process.once("SIGTERM", stop); + + const reuse = await validateReuse(plan, abortController.signal); + if (finished) { + return; + } + plan = buildReleaseExecutionPlanArtifact({ + attemptEvidenceVersion, + sourceAdmissionContract: planInputs.sourceAdmissionContract, + sourceAdmission: planInputs.sourceAdmission, + publicationAdmissionContract: planInputs.publicationAdmissionContract, + publicationAdmission: planInputs.publicationAdmission, + blockers: reuse.blockers, + candidate, + coveragePolicy: planInputs.coveragePolicy, + children: reuse.children, + errors: reuse.errors, + evidenceReuse: evidenceReuseFromInputs(planInputs, reuse.sourceManifest), + expected: { ...expected, candidateRequest, parentRunAttempt: currentAttempt }, + gates: built.gates, + releaseProfile: expected.releaseProfile, + rerunGroup: expected.rerunGroup, + telegramWaiver: planInputs.telegramWaiver, + targetVersion: planInputs.targetVersion, + trustedWorkflow: trustedWorkflowFromInputs(planInputs), + }); + writeExecutionPlan(outputPath, plan); + finished = true; + if ((reuse.blockers?.length ?? 0) > 0 || (reuse.errors?.length ?? 0) > 0) { + throw new Error("release execution plan could not bind reusable evidence"); + } +} + +async function collectMode(mode) { + const expected = { + parentRunAttempt: positiveInteger(process.env.GITHUB_RUN_ATTEMPT, "parent run attempt"), + parentRunId: requiredString(process.env.GITHUB_RUN_ID, "parent run ID"), + repository: requiredString(process.env.GITHUB_REPOSITORY, "GitHub repository"), + targetSha: stringValue(process.env.TARGET_SHA), + workflowRef: requiredString(process.env.GITHUB_REF_NAME, "workflow ref"), + workflowSha: requiredString(process.env.GITHUB_SHA, "workflow SHA"), + }; + const releaseProfile = requiredString(process.env.RELEASE_PROFILE, "release profile"); + const rerunGroup = requiredString(process.env.RERUN_GROUP, "rerun group"); + const outputPath = requiredString(process.env.FULL_RELEASE_STATE_PATH, "state output path"); + const executionPlan = validateReleaseExecutionPlanArtifact( + readArtifact( + requiredString(process.env.FULL_RELEASE_EXECUTION_PLAN_PATH, "execution plan path"), + "execution plan", + ), + { + publicationAdmissionContract: + process.env.FULL_RELEASE_PUBLICATION_ADMISSION_CONTRACT || undefined, + sourceAdmissionContract: process.env.FULL_RELEASE_SOURCE_ADMISSION_CONTRACT || undefined, + parentRunId: expected.parentRunId, + repository: expected.repository, + releaseProfile, + rerunGroup, + targetSha: expected.targetSha, + workflowSha: expected.workflowSha, + }, + ); + const plan = executionPlan.children; + const gateFailures = releasePlanGateFailures(executionPlan.gates); + const failFast = mode === "decision" && process.env.FAIL_FAST === "true"; + const pollIntervalMs = + Number(process.env.FULL_RELEASE_POLL_INTERVAL_MS) || DEFAULT_POLL_INTERVAL_MS; + const heartbeatIntervalMs = + Number(process.env.FULL_RELEASE_HEARTBEAT_INTERVAL_MS) || DEFAULT_HEARTBEAT_INTERVAL_MS; + const cancelledRunIds = new Set(); + let snapshots = plan.map((child) => ({ + ...child, + errors: [], + jobs: [], + status: child.selected && child.runId ? "queued" : child.selected ? "missing" : "skipped", + })); + let finished = false; + const abortController = new AbortController(); + let decisionReuse = { blockers: [], children: plan, errors: [] }; + let transport = { status: "certain" }; + + const writePayload = (decision, cancellation = {}) => { + const payload = buildReleaseStateArtifact({ + cancellation, + children: snapshots, + decision, + executionPlan, + expected, + mode, + releaseProfile, + rerunGroup, + transport, + }); + writeResult(outputPath, payload); + appendSummary(mode, payload); + return payload; + }; + const stop = () => { + if (finished) { + return; + } + abortController.abort(new Error(`${mode} collector cancelled`)); + const decision = classifyReleaseSnapshot({ + cancelled: true, + children: snapshots, + extraBlockers: executionPlan.blockers, + extraErrors: [ + ...executionPlan.errors, + ...(transport.error ? [transport.error] : []), + { + child: "<collector>", + kind: "collector_cancelled", + message: `${mode} collector received a termination signal`, + }, + ], + localFailures: gateFailures, + releaseProfile, + workflowRef: expected.workflowRef, + }); + writePayload(decision, { cancelledRunIds, requested: true }); + finished = true; + process.exit(1); + }; + process.once("SIGINT", stop); + process.once("SIGTERM", stop); + + if (mode === "decision" && executionPlan.evidenceReuse.requested) { + decisionReuse = await validateReuse(executionPlan, abortController.signal); + const exactPlan = JSON.stringify( + plan.map(({ key, runAttempt, runId }) => ({ key, runAttempt, runId })), + ); + const revalidatedPlan = JSON.stringify( + decisionReuse.children.map(({ key, runAttempt, runId }) => ({ key, runAttempt, runId })), + ); + if (exactPlan !== revalidatedPlan) { + decisionReuse = { + ...decisionReuse, + blockers: [ + ...decisionReuse.blockers, + { + child: "<evidence>", + kind: "provenance_mismatch", + message: "revalidated evidence child identities differ from the immutable plan", + runId: executionPlan.evidenceReuse.rootRunId, + url: executionPlan.evidenceReuse.runUrl, + }, + ], + }; + } + if ( + JSON.stringify(sortJsonValueKeys(decisionReuse.sourceManifest)) !== + JSON.stringify(sortJsonValueKeys(executionPlan.evidenceReuse.sourceManifest)) + ) { + decisionReuse = { + ...decisionReuse, + blockers: [ + ...decisionReuse.blockers, + { + child: "<evidence>", + kind: "provenance_mismatch", + message: "revalidated evidence source manifest differs from the immutable plan", + runId: executionPlan.evidenceReuse.rootRunId, + url: executionPlan.evidenceReuse.runUrl, + }, + ], + }; + } + } + + let nextHeartbeat = 0; + while (!finished) { + ghRetryDeadline = transport.deadlineMonotonicMs; + snapshots = await Promise.all( + plan.map((child, index) => readChild(child, snapshots[index], abortController.signal)), + ); + transport = updateReleaseTransportEpisode(transport, snapshots); + const transportReadErrors = transport.error ? [transport.error] : []; + let decision = classifyReleaseSnapshot({ + children: snapshots, + extraBlockers: [...executionPlan.blockers, ...decisionReuse.blockers], + extraErrors: [...transportReadErrors, ...executionPlan.errors, ...decisionReuse.errors], + localFailures: gateFailures, + releaseProfile, + workflowRef: expected.workflowRef, + }); + if (Date.now() >= nextHeartbeat) { + console.log(formatReleaseStateHeartbeat(mode, decision)); + nextHeartbeat = Date.now() + heartbeatIntervalMs; + } + if (failFast && decision.blockers.length > 0) { + const cancellationErrors = await cancelAffectedChildren( + snapshots, + decision.blockers, + cancelledRunIds, + abortController.signal, + ); + if (cancellationErrors.length > 0) { + decision = classifyReleaseSnapshot({ + children: snapshots, + extraBlockers: [ + ...executionPlan.blockers, + ...decisionReuse.blockers, + ...decision.blockers, + ], + extraErrors: [ + ...transportReadErrors, + ...executionPlan.errors, + ...decisionReuse.errors, + ...cancellationErrors, + ], + localFailures: gateFailures, + releaseProfile, + workflowRef: expected.workflowRef, + }); + } + } + const done = + mode === "decision" + ? decision.state !== "qualifying" && + !(decision.state === "passed" && transport.status === "uncertain") + : transport.status !== "uncertain" && + (decision.state === "orchestration_error" || + (decision.state !== "qualifying" && decision.activeRunIds.length === 0)); + if (done) { + const payload = writePayload(decision, { cancelledRunIds, requested: false }); + finished = true; + process.exitCode = + payload.state === "passed" ? 0 : payload.state === "orchestration_error" ? 2 : 1; + return; + } + await abortableSleep( + deadlineDelayMs(pollIntervalMs, transport.deadlineMonotonicMs, performance.now()), + abortController.signal, + ); + } +} + +function readStateCandidates(root, prefix, runId, maxParentRunAttempt, filename) { + const pattern = new RegExp(`^${prefix}-${runId}-([1-9][0-9]*)$`, "u"); + const candidates = readdirSync(root, { withFileTypes: true }) + .filter((entry) => entry.isDirectory()) + .map((entry) => { + const match = pattern.exec(entry.name); + return match ? { attempt: Number(match[1]), name: entry.name } : undefined; + }) + .filter(Boolean) + .filter((entry) => entry.attempt <= maxParentRunAttempt) + .map((entry) => { + const payload = readArtifact(join(root, entry.name, filename), entry.name); + if (Number(payload.parentRunAttempt) !== entry.attempt) { + throw new Error(`${entry.name} payload attempt differs from its artifact name`); + } + return { name: entry.name, payload }; + }); + const directPath = join(root, filename); + if (existsSync(directPath)) { + const payload = readArtifact(directPath, filename); + const attempt = positiveInteger(payload.parentRunAttempt, `${filename} parent run attempt`); + if (attempt <= maxParentRunAttempt) { + candidates.push({ name: `${prefix}-${runId}-${attempt}`, payload }); + } + } + return candidates; +} + +async function validateManifestMode() { + const expected = { + publicationAdmissionContract: + process.env.FULL_RELEASE_PUBLICATION_ADMISSION_CONTRACT || undefined, + sourceAdmissionContract: process.env.FULL_RELEASE_SOURCE_ADMISSION_CONTRACT || undefined, + maxParentRunAttempt: positiveInteger(process.env.GITHUB_RUN_ATTEMPT, "parent run attempt"), + parentRunId: requiredString(process.env.GITHUB_RUN_ID, "parent run ID"), + repository: requiredString(process.env.GITHUB_REPOSITORY, "GitHub repository"), + releaseProfile: requiredString(process.env.RELEASE_PROFILE, "release profile"), + rerunGroup: requiredString(process.env.RERUN_GROUP, "rerun group"), + targetSha: requiredString(process.env.TARGET_SHA, "target SHA"), + workflowRef: requiredString(process.env.GITHUB_REF_NAME, "workflow ref"), + workflowSha: requiredString(process.env.GITHUB_SHA, "workflow SHA"), + }; + const manifestPath = requiredString( + process.env.RELEASE_VALIDATION_MANIFEST_PATH, + "release validation manifest path", + ); + const executionPlanPayload = readArtifact( + requiredString(process.env.RELEASE_EXECUTION_PLAN_PATH, "execution plan path"), + "execution plan", + ); + const executionPlan = validateReleaseExecutionPlanArtifact(executionPlanPayload, expected); + const verified = + executionPlan.attemptEvidenceVersion !== undefined + ? verifyReleaseStateArtifacts( + executionPlanPayload, + readArtifact( + requiredString(process.env.RELEASE_DECISION_PATH, "release decision path"), + "release decision", + ), + readArtifact( + requiredString(process.env.DIAGNOSTIC_DRAIN_PATH, "diagnostic drain path"), + "diagnostic drain", + ), + expected, + ) + : undefined; + const drain = verified?.drain; + const rawManifest = readArtifact(manifestPath, "release validation manifest"); + const { validateParentManifest } = await import("./release-ci-summary.mjs"); + const manifest = validateParentManifest(rawManifest, { + sourceAdmissionContract: expected.sourceAdmissionContract, + candidateBinding: executionPlan.candidate ?? null, + repository: expected.repository, + runAttempt: positiveInteger(process.env.GITHUB_RUN_ATTEMPT, "parent run attempt"), + runId: executionPlan.parentRunId, + workflowRef: executionPlan.workflowRef, + workflowSha: executionPlan.workflowSha, + }); + validatePublicationAdmissionBinding(rawManifest, expected); + if ( + JSON.stringify(rawManifest.sourceAdmission) !== JSON.stringify(executionPlan.sourceAdmission) || + rawManifest.sourceAdmissionContract !== executionPlan.sourceAdmissionContract || + (executionPlan.sourceAdmissionContract && + JSON.stringify(rawManifest.trustedWorkflow) !== JSON.stringify(executionPlan.trustedWorkflow)) + ) { + throw new Error("release manifest source admission differs from its immutable plan"); + } + if ( + rawManifest.publicationAdmissionContract !== executionPlan.publicationAdmissionContract || + JSON.stringify(sortJsonValueKeys(rawManifest.publicationAdmission)) !== + JSON.stringify(sortJsonValueKeys(executionPlan.publicationAdmission)) + ) { + throw new Error("release manifest publication admission differs from its immutable plan"); + } + validateReleaseTelegramWaiverBinding(executionPlan, manifest.validationInputs); + validateReleaseCoveragePolicyBinding(executionPlan, manifest.validationInputs); + const expectedChildRunIds = Object.fromEntries( + executionPlan.children.map((child) => [ + child.key, + child.selected ? stringValue(child.runId) : "", + ]), + ); + const expectedEvidenceReuse = executionPlan.evidenceReuse.requested + ? { + changedPaths: executionPlan.evidenceReuse.changedPaths, + evidenceSha: executionPlan.evidenceReuse.evidenceSha, + policy: executionPlan.evidenceReuse.policy, + runId: executionPlan.evidenceReuse.rootRunId, + selectedRunId: executionPlan.evidenceReuse.selectedRunId, + } + : undefined; + const expectedChildEvidence = drain + ? Object.fromEntries( + Object.entries(drain.children).map(([key, child]) => [ + key, + { + compositeJobsSha256: child.compositeJobsSha256, + dispatchActor: child.dispatchActor, + effectiveRunAttempt: child.runAttempt, + jobs: child.timing.jobs.map((job) => ({ + acceptedRunAttempt: job.acceptedRunAttempt, + completedAt: job.completedAt, + conclusion: job.conclusion, + name: job.name, + startedAt: job.startedAt, + status: job.status, + url: job.url, + })), + observedRunAttempts: child.observedRunAttempts, + plannedRunAttempt: child.plannedRunAttempt, + repository: child.repository, + runId: child.runId, + triggeringActor: child.triggeringActor, + }, + ]), + ) + : undefined; + if ( + manifest.targetSha !== executionPlan.targetSha || + manifest.releaseProfile !== executionPlan.releaseProfile || + manifest.rerunGroup !== executionPlan.rerunGroup || + JSON.stringify(sortJsonValueKeys(manifest.childRunIds)) !== + JSON.stringify(sortJsonValueKeys(expectedChildRunIds)) || + JSON.stringify(sortJsonValueKeys(manifest.evidenceReuse)) !== + JSON.stringify(sortJsonValueKeys(expectedEvidenceReuse)) || + (executionPlan.attemptEvidenceVersion !== undefined && + JSON.stringify(sortJsonValueKeys(rawManifest.childEvidence)) !== + JSON.stringify(sortJsonValueKeys(expectedChildEvidence))) || + rawManifest.executionPlanSha256 !== executionPlan.sha256 || + Number(rawManifest.sourceParentRunAttempt) !== executionPlan.parentRunAttempt + ) { + throw new Error("release validation manifest differs from the immutable execution plan"); + } + rawManifest.advisoryJobs = manifest.advisoryJobs; + writeArtifact(manifestPath, rawManifest); +} + +function selectMode() { + const expected = { + publicationAdmissionContract: + process.env.FULL_RELEASE_PUBLICATION_ADMISSION_CONTRACT || undefined, + sourceAdmissionContract: process.env.FULL_RELEASE_SOURCE_ADMISSION_CONTRACT || undefined, + maxParentRunAttempt: positiveInteger( + process.env.GITHUB_RUN_ATTEMPT, + "current parent run attempt", + ), + parentRunId: requiredString(process.env.GITHUB_RUN_ID, "parent run ID"), + repository: requiredString(process.env.GITHUB_REPOSITORY, "GitHub repository"), + releaseProfile: requiredString(process.env.RELEASE_PROFILE, "release profile"), + rerunGroup: requiredString(process.env.RERUN_GROUP, "rerun group"), + targetSha: requiredString(process.env.TARGET_SHA, "target SHA"), + workflowRef: requiredString(process.env.GITHUB_REF_NAME, "workflow ref"), + workflowSha: requiredString(process.env.GITHUB_SHA, "workflow SHA"), + }; + const selected = selectReleaseStateArtifacts( + readArtifact( + requiredString(process.env.RELEASE_EXECUTION_PLAN_PATH, "execution plan path"), + "execution plan", + ), + readStateCandidates( + requiredString(process.env.RELEASE_DECISION_ATTEMPTS_PATH, "decision attempts path"), + "full-release-decision", + expected.parentRunId, + expected.maxParentRunAttempt, + "full-release-decision.json", + ), + readStateCandidates( + requiredString(process.env.DIAGNOSTIC_DRAIN_ATTEMPTS_PATH, "drain attempts path"), + "full-release-diagnostics", + expected.parentRunId, + expected.maxParentRunAttempt, + "full-release-diagnostic-manifest.json", + ), + expected, + ); + writeArtifact( + requiredString(process.env.RELEASE_DECISION_PATH, "selected decision path"), + selected.decision, + ); + writeArtifact( + requiredString(process.env.DIAGNOSTIC_DRAIN_PATH, "selected drain path"), + selected.drain, + ); + if (process.env.GITHUB_OUTPUT) { + appendFileSync( + process.env.GITHUB_OUTPUT, + `decision_source_attempt=${selected.sourceAttempts.decision}\n`, + ); + appendFileSync( + process.env.GITHUB_OUTPUT, + `drain_source_attempt=${selected.sourceAttempts.drain}\n`, + ); + } +} + +async function main() { + const mode = process.argv[2]; + if (mode === "reuse-publication") { + await publicationReuseMode(); + return; + } + if (["restore-publication", "finalize-publication"].includes(mode)) { + await publicationMode(mode); + return; + } + if (mode === "write-manifest") { + writeManifestMode(); + return; + } + if (mode === "plan") { + await planMode(); + return; + } + if (mode === "verify") { + verifyMode(); + return; + } + if (mode === "select") { + selectMode(); + return; + } + if (mode === "validate-manifest") { + await validateManifestMode(); + return; + } + if (!["decision", "drain"].includes(mode)) { + throw new Error( + "usage: full-release-validation-state.mjs <plan|decision|drain|select|validate-manifest|verify>", + ); + } + await collectMode(mode); +} + +if (process.argv[1]?.endsWith("full-release-validation-state.mjs")) { + try { + await main(); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(2); + } +} diff --git a/scripts/generate-config-doc-baseline.ts b/scripts/generate-config-doc-baseline.ts new file mode 100644 index 0000000000000000000000000000000000000000..4c78bde43b52bb1329e9f49a707bb119ea0bea96 --- /dev/null +++ b/scripts/generate-config-doc-baseline.ts @@ -0,0 +1,58 @@ +#!/usr/bin/env node +// Generate Config Doc Baseline script supports OpenClaw repository automation. +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { writeConfigDocBaselineArtifacts } from "../src/config/doc-baseline.js"; + +const args = new Set(process.argv.slice(2)); +const checkOnly = args.has("--check"); + +if (checkOnly && args.has("--write")) { + console.error("Use either --check or --write, not both."); + process.exit(1); +} + +const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const result = await writeConfigDocBaselineArtifacts({ + repoRoot, + check: checkOnly, +}); + +if (checkOnly) { + if (!result.changed) { + console.log( + `OK ${path.relative(repoRoot, result.hashPath)} and ${path.relative(repoRoot, result.countsPath)}`, + ); + process.exit(0); + } + if (result.countBudgetError) { + console.error( + `Config baseline count budget invalid: ${result.countBudgetError}. Run \`pnpm config:docs:gen\` to recreate ${path.relative(repoRoot, result.countsPath)}.`, + ); + } + for (const violation of result.countViolations) { + console.error(violation.message); + } + if (result.hashChanged) { + console.error( + [ + "Config baseline drift detected.", + `Hash mismatch: ${path.relative(repoRoot, result.hashPath)}`, + "If this config-surface change is intentional, run `pnpm config:docs:gen` and commit the updated hash file.", + "If not intentional, treat this as docs drift or a possible breaking config change and fix the schema/help changes first.", + ].join("\n"), + ); + } + process.exit(1); +} + +console.log( + [ + `Wrote ${path.relative(repoRoot, result.hashPath)}`, + `Wrote ${path.relative(repoRoot, result.countsPath)}`, + `Wrote ${path.relative(repoRoot, result.jsonPaths.combined)} (gitignored, local only)`, + `Wrote ${path.relative(repoRoot, result.jsonPaths.core)} (gitignored, local only)`, + `Wrote ${path.relative(repoRoot, result.jsonPaths.channel)} (gitignored, local only)`, + `Wrote ${path.relative(repoRoot, result.jsonPaths.plugin)} (gitignored, local only)`, + ].join("\n"), +); diff --git a/scripts/generate-host-env-security-policy-swift.mts b/scripts/generate-host-env-security-policy-swift.mts new file mode 100644 index 0000000000000000000000000000000000000000..7a7ad0da7595ec53f5d107309861077e8109ecc8 --- /dev/null +++ b/scripts/generate-host-env-security-policy-swift.mts @@ -0,0 +1,88 @@ +#!/usr/bin/env node +// Generates Swift constants for the host environment security policy. +import fs from "node:fs"; +import path from "node:path"; +import { loadHostEnvSecurityPolicy } from "../src/infra/host-env-security-policy.js"; +import { resolveRepoRoot } from "./lib/repo-root.mjs"; + +const args = new Set(process.argv.slice(2)); +const checkOnly = args.has("--check"); +const writeMode = args.has("--write") || !checkOnly; + +if (checkOnly && args.has("--write")) { + console.error("Use either --check or --write, not both."); + process.exit(1); +} + +const repoRoot = resolveRepoRoot(import.meta.url); +const policyPath = path.join(repoRoot, "src", "infra", "host-env-security-policy.json"); +const outputPath = path.join( + repoRoot, + "apps", + "macos", + "Sources", + "OpenClaw", + "HostEnvSecurityPolicy.generated.swift", +); + +const rawPolicy = JSON.parse(fs.readFileSync(policyPath, "utf8")); +const policy = loadHostEnvSecurityPolicy(rawPolicy); + +const renderSwiftStringArray = (items: readonly string[]) => + items.map((item) => ` "${item}",`).join("\n"); + +const generated = `// Generated file. Do not edit directly. +// Source: src/infra/host-env-security-policy.json +// Regenerate: node --import tsx scripts/generate-host-env-security-policy-swift.mts --write + +import Foundation + +enum HostEnvSecurityPolicy { + static let blockedInheritedKeys: Set<String> = [ +${renderSwiftStringArray(policy.blockedInheritedKeys)} + ] + + static let blockedInheritedPrefixes: [String] = [ +${renderSwiftStringArray(policy.blockedInheritedPrefixes)} + ] + + static let blockedKeys: Set<String> = [ +${renderSwiftStringArray(policy.blockedKeys)} + ] + + static let blockedOverrideKeys: Set<String> = [ +${renderSwiftStringArray(policy.blockedOverrideKeys ?? [])} + ] + + static let blockedOverridePrefixes: [String] = [ +${renderSwiftStringArray(policy.blockedOverridePrefixes ?? [])} + ] + + static let blockedPrefixes: [String] = [ +${renderSwiftStringArray(policy.blockedPrefixes)} + ] +} +`; + +const current = fs.existsSync(outputPath) ? fs.readFileSync(outputPath, "utf8") : null; + +if (checkOnly) { + if (current === generated) { + console.log(`OK ${path.relative(repoRoot, outputPath)}`); + process.exit(0); + } + console.error( + [ + `Out of date ${path.relative(repoRoot, outputPath)}.`, + "Run: node --import tsx scripts/generate-host-env-security-policy-swift.mts --write", + ].join("\n"), + ); + process.exit(1); +} + +if (writeMode) { + if (current !== generated) { + fs.writeFileSync(outputPath, generated); + } + console.log(`Wrote ${path.relative(repoRoot, outputPath)}`); +} diff --git a/scripts/generate-mac-app-icons.sh b/scripts/generate-mac-app-icons.sh new file mode 100644 index 0000000000000000000000000000000000000000..a8b9299679586537b653362e5ffc1302cc5c526c --- /dev/null +++ b/scripts/generate-mac-app-icons.sh @@ -0,0 +1,70 @@ +#!/bin/bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +OUTPUT_DIR="$ROOT_DIR/apps/macos/Sources/OpenClaw/Resources/AppIcons" +MODE="${1:---write}" +if [[ "$MODE" != "--write" && "$MODE" != "--check" ]]; then + echo "Usage: /bin/bash scripts/generate-mac-app-icons.sh [--write|--check]" >&2 + exit 1 +fi +WORK_DIR="$(mktemp -d)" +trap 'rm -rf "$WORK_DIR"' EXIT +mkdir -p "$OUTPUT_DIR" + +# Each family has editable vector artwork; actool owns the macOS mask and padding. +/usr/bin/python3 - "$ROOT_DIR/apps/macos" "$WORK_DIR" <<'PY' +import json +import shutil +import sys +import xml.etree.ElementTree as ET +from pathlib import Path + +root, work = map(Path, sys.argv[1:]) +for name, (source, dark_fill, dark_foreground) in { + "paper": ("Icon.icon", {"linear-gradient": ["srgb:0.19200,0.19200,0.19200,1.00000", "srgb:0.07800,0.07800,0.07800,1.00000"]}, None), + "heritage": ("AppIconDesigns/Heritage.icon", {"solid": "srgb:0.13333,0.13333,0.13333,1.00000"}, None), + "clawmark": ("AppIconDesigns/Clawmark.icon", {"solid": "srgb:0.10196,0.10980,0.12157,1.00000"}, None), + "origami": ("AppIconDesigns/Origami.icon", {"solid": "srgb:0.07451,0.12549,0.20000,1.00000"}, None), + "pincer": ("AppIconDesigns/Pincer.icon", {"solid": "srgb:0.16863,0.18039,0.18824,1.00000"}, "#f8f5ec"), + "openC": ("AppIconDesigns/OpenC.icon", {"solid": "srgb:0.16863,0.18039,0.18824,1.00000"}, "#f8f5ec"), +}.items(): + for appearance in ("light", "dark"): + icon = work / f"{name}-{appearance}" / "Icon.icon" + shutil.copytree(root / source, icon) + if appearance == "dark": + document = json.loads((icon / "icon.json").read_text()) + document["fill"] = dark_fill + (icon / "icon.json").write_text(json.dumps(document, indent=2) + "\n") + if dark_foreground: + # Monochrome pairs invert the glyph as well as the tile. + ET.register_namespace("", "http://www.w3.org/2000/svg") + artwork = icon / "Assets" / "molty.svg" + tree = ET.parse(artwork) + tree.getroot().set("fill", dark_foreground) + tree.write(artwork, encoding="unicode") +PY + +for style in paper-light paper-dark heritage-light heritage-dark clawmark-light clawmark-dark origami-light origami-dark pincer-light pincer-dark openC-light openC-dark; do + mkdir "$WORK_DIR/$style/compiled" + xcrun actool "$WORK_DIR/$style/Icon.icon" \ + --compile "$WORK_DIR/$style/compiled" \ + --output-format human-readable-text --notices --warnings --errors \ + --output-partial-info-plist "$WORK_DIR/$style/compiled/icon.plist" \ + --app-icon Icon --include-all-app-icons --enable-on-demand-resources NO \ + --development-region en --target-device mac --minimum-deployment-target 15.0 --platform macosx + if [[ "$MODE" == "--check" ]]; then + if ! cmp -s "$WORK_DIR/$style/compiled/Icon.icns" "$OUTPUT_DIR/$style.icns"; then + echo "Stale $style icon. Run /bin/bash scripts/generate-mac-app-icons.sh with the current Xcode toolchain." >&2 + exit 1 + fi + else + cp "$WORK_DIR/$style/compiled/Icon.icns" "$OUTPUT_DIR/$style.icns" + fi +done + +if [[ "$MODE" == "--check" ]]; then + cmp "$OUTPUT_DIR/paper-light.icns" "$OUTPUT_DIR/../OpenClaw.icns" +else + cp "$OUTPUT_DIR/paper-light.icns" "$OUTPUT_DIR/../OpenClaw.icns" +fi diff --git a/scripts/generate-npm-package-lock.mjs b/scripts/generate-npm-package-lock.mjs new file mode 100644 index 0000000000000000000000000000000000000000..e7622ec7d0f8e747ec30b7aff589422610f60482 --- /dev/null +++ b/scripts/generate-npm-package-lock.mjs @@ -0,0 +1,6 @@ +#!/usr/bin/env node +import { runTsxCliShim } from "./lib/tsx-cli-shim.mjs"; + +await runTsxCliShim(import.meta.url, { + implementation: "./generate-npm-package-lock.mts", +}); diff --git a/scripts/generate-plugin-inventory-doc.mts b/scripts/generate-plugin-inventory-doc.mts new file mode 100644 index 0000000000000000000000000000000000000000..4264ce0b989fbb88383410bd10863180238a59bb --- /dev/null +++ b/scripts/generate-plugin-inventory-doc.mts @@ -0,0 +1,835 @@ +#!/usr/bin/env node +// Generates the plugin inventory documentation page. +import fs from "node:fs"; +import path from "node:path"; +import process from "node:process"; +import type { PluginManifest as RuntimePluginManifest } from "../src/plugins/manifest-types.js"; +import type { PackageManifest as RuntimePackageManifest } from "../src/plugins/package-manifest.js"; +import { collectExcludedPackagedExtensionDirs } from "./lib/packaged-extension-dirs.mts"; +import { + assertPluginInventoryCoverage, + resolvePluginSurface, +} from "./lib/plugin-inventory-doc.mts"; + +const DOC_PATH = "docs/plugins/plugin-inventory.md"; +const REFERENCE_INDEX_PATH = "docs/plugins/reference.md"; +const REFERENCE_DIR = "docs/plugins/reference"; +const ROOT = process.cwd(); +const EXTENSIONS_DIR = path.join(ROOT, "extensions"); + +const PROVIDER_DOC_ALIASES = new Map([ + ["amazon-bedrock", "/providers/bedrock"], + ["amazon-bedrock-mantle", "/providers/bedrock-mantle"], + ["kimi", "/providers/moonshot"], + ["perplexity", "/providers/perplexity-provider"], +]); +const PLUGIN_DOC_ALIASES = new Map([ + ["acpx", "/tools/acp-agents-setup"], + ["brave", "/tools/brave-search"], + ["browser", "/tools/browser"], + ["codex", "/plugins/codex-harness"], + ["document-extract", "/tools/pdf"], + ["geolocation", "/plugins/geolocation"], + ["duckduckgo", "/tools/duckduckgo-search"], + ["exa", "/tools/exa-search"], + ["firecrawl", "/tools/firecrawl"], + ["imap", "/automation/imap"], + ["parallel", "/tools/parallel-search"], + ["perplexity", "/tools/perplexity-search"], + ["policy", "/cli/policy"], + ["tavily", "/tools/tavily"], + ["tokenjuice", "/tools/tokenjuice"], +]); +const SKIPPED_REFERENCE_PAGE_IDS = new Set(["parallel"]); +const MANUAL_SECTION_START = "<!-- openclaw-plugin-reference:manual-start -->"; +const MANUAL_SECTION_END = "<!-- openclaw-plugin-reference:manual-end -->"; +const GENERATED_NOTICE = `<!-- Generated file. Do not edit by hand. +Run \`pnpm plugins:inventory:gen\` to rebuild it. -->`; +// Keep the marker names in this notice unbracketed. A bracketed copy would make +// extractManualReferenceSections match the notice instead of the real marker. +const GENERATED_REFERENCE_NOTICE = `<!-- Generated file. Do not edit by hand. +Run \`pnpm plugins:inventory:gen\` to rebuild it. Hand-written text survives only +between the openclaw-plugin-reference:manual-start and +openclaw-plugin-reference:manual-end comment markers. -->`; +// Generated link labels are user-visible product names and translation source. +const RELATED_DOC_PRODUCT_IDS = new Set([ + "chutes", + "discord", + "fireworks", + "googlechat", + "imessage", + "line", + "matrix", + "meta", + "msteams", + "raft", + "runway", + "signal", + "slack", + "synthetic", + "telegram", + "tokenjuice", + "whatsapp", +]); + +type PluginManifest = Partial<RuntimePluginManifest>; +type PluginPackageJson = Partial<RuntimePackageManifest> & { + openclaw?: RuntimePackageManifest["openclaw"] & { + release?: Partial<Record<"publishToClawHub" | "publishToNpm", boolean>>; + }; +}; +type DocLink = { label: string; href: string }; +type PluginStatus = "core" | "external" | "source"; +type PluginSourceEntry = { + dirName: string; + id: string; + manifest: PluginManifest; + packageJson: PluginPackageJson; +}; + +function createPluginRecord(entry: PluginSourceEntry, excludedDirs: Set<string>) { + const { id, manifest, packageJson } = entry; + const status = resolveStatus(entry, excludedDirs); + return { + description: resolveDescription(entry), + docs: resolveDocs(entry), + id, + installRoute: resolveInstallRoute(packageJson, status), + name: humanizeId(id), + packageName: packageJson.name ?? (status === "core" ? "openclaw" : "-"), + status, + surface: resolvePluginSurface(manifest), + }; +} + +type PluginRecord = ReturnType<typeof createPluginRecord>; + +function readJsonPath(filePath: string): unknown { + return JSON.parse(fs.readFileSync(filePath, "utf8")) as unknown; +} + +function fileExists(relativePath: string) { + return fs.existsSync(path.join(ROOT, relativePath)); +} + +function normalizeDocPath(value: unknown) { + if (typeof value !== "string" || !value.startsWith("/")) { + return null; + } + return value.replace(/\.mdx?$/u, ""); +} + +function docLink({ label, href }: DocLink) { + return `[${label}](${href})`; +} + +function relatedDocLabel(value: string) { + return RELATED_DOC_PRODUCT_IDS.has(value) ? humanizeId(value) : value; +} + +function pluginReferencePath(id: string) { + return `/plugins/reference/${id}`; +} + +function hasGeneratedReferencePage(record: PluginRecord) { + if (!SKIPPED_REFERENCE_PAGE_IDS.has(record.id)) { + return true; + } + if (PLUGIN_DOC_ALIASES.has(record.id)) { + return false; + } + throw new Error(`skipped plugin reference page ${record.id} needs a plugin doc alias`); +} + +function pluginInventoryHref(record: PluginRecord) { + if (hasGeneratedReferencePage(record)) { + return pluginReferencePath(record.id); + } + return PLUGIN_DOC_ALIASES.get(record.id) ?? null; +} + +function pluginReferenceLabel(record: PluginRecord) { + const label = escapeInventoryText(record.id); + const href = pluginInventoryHref(record); + return href ? docLink({ href, label }) : label; +} + +function humanizeId(value: string) { + if (value === "teams-meetings") { + return "Microsoft Teams meetings"; + } + if (value === "zoom-meetings") { + return "Zoom meetings"; + } + const names = new Map([ + ["acpx", "ACPx"], + ["ai", "AI"], + ["api", "API"], + ["aws", "AWS"], + ["azure", "Azure"], + ["byteplus", "BytePlus"], + ["clawrouter", "ClawRouter"], + ["codex", "Codex"], + ["cli", "CLI"], + ["comfy", "ComfyUI"], + ["dashscope", "DashScope"], + ["deepgram", "Deepgram"], + ["deepinfra", "DeepInfra"], + ["deepseek", "DeepSeek"], + ["duckduckgo", "DuckDuckGo"], + ["exa", "Exa"], + ["fal", "fal"], + ["feishu", "Feishu"], + ["github", "GitHub"], + ["googlechat", "Google Chat"], + ["gpt", "GPT"], + ["groq", "Groq"], + ["huggingface", "Hugging Face"], + ["imessage", "iMessage"], + ["irc", "IRC"], + ["kimi", "Kimi"], + ["line", "LINE"], + ["litellm", "LiteLLM"], + ["llm", "LLM"], + ["lmstudio", "LM Studio"], + ["longcat", "LongCat"], + ["mdns", "mDNS"], + ["minimax", "MiniMax"], + ["modelstudio", "Model Studio"], + ["msteams", "Microsoft Teams"], + ["nextcloud", "Nextcloud"], + ["nvidia", "NVIDIA"], + ["openai", "OpenAI"], + ["opencode", "OpenCode"], + ["openrouter", "OpenRouter"], + ["otel", "OpenTelemetry"], + ["pixverse", "PixVerse"], + ["qa", "QA"], + ["qqbot", "QQ Bot"], + ["qwen", "Qwen"], + ["qwencloud", "Qwen Cloud"], + ["raft", "Raft"], + ["searxng", "SearXNG"], + ["sglang", "SGLang"], + ["stepfun", "StepFun"], + ["tokenhub", "TokenHub"], + ["tts", "TTS"], + ["twitch", "Twitch"], + ["ui", "UI"], + ["vllm", "vLLM"], + ["whatsapp", "WhatsApp"], + ["xai", "xAI"], + ["zai", "Z.AI"], + ["zalouser", "Zalo Personal"], + ]); + return value + .split("-") + .map((part) => names.get(part) ?? part.charAt(0).toUpperCase() + part.slice(1)) + .join(" "); +} + +function displayList(values: unknown[]) { + return values + .filter((value): value is string => typeof value === "string" && value.length > 0) + .map(humanizeId) + .join(", "); +} + +function normalizePackageDescription(value: unknown) { + if (typeof value !== "string") { + return null; + } + return value.trim().replace(/\s+/gu, " ").replace(/\.$/u, ""); +} + +function resolveDescription({ manifest, packageJson }: PluginSourceEntry) { + const manifestDescription = normalizePackageDescription(manifest.description); + if (manifestDescription) { + return `${manifestDescription}.`; + } + + const channels = Array.isArray(manifest.channels) ? manifest.channels : []; + if (channels.length > 0) { + const channelLabel = displayList(channels); + const channelNoun = channelLabel.toLowerCase().includes("channel") ? "" : " channel"; + return `Adds the ${channelLabel}${channelNoun} surface for sending and receiving OpenClaw messages.`; + } + + const providers = Array.isArray(manifest.providers) ? manifest.providers : []; + if (providers.length > 0) { + return `Adds ${displayList(providers)} model provider support to OpenClaw.`; + } + + const contracts = Object.keys(manifest.contracts ?? {}).toSorted((left, right) => + left.localeCompare(right), + ); + const contractDescriptions: Record<string, string> = { + agentToolResultMiddleware: "Adds agent tool-result middleware.", + documentExtractors: "Adds document extraction for local attachments.", + imageGenerationProviders: "Adds image generation provider support.", + mediaUnderstandingProviders: "Adds media understanding provider support.", + embeddingProviders: "Adds embedding provider support, including memory search.", + migrationProviders: "Adds migration import support.", + musicGenerationProviders: "Adds music generation provider support.", + realtimeTranscriptionProviders: "Adds realtime transcription provider support.", + realtimeVoiceProviders: "Adds realtime voice provider support.", + speechProviders: "Adds text-to-speech provider support.", + tools: "Adds agent-callable tools.", + videoGenerationProviders: "Adds video generation provider support.", + webContentExtractors: "Adds readable web content extraction.", + webFetchProviders: "Adds web fetch provider support.", + webSearchProviders: "Adds web search provider support.", + workerProviders: "Adds cloud worker provider support.", + }; + const describedContracts = contracts + .map((contract) => contractDescriptions[contract]) + .filter((value) => typeof value === "string"); + if (describedContracts.length > 0) { + return describedContracts.join(" "); + } + + const packageDescription = normalizePackageDescription(packageJson.description); + return packageDescription ? `${packageDescription}.` : "Provides an OpenClaw plugin."; +} + +function pushUniqueDocLink(values: DocLink[], value: DocLink | null) { + if ( + value && + !values.some((existing) => existing.label === value.label && existing.href === value.href) + ) { + values.push(value); + } +} + +function resolveDocs({ dirName, manifest, packageJson }: PluginSourceEntry) { + const links: DocLink[] = []; + const manifestId = typeof manifest.id === "string" ? manifest.id : dirName; + const pluginAlias = PLUGIN_DOC_ALIASES.get(manifestId) ?? PLUGIN_DOC_ALIASES.get(dirName); + if (pluginAlias) { + const pluginAliasLabel = relatedDocLabel(manifestId); + pushUniqueDocLink(links, { href: pluginAlias, label: pluginAliasLabel }); + } + + const channelDoc = normalizeDocPath(packageJson.openclaw?.channel?.docsPath); + if (channelDoc) { + pushUniqueDocLink(links, { + href: channelDoc, + label: relatedDocLabel(channelDoc.replace(/^\/channels\//u, "")), + }); + } + + for (const channel of manifest.channels ?? []) { + if (typeof channel !== "string") { + continue; + } + const relativePath = `docs/channels/${channel}.md`; + if (fileExists(relativePath)) { + pushUniqueDocLink(links, { href: `/channels/${channel}`, label: relatedDocLabel(channel) }); + } + } + + for (const provider of manifest.providers ?? []) { + if (typeof provider !== "string") { + continue; + } + const alias = PROVIDER_DOC_ALIASES.get(provider); + if (alias) { + pushUniqueDocLink(links, { href: alias, label: relatedDocLabel(provider) }); + continue; + } + const relativePath = `docs/providers/${provider}.md`; + if (fileExists(relativePath)) { + pushUniqueDocLink(links, { + href: `/providers/${provider}`, + label: relatedDocLabel(provider), + }); + } + } + + for (const candidate of [manifest.id, dirName, ...(manifest.legacyPluginIds ?? [])]) { + if (typeof candidate !== "string") { + continue; + } + if (fileExists(`docs/channels/${candidate}.md`)) { + pushUniqueDocLink(links, { + href: `/channels/${candidate}`, + label: relatedDocLabel(candidate), + }); + } + if (fileExists(`docs/providers/${candidate}.md`)) { + pushUniqueDocLink(links, { + href: `/providers/${candidate}`, + label: relatedDocLabel(candidate), + }); + } + if (fileExists(`docs/plugins/${candidate}.md`)) { + pushUniqueDocLink(links, { + href: `/plugins/${candidate}`, + label: relatedDocLabel(candidate), + }); + } + } + + return links; +} + +function resolveInstallRoute(packageJson: PluginPackageJson, status: PluginStatus): string { + if (status === "source") { + return "source checkout only"; + } + if (status === "core") { + // Explicit bundle ownership describes the current install surface; release flags may stage future publication. + if (packageJson.openclaw?.build?.bundledDist === true) { + return "included in OpenClaw"; + } + const release = packageJson.openclaw?.release; + if (release?.publishToClawHub === true || release?.publishToNpm === true) { + return `included in OpenClaw, and also from ${resolveInstallRoute(packageJson, "external")}`; + } + return "included in OpenClaw"; + } + const install = packageJson.openclaw?.install; + const release = packageJson.openclaw?.release; + const clawhubSpec = + typeof install?.clawhubSpec === "string" ? `: \`${install.clawhubSpec}\`` : ""; + const npmSpec = + typeof install?.npmSpec === "string" && install.npmSpec !== packageJson.name + ? `: \`${install.npmSpec}\`` + : ""; + if (release?.publishToClawHub === true && release?.publishToNpm === true) { + if (install?.defaultChoice === "clawhub") { + return clawhubSpec ? `ClawHub${clawhubSpec} or npm${npmSpec}` : `ClawHub + npm${npmSpec}`; + } + return clawhubSpec ? `npm${npmSpec} or ClawHub${clawhubSpec}` : `npm${npmSpec} or ClawHub`; + } + if (release?.publishToClawHub === true) { + return `ClawHub${clawhubSpec || npmSpec}`; + } + if (release?.publishToNpm === true || typeof install?.npmSpec === "string") { + return `npm${npmSpec}`; + } + return "installable plugin"; +} + +function resolveStatus( + { dirName, packageJson }: PluginSourceEntry, + excludedDirs: Set<string>, +): PluginStatus { + const release = packageJson.openclaw?.release; + const hasInstallSpec = + typeof packageJson.openclaw?.install?.clawhubSpec === "string" || + typeof packageJson.openclaw?.install?.npmSpec === "string"; + if (!excludedDirs.has(dirName)) { + return "core"; + } + if (release?.publishToClawHub === true || release?.publishToNpm === true || hasInstallSpec) { + return "external"; + } + return "source"; +} + +function escapeInventoryText(value: unknown) { + return String(value).replaceAll("\n", " ").trim(); +} + +function renderInventoryList(records: PluginRecord[]) { + if (records.length === 0) { + return "_None._"; + } + + return records + .map( + (record) => + `- **${pluginReferenceLabel(record)}** (\`${escapeInventoryText(record.packageName)}\`) - ${escapeInventoryText(record.installRoute)}. ${escapeInventoryText(record.description)}`, + ) + .join("\n\n"); +} + +function renderRelatedDocs(record: PluginRecord) { + if (record.docs.length === 0) { + return ""; + } + return `## Related docs + +${record.docs.map((link) => `- ${docLink(link)}`).join("\n")}`; +} + +function stripGeneratedNotice(value: string) { + const noticeStart = value.indexOf(GENERATED_REFERENCE_NOTICE); + return noticeStart === -1 ? value : value.slice(noticeStart + GENERATED_REFERENCE_NOTICE.length); +} + +function extractManualReferenceSections(rawContent: string) { + const content = stripGeneratedNotice(rawContent); + const markerStart = content.indexOf(MANUAL_SECTION_START); + if (markerStart !== -1) { + const contentStart = markerStart + MANUAL_SECTION_START.length; + const markerEnd = content.indexOf(MANUAL_SECTION_END, contentStart); + if (markerEnd !== -1) { + return content.slice(contentStart, markerEnd).trim(); + } + } + + // The surface block is a list, so consume every non-blank line under the heading. + // A single-line pattern here would treat later surface bullets as manual text. + const surfaceMatch = /\n## Surface\n\n(?:[^\n]+\n)*/u.exec(content); + if (!surfaceMatch?.index) { + return ""; + } + const manualStart = surfaceMatch.index + surfaceMatch[0].length; + const relatedDocsStart = content.indexOf("\n## Related docs\n", manualStart); + const manualEnd = relatedDocsStart === -1 ? content.length : relatedDocsStart; + return content.slice(manualStart, manualEnd).trim(); +} + +function readManualReferenceSections(relativePath: string) { + const fullPath = path.join(ROOT, relativePath); + if (!fs.existsSync(fullPath)) { + return ""; + } + return extractManualReferenceSections(fs.readFileSync(fullPath, "utf8")); +} + +function renderManualReferenceSections(manualSections: string) { + if (!manualSections) { + return ""; + } + return `${MANUAL_SECTION_START} + +${manualSections} + +${MANUAL_SECTION_END}`; +} + +// Generated reference titles carry a "reference" suffix so they never collide with a +// hand-written plugin guide title such as "Beam plugin" in docs/plugins/<id>.md. +function referencePageTitle(record: PluginRecord) { + return `${record.name} plugin reference`; +} + +function renderSurface(surface: string[]) { + if (surface.length === 0) { + return "This plugin declares no channels, providers, commands, or contracts."; + } + return surface.map((part) => `- ${part}`).join("\n"); +} + +function renderReferencePage(record: PluginRecord, manualSections = "") { + const relatedDocs = renderRelatedDocs(record); + const manualBlock = renderManualReferenceSections(manualSections); + return `--- +summary: "${record.description.replaceAll('"', '\\"')}" +read_when: + - You are installing, configuring, or auditing the ${record.id} plugin +title: "${referencePageTitle(record)}" +--- + +${GENERATED_REFERENCE_NOTICE} + +${record.description} + +## Distribution + +- Package: \`${record.packageName}\` +- Install route: ${record.installRoute} + +## Surface + +${renderSurface(record.surface)}${manualBlock ? `\n\n${manualBlock}` : ""}${relatedDocs ? `\n\n${relatedDocs}` : ""} +`; +} + +function renderReferenceIndex(records: PluginRecord[]) { + const referenceCount = records.filter(hasGeneratedReferencePage).length; + return `--- +summary: "Pointer to the generated OpenClaw plugin reference pages" +read_when: + - You need a reference page for a specific OpenClaw plugin + - You are auditing plugin docs coverage +title: "Plugin reference" +--- + +${GENERATED_NOTICE} + +This section holds one reference page for each OpenClaw plugin. Each page states +the package, the install route, and the surface the plugin adds. + +This page is a pointer, not the index. The browsable list of all +${referenceCount} generated plugin reference pages lives in +[Plugin inventory](/plugins/plugin-inventory), sorted by distribution, package, +and description. + +## How this page is built + +OpenClaw generates this page from the top-level +\`extensions/*/openclaw.plugin.json\` manifests. Package metadata enriches +entries when \`package.json\` is present. Regenerate the page with: + +\`\`\`bash +pnpm plugins:inventory:gen +\`\`\` +`; +} + +function collectPluginSourceEntries(): PluginSourceEntry[] { + const entries: PluginSourceEntry[] = []; + for (const dirName of fs + .readdirSync(EXTENSIONS_DIR) + .toSorted((left, right) => left.localeCompare(right))) { + const packagePath = path.join(EXTENSIONS_DIR, dirName, "package.json"); + const manifestPath = path.join(EXTENSIONS_DIR, dirName, "openclaw.plugin.json"); + if (!fs.existsSync(manifestPath)) { + continue; + } + const packageJson = fs.existsSync(packagePath) + ? (readJsonPath(packagePath) as PluginPackageJson) + : {}; + const manifest = readJsonPath(manifestPath) as PluginManifest; + const id = typeof manifest.id === "string" && manifest.id ? manifest.id : dirName; + entries.push({ dirName, id, manifest, packageJson }); + } + return entries; +} + +function enumerateTopLevelPluginManifests() { + return fs + .readdirSync(EXTENSIONS_DIR) + .toSorted((left, right) => left.localeCompare(right)) + .flatMap((dirName) => { + const manifestPath = path.join(EXTENSIONS_DIR, dirName, "openclaw.plugin.json"); + if (!fs.existsSync(manifestPath)) { + return []; + } + const manifest = readJsonPath(manifestPath) as PluginManifest; + const id = typeof manifest.id === "string" && manifest.id ? manifest.id : dirName; + return [{ dirName, id }]; + }); +} + +type ExternalPluginDocsInventorySeedEntry = { + openclaw?: { + channel?: NonNullable<PluginPackageJson["openclaw"]>["channel"]; + channelHostConfig?: { + docsInventory?: { + package?: PluginPackageJson; + manifest?: PluginManifest; + }; + }; + }; +}; + +function collectExternalPluginDocsInventoryEntries(): PluginSourceEntry[] { + const seed = readJsonPath(path.join(ROOT, "scripts/lib/official-external-channel-seed.json")) as { + entries?: ExternalPluginDocsInventorySeedEntry[]; + }; + const entries: PluginSourceEntry[] = []; + for (const entry of Array.isArray(seed.entries) ? seed.entries : []) { + const inventory = entry?.openclaw?.channelHostConfig?.docsInventory; + const packageMetadata = inventory?.package; + const manifest = inventory?.manifest; + if (!inventory) { + continue; + } + if ( + typeof packageMetadata?.name !== "string" || + typeof manifest?.id !== "string" || + !entry?.openclaw?.channel + ) { + throw new Error("external plugin docs inventory metadata is incomplete"); + } + entries.push({ + dirName: manifest.id, + id: manifest.id, + manifest, + packageJson: { + ...packageMetadata, + openclaw: { + ...packageMetadata.openclaw, + channel: entry.openclaw.channel, + }, + }, + }); + } + return entries; +} + +function collectPluginRecords() { + const rootPackageJson = readJsonPath(path.join(ROOT, "package.json")) as { files?: unknown[] }; + const excludedDirs = collectExcludedPackagedExtensionDirs(rootPackageJson); + const sourceEntries = collectPluginSourceEntries(); + assertPluginInventoryCoverage(sourceEntries, enumerateTopLevelPluginManifests()); + const records = sourceEntries.map((entry) => createPluginRecord(entry, excludedDirs)); + + const sourceIds = new Set(sourceEntries.map((entry) => entry.id)); + for (const { + dirName, + id, + manifest, + packageJson, + } of collectExternalPluginDocsInventoryEntries()) { + if (sourceIds.has(id)) { + continue; + } + records.push({ + description: resolveDescription({ dirName, id, manifest, packageJson }), + docs: resolveDocs({ dirName, id, manifest, packageJson }), + id, + installRoute: resolveInstallRoute(packageJson, "external"), + name: humanizeId(id), + packageName: packageJson.name ?? "-", + status: "external", + surface: resolvePluginSurface(manifest), + }); + } + return records.toSorted((left, right) => left.id.localeCompare(right.id)); +} + +function writeGeneratedDocs(records: PluginRecord[]) { + fs.mkdirSync(path.join(ROOT, REFERENCE_DIR), { recursive: true }); + for (const record of records.filter(hasGeneratedReferencePage)) { + const relativePath = path.join(REFERENCE_DIR, `${record.id}.md`); + const manualSections = readManualReferenceSections(relativePath); + fs.writeFileSync( + path.join(ROOT, relativePath), + renderReferencePage(record, manualSections), + "utf8", + ); + } + fs.writeFileSync(path.join(ROOT, REFERENCE_INDEX_PATH), renderReferenceIndex(records), "utf8"); +} + +function readGeneratedDocs(records: PluginRecord[]) { + return [ + [REFERENCE_INDEX_PATH, renderReferenceIndex(records)] satisfies [string, string], + ...records.filter(hasGeneratedReferencePage).map((record) => { + const relativePath = path.join(REFERENCE_DIR, `${record.id}.md`); + return [ + relativePath, + renderReferencePage(record, readManualReferenceSections(relativePath)), + ] satisfies [string, string]; + }), + ]; +} + +function renderDocument(records: PluginRecord[]) { + const groups = { + core: records.filter((record) => record.status === "core"), + external: records.filter((record) => record.status === "external"), + source: records.filter((record) => record.status === "source"), + }; + + return `--- +summary: "Generated inventory of OpenClaw plugins shipped in core, published externally, or kept source-only" +read_when: + - You are deciding whether a plugin ships in the core npm package or installs separately + - You are updating bundled plugin package metadata or release automation + - You need the canonical internal vs external plugin list +title: "Plugin inventory" +--- + +${GENERATED_NOTICE} + +This page lists every OpenClaw plugin with its package, install route, and +description. Operators use it to find a plugin and to see whether that plugin +needs a separate install. Maintainers use it to check bundled plugin metadata +and release automation. + +## Definitions + +- **Core npm package:** built into the \`openclaw\` npm package and available without a separate plugin install. +- **Official external package:** OpenClaw-maintained plugin omitted from the core npm package, kept in this official inventory, and installed on demand through ClawHub and/or npm. +- **Source checkout only:** repo-local plugin omitted from published npm artifacts and not advertised as an installable package. + +Source checkouts are different from npm installs: after \`pnpm install\`, bundled +plugins load from \`extensions/<id>\` so local edits and package-local workspace +dependencies are available. + +## Install a plugin + +Use the install route in each entry to decide whether install is needed. Plugins +that say \`included in OpenClaw\` are already present in the core package. +Official external packages need one install. Installation applies to the running +local Gateway without restarting it; start the Gateway if it was stopped. + +For example, Discord is an official external package: + +\`\`\`bash +openclaw plugins install @openclaw/discord +openclaw plugins inspect discord --runtime --json +\`\`\` + +Ordinary bare package specs install from npm. Use \`clawhub:@openclaw/discord\` +or \`npm:@openclaw/discord\` when you need an explicit source. After install, +follow the plugin's setup doc, such as [Discord](/channels/discord), to add +credentials and channel config. See +[Manage plugins](/plugins/manage-plugins) for update, uninstall, and publishing +commands. + +Each entry lists the package, distribution route, and description. + +## Core npm package + +${groups.core.length} plugins + +${renderInventoryList(groups.core)} + +## Official external packages + +${groups.external.length} plugins + +${renderInventoryList(groups.external)} + +## Source checkout only + +${groups.source.length} plugins + +${renderInventoryList(groups.source)} + +## How this page is built + +OpenClaw generates this page from the top-level +\`extensions/*/openclaw.plugin.json\` manifests and the root npm package +\`files\` exclusions. Optional \`package.json\` metadata enriches package and +distribution details. Regenerate the page with: + +\`\`\`bash +pnpm plugins:inventory:gen +\`\`\` +`; +} + +function main(argv = process.argv.slice(2)) { + const write = argv.includes("--write"); + const check = argv.includes("--check"); + if (write === check) { + console.error( + "usage: node --import tsx scripts/generate-plugin-inventory-doc.mts --write|--check", + ); + process.exit(2); + } + + const records = collectPluginRecords(); + const next = renderDocument(records); + const docPath = path.join(ROOT, DOC_PATH); + if (write) { + fs.writeFileSync(docPath, next, "utf8"); + writeGeneratedDocs(records); + return; + } + + const current = fs.existsSync(docPath) ? fs.readFileSync(docPath, "utf8") : ""; + if (current !== next) { + console.error(`${DOC_PATH} is stale. Run \`pnpm plugins:inventory:gen\`.`); + process.exit(1); + } + for (const [relativePath, expected] of readGeneratedDocs(records)) { + const fullPath = path.join(ROOT, relativePath); + const actual = fs.existsSync(fullPath) ? fs.readFileSync(fullPath, "utf8") : ""; + if (actual !== expected) { + console.error(`${relativePath} is stale. Run \`pnpm plugins:inventory:gen\`.`); + process.exit(1); + } + } +} + +main(); diff --git a/scripts/generate-prompt-snapshots.ts b/scripts/generate-prompt-snapshots.ts new file mode 100644 index 0000000000000000000000000000000000000000..0c2d1d0d2a120504dbd369bd7b5ef308f6e17c23 --- /dev/null +++ b/scripts/generate-prompt-snapshots.ts @@ -0,0 +1,392 @@ +// Generate Prompt Snapshots script supports OpenClaw repository automation. +import { execFile } from "node:child_process"; +import { createHash } from "node:crypto"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { promisify } from "node:util"; +import { applyPatch, createTwoFilesPatch, FILE_HEADERS_ONLY, parsePatch } from "diff"; +import { createHappyPathPromptSnapshotFiles } from "../test/helpers/agents/happy-path-prompt-snapshots.js"; +import { + CODEX_PROMPT_SNAPSHOT_BASE_SCENARIO, + CODEX_PROMPT_SNAPSHOT_FILES, + CODEX_RUNTIME_HAPPY_PATH_PROMPT_SNAPSHOT_DIR, + type CodexPromptSnapshotScenario, +} from "../test/helpers/agents/prompt-snapshot-paths.js"; +import { coerceErrorMessage as describeError } from "./lib/error-format.mts"; +import { + deleteStalePromptSnapshotFiles, + listCommittedSnapshotArtifactPaths, +} from "./prompt-snapshot-files.js"; + +const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const oxfmtPath = path.resolve( + repoRoot, + "node_modules", + ".bin", + process.platform === "win32" ? "oxfmt.cmd" : "oxfmt", +); +const execFileAsync = promisify(execFile); + +type PromptSnapshotFile = Awaited<ReturnType<typeof createHappyPathPromptSnapshotFiles>>[number]; +type CodexDynamicToolSnapshotSpec = { name: string }; +type CodexDynamicToolSnapshotOverrides = { + base: string; + replace: Record<string, CodexDynamicToolSnapshotSpec>; +}; + +const CODEX_DYNAMIC_TOOL_SNAPSHOT_PREFIX = "codex-dynamic-tools."; +const CODEX_DYNAMIC_TOOL_BASE_SNAPSHOT = `${CODEX_DYNAMIC_TOOL_SNAPSHOT_PREFIX}telegram-direct.json`; +const SHA256_HEADER_PATTERN = /^sha256=([a-f0-9]{64})$/u; + +function promptSnapshotEntries(): Array<[CodexPromptSnapshotScenario, string]> { + return Object.entries(CODEX_PROMPT_SNAPSHOT_FILES) as Array< + [CodexPromptSnapshotScenario, string] + >; +} + +function resolvePromptSnapshotFileName(scenario: string): string { + if (!Object.hasOwn(CODEX_PROMPT_SNAPSHOT_FILES, scenario)) { + throw new Error(`Unknown Codex prompt snapshot scenario: ${scenario}`); + } + return CODEX_PROMPT_SNAPSHOT_FILES[scenario as CodexPromptSnapshotScenario]; +} + +function sha256(content: string): string { + return createHash("sha256").update(content).digest("hex"); +} + +function readSha256Header(header: string | undefined, label: string): string { + const hash = SHA256_HEADER_PATTERN.exec(header ?? "")?.[1]; + if (!hash) { + throw new Error(`Invalid Codex prompt snapshot ${label} hash header`); + } + return hash; +} + +function createCodexPromptSnapshotDelta(scenario: string, base: string, target: string): string { + const targetFileName = resolvePromptSnapshotFileName(scenario); + const baseFileName = CODEX_PROMPT_SNAPSHOT_FILES[CODEX_PROMPT_SNAPSHOT_BASE_SCENARIO]; + if (targetFileName === baseFileName) { + throw new Error("The canonical Codex prompt snapshot does not use a delta"); + } + return createTwoFilesPatch( + baseFileName, + targetFileName, + base, + target, + `sha256=${sha256(base)}`, + `sha256=${sha256(target)}`, + { context: 0, headerOptions: FILE_HEADERS_ONLY }, + ); +} + +/** Strictly materialize one SHA-bound, zero-context prompt snapshot delta. */ +export function materializeCodexPromptSnapshotDelta(params: { + scenario: string; + base: string; + delta: string; +}): string { + const targetFileName = resolvePromptSnapshotFileName(params.scenario); + const baseFileName = CODEX_PROMPT_SNAPSHOT_FILES[CODEX_PROMPT_SNAPSHOT_BASE_SCENARIO]; + let patches; + try { + patches = parsePatch(params.delta); + } catch (error) { + throw new Error("Invalid Codex prompt snapshot delta", { cause: error }); + } + const patch = patches[0]; + if (patches.length !== 1 || !patch) { + throw new Error("Codex prompt snapshot delta must contain exactly one patch"); + } + if (patch.oldFileName !== baseFileName || patch.newFileName !== targetFileName) { + throw new Error("Codex prompt snapshot delta file names do not match the scenario"); + } + const sourceHash = readSha256Header(patch.oldHeader, "source"); + const targetHash = readSha256Header(patch.newHeader, "target"); + if (sha256(params.base) !== sourceHash) { + throw new Error("Codex prompt snapshot delta source hash does not match the canonical base"); + } + const materialized = applyPatch(params.base, patch, { + fuzzFactor: 0, + autoConvertLineEndings: false, + }); + if (materialized === false || sha256(materialized) !== targetHash) { + throw new Error("Codex prompt snapshot delta did not materialize its target hash"); + } + if (createCodexPromptSnapshotDelta(params.scenario, params.base, materialized) !== params.delta) { + throw new Error("Codex prompt snapshot delta is not canonical zero-context output"); + } + return materialized; +} + +function factorCodexPromptSnapshotFiles(files: PromptSnapshotFile[]): PromptSnapshotFile[] { + const byFileName = new Map(files.map((file) => [path.basename(file.path), file])); + const baseFileName = CODEX_PROMPT_SNAPSHOT_FILES[CODEX_PROMPT_SNAPSHOT_BASE_SCENARIO]; + const base = byFileName.get(baseFileName); + if (!base) { + throw new Error(`Missing canonical Codex prompt snapshot: ${baseFileName}`); + } + return files.map((file) => { + const scenario = promptSnapshotEntries().find( + ([, fileName]) => fileName === path.basename(file.path), + ); + if (!scenario || scenario[0] === CODEX_PROMPT_SNAPSHOT_BASE_SCENARIO) { + return file; + } + const delta = createCodexPromptSnapshotDelta(scenario[0], base.content, file.content); + if ( + materializeCodexPromptSnapshotDelta({ + scenario: scenario[0], + base: base.content, + delta, + }) !== file.content + ) { + throw new Error(`Codex prompt snapshot delta roundtrip failed: ${scenario[0]}`); + } + return { path: `${file.path}.diff`, content: delta }; + }); +} + +async function writeSnapshotFiles(root: string, files: PromptSnapshotFile[]) { + await Promise.all( + files.map(async (file) => { + const filePath = path.resolve(root, file.path); + await fs.mkdir(path.dirname(filePath), { recursive: true }); + await fs.writeFile(filePath, file.content); + }), + ); +} + +async function formatSnapshotFiles(root: string, files: PromptSnapshotFile[]) { + const filePaths = files + .filter((file) => file.path.endsWith(".md") || file.path.endsWith(".json")) + .map((file) => path.resolve(root, file.path)); + if (filePaths.length === 0) { + return; + } + await execFileAsync(oxfmtPath, ["--write", "--threads=1", ...filePaths], { + cwd: repoRoot, + }); +} + +async function readSnapshotFiles(root: string, files: PromptSnapshotFile[]) { + return await Promise.all( + files.map(async (file) => ({ + ...file, + content: await fs.readFile(path.resolve(root, file.path), "utf8"), + })), + ); +} + +/** Keep complete Codex tool specs on the wire; deduplicate only their committed review fixtures. */ +function factorCodexDynamicToolSnapshotFiles(files: PromptSnapshotFile[]): PromptSnapshotFile[] { + const base = files.find((file) => path.basename(file.path) === CODEX_DYNAMIC_TOOL_BASE_SNAPSHOT); + if (!base) { + return files; + } + const baseTools = JSON.parse(base.content) as CodexDynamicToolSnapshotSpec[]; + if (new Set(baseTools.map((tool) => tool.name)).size !== baseTools.length) { + return files; + } + + return files.map((file) => { + const fileName = path.basename(file.path); + if ( + fileName === CODEX_DYNAMIC_TOOL_BASE_SNAPSHOT || + !fileName.startsWith(CODEX_DYNAMIC_TOOL_SNAPSHOT_PREFIX) || + !fileName.endsWith(".json") + ) { + return file; + } + const tools = JSON.parse(file.content) as CodexDynamicToolSnapshotSpec[]; + if ( + tools.length !== baseTools.length || + tools.some((tool, index) => tool.name !== baseTools[index]?.name) + ) { + return file; + } + const replace = Object.fromEntries( + tools.flatMap((tool, index) => + JSON.stringify(tool) === JSON.stringify(baseTools[index]) ? [] : [[tool.name, tool]], + ), + ); + const overrides: CodexDynamicToolSnapshotOverrides = { + base: CODEX_DYNAMIC_TOOL_BASE_SNAPSHOT, + replace, + }; + return { ...file, content: `${JSON.stringify(overrides, null, 2)}\n` }; + }); +} + +async function formatPromptSnapshotFiles( + files: PromptSnapshotFile[], +): Promise<PromptSnapshotFile[]> { + const tmpRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-prompt-snapshots-")); + try { + await writeSnapshotFiles(tmpRoot, files); + await formatSnapshotFiles(tmpRoot, files); + return await readSnapshotFiles(tmpRoot, files); + } finally { + await fs.rm(tmpRoot, { recursive: true, force: true }); + } +} + +async function createFormattedPromptSnapshotFiles(): Promise<PromptSnapshotFile[]> { + const files = await createHappyPathPromptSnapshotFiles(); + const formatted = await formatPromptSnapshotFiles(factorCodexDynamicToolSnapshotFiles(files)); + return factorCodexPromptSnapshotFiles(formatted); +} + +/** Materialize one complete, formatted Codex prompt snapshot for human review. */ +export async function materializeCodexPromptSnapshot(scenario: string): Promise<string> { + const targetFileName = resolvePromptSnapshotFileName(scenario); + const baseFileName = CODEX_PROMPT_SNAPSHOT_FILES[CODEX_PROMPT_SNAPSHOT_BASE_SCENARIO]; + const base = await fs.readFile( + path.resolve(repoRoot, CODEX_RUNTIME_HAPPY_PATH_PROMPT_SNAPSHOT_DIR, baseFileName), + "utf8", + ); + if (targetFileName === baseFileName) { + return base; + } + const delta = await fs.readFile( + path.resolve(repoRoot, CODEX_RUNTIME_HAPPY_PATH_PROMPT_SNAPSHOT_DIR, `${targetFileName}.diff`), + "utf8", + ); + return materializeCodexPromptSnapshotDelta({ scenario, base, delta }); +} + +/** Materialize one complete, formatted Codex dynamic-tool catalog for human review. */ +export async function materializeCodexDynamicToolSnapshot(scenario: string): Promise<string> { + if (!/^[a-z][a-z0-9-]*$/u.test(scenario)) { + throw new Error(`Invalid Codex dynamic-tool snapshot scenario: ${scenario}`); + } + const snapshotPath = path.join( + CODEX_RUNTIME_HAPPY_PATH_PROMPT_SNAPSHOT_DIR, + `${CODEX_DYNAMIC_TOOL_SNAPSHOT_PREFIX}${scenario}.json`, + ); + const snapshot = JSON.parse(await fs.readFile(path.resolve(repoRoot, snapshotPath), "utf8")) as + | CodexDynamicToolSnapshotSpec[] + | CodexDynamicToolSnapshotOverrides; + let tools: CodexDynamicToolSnapshotSpec[]; + if (Array.isArray(snapshot)) { + tools = snapshot; + } else { + if (snapshot.base !== CODEX_DYNAMIC_TOOL_BASE_SNAPSHOT) { + throw new Error(`Unsupported Codex dynamic-tool snapshot base: ${snapshot.base}`); + } + const basePath = path.join(CODEX_RUNTIME_HAPPY_PATH_PROMPT_SNAPSHOT_DIR, snapshot.base); + const base = JSON.parse(await fs.readFile(path.resolve(repoRoot, basePath), "utf8")) as + | CodexDynamicToolSnapshotSpec[] + | CodexDynamicToolSnapshotOverrides; + if (!Array.isArray(base)) { + throw new Error("The canonical Codex dynamic-tool snapshot must contain complete tools"); + } + const baseNames = new Set(base.map((tool) => tool.name)); + for (const [name, replacement] of Object.entries(snapshot.replace)) { + if (!baseNames.has(name) || replacement.name !== name) { + throw new Error(`Invalid Codex dynamic-tool snapshot replacement: ${name}`); + } + } + tools = base.map((tool) => + Object.hasOwn(snapshot.replace, tool.name) ? snapshot.replace[tool.name]! : tool, + ); + } + const [formatted] = await formatPromptSnapshotFiles([ + { path: snapshotPath, content: `${JSON.stringify(tools, null, 2)}\n` }, + ]); + if (!formatted) { + throw new Error(`Failed to materialize Codex dynamic-tool snapshot: ${scenario}`); + } + return formatted.content; +} + +async function writeSnapshots() { + const files = await createFormattedPromptSnapshotFiles(); + await fs.mkdir(path.resolve(repoRoot, CODEX_RUNTIME_HAPPY_PATH_PROMPT_SNAPSHOT_DIR), { + recursive: true, + }); + const deleted = await deleteStalePromptSnapshotFiles(repoRoot, files); + await writeSnapshotFiles(repoRoot, files); + const deletedSummary = deleted.length > 0 ? ` Deleted ${deleted.length} stale file(s).` : ""; + console.log(`Wrote ${files.length} prompt snapshot files.${deletedSummary}`); +} + +async function checkSnapshots() { + const files = await createFormattedPromptSnapshotFiles(); + const expectedPaths = new Set(files.map((file) => file.path)); + const mismatches: string[] = []; + for (const file of files) { + const filePath = path.resolve(repoRoot, file.path); + let actual: string; + try { + actual = await fs.readFile(filePath, "utf8"); + } catch (error) { + mismatches.push(`${file.path}: missing (${describeError(error)})`); + continue; + } + if (actual !== file.content) { + mismatches.push(`${file.path}: differs from generated output`); + } + } + for (const snapshotPath of await listCommittedSnapshotArtifactPaths(repoRoot)) { + if (!expectedPaths.has(snapshotPath)) { + mismatches.push(`${snapshotPath}: stale file (not generated)`); + } + } + if (mismatches.length > 0) { + console.error("Prompt snapshot drift detected. Run `pnpm prompt:snapshots:gen`."); + for (const mismatch of mismatches) { + console.error(`- ${mismatch}`); + } + process.exitCode = 1; + return; + } + console.log(`Prompt snapshots are current (${files.length} files).`); +} + +async function runPromptSnapshotGenerator(argv = process.argv.slice(2)) { + if (argv[0] === "--materialize-prompt") { + const scenario = argv[1]; + if (!scenario || argv.length !== 2) { + console.error( + "Usage: node --import tsx scripts/generate-prompt-snapshots.ts --materialize-prompt <scenario>", + ); + process.exitCode = 2; + return; + } + process.stdout.write(await materializeCodexPromptSnapshot(scenario)); + return; + } + if (argv[0] === "--materialize") { + const scenario = argv[1]; + if (!scenario || argv.length !== 2) { + console.error( + "Usage: node --import tsx scripts/generate-prompt-snapshots.ts --materialize <scenario>", + ); + process.exitCode = 2; + return; + } + process.stdout.write(await materializeCodexDynamicToolSnapshot(scenario)); + return; + } + const mode = argv.includes("--write") ? "write" : argv.includes("--check") ? "check" : undefined; + + if (!mode) { + console.error("Usage: pnpm prompt:snapshots:gen | pnpm prompt:snapshots:check"); + process.exitCode = 2; + return; + } + + if (mode === "write") { + await writeSnapshots(); + } else { + await checkSnapshots(); + } +} + +const invokedPath = process.argv[1] ? pathToFileURL(path.resolve(process.argv[1])).href : ""; +if (import.meta.url === invokedPath) { + await runPromptSnapshotGenerator(); +} diff --git a/scripts/gh-read.ts b/scripts/gh-read.ts new file mode 100644 index 0000000000000000000000000000000000000000..8b88bf3a0d1b34224d7c69088d8e114ddf8a6146 --- /dev/null +++ b/scripts/gh-read.ts @@ -0,0 +1,391 @@ +// Gh Read script supports OpenClaw repository automation. +import { spawnSync } from "node:child_process"; +import { createPrivateKey, createSign } from "node:crypto"; +import { pathToFileURL } from "node:url"; +import { readSecretFileSync } from "@openclaw/fs-safe/secret"; +import { expectDefined } from "../packages/normalization-core/src/expect.js"; +import { truncateUtf16Safe } from "../packages/normalization-core/src/utf16-slice.js"; +import { cancelResponseReaderSoon, readBoundedResponseText } from "./lib/bounded-response.mjs"; +import { parseStrictIntegerOption } from "./lib/dev-tooling-safety.ts"; +import { + normalizeGitHubRepo as normalizeRepo, + resolveGitHubRepoFromOrigin, +} from "./lib/github-repo.ts"; + +export { normalizeRepo }; + +const APP_ID_ENV = "OPENCLAW_GH_READ_APP_ID"; +const KEY_FILE_ENV = "OPENCLAW_GH_READ_PRIVATE_KEY_FILE"; +const INSTALLATION_ID_ENV = "OPENCLAW_GH_READ_INSTALLATION_ID"; +const PERMISSIONS_ENV = "OPENCLAW_GH_READ_PERMISSIONS"; +const API_VERSION = "2022-11-28"; +const DEFAULT_GITHUB_FETCH_TIMEOUT_MS = 30_000; +const GITHUB_ERROR_BODY_MAX_CHARS = 4096; +const GITHUB_JSON_BODY_MAX_BYTES = 1024 * 1024; +const GITHUB_APP_PRIVATE_KEY_MAX_BYTES = 64 * 1024; +const DEFAULT_READ_PERMISSION_KEYS = [ + "actions", + "checks", + "contents", + "issues", + "metadata", + "pull_requests", + "statuses", +] as const; + +type GrantedPermissionLevel = "read" | "write" | "admin" | null | undefined; +type RequestedPermissionLevel = "read" | "write"; +type GrantedPermissions = Record<string, GrantedPermissionLevel>; +type RequestedPermissions = Record<string, RequestedPermissionLevel>; + +type InstallationResponse = { + id: number; + permissions?: GrantedPermissions; +}; + +type AccessTokenResponse = { + token: string; +}; + +type GitHubJsonOptions = { + fetchImpl?: typeof fetch; + timeoutMs?: number; +}; + +type GitHubBodyReadOptions = { + signal?: AbortSignal; + timeoutPromise?: Promise<never>; +}; + +export function parseRepoArg(args: string[]): string | null { + for (let i = 0; i < args.length; i += 1) { + const arg = expectDefined(args[i], `GitHub CLI argument at index ${i}`); + if (arg === "-R" || arg === "--repo") { + return normalizeRepo(args[i + 1] ?? null); + } + if (arg.startsWith("--repo=")) { + return normalizeRepo(arg.slice("--repo=".length)); + } + if (arg.startsWith("-R") && arg.length > 2) { + return normalizeRepo(arg.slice(2)); + } + } + return null; +} + +export function parsePermissionKeys(raw: string | null | undefined): string[] { + const trimmed = raw?.trim(); + if (!trimmed) { + return [...DEFAULT_READ_PERMISSION_KEYS]; + } + + return trimmed + .split(",") + .map((value) => value.trim()) + .filter(Boolean); +} + +export function buildReadPermissions( + grantedPermissions: GrantedPermissions | null | undefined, + requestedKeys: readonly string[], +): RequestedPermissions { + const permissions: RequestedPermissions = {}; + for (const key of requestedKeys) { + const granted = grantedPermissions?.[key]; + if (granted === "read" || granted === "write") { + permissions[key] = "read"; + } + } + return permissions; +} + +export function resolveGitHubFetchTimeoutMs(raw = process.env.OPENCLAW_GH_READ_FETCH_TIMEOUT_MS) { + return parseStrictIntegerOption({ + fallback: DEFAULT_GITHUB_FETCH_TIMEOUT_MS, + label: "OPENCLAW_GH_READ_FETCH_TIMEOUT_MS", + min: 1, + raw, + }); +} + +function isMainModule() { + const entry = process.argv[1]; + return entry ? import.meta.url === pathToFileURL(entry).href : false; +} + +function fail(message: string): never { + console.error(`gh-read: ${message}`); + process.exit(1); +} + +function readRequiredEnv(name: string): string { + const value = process.env[name]?.trim(); + if (!value) { + fail(`missing ${name}`); + } + return value; +} + +function resolveRepo(args: string[]): string | null { + const fromArgs = parseRepoArg(args); + if (fromArgs) { + return fromArgs; + } + + const fromEnv = normalizeRepo(process.env.GH_REPO); + if (fromEnv) { + return fromEnv; + } + + try { + return resolveGitHubRepoFromOrigin(); + } catch { + return null; + } +} + +function base64UrlEncode(value: string | Uint8Array) { + return Buffer.from(value) + .toString("base64") + .replace(/\+/g, "-") + .replace(/\//g, "_") + .replace(/=+$/g, ""); +} + +function createAppJwt(appId: string, privateKeyPem: string) { + const now = Math.floor(Date.now() / 1000); + const header = base64UrlEncode(JSON.stringify({ alg: "RS256", typ: "JWT" })); + const payload = base64UrlEncode(JSON.stringify({ iat: now - 60, exp: now + 9 * 60, iss: appId })); + const signingInput = `${header}.${payload}`; + const signer = createSign("RSA-SHA256"); + signer.update(signingInput); + signer.end(); + const signature = signer.sign(createPrivateKey(privateKeyPem)); + return `${signingInput}.${base64UrlEncode(signature)}`; +} + +async function withGitHubFetchTimeout<T>( + label: string, + timeoutMs: number, + run: (signal: AbortSignal, timeoutPromise: Promise<never>) => Promise<T>, +): Promise<T> { + const controller = new AbortController(); + let timeout: ReturnType<typeof setTimeout> | undefined; + const timeoutPromise = new Promise<never>((_resolve, reject) => { + timeout = setTimeout(() => { + const error = new Error(`${label} exceeded timeout of ${timeoutMs}ms`); + reject(error); + controller.abort(error); + }, timeoutMs); + }); + try { + return await Promise.race([run(controller.signal, timeoutPromise), timeoutPromise]); + } finally { + if (timeout) { + clearTimeout(timeout); + } + } +} + +async function readGitHubErrorChunk( + reader: ReadableStreamDefaultReader<Uint8Array>, + timeoutPromise: Promise<never> | undefined, + markCanceled: () => void, +): Promise<ReadableStreamReadResult<Uint8Array>> { + const read = reader.read(); + if (!timeoutPromise) { + return await read; + } + return await Promise.race([ + read, + timeoutPromise.catch((error: unknown) => { + markCanceled(); + cancelResponseReaderSoon(reader); + throw error; + }), + ]); +} + +export async function readBoundedGitHubErrorText( + response: Response, + maxChars = GITHUB_ERROR_BODY_MAX_CHARS, + options: Pick<GitHubBodyReadOptions, "timeoutPromise"> = {}, +): Promise<string> { + if (!response.body) { + return ""; + } + + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let text = ""; + let truncated = false; + let canceled = false; + + try { + while (text.length <= maxChars) { + const { done, value } = await readGitHubErrorChunk(reader, options.timeoutPromise, () => { + canceled = true; + }); + if (done) { + text += decoder.decode(); + break; + } + + text += decoder.decode(value, { stream: true }); + if (text.length > maxChars) { + text = truncateUtf16Safe(text, maxChars); + truncated = true; + break; + } + } + } finally { + if (truncated) { + await reader.cancel().catch(() => undefined); + } else if (!canceled) { + reader.releaseLock(); + } + } + + return truncated ? `${text}\n[truncated]` : text; +} + +export async function readBoundedGitHubJson<T>( + response: Response, + maxBytes = GITHUB_JSON_BODY_MAX_BYTES, + options: GitHubBodyReadOptions = {}, +): Promise<T> { + const text = await readBoundedResponseText(response, "GitHub API", maxBytes, { + createTooLargeError: (message: string) => + Object.assign(new Error(message), { + code: "ETOOBIG", + }), + signal: options.signal, + timeoutPromise: options.timeoutPromise, + }); + return JSON.parse(text) as T; +} + +export async function githubJson<T>( + path: string, + bearerToken: string, + init?: { + method?: "GET" | "POST"; + body?: unknown; + }, + options: GitHubJsonOptions = {}, +): Promise<T> { + const fetchImpl = options.fetchImpl ?? fetch; + const timeoutMs = options.timeoutMs ?? resolveGitHubFetchTimeoutMs(); + return await withGitHubFetchTimeout( + `GitHub API ${init?.method ?? "GET"} ${path}`, + timeoutMs, + async (signal, timeoutPromise) => { + const response = await fetchImpl(`https://api.github.com${path}`, { + method: init?.method ?? "GET", + headers: { + Accept: "application/vnd.github+json", + Authorization: `Bearer ${bearerToken}`, + "Content-Type": "application/json", + "User-Agent": "openclaw-gh-read", + "X-GitHub-Api-Version": API_VERSION, + }, + body: init?.body === undefined ? undefined : JSON.stringify(init.body), + signal, + }); + + if (!response.ok) { + const text = await readBoundedGitHubErrorText(response, undefined, { timeoutPromise }); + fail(`${init?.method ?? "GET"} ${path} failed (${response.status}): ${text}`); + } + + return await readBoundedGitHubJson<T>(response, undefined, { signal, timeoutPromise }); + }, + ); +} + +async function resolveInstallation( + appJwt: string, + repo: string | null, +): Promise<InstallationResponse> { + const installationId = process.env[INSTALLATION_ID_ENV]?.trim(); + if (repo) { + return githubJson<InstallationResponse>(`/repos/${repo}/installation`, appJwt); + } + if (installationId) { + return githubJson<InstallationResponse>(`/app/installations/${installationId}`, appJwt); + } + fail( + `missing repo context; pass -R owner/repo, set GH_REPO, or set ${INSTALLATION_ID_ENV} for a direct installation lookup`, + ); + throw new Error("unreachable"); +} + +async function createInstallationToken( + appJwt: string, + installation: InstallationResponse, + repo: string | null, +): Promise<string> { + const repoName = repo?.split("/")[1] ?? null; + const requestedPermissionKeys = parsePermissionKeys(process.env[PERMISSIONS_ENV]); + const permissions = buildReadPermissions(installation.permissions, requestedPermissionKeys); + const body: { + repositories?: string[]; + permissions?: RequestedPermissions; + } = {}; + + if (repoName) { + body.repositories = [repoName]; + } + if (Object.keys(permissions).length > 0) { + body.permissions = permissions; + } + + const tokenResponse = await githubJson<AccessTokenResponse>( + `/app/installations/${installation.id}/access_tokens`, + appJwt, + { method: "POST", body }, + ); + return tokenResponse.token; +} + +export function readGitHubAppPrivateKey(filePath: string): string { + return readSecretFileSync(filePath, "GitHub App private key", { + maxBytes: GITHUB_APP_PRIVATE_KEY_MAX_BYTES, + rejectHardlinks: false, + }); +} + +async function main() { + if (process.argv.length <= 2) { + fail( + "usage: scripts/gh-read <gh args...>\nset OPENCLAW_GH_READ_APP_ID and OPENCLAW_GH_READ_PRIVATE_KEY_FILE first", + ); + } + + const ghArgs = process.argv.slice(2); + const appId = readRequiredEnv(APP_ID_ENV); + const privateKeyPath = readRequiredEnv(KEY_FILE_ENV); + const privateKeyPem = readGitHubAppPrivateKey(privateKeyPath); + const repo = resolveRepo(ghArgs); + const appJwt = createAppJwt(appId, privateKeyPem); + const installation = await resolveInstallation(appJwt, repo); + const token = await createInstallationToken(appJwt, installation, repo); + const child = spawnSync("gh", ghArgs, { + stdio: "inherit", + env: { + ...process.env, + GH_TOKEN: token, + GITHUB_TOKEN: token, + }, + }); + + if (child.error) { + fail(child.error.message); + } + + process.exit(child.status ?? 1); +} + +if (isMainModule()) { + await main(); +} diff --git a/scripts/install-periphery.sh b/scripts/install-periphery.sh new file mode 100644 index 0000000000000000000000000000000000000000..829898f0025073ddcc5c82ab8cde8dd55f6bf72f --- /dev/null +++ b/scripts/install-periphery.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash + +set -euo pipefail + +temp_dir="" +cleanup() { + local exit_code="$?" + if [[ -n "$temp_dir" ]]; then + rm -rf "$temp_dir" + fi + if [[ "$exit_code" -ne 0 ]]; then + printf '[install-periphery] FAILED (exit %s)\n' "$exit_code" >&2 + fi +} +trap cleanup EXIT + +if [[ "$#" -ne 1 ]]; then + echo "usage: $0 <install-directory>" >&2 + exit 2 +fi + +# The archived OSS release stays pinned until native CI validates an owner-approved successor. +readonly periphery_version="3.8.0" +readonly periphery_checksum="07d4e286e31dd79164df39097e0b59f533c94badbe18158464a455ea88a166d7" + +install_dir="$1" +temp_dir="$(mktemp -d)" +archive="$temp_dir/periphery.zip" +extract_dir="$temp_dir/extract" + +curl --fail --location --silent --show-error \ + --connect-timeout 10 --max-time 120 \ + --retry 3 --retry-max-time 120 \ + --output "$archive" \ + "https://github.com/peripheryapp/periphery/releases/download/$periphery_version/periphery-$periphery_version.zip" +if [[ "$(shasum -a 256 "$archive" | awk '{print $1}')" != "$periphery_checksum" ]]; then + echo "periphery archive checksum mismatch" >&2 + exit 1 +fi + +mkdir -p "$install_dir" "$extract_dir" +unzip -q "$archive" -d "$extract_dir" + +# The signed release resolves libIndexStore through @loader_path, beside the executable. +install -m 0755 "$extract_dir/periphery" "$install_dir/periphery" +install -m 0644 "$extract_dir/libIndexStore.dylib" "$install_dir/libIndexStore.dylib" +install -m 0644 "$extract_dir/LICENSE.md" "$install_dir/LICENSE.md" + +installed_version="$("$install_dir/periphery" version)" +if [[ "$installed_version" != "$periphery_version" ]]; then + echo "error: expected Periphery $periphery_version, got $installed_version" >&2 + exit 1 +fi +printf 'Periphery %s\n' "$installed_version" diff --git a/scripts/install-smoke-candidate-payload.mts b/scripts/install-smoke-candidate-payload.mts new file mode 100644 index 0000000000000000000000000000000000000000..4c366d4f3f04f9bff26aacf0b526794ded85269a --- /dev/null +++ b/scripts/install-smoke-candidate-payload.mts @@ -0,0 +1,429 @@ +#!/usr/bin/env node +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const MANIFEST_NAME = "install-smoke-candidate-payload.json"; +const SCHEMA = "openclaw.install-smoke-candidate-payload/v1"; +const PAYLOAD_FILES = [ + { name: "candidate.tgz", role: "package" }, + { name: "candidate-pack.json", role: "package-metadata" }, + { name: "install.sh", role: "installer" }, + { name: "install-cli.sh", role: "cli-installer" }, +] as const; + +type PayloadIdentity = { + harnessRepository: string; + harnessSha: string; + repository: string; + runAttempt: string; + runId: string; + targetSha: string; +}; + +type PayloadFile = { + name: string; + role: (typeof PAYLOAD_FILES)[number]["role"]; + sha256: string; + size: number; +}; + +type InstallSmokeCandidatePayloadManifest = PayloadIdentity & { + files: PayloadFile[]; + packageVersion: string; + schema: typeof SCHEMA; + sourceArchiveSha256: string; +}; + +type SealInstallSmokeCandidatePayloadOptions = PayloadIdentity & { + archivePath: string; + outputDir: string; + packageDir: string; +}; + +type VerifyInstallSmokeCandidatePayloadOptions = PayloadIdentity & { + expectedManifestSha256: string; + expectedPackageVersion: string; + expectedSourceArchiveSha256: string; + payloadDir: string; +}; + +function assertRepository(value: string, label: string): string { + if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(value)) { + throw new Error(`${label} must be an owner/repository slug`); + } + return value; +} + +function assertSha(value: string, label: string): string { + if (!/^[0-9a-f]{40}$/u.test(value)) { + throw new Error(`${label} must be a full lowercase commit SHA`); + } + return value; +} + +function assertPositiveInteger(value: string, label: string): string { + if (!/^[1-9][0-9]*$/u.test(value)) { + throw new Error(`${label} must be a positive integer`); + } + return value; +} + +function assertVersion(value: unknown, label: string): string { + if (typeof value !== "string" || !/^[0-9A-Za-z][0-9A-Za-z.+-]*$/u.test(value)) { + throw new Error(`${label} must be a package version`); + } + return value; +} + +function validateIdentity(identity: PayloadIdentity): PayloadIdentity { + return { + harnessRepository: assertRepository(identity.harnessRepository, "harness repository"), + harnessSha: assertSha(identity.harnessSha, "harness SHA"), + repository: assertRepository(identity.repository, "repository"), + runAttempt: assertPositiveInteger(identity.runAttempt, "run attempt"), + runId: assertPositiveInteger(identity.runId, "run ID"), + targetSha: assertSha(identity.targetSha, "target SHA"), + }; +} + +async function assertRegularFile(filePath: string, label: string): Promise<void> { + const stat = await fs.lstat(filePath); + if (!stat.isFile() || stat.isSymbolicLink()) { + throw new Error(`${label} must be a regular file`); + } +} + +async function sha256File(filePath: string): Promise<string> { + const hash = createHash("sha256"); + hash.update(await fs.readFile(filePath)); + return hash.digest("hex"); +} + +async function describeFile( + filePath: string, + descriptor: (typeof PAYLOAD_FILES)[number], +): Promise<PayloadFile> { + await assertRegularFile(filePath, descriptor.name); + const stat = await fs.stat(filePath); + return { + name: descriptor.name, + role: descriptor.role, + sha256: await sha256File(filePath), + size: stat.size, + }; +} + +async function readJson(filePath: string, label: string): Promise<unknown> { + await assertRegularFile(filePath, label); + try { + return JSON.parse(await fs.readFile(filePath, "utf8")) as unknown; + } catch (error) { + throw new Error(`${label} must contain valid JSON`, { cause: error }); + } +} + +function runPythonTarReader(args: string[]): Buffer { + const script = String.raw` +import json +import pathlib +import sys +import tarfile + +mode, archive_path, member_name = sys.argv[1:] +with tarfile.open(archive_path, "r:gz") as archive: + members = archive.getmembers() + if mode == "package-metadata": + files = [member for member in members if member.isfile()] + print(json.dumps({ + "entryCount": len(files), + "unpackedSize": sum(member.size for member in files), + })) + sys.exit(0) + if mode == "repo-file": + requested_name = member_name + roots = { + member.name.split("/", 1)[0] + for member in members + if ( + member.name + and not member.name.startswith("/") + and not member.name.split("/", 1)[0].startswith("._") + ) + } + if len(roots) != 1: + raise RuntimeError("candidate archive must contain one repository root") + member_name = next(iter(roots)) + "/" + member_name + member = archive.getmember(member_name) + if not member.isfile(): + label = requested_name if mode == "repo-file" else member_name + raise RuntimeError(label + " must be a regular file in the candidate archive") + extracted = archive.extractfile(member) + if extracted is None: + raise RuntimeError("failed to read " + member_name) + sys.stdout.buffer.write(extracted.read()) +`; + const result = spawnSync("python3", ["-c", script, ...args], { + encoding: "buffer", + maxBuffer: 16 * 1024 * 1024, + }); + if (result.status !== 0) { + const stderr = result.stderr.toString("utf8").trim(); + throw new Error(`candidate archive read failed: ${stderr || `exit ${result.status}`}`); + } + return result.stdout; +} + +function readPackageJsonFromTarball(tarballPath: string): Record<string, unknown> { + const raw = runPythonTarReader(["member", tarballPath, "package/package.json"]); + try { + const value = JSON.parse(raw.toString("utf8")) as unknown; + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error("package/package.json must be an object"); + } + return value as Record<string, unknown>; + } catch (error) { + throw new Error("candidate package tarball has invalid package/package.json", { cause: error }); + } +} + +function readPackageTarballMetadata(tarballPath: string): { + entryCount: number; + unpackedSize: number; +} { + const raw = runPythonTarReader(["package-metadata", tarballPath, ""]); + try { + const value = JSON.parse(raw.toString("utf8")) as Record<string, unknown>; + if ( + typeof value.entryCount !== "number" || + !Number.isSafeInteger(value.entryCount) || + value.entryCount < 1 || + typeof value.unpackedSize !== "number" || + !Number.isSafeInteger(value.unpackedSize) || + value.unpackedSize < 0 + ) { + throw new Error("package metadata has invalid archive sizes"); + } + return { entryCount: value.entryCount, unpackedSize: value.unpackedSize }; + } catch (error) { + throw new Error("candidate package tarball metadata is invalid", { cause: error }); + } +} + +async function writeExclusive(filePath: string, contents: string | Uint8Array): Promise<void> { + await fs.writeFile(filePath, contents, { flag: "wx", mode: 0o644 }); +} + +export async function sealInstallSmokeCandidatePayload( + options: SealInstallSmokeCandidatePayloadOptions, +): Promise<InstallSmokeCandidatePayloadManifest> { + const identity = validateIdentity(options); + await assertRegularFile(options.archivePath, "candidate source archive"); + await fs.mkdir(options.outputDir, { recursive: true }); + const existing = await fs.readdir(options.outputDir); + if (existing.length > 0) { + throw new Error("candidate payload output directory must be empty"); + } + + const sourceTarballPath = path.join(options.packageDir, "candidate.tgz"); + await assertRegularFile(sourceTarballPath, "candidate package tarball"); + const packageJson = readPackageJsonFromTarball(sourceTarballPath); + if (packageJson.name !== "openclaw") { + throw new Error("candidate package tarball must contain the openclaw package"); + } + const packageVersion = assertVersion(packageJson.version, "candidate package version"); + const packageMetadata = readPackageTarballMetadata(sourceTarballPath); + const packageSize = (await fs.stat(sourceTarballPath)).size; + + // Re-read installers from the immutable source archive after candidate execution. Candidate + // build hooks never get a writable handle to the sealed scripts consumed by privileged jobs. + const installScript = runPythonTarReader([ + "repo-file", + options.archivePath, + "scripts/install.sh", + ]); + const cliInstallScript = runPythonTarReader([ + "repo-file", + options.archivePath, + "scripts/install-cli.sh", + ]); + await fs.copyFile(sourceTarballPath, path.join(options.outputDir, "candidate.tgz")); + await writeExclusive( + path.join(options.outputDir, "candidate-pack.json"), + `${JSON.stringify( + [ + { + entryCount: packageMetadata.entryCount, + filename: "candidate.tgz", + name: "openclaw", + size: packageSize, + unpackedSize: packageMetadata.unpackedSize, + version: packageVersion, + }, + ], + null, + 2, + )}\n`, + ); + await writeExclusive(path.join(options.outputDir, "install.sh"), installScript); + await writeExclusive(path.join(options.outputDir, "install-cli.sh"), cliInstallScript); + + const files = await Promise.all( + PAYLOAD_FILES.map((descriptor) => + describeFile(path.join(options.outputDir, descriptor.name), descriptor), + ), + ); + const manifest: InstallSmokeCandidatePayloadManifest = { + ...identity, + files, + packageVersion, + schema: SCHEMA, + sourceArchiveSha256: await sha256File(options.archivePath), + }; + await writeExclusive( + path.join(options.outputDir, MANIFEST_NAME), + `${JSON.stringify(manifest, null, 2)}\n`, + ); + return manifest; +} + +function validateManifest( + value: unknown, + expectedIdentity: PayloadIdentity, + expectedPackageVersion: string, + expectedSourceArchiveSha256: string, +): InstallSmokeCandidatePayloadManifest { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error("candidate payload manifest must be an object"); + } + const manifest = value as Partial<InstallSmokeCandidatePayloadManifest>; + if (manifest.schema !== SCHEMA) { + throw new Error("candidate payload manifest schema is invalid"); + } + for (const [key, expected] of Object.entries(expectedIdentity)) { + if (manifest[key as keyof PayloadIdentity] !== expected) { + throw new Error(`candidate payload manifest ${key} does not match the expected tuple`); + } + } + if (manifest.packageVersion !== expectedPackageVersion) { + throw new Error("candidate payload package version does not match the expected version"); + } + if (manifest.sourceArchiveSha256 !== expectedSourceArchiveSha256) { + throw new Error("candidate payload source archive digest does not match producer output"); + } + if (!Array.isArray(manifest.files) || manifest.files.length !== PAYLOAD_FILES.length) { + throw new Error("candidate payload manifest file inventory is invalid"); + } + return manifest as InstallSmokeCandidatePayloadManifest; +} + +export async function verifyInstallSmokeCandidatePayload( + options: VerifyInstallSmokeCandidatePayloadOptions, +): Promise<InstallSmokeCandidatePayloadManifest> { + const identity = validateIdentity(options); + const expectedPackageVersion = assertVersion( + options.expectedPackageVersion, + "expected package version", + ); + if (!/^[0-9a-f]{64}$/u.test(options.expectedManifestSha256)) { + throw new Error("expected manifest SHA-256 must be lowercase hexadecimal"); + } + if (!/^[0-9a-f]{64}$/u.test(options.expectedSourceArchiveSha256)) { + throw new Error("expected source archive SHA-256 must be lowercase hexadecimal"); + } + const entries = (await fs.readdir(options.payloadDir)).toSorted(); + const expectedEntries = [...PAYLOAD_FILES.map(({ name }) => name), MANIFEST_NAME].toSorted(); + if (JSON.stringify(entries) !== JSON.stringify(expectedEntries)) { + throw new Error("candidate payload contains missing or unexpected files"); + } + const manifestPath = path.join(options.payloadDir, MANIFEST_NAME); + await assertRegularFile(manifestPath, "candidate payload manifest"); + if ((await sha256File(manifestPath)) !== options.expectedManifestSha256) { + throw new Error("candidate payload manifest digest does not match producer output"); + } + const manifest = validateManifest( + await readJson(manifestPath, "candidate payload manifest"), + identity, + expectedPackageVersion, + options.expectedSourceArchiveSha256, + ); + for (let index = 0; index < PAYLOAD_FILES.length; index += 1) { + const descriptor = PAYLOAD_FILES[index]!; + const expected = manifest.files[index]; + if ( + !expected || + expected.name !== descriptor.name || + expected.role !== descriptor.role || + !/^[0-9a-f]{64}$/u.test(expected.sha256) || + !Number.isSafeInteger(expected.size) || + expected.size < 0 + ) { + throw new Error(`candidate payload manifest entry is invalid for ${descriptor.name}`); + } + const actual = await describeFile(path.join(options.payloadDir, descriptor.name), descriptor); + if (actual.sha256 !== expected.sha256 || actual.size !== expected.size) { + throw new Error(`candidate payload digest does not match for ${descriptor.name}`); + } + } + return manifest; +} + +function readOption(argv: string[], name: string): string { + const index = argv.indexOf(name); + if (index === -1 || !argv[index + 1] || argv[index + 1]!.startsWith("--")) { + throw new Error(`${name} is required`); + } + if (argv.includes(name, index + 1)) { + throw new Error(`${name} may only be provided once`); + } + return argv[index + 1]!; +} + +function readIdentity(argv: string[]): PayloadIdentity { + return { + harnessRepository: readOption(argv, "--harness-repository"), + harnessSha: readOption(argv, "--harness-sha"), + repository: readOption(argv, "--repository"), + runAttempt: readOption(argv, "--run-attempt"), + runId: readOption(argv, "--run-id"), + targetSha: readOption(argv, "--target-sha"), + }; +} + +async function main(): Promise<void> { + const [command, ...argv] = process.argv.slice(2); + if (command === "seal") { + const manifest = await sealInstallSmokeCandidatePayload({ + ...readIdentity(argv), + archivePath: readOption(argv, "--archive"), + outputDir: readOption(argv, "--output-dir"), + packageDir: readOption(argv, "--package-dir"), + }); + process.stdout.write(`${JSON.stringify(manifest)}\n`); + return; + } + if (command === "verify") { + const manifest = await verifyInstallSmokeCandidatePayload({ + ...readIdentity(argv), + expectedManifestSha256: readOption(argv, "--manifest-sha256"), + expectedPackageVersion: readOption(argv, "--package-version"), + expectedSourceArchiveSha256: readOption(argv, "--source-archive-sha256"), + payloadDir: readOption(argv, "--payload-dir"), + }); + process.stdout.write(`${JSON.stringify(manifest)}\n`); + return; + } + throw new Error("usage: install-smoke-candidate-payload.mts <seal|verify> [options]"); +} + +if ( + process.argv[1] && + (await fs.realpath(process.argv[1])) === (await fs.realpath(fileURLToPath(import.meta.url))) +) { + await main().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); + }); +} diff --git a/scripts/install.ps1 b/scripts/install.ps1 new file mode 100644 index 0000000000000000000000000000000000000000..4ab77160bcdcd25f4efb22f7ade91ad164e77204 --- /dev/null +++ b/scripts/install.ps1 @@ -0,0 +1,2473 @@ +# OpenClaw Installer for Windows +# Usage: powershell -c "irm https://openclaw.ai/install.ps1 | iex" +# powershell -c "& ([scriptblock]::Create((irm https://openclaw.ai/install.ps1))) -Tag beta -NoOnboard -DryRun" + +[CmdletBinding(PositionalBinding = $false)] +param( + [string]$Tag = "latest", + [ValidateSet("npm", "git")] + [string]$InstallMethod = "npm", + [string]$GitDir, + [switch]$NoOnboard, + [switch]$NoGitUpdate, + [switch]$DryRun, + [switch]$NodeOnly, + [string]$NodePrefix, + [switch]$Help +) + +$ErrorActionPreference = "Stop" + +# BEGIN GENERATED UPDATE NETWORK BUDGET +# Source: src/infra/update-network-budget.ts; regenerate: node scripts/generate-update-network-budget.mjs +$script:UpdateNetworkTimeoutSeconds = 300 +# END GENERATED UPDATE NETWORK BUDGET + +if ($Help) { + @" +Usage: + powershell -File install.ps1 [options] + & ([scriptblock]::Create((irm https://openclaw.ai/install.ps1))) [options] + +Options: + -InstallMethod npm|git Install method (default: npm) + -Tag <tag|version> OpenClaw version or dist-tag (default: latest) + -GitDir <path> Git checkout directory + -NoOnboard Skip onboarding + -NoGitUpdate Skip git pull + -DryRun Print actions only + -NodeOnly Install only a private Node.js runtime; do not change PATH + -NodePrefix <path> Absolute private directory for -NodeOnly (required) + -Help Show this help +"@ | Write-Output + return +} + +$script:InstallExitCode = 0 + +function Fail-Install { + param([int]$Code = 1) + + $script:InstallExitCode = $Code +} + +function Test-BooleanSuccessResult { + param([object[]]$Results) + + return ($Results.Count -gt 0 -and $Results[-1] -eq $true) +} + +function Remove-MatchingQuotes { + param([string]$Value) + + if ([string]::IsNullOrEmpty($Value) -or $Value.Length -lt 2) { + return $Value + } + + $first = $Value[0] + $last = $Value[$Value.Length - 1] + if (($first -eq '"' -and $last -eq '"') -or ($first -eq "'" -and $last -eq "'")) { + return $Value.Substring(1, $Value.Length - 2) + } + return $Value +} + +function Resolve-NodeOptionsWithMinOldSpace { + param( + [string]$NodeOptions, + [int]$MinOldSpaceMb = 8192 + ) + + $parts = @($NodeOptions -split '\s+' | Where-Object { -not [string]::IsNullOrWhiteSpace($_) }) + $resolved = New-Object System.Collections.Generic.List[string] + $foundOldSpace = $false + for ($index = 0; $index -lt $parts.Count; $index++) { + $part = $parts[$index] + $normalizedPart = Remove-MatchingQuotes -Value $part + if ($normalizedPart -match '^--max[-_]old[-_]space[-_]size=(?<value>.+)$') { + $parsed = 0 + $parsedValue = Remove-MatchingQuotes -Value $Matches["value"] + if (-not [int]::TryParse($parsedValue, [ref]$parsed)) { + $parsed = $MinOldSpaceMb + } + $foundOldSpace = $true + $value = [Math]::Max($parsed, $MinOldSpaceMb) + $resolved.Add("--max-old-space-size=$value") + continue + } + if ($normalizedPart -match '^--max[-_]old[-_]space[-_]size$') { + $foundOldSpace = $true + $next = if ($index + 1 -lt $parts.Count) { $parts[$index + 1] } else { $null } + $parsed = 0 + $parsedNext = Remove-MatchingQuotes -Value $next + if (-not [int]::TryParse($parsedNext, [ref]$parsed)) { + $parsed = $MinOldSpaceMb + } + $value = [Math]::Max($parsed, $MinOldSpaceMb) + $resolved.Add("--max-old-space-size=$value") + if ($next) { + $index += 1 + } + continue + } + $resolved.Add($part) + } + if (-not $foundOldSpace) { + $resolved.Add("--max-old-space-size=$MinOldSpaceMb") + } + return ($resolved -join " ") +} + +function Complete-Install { + if ($script:InstallExitCode -eq 0) { + return + } + + if ($PSCommandPath) { + exit $script:InstallExitCode + } + + throw "OpenClaw installation failed with exit code $($script:InstallExitCode)." +} + +function Resolve-InstallerTempDirectory { + param([scriptblock]$LongPathResolver) + + $candidates = New-Object System.Collections.Generic.List[string] + foreach ($candidate in @($env:TEMP, $env:TMP, [System.IO.Path]::GetTempPath())) { + if (-not [string]::IsNullOrWhiteSpace($candidate)) { + $candidates.Add($candidate) + } + } + + $localAppData = [Environment]::GetFolderPath("LocalApplicationData") + if (-not [string]::IsNullOrWhiteSpace($localAppData)) { + $candidates.Add((Join-Path $localAppData "Temp")) + } + + $userHome = [Environment]::GetFolderPath("UserProfile") + if (-not [string]::IsNullOrWhiteSpace($userHome)) { + $candidates.Add((Join-Path $userHome "AppData\Local\Temp")) + } + + foreach ($candidate in $candidates) { + $pathToResolve = $candidate + $candidateHasShortAlias = $candidate -match '(?i)~\d' + $requiresCanonicalization = ( + $candidateHasShortAlias -or + $candidate.StartsWith('\\?\', [System.StringComparison]::OrdinalIgnoreCase) + ) + if ($pathToResolve.StartsWith('\\?\UNC\', [System.StringComparison]::OrdinalIgnoreCase)) { + $pathToResolve = "\\" + $pathToResolve.Substring(8) + } elseif ($pathToResolve.StartsWith('\\?\', [System.StringComparison]::OrdinalIgnoreCase)) { + $pathToResolve = $pathToResolve.Substring(4) + } + try { + if ($LongPathResolver) { + $resolvedCandidate = & $LongPathResolver $pathToResolve + } else { + # Windows PowerShell 5.1: FSO Folder.Path echoes 8.3 aliases; Get-Item.FullName expands them. + $resolvedCandidate = (Get-Item -LiteralPath $pathToResolve -ErrorAction Stop).FullName + } + } catch { + if ($requiresCanonicalization) { + continue + } + $resolvedCandidate = $pathToResolve + } + if ([string]::IsNullOrWhiteSpace($resolvedCandidate)) { + continue + } + if ($candidateHasShortAlias -and $resolvedCandidate -match '(?i)~\d') { + continue + } + if ($resolvedCandidate.StartsWith('\\?\UNC\', [System.StringComparison]::OrdinalIgnoreCase)) { + $resolvedCandidate = "\\" + $resolvedCandidate.Substring(8) + } elseif ($resolvedCandidate.StartsWith('\\?\', [System.StringComparison]::OrdinalIgnoreCase)) { + $resolvedCandidate = $resolvedCandidate.Substring(4) + } + if ( + Test-Path -LiteralPath $resolvedCandidate -PathType Container + ) { + return $resolvedCandidate + } + # Try the next existing root; archive and extraction children do not exist yet. + } + + throw "Could not find a usable Windows temporary directory." +} + +function Initialize-InstallerTempDirectory { + param([scriptblock]$LongPathResolver) + + $tempDirectory = Resolve-InstallerTempDirectory -LongPathResolver $LongPathResolver + $script:InstallerTempDirectory = $tempDirectory + $env:TEMP = $tempDirectory + $env:TMP = $tempDirectory +} + +# Check if running in PowerShell +if ($PSVersionTable.PSVersion.Major -lt 5) { + Write-Host "Error: PowerShell 5+ required" -ForegroundColor Red + Fail-Install + Complete-Install + return +} + +if (-not $PSBoundParameters.ContainsKey("InstallMethod")) { + if (-not [string]::IsNullOrWhiteSpace($env:OPENCLAW_INSTALL_METHOD)) { + $InstallMethod = $env:OPENCLAW_INSTALL_METHOD + } +} +if (-not $PSBoundParameters.ContainsKey("GitDir")) { + if (-not [string]::IsNullOrWhiteSpace($env:OPENCLAW_GIT_DIR)) { + $GitDir = $env:OPENCLAW_GIT_DIR + } +} +if (-not $PSBoundParameters.ContainsKey("NoOnboard")) { + if ($env:OPENCLAW_NO_ONBOARD -eq "1") { + $NoOnboard = $true + } +} +if (-not $PSBoundParameters.ContainsKey("NoGitUpdate")) { + if ($env:OPENCLAW_GIT_UPDATE -eq "0") { + $NoGitUpdate = $true + } +} +if (-not $PSBoundParameters.ContainsKey("DryRun")) { + if ($env:OPENCLAW_DRY_RUN -eq "1") { + $DryRun = $true + } +} + +if ([string]::IsNullOrWhiteSpace($GitDir)) { + $userHome = [Environment]::GetFolderPath("UserProfile") + $GitDir = (Join-Path $userHome "openclaw") +} + +Initialize-InstallerTempDirectory + +Write-Host "" +Write-Host " OpenClaw Installer" -ForegroundColor Cyan +Write-Host "" +Write-Host "[OK] Windows detected" -ForegroundColor Green + +# Check for Node.js +function Test-NodeVersionSupported { + param([string]$Version) + + if ([string]::IsNullOrWhiteSpace($Version)) { + return $false + } + # This standalone installer runs before OpenClaw exists on disk. Mirror the + # release grammar in node-version.mjs; parity cases guard this boundary. + $versionMatch = [regex]::Match( + $Version, + '^\s*v?(?<major>0|[1-9]\d*)\.(?<minor>0|[1-9]\d*)\.(?<patch>0|[1-9]\d*)(?:\+[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?\s*$' + ) + if (-not $versionMatch.Success) { + return $false + } + $major = 0L + $minor = 0L + $patch = 0L + $maxSafeInteger = 9007199254740991L + if ( + -not [long]::TryParse($versionMatch.Groups["major"].Value, [ref]$major) -or + -not [long]::TryParse($versionMatch.Groups["minor"].Value, [ref]$minor) -or + -not [long]::TryParse($versionMatch.Groups["patch"].Value, [ref]$patch) -or + $major -gt $maxSafeInteger -or + $minor -gt $maxSafeInteger -or + $patch -gt $maxSafeInteger + ) { + return $false + } + if ($major -eq 24) { + return ($minor -ge 16) + } + if ($major -eq 26) { + return ($minor -ge 1) + } + return ($major -gt 26) +} + +function Test-NodeSqliteSupported { + param([string]$Version) + + if ([string]::IsNullOrWhiteSpace($Version)) { + return $false + } + $versionMatch = [regex]::Match($Version, '^(?<major>\d+)\.(?<minor>\d+)\.(?<patch>\d+)$') + if (-not $versionMatch.Success) { + return $false + } + $major = [int]$versionMatch.Groups["major"].Value + $minor = [int]$versionMatch.Groups["minor"].Value + $patch = [int]$versionMatch.Groups["patch"].Value + return ( + $major -gt 3 -or + ( + $major -eq 3 -and + ( + $minor -gt 51 -or + ($minor -eq 51 -and $patch -ge 3) -or + ($minor -eq 50 -and $patch -ge 7) -or + ($minor -eq 44 -and $patch -ge 6) + ) + ) + ) +} + +function Check-Node { + param([string]$NodePath) + + try { + if ([string]::IsNullOrWhiteSpace($NodePath)) { + $nodeCommand = Get-Command node -CommandType Application -ErrorAction Stop | Select-Object -First 1 + $NodePath = $nodeCommand.Source + } + $nodeVersion = (& $nodePath -v 2>$null) + $sqliteProbe = @' +const result = { available: false, version: null, text: false, blob: false, json: false }; +let db; +try { + const { DatabaseSync } = require("node:sqlite"); + db = new DatabaseSync(":memory:"); + result.available = true; + result.version = db.prepare("SELECT sqlite_version() AS version").get()?.version ?? null; + const text = "a\u0000b\u0000"; + const bytes = Buffer.from(text, "utf8"); + const json = JSON.stringify({ value: text }); + db.exec("CREATE TABLE probe (text_value TEXT, blob_value BLOB, json_value TEXT)"); + db.prepare("INSERT INTO probe VALUES (?, ?, ?)").run(text, bytes, json); + const row = db.prepare("SELECT text_value, blob_value, json_value FROM probe").get(); + result.text = typeof row?.text_value === "string" && row.text_value.length === text.length && Buffer.from(row.text_value, "utf8").equals(bytes); + result.blob = row?.blob_value instanceof Uint8Array && Buffer.from(row.blob_value).equals(bytes); + result.json = row?.json_value === json && JSON.parse(row.json_value).value === text; +} catch (error) { + result.error = error instanceof Error ? error.message : String(error); +} finally { + db?.close(); +} +process.stdout.write(JSON.stringify(result)); +'@ + $sqliteOutput = ($sqliteProbe | & $nodePath - 2>$null) + $sqlite = $null + if ($LASTEXITCODE -ne 0) { + $sqliteOutput = $null + } + if ($sqliteOutput) { + $sqlite = $sqliteOutput | ConvertFrom-Json + } + $sqliteVersion = $sqlite.version + if ($nodeVersion) { + if ( + (Test-NodeVersionSupported -Version $nodeVersion) -and + (Test-NodeSqliteSupported -Version $sqliteVersion) -and + $sqlite.text -and $sqlite.blob -and $sqlite.json -and -not $sqlite.error + ) { + Write-Host "[OK] Node.js $nodeVersion found" -ForegroundColor Green + return $true + } elseif ($sqlite.available -and -not $sqlite.text -and -not $sqlite.error) { + Write-Host "[!] Node $nodeVersion`: node:sqlite truncates TEXT at embedded NUL (nodejs/node#61954); use 24.16+/26.1+ or a build with the fix" -ForegroundColor Yellow + return $false + } elseif ($sqlite.error -or -not $sqlite.blob -or -not $sqlite.json) { + Write-Host "[!] Node $nodeVersion`: node:sqlite NUL round-trip capability probe failed; use 24.16+/26.1+ or a build with the fix" -ForegroundColor Yellow + return $false + } elseif (Test-NodeVersionSupported -Version $nodeVersion) { + $sqliteVersionLabel = if ([string]::IsNullOrWhiteSpace($sqliteVersion)) { + "unavailable" + } else { + $sqliteVersion + } + Write-Host "[!] Node.js $nodeVersion uses SQLite $sqliteVersionLabel; SQLite 3.51.3+, 3.50.7+ within 3.50.x, or 3.44.6+ within 3.44.x is required" -ForegroundColor Yellow + return $false + } else { + Write-Host "[!] Node.js $nodeVersion found, but Node 24.16.0+ or Node 26.1.0+ is required" -ForegroundColor Yellow + return $false + } + } + } catch { + Write-Host "[!] Node.js not found" -ForegroundColor Yellow + return $false + } + return $false +} + +function Get-WindowsPortableArchitecture { + # Native ARM64 Windows may run Windows PowerShell under x64 emulation, so + # process env vars can report AMD64 even when the host CPU is ARM64. + try { + $processor = Get-CimInstance -ClassName Win32_Processor -ErrorAction Stop | Select-Object -First 1 + if ($processor -and $processor.Architecture -eq 12) { + return "arm64" + } + } catch {} + + try { + $computerSystem = Get-CimInstance -ClassName Win32_ComputerSystem -ErrorAction Stop | Select-Object -First 1 + if ($computerSystem -and $computerSystem.SystemType -match "ARM64") { + return "arm64" + } + } catch {} + + foreach ($architecture in @($env:PROCESSOR_ARCHITEW6432, $env:PROCESSOR_ARCHITECTURE)) { + if ($architecture -match "ARM64") { + return "arm64" + } + } + return "x64" +} + +function Get-OpenClawDepsRoot { + $localAppData = $env:LOCALAPPDATA + if ([string]::IsNullOrWhiteSpace($localAppData)) { + $localAppData = [Environment]::GetFolderPath("LocalApplicationData") + } + if ([string]::IsNullOrWhiteSpace($localAppData)) { + $localAppData = Join-Path ([Environment]::GetFolderPath("UserProfile")) "AppData\Local" + } + return (Join-Path $localAppData "OpenClaw\deps") +} + +function Get-PortableNodeRoot { + return (Join-Path (Get-OpenClawDepsRoot) "portable-node") +} + +function Get-PortableNodeCommandPath { + $root = Get-PortableNodeRoot + $candidate = Join-Path $root "node.exe" + if (Test-Path $candidate) { + return $candidate + } + return $null +} + +function Use-PortableNodeIfPresent { + $nodeExe = Get-PortableNodeCommandPath + if (-not $nodeExe) { + return $false + } + + Add-ToProcessPath (Split-Path -Parent $nodeExe) + return (Check-Node) +} + +function Ensure-PortableNodeOnUserPath { + $nodeExe = Get-PortableNodeCommandPath + if (-not $nodeExe) { + return + } + + $nodeDir = Split-Path -Parent $nodeExe + if (Add-ToUserPath $nodeDir) { + Write-Host "[!] Added $nodeDir to user PATH (restart terminal if node or openclaw is not found)" -ForegroundColor Yellow + } +} + +function Get-WebRequestTimeoutParameters { + param( + [Parameter(Mandatory = $true)] + [string]$CommandName + ) + + $command = Get-Command $CommandName -ErrorAction Stop + if ($command.Parameters.ContainsKey("OperationTimeoutSeconds")) { + $timeouts = @{ OperationTimeoutSeconds = $script:UpdateNetworkTimeoutSeconds } + if ($command.Parameters.ContainsKey("ConnectionTimeoutSeconds")) { + $timeouts.ConnectionTimeoutSeconds = $script:UpdateNetworkTimeoutSeconds + } + return $timeouts + } + + return @{ TimeoutSec = $script:UpdateNetworkTimeoutSeconds } +} + +function Save-InstallerDownload { + param( + [Parameter(Mandatory = $true)] + [string]$Uri, + [Parameter(Mandatory = $true)] + [string]$OutFile + ) + + $command = Get-Command Invoke-WebRequest -ErrorAction Stop + if ($command.Parameters.ContainsKey("OperationTimeoutSeconds")) { + $timeouts = Get-WebRequestTimeoutParameters -CommandName "Invoke-WebRequest" + Invoke-WebRequest -UseBasicParsing -Uri $Uri -OutFile $OutFile @timeouts + return + } + + # Older PowerShell has only a total deadline. Synchronous stream reads renew + # ReadWriteTimeout, preserving slow downloads with the system TLS/proxy defaults. + $request = [System.Net.WebRequest]::CreateHttp($Uri) + $request.Timeout = $script:UpdateNetworkTimeoutSeconds * 1000 + $request.ReadWriteTimeout = $request.Timeout + $response = $null + $responseStream = $null + $outputStream = $null + try { + $response = $request.GetResponse() + $responseStream = $response.GetResponseStream() + $outputStream = [System.IO.File]::Create($OutFile) + $responseStream.CopyTo($outputStream) + } finally { + if ($outputStream) { $outputStream.Dispose() } + if ($responseStream) { $responseStream.Dispose() } + if ($response) { $response.Dispose() } + $request.Abort() + } +} + +function Resolve-PortableNodeDownload { + $architecture = Get-WindowsPortableArchitecture + $requestTimeouts = Get-WebRequestTimeoutParameters -CommandName "Invoke-RestMethod" + $index = Invoke-RestMethod -Uri "https://nodejs.org/dist/index.json" @requestTimeouts + $release = $index | + Where-Object { $_.version -match '^v26\.' } | + Select-Object -First 1 + + if (-not $release -or -not $release.version) { + throw "Could not resolve latest Node.js 26 release metadata." + } + + $fileKey = "win-$architecture-zip" + if ($release.files -and -not ($release.files -contains $fileKey)) { + throw "Node.js $($release.version) does not publish $fileKey." + } + + $name = "node-$($release.version)-win-$architecture.zip" + return @{ + Version = $release.version + Name = $name + Url = "https://nodejs.org/dist/$($release.version)/$name" + } +} + +function Expand-PortableNodeArchive { + param( + [Parameter(Mandatory = $true)] + [string]$ZipPath, + [Parameter(Mandatory = $true)] + [string]$DestinationPath + ) + + $tarCommand = Get-Command tar -ErrorAction SilentlyContinue + if ($tarCommand -and $tarCommand.Source) { + New-Item -ItemType Directory -Force -Path $DestinationPath | Out-Null + Invoke-CommandFromWindowsSafeDirectory -CommandPath $tarCommand.Source -Arguments @("-xf", $ZipPath, "-C", $DestinationPath, "--strip-components", "1") + if ($LASTEXITCODE -eq 0) { + return + } + + $tarExitCode = $LASTEXITCODE + if (Test-Path $DestinationPath) { + Remove-Item -Recurse -Force $DestinationPath + } + Write-Host "[!] tar extraction failed with exit code $tarExitCode; trying .NET zip extraction." -ForegroundColor Yellow + } + + $fallbackExtract = Join-Path (Split-Path -Parent $DestinationPath) ("portable-node-extract-" + [guid]::NewGuid().ToString("N")) + try { + Add-Type -AssemblyName System.IO.Compression.FileSystem + [System.IO.Compression.ZipFile]::ExtractToDirectory($ZipPath, $fallbackExtract) + + $nodeDir = Get-ChildItem -Path $fallbackExtract -Directory | + Where-Object { Test-Path (Join-Path $_.FullName "node.exe") } | + Select-Object -First 1 + if (-not $nodeDir) { + throw "Node.js archive did not contain node.exe." + } + Copy-Item -LiteralPath $nodeDir.FullName -Destination $DestinationPath -Recurse -Force + } finally { + if (Test-Path $fallbackExtract) { + Remove-Item -Recurse -Force $fallbackExtract + } + } +} + +function Install-PortableNode { + if (Use-PortableNodeIfPresent) { + Ensure-PortableNodeOnUserPath + $nodeVersion = (& node -v 2>$null) + if ($nodeVersion) { + Write-Host "[OK] User-local Node.js already available: $nodeVersion" -ForegroundColor Green + } + return + } + + Write-Host " No package manager found; bootstrapping user-local portable Node.js..." -ForegroundColor Gray + + $download = Resolve-PortableNodeDownload + $portableRoot = Get-PortableNodeRoot + $portableParent = Split-Path -Parent $portableRoot + $tmpZip = Join-Path $script:InstallerTempDirectory $download.Name + + New-Item -ItemType Directory -Force -Path $portableParent | Out-Null + if (Test-Path $portableRoot) { + Remove-Item -Recurse -Force $portableRoot + } + + try { + Write-Host " Downloading Node.js $($download.Version)..." -ForegroundColor Gray + Save-InstallerDownload -Uri $download.Url -OutFile $tmpZip + Expand-PortableNodeArchive -ZipPath $tmpZip -DestinationPath $portableRoot + } finally { + if (Test-Path $tmpZip) { + Remove-Item -Force $tmpZip + } + } + + if (-not (Use-PortableNodeIfPresent)) { + throw "Portable Node.js bootstrap completed, but node is still unavailable." + } + Ensure-PortableNodeOnUserPath + + $nodeVersion = (& node -v 2>$null) + Write-Host "[OK] User-local Node.js ready: $nodeVersion" -ForegroundColor Green +} + +function Install-PrivateNode { + param([Parameter(Mandatory = $true)][string]$Prefix) + + $download = Resolve-PortableNodeDownload + $temporaryRoot = Join-Path $script:InstallerTempDirectory ("openclaw-private-node-" + [guid]::NewGuid().ToString("N")) + $archive = Join-Path $temporaryRoot $download.Name + $checksums = Join-Path $temporaryRoot "SHASUMS256.txt" + $parent = Split-Path -Parent $Prefix + $extracted = Join-Path $parent (".openclaw-node-" + [guid]::NewGuid().ToString("N")) + $backup = $null + try { + New-Item -ItemType Directory -Path $temporaryRoot | Out-Null + Save-InstallerDownload -Uri $download.Url -OutFile $archive + Save-InstallerDownload -Uri "https://nodejs.org/dist/$($download.Version)/SHASUMS256.txt" -OutFile $checksums + $checksumPattern = '^(?<hash>[0-9a-fA-F]{64})\s+\*?' + [regex]::Escape($download.Name) + '$' + $expected = @(Get-Content -LiteralPath $checksums | ForEach-Object { + if ($_ -match $checksumPattern) { $Matches["hash"] } + }) + if ($expected.Count -ne 1 -or (Get-FileHash -LiteralPath $archive -Algorithm SHA256).Hash -ne $expected[0]) { + throw "Node.js archive checksum verification failed." + } + + New-Item -ItemType Directory -Force -Path $parent | Out-Null + Expand-PortableNodeArchive -ZipPath $archive -DestinationPath $extracted + $nodeExe = Join-Path $extracted "node.exe" + if (-not (Check-Node -NodePath $nodeExe)) { + throw "Downloaded Node.js does not satisfy OpenClaw runtime requirements." + } + + # Keep the matching npm/npx alongside node.exe, without touching global packages or PATH. + # Stage on the destination volume and preserve the previous private runtime until publication. + if (Test-Path -LiteralPath $Prefix) { + $backup = Join-Path $parent (".openclaw-node-backup-" + [guid]::NewGuid().ToString("N")) + [System.IO.Directory]::Move($Prefix, $backup) + } + try { + [System.IO.Directory]::Move($extracted, $Prefix) + } catch { + if ($backup) { + [System.IO.Directory]::Move($backup, $Prefix) + $backup = $null + } + throw + } + if ($backup) { + Remove-Item -LiteralPath $backup -Recurse -Force + $backup = $null + } + Write-Host "[OK] Private Node.js ready: $(Join-Path $Prefix 'node.exe')" -ForegroundColor Green + } finally { + if (Test-Path -LiteralPath $extracted) { + Remove-Item -LiteralPath $extracted -Recurse -Force + } + if (Test-Path -LiteralPath $temporaryRoot) { + Remove-Item -LiteralPath $temporaryRoot -Recurse -Force + } + } +} + +# Install Node.js +function Install-Node { + Write-Host "[*] Installing Node.js..." -ForegroundColor Yellow + + # Try winget first (Windows 11 / Windows 10 with App Installer) + if (Get-Command winget -ErrorAction SilentlyContinue) { + Write-Host " Using winget..." -ForegroundColor Gray + winget install OpenJS.NodeJS.LTS --source winget --accept-package-agreements --accept-source-agreements | Out-Host + + # Refresh PATH + Refresh-ProcessPath + Add-InstalledNodeToProcessPath | Out-Null + if (Check-Node) { + Write-Host "[OK] Node.js installed via winget" -ForegroundColor Green + return $true + } + Write-Host "[!] winget completed, but Node.js is still unavailable in this shell" -ForegroundColor Yellow + Write-Host "Restart PowerShell and re-run the installer if Node.js was installed successfully." -ForegroundColor Yellow + return $false + } + + # Try Chocolatey + if (Get-Command choco -ErrorAction SilentlyContinue) { + Write-Host " Using Chocolatey..." -ForegroundColor Gray + choco upgrade nodejs-lts -y --install-if-not-installed | Out-Host + + # Refresh PATH + Refresh-ProcessPath + if (Check-Node) { + Write-Host "[OK] Node.js installed via Chocolatey" -ForegroundColor Green + return $true + } + Write-Host "[!] Chocolatey completed, but the installed Node.js runtime is unsupported" -ForegroundColor Yellow + return $false + } + + # Try Scoop + if (Get-Command scoop -ErrorAction SilentlyContinue) { + Write-Host " Using Scoop..." -ForegroundColor Gray + scoop update | Out-Host + scoop install nodejs-lts | Out-Host + scoop update nodejs-lts | Out-Host + + # Refresh PATH + Refresh-ProcessPath + if (Check-Node) { + Write-Host "[OK] Node.js installed via Scoop" -ForegroundColor Green + return $true + } + Write-Host "[!] Scoop completed, but the installed Node.js runtime is unsupported" -ForegroundColor Yellow + return $false + } + + try { + Install-PortableNode + if (Check-Node) { + return $true + } + } catch { + Write-Host "[!] Portable Node.js bootstrap failed: $($_.Exception.Message)" -ForegroundColor Yellow + } + + # Manual download fallback + Write-Host "" + Write-Host "Error: Could not install Node.js automatically." -ForegroundColor Red + Write-Host "" + Write-Host "Please install Node.js 26 manually:" -ForegroundColor Yellow + Write-Host " https://nodejs.org/en/download/" -ForegroundColor Cyan + Write-Host "" + Write-Host "Or install winget (App Installer) from the Microsoft Store." -ForegroundColor Gray + return $false +} + +# Check for existing OpenClaw installation +function Check-ExistingOpenClaw { + if (Get-OpenClawCommandPath) { + Write-Host "[*] Existing OpenClaw installation detected" -ForegroundColor Yellow + return $true + } + return $false +} + +function Check-Git { + try { + $null = Get-Command git -ErrorAction Stop + return $true + } catch { + return $false + } +} + +function Add-ToProcessPath { + param( + [Parameter(Mandatory = $true)] + [string]$PathEntry + ) + + if ([string]::IsNullOrWhiteSpace($PathEntry)) { + return + } + + $currentEntries = @($env:Path -split ";" | Where-Object { -not [string]::IsNullOrWhiteSpace($_) }) + if ($currentEntries | Where-Object { $_ -ieq $PathEntry }) { + return + } + + $env:Path = "$PathEntry;$env:Path" +} + +function Add-InstalledNodeToProcessPath { + foreach ($programFilesRoot in @($env:ProgramW6432, $env:ProgramFiles, ${env:ProgramFiles(x86)})) { + if ([string]::IsNullOrWhiteSpace($programFilesRoot)) { + continue + } + $nodeDir = Join-Path $programFilesRoot "nodejs" + if (Test-Path (Join-Path $nodeDir "node.exe")) { + Add-ToProcessPath $nodeDir + return $true + } + } + return $false +} + +function Refresh-ProcessPath { + $entries = New-Object System.Collections.Generic.List[string] + $pathValues = @( + [Environment]::GetEnvironmentVariable("Path", "Machine"), + [Environment]::GetEnvironmentVariable("Path", "User"), + $env:Path + ) + + foreach ($pathValue in $pathValues) { + foreach ($entry in @($pathValue -split ";")) { + if ( + -not [string]::IsNullOrWhiteSpace($entry) -and + -not ($entries | Where-Object { $_ -ieq $entry }) + ) { + $entries.Add($entry) + } + } + } + $env:Path = $entries -join ";" +} + +function Add-ToUserPath { + param( + [Parameter(Mandatory = $true)] + [string]$PathEntry + ) + + if ([string]::IsNullOrWhiteSpace($PathEntry)) { + return $false + } + + Add-ToProcessPath $PathEntry + + $userPath = [Environment]::GetEnvironmentVariable("Path", "User") + $userEntries = @($userPath -split ";" | Where-Object { -not [string]::IsNullOrWhiteSpace($_) }) + if ($userEntries | Where-Object { $_ -ieq $PathEntry }) { + return $false + } + + $newUserPath = if ([string]::IsNullOrWhiteSpace($userPath)) { + $PathEntry + } else { + "$userPath;$PathEntry" + } + [Environment]::SetEnvironmentVariable("Path", $newUserPath, "User") + return $true +} + +function Get-PortableGitRoot { + return (Join-Path (Get-OpenClawDepsRoot) "portable-git") +} + +function Get-PortableGitCommandPath { + $root = Get-PortableGitRoot + foreach ($candidate in @( + (Join-Path $root "mingw64\bin\git.exe"), + (Join-Path $root "cmd\git.exe"), + (Join-Path $root "bin\git.exe"), + (Join-Path $root "git.exe") + )) { + if (Test-Path $candidate) { + return $candidate + } + } + return $null +} + +function Use-PortableGitIfPresent { + $gitExe = Get-PortableGitCommandPath + if (-not $gitExe) { + return $false + } + + foreach ($pathEntry in (Get-PortableGitPathEntries)) { + Add-ToProcessPath $pathEntry + } + if (Check-Git) { + return $true + } + return $false +} + +function Get-PortableGitPathEntries { + $gitExe = Get-PortableGitCommandPath + if (-not $gitExe) { + return @() + } + + $portableRoot = Get-PortableGitRoot + $pathEntries = @( + (Join-Path $portableRoot "mingw64\bin"), + (Join-Path $portableRoot "usr\bin"), + (Split-Path -Parent $gitExe) + ) + return ($pathEntries | Where-Object { Test-Path $_ } | Select-Object -Unique) +} + +function Ensure-PortableGitOnUserPath { + $added = @() + foreach ($pathEntry in (Get-PortableGitPathEntries)) { + if (Add-ToUserPath $pathEntry) { + $added += $pathEntry + } + } + + if ($added.Count -gt 0) { + Write-Host "[!] Added user-local Git to user PATH (restart terminal if git or git-backed updates are not found)" -ForegroundColor Yellow + } +} + +function Resolve-PortableGitDownload { + $releaseApi = "https://api.github.com/repos/git-for-windows/git/releases/latest" + $headers = @{ + "User-Agent" = "openclaw-installer" + "Accept" = "application/vnd.github+json" + } + $requestTimeouts = Get-WebRequestTimeoutParameters -CommandName "Invoke-RestMethod" + $release = Invoke-RestMethod -Uri $releaseApi -Headers $headers @requestTimeouts + if (-not $release -or -not $release.assets) { + throw "Could not resolve latest git-for-windows release metadata." + } + + $architecture = Get-WindowsPortableArchitecture + $assetPattern = if ($architecture -eq "arm64") { '^MinGit-.*-arm64\.zip$' } else { '^MinGit-.*-64-bit\.zip$' } + $asset = $release.assets | + Where-Object { $_.name -match $assetPattern -and $_.name -notmatch 'busybox' } | + Select-Object -First 1 + if (-not $asset -and $architecture -eq "arm64") { + $asset = $release.assets | + Where-Object { $_.name -match '^MinGit-.*-64-bit\.zip$' -and $_.name -notmatch 'busybox' } | + Select-Object -First 1 + } + + if (-not $asset) { + throw "Could not find a MinGit zip asset in the latest git-for-windows release." + } + + return @{ + Tag = $release.tag_name + Name = $asset.name + Url = $asset.browser_download_url + } +} + +function Install-PortableGit { + if (Use-PortableGitIfPresent) { + Ensure-PortableGitOnUserPath + $portableVersion = (& git --version 2>$null) + if ($portableVersion) { + Write-Host "[OK] User-local Git already available: $portableVersion" -ForegroundColor Green + } + return + } + + Write-Host "[*] Git not found; bootstrapping user-local portable Git..." -ForegroundColor Yellow + + $download = Resolve-PortableGitDownload + $portableRoot = Get-PortableGitRoot + $portableParent = Split-Path -Parent $portableRoot + $tempName = "openclaw-portable-git-" + [guid]::NewGuid().ToString("N") + $tmpZip = Join-Path $script:InstallerTempDirectory ($tempName + ".zip") + $tmpExtract = Join-Path $script:InstallerTempDirectory $tempName + + New-Item -ItemType Directory -Force -Path $portableParent | Out-Null + if (Test-Path $portableRoot) { + Remove-Item -Recurse -Force $portableRoot + } + if (Test-Path $tmpExtract) { + Remove-Item -Recurse -Force $tmpExtract + } + New-Item -ItemType Directory -Force -Path $tmpExtract | Out-Null + + try { + Write-Host " Downloading $($download.Tag)..." -ForegroundColor Gray + Save-InstallerDownload -Uri $download.Url -OutFile $tmpZip + Expand-Archive -Path $tmpZip -DestinationPath $tmpExtract -Force + New-Item -ItemType Directory -Force -Path $portableRoot | Out-Null + Move-Item -Path (Join-Path $tmpExtract "*") -Destination $portableRoot -Force + } finally { + if (Test-Path $tmpZip) { + Remove-Item -Force $tmpZip + } + if (Test-Path $tmpExtract) { + Remove-Item -Recurse -Force $tmpExtract + } + } + + if (-not (Use-PortableGitIfPresent)) { + throw "Portable Git bootstrap completed, but git is still unavailable." + } + Ensure-PortableGitOnUserPath + + $portableVersion = (& git --version 2>$null) + Write-Host "[OK] User-local Git ready: $portableVersion" -ForegroundColor Green +} + +function Ensure-Git { + if (Check-Git) { return $true } + if (Use-PortableGitIfPresent) { + Ensure-PortableGitOnUserPath + return $true + } + try { + Install-PortableGit + if (Check-Git) { + return $true + } + } catch { + Write-Host "[!] Portable Git bootstrap failed: $($_.Exception.Message)" -ForegroundColor Yellow + } + + Write-Host "" + Write-Host "Error: Git is required to install OpenClaw." -ForegroundColor Red + Write-Host "Auto-bootstrap of user-local Git did not succeed." -ForegroundColor Yellow + Write-Host "Install Git for Windows manually, then re-run this installer:" -ForegroundColor Yellow + Write-Host " https://git-scm.com/download/win" -ForegroundColor Cyan + return $false +} + +function Test-GitFilterSupport { + try { + $help = (& git clone -h 2>&1 | Out-String) + return $help -match '(?m)\s--(?:\[no-\])?filter(?:[ =]|$)' + } catch { + return $false + } +} + +function Get-OpenClawCommandPath { + $openclawCmd = Get-Command openclaw.cmd -ErrorAction SilentlyContinue + if ($openclawCmd -and $openclawCmd.Source) { + return $openclawCmd.Source + } + + $openclaw = Get-Command openclaw -ErrorAction SilentlyContinue + if ($openclaw -and $openclaw.Source) { + return $openclaw.Source + } + + return $null +} + +function Invoke-OpenClawCommand { + param( + [Parameter(ValueFromRemainingArguments = $true)] + [string[]]$Arguments + ) + + $commandPath = Get-OpenClawCommandPath + if (-not $commandPath) { + throw "openclaw command not found on PATH." + } + + & $commandPath @Arguments + $exitCode = $LASTEXITCODE + if ($exitCode -ne 0) { + throw "openclaw $($Arguments -join ' ') failed with exit code $exitCode." + } +} + +function Invoke-InteractiveOpenClawCommand { + param( + [Parameter(ValueFromRemainingArguments = $true)] + [string[]]$Arguments + ) + + $commandPath = Get-OpenClawCommandPath + if (-not $commandPath) { + throw "openclaw command not found on PATH." + } + + $process = Start-Process -FilePath $commandPath -ArgumentList $Arguments -NoNewWindow -Wait -PassThru + if ($process.ExitCode -ne 0) { + throw "openclaw $($Arguments -join ' ') failed with exit code $($process.ExitCode)." + } +} + +function Resolve-CommandPath { + param( + [Parameter(Mandatory = $true)] + [string[]]$Candidates + ) + + foreach ($candidate in $Candidates) { + $command = Get-Command $candidate -ErrorAction SilentlyContinue + if ($command -and $command.Source) { + return $command.Source + } + } + + return $null +} + +function Get-NpmCommandPath { + $path = Resolve-CommandPath -Candidates @("npm.cmd", "npm.exe", "npm") + if (-not $path) { + throw "npm not found on PATH." + } + return $path +} + +function Get-CorepackCommandPath { + return (Resolve-CommandPath -Candidates @("corepack.cmd", "corepack.exe", "corepack")) +} + +function Get-WindowsCommandSafeDirectory { + $userHome = [Environment]::GetFolderPath("UserProfile") + if (-not [string]::IsNullOrWhiteSpace($userHome) -and (Test-Path $userHome)) { + return $userHome + } + if (Test-Path -LiteralPath $script:InstallerTempDirectory -PathType Container) { + return $script:InstallerTempDirectory + } + return $null +} + +function Invoke-CommandFromWindowsSafeDirectory { + param( + [Parameter(Mandatory = $true)] + [string]$CommandPath, + [string[]]$Arguments = @(), + [string]$WorkingDirectory + ) + + $safeDir = if ([string]::IsNullOrWhiteSpace($WorkingDirectory)) { Get-WindowsCommandSafeDirectory } else { $WorkingDirectory } + $pushedLocation = $false + $previousErrorActionPreference = $ErrorActionPreference + try { + if (-not [string]::IsNullOrWhiteSpace($safeDir)) { + Push-Location -LiteralPath $safeDir + $pushedLocation = $true + } + # Windows PowerShell 5.1 treats native stderr warnings as PowerShell errors. + $ErrorActionPreference = "Continue" + & $CommandPath @Arguments + } finally { + $ErrorActionPreference = $previousErrorActionPreference + if ($pushedLocation) { + Pop-Location + } + } +} + +function Invoke-NpmCommand { + param( + [string[]]$Arguments = @(), + [string]$CommandPath, + [string]$WorkingDirectory + ) + if ([string]::IsNullOrWhiteSpace($CommandPath)) { + $CommandPath = Get-NpmCommandPath + } + Invoke-CommandFromWindowsSafeDirectory -CommandPath $CommandPath -Arguments $Arguments -WorkingDirectory $WorkingDirectory +} + +function Get-NpmGlobalBinCandidates { + param( + [string]$NpmPrefix + ) + + $candidates = @() + if (-not [string]::IsNullOrWhiteSpace($NpmPrefix)) { + $candidates += $NpmPrefix + $candidates += (Join-Path $NpmPrefix "bin") + } + if (-not [string]::IsNullOrWhiteSpace($env:APPDATA)) { + $candidates += (Join-Path $env:APPDATA "npm") + } + + return $candidates | Where-Object { -not [string]::IsNullOrWhiteSpace($_) } | Select-Object -Unique +} + +function Ensure-OpenClawOnPath { + if (Get-OpenClawCommandPath) { + return $true + } + + $npmPrefix = $null + try { + $npmPrefix = (Invoke-NpmCommand -Arguments @("config", "get", "prefix") 2>$null).Trim() + } catch { + $npmPrefix = $null + } + + $npmBins = Get-NpmGlobalBinCandidates -NpmPrefix $npmPrefix + foreach ($npmBin in $npmBins) { + if (-not (Test-Path (Join-Path $npmBin "openclaw.cmd"))) { + continue + } + + if (Add-ToUserPath $npmBin) { + Write-Host "[!] Added $npmBin to user PATH (restart terminal if command not found)" -ForegroundColor Yellow + } + return $true + } + + Write-Host "[!] openclaw is not on PATH yet." -ForegroundColor Yellow + Write-Host "Restart PowerShell or add the npm global install folder to PATH." -ForegroundColor Yellow + if ($npmBins.Count -gt 0) { + Write-Host "Expected path (one of):" -ForegroundColor Gray + foreach ($npmBin in $npmBins) { + Write-Host " $npmBin" -ForegroundColor Cyan + } + } else { + Write-Host "Hint: run \"npm config get prefix\" to find your npm global path." -ForegroundColor Gray + } + return $false +} + +function Get-RepoPnpmVersion { + param([string]$RepoDir) + + if ([string]::IsNullOrWhiteSpace($RepoDir)) { + return $null + } + + $packageJsonPath = Join-Path $RepoDir "package.json" + if (-not (Test-Path $packageJsonPath)) { + return $null + } + + try { + $packageJson = Get-Content -LiteralPath $packageJsonPath -Raw | ConvertFrom-Json + if ($packageJson.packageManager -match '^pnpm@(?<version>[^+]+)') { + return $Matches["version"] + } + if ($packageJson.devEngines -and $packageJson.devEngines.packageManager) { + $packageManager = $packageJson.devEngines.packageManager + if ($packageManager.name -eq "pnpm" -and -not [string]::IsNullOrWhiteSpace($packageManager.version)) { + return $packageManager.version + } + } + } catch { + return $null + } + + return $null +} + +function Test-PnpmCommandMatchesVersion { + param( + [string]$PnpmVersion, + [string]$RepoDir, + [string]$PnpmCommand + ) + + if (-not (Test-Path -LiteralPath $PnpmCommand)) { + return $false + } + if ([string]::IsNullOrWhiteSpace($PnpmVersion)) { + return $true + } + + $pushedLocation = $false + try { + if (-not [string]::IsNullOrWhiteSpace($RepoDir) -and (Test-Path $RepoDir)) { + Push-Location -LiteralPath $RepoDir + $pushedLocation = $true + } + $currentVersion = (& $pnpmCommand --version 2>$null) + return ($LASTEXITCODE -eq 0 -and $currentVersion -and $currentVersion.Trim() -eq $PnpmVersion) + } catch { + return $false + } finally { + if ($pushedLocation) { + Pop-Location + } + } +} + +function Ensure-Pnpm { + param([string]$RepoDir, [string]$InstallDirectory) + + $pnpmVersion = Get-RepoPnpmVersion -RepoDir $RepoDir + $pnpmSpec = if ([string]::IsNullOrWhiteSpace($pnpmVersion)) { "pnpm@latest" } else { "pnpm@$pnpmVersion" } + $corepackCommand = Get-CorepackCommandPath + if ($corepackCommand) { + try { + Invoke-CommandFromWindowsSafeDirectory -CommandPath $corepackCommand -Arguments @("enable", "--install-directory", $InstallDirectory, "pnpm") | Out-Host + $pnpmCommand = Join-Path $InstallDirectory "pnpm.cmd" + if ($LASTEXITCODE -eq 0 -and (Test-PnpmCommandMatchesVersion -PnpmVersion $pnpmVersion -RepoDir $RepoDir -PnpmCommand $pnpmCommand)) { + Write-Host "[OK] pnpm selected via Corepack ($pnpmSpec)" -ForegroundColor Green + return $pnpmCommand + } + } catch { + # A stale Corepack signature set must still reach npm provisioning. + } + Write-Host "[!] Corepack could not provision pnpm; falling back to npm." -ForegroundColor Yellow + } + Write-Host "[*] Installing pnpm..." -ForegroundColor Yellow + $npmCommand = Get-NpmCommandPath + $npmPrefix = Join-Path $InstallDirectory "npm" + $lifecycleIdentity = if ($pnpmVersion) { $pnpmSpec } else { "pnpm" } + $lifecycleArgument = Get-NpmLifecycleAllowArgument -NpmCommand $npmCommand -InstallSpec $pnpmSpec -NpmCwd $RepoDir -ExactIdentity $lifecycleIdentity + $installArgs = @("install", "-g", "--prefix", $npmPrefix, $pnpmSpec) + if ($lifecycleArgument) { $installArgs += $lifecycleArgument } + Invoke-NpmCommand -CommandPath $npmCommand -Arguments $installArgs | Out-Host + if ($LASTEXITCODE -ne 0) { + throw "Could not install $pnpmSpec into the installer prefix." + } + # npm's Windows global shim lives directly in the explicit prefix, not PATH. + $pnpmCommand = Join-Path $npmPrefix "pnpm.cmd" + if (-not (Test-PnpmCommandMatchesVersion -PnpmVersion $pnpmVersion -RepoDir $RepoDir -PnpmCommand $pnpmCommand)) { + throw "Could not provision $pnpmSpec for $RepoDir." + } + Write-Host "[OK] pnpm installed" -ForegroundColor Green + return $pnpmCommand +} + +# Install OpenClaw +function Resolve-LocalNpmPackagePath { + param([string]$PackagePath) + + try { + return (Resolve-Path -LiteralPath $PackagePath -ErrorAction Stop).ProviderPath + } catch { + return [System.IO.Path]::GetFullPath($PackagePath) + } +} + +function Resolve-LocalNpmPackageInstallSpec { + param([string]$InstallSpec) + + if ([string]::IsNullOrWhiteSpace($InstallSpec)) { + return $InstallSpec + } + if ($InstallSpec -match '^file:(?<path>.+)$') { + $filePath = $Matches["path"] + if ( + $filePath -match '^/' -or + $filePath -match '^\\\\' -or + $filePath -match '^[A-Za-z]:[\\/]' + ) { + return $InstallSpec + } + return ([System.Uri](Resolve-LocalNpmPackagePath -PackagePath $filePath)).AbsoluteUri + } + if ( + $InstallSpec -match '^https?:' -or + $InstallSpec -match '^(git\+|github:)' -or + $InstallSpec -match '^[A-Za-z]:[\\/]' -or + $InstallSpec -match '^\\\\' + ) { + return $InstallSpec + } + if ($InstallSpec -notmatch '^\.\.?[\\/]' -and $InstallSpec -notmatch '\.tgz$') { + return $InstallSpec + } + + try { + return (Resolve-LocalNpmPackagePath -PackagePath $InstallSpec) + } catch { + return $InstallSpec + } +} + +function Resolve-NpmOpenClawInstallSpec { + param( + [string]$PackageName, + [string]$RequestedTag + ) + + if ([string]::IsNullOrWhiteSpace($RequestedTag)) { + return "$PackageName@latest" + } + + $trimmedTag = $RequestedTag.Trim() + if ( + $trimmedTag -match '^(https?|file):' -or + $trimmedTag -match '^(git\+|github:)' -or + $trimmedTag -match '^[A-Za-z]:[\\/]' -or + $trimmedTag -match '^\\\\' -or + $trimmedTag -match '^\.\.?[\\/]' -or + $trimmedTag -match '\.tgz($|[?#])' + ) { + return (Resolve-LocalNpmPackageInstallSpec -InstallSpec $trimmedTag) + } + + return "$PackageName@$trimmedTag" +} + +function Test-OpenClawSourcePackageInstallSpec { + param([string]$RequestedTag) + + if ([string]::IsNullOrWhiteSpace($RequestedTag)) { + return $false + } + + $normalizedTag = $RequestedTag.Trim().ToLowerInvariant() + if ($normalizedTag.StartsWith("openclaw@")) { + $normalizedTag = $normalizedTag.Substring("openclaw@".Length) + } + + if ($normalizedTag -eq "main") { + return $true + } + if ($normalizedTag -match '^github:openclaw/openclaw($|[#/])') { + return $true + } + + if ($normalizedTag.StartsWith("git+")) { + $normalizedTag = $normalizedTag.Substring("git+".Length) + } + return ( + $normalizedTag -match '^https?://github\.com/openclaw/openclaw(\.git)?($|[?#])' -or + $normalizedTag -match '^ssh://git@github\.com[:/]openclaw/openclaw(\.git)?($|[?#])' -or + $normalizedTag -match '^git://github\.com/openclaw/openclaw(\.git)?($|[?#])' -or + $normalizedTag -match '^git@github\.com:openclaw/openclaw(\.git)?($|[?#])' + ) +} + +function Resolve-NpmConfigPath { + param([string]$RawPath) + if ([string]::IsNullOrWhiteSpace($RawPath) -or $RawPath -eq "null" -or $RawPath -eq "undefined") { + return $null + } + if (($RawPath.StartsWith("~/") -or $RawPath.StartsWith("~\")) -and -not [string]::IsNullOrWhiteSpace($HOME)) { + return (Join-Path $HOME $RawPath.Substring(2)) + } + if (($RawPath.StartsWith('${HOME}/') -or $RawPath.StartsWith('${HOME}\')) -and -not [string]::IsNullOrWhiteSpace($HOME)) { + return (Join-Path $HOME $RawPath.Substring(8)) + } + return $RawPath +} + +function Test-NpmConfigFileKey { + param( + [string]$Path, + [string]$Key + ) + if ([string]::IsNullOrWhiteSpace($Path) -or -not (Test-Path -LiteralPath $Path -PathType Leaf)) { + return $false + } + $escapedKey = [regex]::Escape($Key) + return [bool](Select-String -LiteralPath $Path -Pattern "^\s*$escapedKey\s*=" -Quiet) +} + +function Test-NpmConfigRawKey { + param( + [string]$Key, + [string]$ProjectDir + ) + $files = New-Object System.Collections.Generic.List[string] + if (-not [string]::IsNullOrWhiteSpace($ProjectDir)) { + $files.Add((Join-Path $ProjectDir ".npmrc")) + } + $userConfig = if ($env:NPM_CONFIG_USERCONFIG) { + $env:NPM_CONFIG_USERCONFIG + } else { + $env:npm_config_userconfig + } + if ($userConfig) { + $resolvedUserConfig = Resolve-NpmConfigPath $userConfig + if ($resolvedUserConfig) { $files.Add($resolvedUserConfig) } + } elseif (-not [string]::IsNullOrWhiteSpace($HOME)) { + $files.Add((Join-Path $HOME ".npmrc")) + } + + $globalConfig = if ($env:NPM_CONFIG_GLOBALCONFIG) { $env:NPM_CONFIG_GLOBALCONFIG } else { $env:npm_config_globalconfig } + if ($globalConfig) { + $resolvedGlobalConfig = Resolve-NpmConfigPath $globalConfig + if ($resolvedGlobalConfig) { $files.Add($resolvedGlobalConfig) } + } + + $detectedGlobalConfig = (Invoke-NpmCommand -Arguments @("config", "get", "globalconfig", "--global") 2>$null) + if ($LASTEXITCODE -eq 0) { + $resolvedDetectedGlobalConfig = Resolve-NpmConfigPath $detectedGlobalConfig + if ($resolvedDetectedGlobalConfig) { $files.Add($resolvedDetectedGlobalConfig) } + } + + foreach ($file in ($files | Select-Object -Unique)) { + if (Test-NpmConfigFileKey -Path $file -Key $Key) { + return $true + } + } + return $false +} + +function Test-ShouldPreferOfflinePnpmInstall { + param([string]$ProjectDir, [string]$PnpmCommand) + if ( + (Test-Path -LiteralPath "Env:PNPM_CONFIG_PREFER_OFFLINE") -or + (Test-Path -LiteralPath "Env:pnpm_config_prefer_offline") + ) { + return $false + } + $pushedLocation = $false + try { + Push-Location -LiteralPath $ProjectDir + $pushedLocation = $true + $configured = (& $pnpmCommand config get prefer-offline 2>$null) + $configExitCode = $LASTEXITCODE + } catch { return $false } finally { + if ($pushedLocation) { Pop-Location } + } + if ($configExitCode -ne 0) { + return $false + } + $value = "$configured".Trim() + return [string]::IsNullOrWhiteSpace($value) -or $value -eq "undefined" -or $value -eq "null" +} + +function Add-NpmCacheCandidate { + param( + [System.Collections.Generic.List[string]]$Candidates, + [object]$Path + ) + foreach ($rawPath in @($Path)) { + $trimmed = "$rawPath".Trim() + if (-not [string]::IsNullOrWhiteSpace($trimmed) -and -not $Candidates.Contains($trimmed)) { + $Candidates.Add($trimmed) + } + } +} + +function Get-NpmDebugLogRootCandidates { + $candidates = New-Object System.Collections.Generic.List[string] + Add-NpmCacheCandidate -Candidates $candidates -Path $env:NPM_CONFIG_CACHE + + $detectedCache = (Invoke-NpmCommand -Arguments @("config", "get", "cache") 2>$null) + if ($LASTEXITCODE -eq 0) { + Add-NpmCacheCandidate -Candidates $candidates -Path $detectedCache + } + + if ($env:LOCALAPPDATA) { + Add-NpmCacheCandidate -Candidates $candidates -Path (Join-Path $env:LOCALAPPDATA "npm-cache") + } + if ($env:APPDATA) { + Add-NpmCacheCandidate -Candidates $candidates -Path (Join-Path $env:APPDATA "npm-cache") + } + if ($env:USERPROFILE) { + Add-NpmCacheCandidate -Candidates $candidates -Path (Join-Path $env:USERPROFILE "AppData\Local\npm-cache") + } + + return $candidates +} + +function Get-LatestNpmDebugLogPath { + param([object[]]$CacheRoots) + foreach ($cacheDir in @($CacheRoots)) { + $logDir = Join-Path $cacheDir "_logs" + if (-not (Test-Path -LiteralPath $logDir -PathType Container)) { + continue + } + $latestLog = Get-ChildItem -LiteralPath $logDir -Filter "*-debug-*.log" -File -ErrorAction SilentlyContinue | + Sort-Object LastWriteTimeUtc -Descending | + Select-Object -First 1 + if ($latestLog) { + return $latestLog.FullName + } + } + return $null +} + +function Write-NpmInstallFailureDetails { + param( + [object[]]$Output, + [object[]]$CacheRoots + ) + $printedOutput = $false + foreach ($line in @($Output)) { + if ($null -eq $line) { + continue + } + $text = "$line" + if ([string]::IsNullOrWhiteSpace($text)) { + continue + } + Write-Host $text + $printedOutput = $true + } + + $latestLog = Get-LatestNpmDebugLogPath -CacheRoots $CacheRoots + if ($latestLog) { + Write-Host "Latest npm debug log ($latestLog):" -ForegroundColor Yellow + Get-Content -LiteralPath $latestLog -Tail 120 -ErrorAction SilentlyContinue | ForEach-Object { Write-Host $_ } + return + } + + if (-not $printedOutput) { + Write-Host "npm produced no console output and no npm debug log was found." -ForegroundColor Yellow + } +} + +function Get-NpmLifecycleAllowArgument { + param( + [string]$NpmCommand, + [string]$InstallSpec, + [string]$NpmCwd, + [string]$ExactIdentity + ) + $versionOutput = @(Invoke-NpmCommand -CommandPath $NpmCommand -WorkingDirectory $NpmCwd -Arguments @("--version") 2>$null) + if ($LASTEXITCODE -ne 0 -or $versionOutput.Count -eq 0) { + throw "Unable to determine npm version; no package changes were made." + } + $nodeCommand = (Get-Command node -CommandType Application -ErrorAction Stop | Select-Object -First 1).Source + $kernel = @' +const path = require("node:path"); +const [versionOutput, spec, cwd, exactIdentity] = process.argv.slice(2); +const version = versionOutput.trim().split(/\r?\n/).at(-1) ?? ""; +const parsed = version.match(/^[vV]?(\d+)\.(\d+)\.(\d+)(?:[-+][0-9A-Za-z.-]+)?$/); +const fail = (message) => { process.stderr.write(`${message}\n`); process.exit(1); }; +if (!parsed) fail("Unable to determine npm version; no package changes were made."); +if (+parsed[1] < 12 && (+parsed[1] !== 11 || +parsed[2] < 16)) process.exit(0); +const normalized = spec.trim(); +const unaliased = normalized.toLowerCase().startsWith("openclaw@") ? normalized.slice(9).trim() : normalized; +const explicit = (value) => /\.(?:tgz|tar\.gz)$/i.test(value) || value.includes("://") || value.includes("#") || /^(?:file|github|git\+(?:ssh|https|http|file)|npm):/i.test(value); +let identity = !normalized || explicit(normalized) || explicit(unaliased) || /^\.{1,2}(?:[\\/]|$)/.test(unaliased) || path.isAbsolute(normalized) || path.isAbsolute(unaliased) ? unaliased : "openclaw"; +const alias = /^npm:/i.test(identity); +if (alias) identity = /^npm:(@[^/]+\/[^@]+|[^@]+?)(?:@.*)?$/i.exec(identity)?.[1] ?? ""; +const filePrefix = /^file:/i.test(identity) ? "file:" : ""; +const archivePath = identity.slice(filePrefix.length); +const gitShorthand = !/^~[\\/]/.test(identity) && /^[^./@\s:#][^/\s:@#]*\/[^/\s:@#]+(?:#[\s\S]*)?$/.test(identity); +const localArchive = !alias && !gitShorthand && /\.(?:tgz|tar\.gz|tar)$/i.test(archivePath) && (filePrefix || path.isAbsolute(archivePath) || !/^[a-z][a-z0-9+.-]*:/i.test(archivePath)); +let absoluteArchive = ""; +if (localArchive) { + const npmPath = process.platform === "win32" ? archivePath.replaceAll("\\", "/") : archivePath; + // Escape raw paths before URL normalization so literal %, #, and ? retain their identity. + let fileUrl = `file:${encodeURI(npmPath).replace(/[?#]/g, encodeURIComponent)}`; + fileUrl = fileUrl.replace(/^file:\/\/(?=[^/])/, "file:/").replace(/^file:\/{1,3}(?=\.\.?(?:\/|$))/, "file:"); + const specPath = decodeURIComponent(new URL(fileUrl).pathname); + let resolvedPath = decodeURIComponent(new URL(fileUrl, `${require("node:url").pathToFileURL(path.resolve(cwd || process.cwd())).href}/`).pathname); + if (process.platform === "win32") resolvedPath = resolvedPath.replace(/^\/+([a-z]:\/)/i, "$1"); + absoluteArchive = /^\/~(?:\/|$)/.test(specPath) ? path.resolve(require("node:os").homedir(), specPath.slice(3)) : path.resolve(cwd || process.cwd(), resolvedPath); +} +// Tarballs match the absolute npm resolved identity; directory links accept relative paths. +// Keep the npm 11 comma-path identity: its advisory/strict decision stays npm-owned. +if (absoluteArchive && (+parsed[1] >= 12 || !absoluteArchive.includes(","))) identity = `${filePrefix}${absoluteArchive}`; +else { + const relative = cwd && path.isAbsolute(identity) ? path.relative(cwd, identity) || "." : ""; + if (relative) identity = path.isAbsolute(relative) || relative === "." || relative === ".." || relative.startsWith(`..${path.sep}`) ? relative : `.${path.sep}${relative}`; +} +if (exactIdentity) identity = exactIdentity; +if (!identity || identity.includes(",")) fail(`npm cannot allow lifecycle scripts for install target '${spec}'; use a package URL or local path without commas.`); +process.stdout.write(`--allow-scripts=${identity}\n`); +'@ + $kernelOutput = @($kernel | & $nodeCommand - $versionOutput[-1].ToString() $InstallSpec $NpmCwd $ExactIdentity 2>&1) + if ($LASTEXITCODE -ne 0) { + throw $kernelOutput[-1].ToString() + } + if ($kernelOutput.Count -eq 0) { + return $null + } + return $kernelOutput[-1].ToString() +} + +function Test-NpmLifecycleCompleted { + param( + [string]$NpmCommand, + [string]$NpmCwd + ) + $rootOutput = @(Invoke-NpmCommand -CommandPath $NpmCommand -WorkingDirectory $NpmCwd -Arguments @("root", "-g") 2>$null) + if ($LASTEXITCODE -ne 0 -or $rootOutput.Count -eq 0) { + return $false + } + $npmRoot = $rootOutput[-1].ToString().Trim() + if ([string]::IsNullOrWhiteSpace($npmRoot)) { + return $false + } + $entryPath = Join-Path $npmRoot "openclaw\dist\entry.js" + $pendingPath = Join-Path $npmRoot "openclaw\.openclaw-lifecycle-pending" + $legacyGuardPath = Join-Path $npmRoot "openclaw\dist\openclaw-install-guard" + return (Test-Path -LiteralPath $entryPath -PathType Leaf) -and -not (Test-Path -LiteralPath $pendingPath) -and -not (Test-Path -LiteralPath $legacyGuardPath) +} + +function Format-OpenClawGitWrapper { + param([string]$EntryPath) + return "@echo off`r`nnode `"$EntryPath`" %*`r`n" +} + +function Publish-TextFileAtomically { + param( + [string]$Path, + [string]$Contents + ) + $directory = Split-Path -Parent $Path + New-Item -ItemType Directory -Force -Path $directory | Out-Null + $temporaryPath = Join-Path $directory (".openclaw-wrapper-" + [guid]::NewGuid().ToString("N") + ".cmd") + $encoding = New-Object System.Text.UTF8Encoding($false) + [System.IO.File]::WriteAllText($temporaryPath, $Contents, $encoding) + try { + if (Test-Path -LiteralPath $Path) { + [System.IO.File]::Replace($temporaryPath, $Path, $null) + } else { + [System.IO.File]::Move($temporaryPath, $Path) + } + } finally { + if (Test-Path -LiteralPath $temporaryPath) { + Remove-Item -LiteralPath $temporaryPath -Force -ErrorAction SilentlyContinue + } + } +} + +function Install-OpenClaw { + if ([string]::IsNullOrWhiteSpace($Tag)) { + $Tag = "latest" + } + if (Test-OpenClawSourcePackageInstallSpec -RequestedTag $Tag) { + Write-Host "Error: npm installs do not support OpenClaw GitHub source targets like '$Tag'." -ForegroundColor Red + Write-Host "Use -InstallMethod git -Tag main for the moving main checkout, or use latest, beta, an exact version, or a built .tgz package." -ForegroundColor Yellow + return $false + } + if (-not (Ensure-Git)) { + return $false + } + + # Use openclaw package for beta, openclaw for stable + $packageName = "openclaw" + if ($Tag -eq "beta" -or $Tag -match "^beta\.") { + $packageName = "openclaw" + } + $installSpec = Resolve-NpmOpenClawInstallSpec -PackageName $packageName -RequestedTag $Tag + $npmCommand = Get-NpmCommandPath + $npmCwd = Get-WindowsCommandSafeDirectory + $lifecycleArgument = Get-NpmLifecycleAllowArgument -NpmCommand $npmCommand -InstallSpec $installSpec -NpmCwd $npmCwd + Write-Host "[*] Installing OpenClaw ($installSpec)..." -ForegroundColor Yellow + $freshnessArgs = @("--min-release-age=0") + $minReleaseAge = (Invoke-NpmCommand -CommandPath $npmCommand -WorkingDirectory $npmCwd -Arguments @("config", "get", "min-release-age", "--global") 2>$null) + $minReleaseAgeStatus = $LASTEXITCODE + if (Test-NpmConfigRawKey -Key "min-release-age") { + $freshnessArgs = @("--min-release-age=0") + } elseif ($minReleaseAgeStatus -ne 0 -or -not $minReleaseAge -or $minReleaseAge.Trim() -eq "null" -or $minReleaseAge.Trim() -eq "undefined") { + $beforeValue = (Invoke-NpmCommand -CommandPath $npmCommand -WorkingDirectory $npmCwd -Arguments @("config", "get", "before", "--global") 2>$null) + if ($LASTEXITCODE -eq 0 -and $beforeValue -and $beforeValue.Trim() -ne "null" -and $beforeValue.Trim() -ne "undefined") { + $freshnessArgs = @("--before=$((Get-Date).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ss.fffZ"))") + } + } + $prevLogLevel = $env:NPM_CONFIG_LOGLEVEL + $prevUpdateNotifier = $env:NPM_CONFIG_UPDATE_NOTIFIER + $prevFund = $env:NPM_CONFIG_FUND + $prevAudit = $env:NPM_CONFIG_AUDIT + $prevBefore = $env:NPM_CONFIG_BEFORE + $prevMinReleaseAge = $env:NPM_CONFIG_MIN_RELEASE_AGE + $env:NPM_CONFIG_LOGLEVEL = "error" + $env:NPM_CONFIG_UPDATE_NOTIFIER = "false" + $env:NPM_CONFIG_FUND = "false" + $env:NPM_CONFIG_AUDIT = "false" + Remove-Item Env:NPM_CONFIG_BEFORE -ErrorAction SilentlyContinue + Remove-Item Env:NPM_CONFIG_MIN_RELEASE_AGE -ErrorAction SilentlyContinue + try { + # Resolve cache roots before the install so failure reporting cannot create a newer npm log. + $npmDebugLogRoots = @(Get-NpmDebugLogRootCandidates) + $lifecycleArguments = if ($lifecycleArgument) { @($lifecycleArgument) } else { @() } + $npmInstallArguments = @("install", "-g") + $freshnessArgs + $lifecycleArguments + @("$installSpec") + $npmOutput = Invoke-NpmCommand -CommandPath $npmCommand -WorkingDirectory $npmCwd -Arguments $npmInstallArguments 2>&1 + $npmInstallStatus = $LASTEXITCODE + if ($npmInstallStatus -ne 0) { + Write-Host "[!] npm install failed; retrying once" -ForegroundColor Yellow + $npmOutput = Invoke-NpmCommand -CommandPath $npmCommand -WorkingDirectory $npmCwd -Arguments $npmInstallArguments 2>&1 + $npmInstallStatus = $LASTEXITCODE + } + if ($npmInstallStatus -ne 0) { + Write-Host "[!] npm install failed" -ForegroundColor Red + if ($npmOutput -match "spawn git" -or $npmOutput -match "ENOENT.*git") { + Write-Host "Error: git is missing from PATH." -ForegroundColor Red + Write-Host "Install Git for Windows, then reopen PowerShell and retry:" -ForegroundColor Yellow + Write-Host " https://git-scm.com/download/win" -ForegroundColor Cyan + } else { + Write-Host "Re-run with verbose output to see the full error:" -ForegroundColor Yellow + Write-Host ' powershell -c "irm https://openclaw.ai/install.ps1 | iex"' -ForegroundColor Cyan + } + Write-NpmInstallFailureDetails -Output $npmOutput -CacheRoots $npmDebugLogRoots + return $false + } + if (-not (Test-NpmLifecycleCompleted -NpmCommand $npmCommand -NpmCwd $npmCwd)) { + Write-Host "[!] npm install did not produce a usable OpenClaw package; lifecycle scripts may not have completed." -ForegroundColor Red + return $false + } + } finally { + $env:NPM_CONFIG_LOGLEVEL = $prevLogLevel + $env:NPM_CONFIG_UPDATE_NOTIFIER = $prevUpdateNotifier + $env:NPM_CONFIG_FUND = $prevFund + $env:NPM_CONFIG_AUDIT = $prevAudit + $env:NPM_CONFIG_BEFORE = $prevBefore + $env:NPM_CONFIG_MIN_RELEASE_AGE = $prevMinReleaseAge + } + Write-Host "[OK] OpenClaw installed" -ForegroundColor Green + return $true +} + +# Install OpenClaw from GitHub +function Assert-GitCheckoutHasCommit { + param([string]$RepoDir) + + $gitDir = Join-Path $RepoDir ".git" + if (-not (Test-Path -LiteralPath $gitDir -PathType Container)) { + return + } + + $hasHead = $false + try { + git "--git-dir=$gitDir" "--work-tree=$RepoDir" rev-parse --verify --quiet "HEAD^{commit}" 2>$null | Out-Null + $hasHead = ($LASTEXITCODE -eq 0) + } catch {} + if (-not $hasHead) { + throw "Git checkout has no commit: $RepoDir. Move or remove this incomplete checkout, then retry." + } +} + +function Resolve-PhysicalDirectoryPath { + param([string]$Path) + + # Legacy PowerShell/.cmd argument passing strips embedded quotes; keep code on stdin. + $resolved = 'process.stdout.write(require("node:fs").realpathSync(process.argv[2]))' | & node - $Path + if ($LASTEXITCODE -ne 0 -or -not $resolved) { + throw "Could not resolve directory path: $Path" + } + return [string]$resolved +} + +function Resolve-GitCheckoutPath { + param([string]$RepoDir) + + $parentDir = Split-Path -Parent $RepoDir + if (-not $parentDir) { + $parentDir = (Get-Location).Path + } + New-Item -ItemType Directory -Force -Path $parentDir | Out-Null + $parentDir = Resolve-PhysicalDirectoryPath -Path $parentDir + $repoEntry = Get-Item -LiteralPath $RepoDir -Force -ErrorAction SilentlyContinue + if ($repoEntry -and $repoEntry.PSIsContainer) { + return (Resolve-PhysicalDirectoryPath -Path $RepoDir) + } + return (Join-Path $parentDir (Split-Path -Leaf $RepoDir)) +} + +function New-TransactionalGitCheckout { + param( + [string]$RepoUrl, + [string]$RepoDir + ) + + $RepoDir = Resolve-GitCheckoutPath -RepoDir $RepoDir + $parentDir = Split-Path -Parent $RepoDir + $preserveRepoDir = (Test-Path -LiteralPath $RepoDir -PathType Container) -and + @(Get-ChildItem -LiteralPath $RepoDir -Force).Count -eq 0 + $stagingParent = if ($preserveRepoDir) { $RepoDir } else { $parentDir } + $stagingDir = Join-Path $stagingParent (".openclaw-clone-" + [guid]::NewGuid().ToString("N")) + New-Item -ItemType Directory -Path $stagingDir | Out-Null + $retainStaging = $false + + try { + # Keep ref metadata for later updates while avoiding file blobs when + # the local Git client supports partial clones. Git warns and falls + # back to a full clone when the server cannot filter. + $cloneArgs = @("clone") + if (Test-GitFilterSupport) { + $cloneArgs += "--filter=blob:none" + } + $cloneArgs += @($RepoUrl, $stagingDir) + & git @cloneArgs + if ($LASTEXITCODE -ne 0) { + throw "git clone failed with exit code $LASTEXITCODE" + } + $destinationEntry = Get-Item -LiteralPath $RepoDir -Force -ErrorAction SilentlyContinue + if (-not $preserveRepoDir -and $destinationEntry) { + throw "Git install dir appeared while cloning: $RepoDir. The existing path was left unchanged; move it or choose another -GitDir, then retry." + } + if (-not $preserveRepoDir) { + [System.IO.Directory]::Move($stagingDir, $RepoDir) + } else { + $destinationEntries = @(Get-ChildItem -LiteralPath $RepoDir -Force) + if ($destinationEntries.Count -ne 1 -or $destinationEntries[0].Name -ne (Split-Path -Leaf $stagingDir)) { + throw "Git install dir changed while cloning: $RepoDir. The existing path was left unchanged; move it or choose another -GitDir, then retry." + } + $entries = @(Get-ChildItem -LiteralPath $stagingDir -Force | Sort-Object { if ($_.Name -eq ".git") { 1 } else { 0 } }) + $moved = [System.Collections.Generic.List[System.IO.FileSystemInfo]]::new() + try { + foreach ($entry in $entries) { + Move-Item -LiteralPath $entry.FullName -Destination $RepoDir -ErrorAction Stop + $moved.Add($entry) + } + } catch { + $publicationError = $_.Exception + $rollbackErrors = [System.Collections.Generic.List[System.Exception]]::new() + for ($i = $moved.Count - 1; $i -ge 0; $i--) { + try { + Move-Item -LiteralPath (Join-Path $RepoDir $moved[$i].Name) -Destination $stagingDir -ErrorAction Stop + } catch { + $rollbackErrors.Add($_.Exception) + } + } + if ($rollbackErrors.Count -gt 0) { + $retainStaging = $true + throw "Could not publish or fully roll back the cloned checkout at $RepoDir; recovery files remain at $stagingDir. $($publicationError.Message)" + } + throw $publicationError + } + } + } finally { + if (-not $retainStaging -and (Test-Path -LiteralPath $stagingDir)) { + Remove-Item -LiteralPath $stagingDir -Recurse -Force -ErrorAction SilentlyContinue + } + } +} + +function Install-OpenClawFromGit { + param( + [string]$RepoDir, + [switch]$SkipUpdate + ) + if (-not (Ensure-Git)) { + return $false + } + + $RepoDir = Resolve-GitCheckoutPath -RepoDir $RepoDir + $repoUrl = "https://github.com/openclaw/openclaw.git" + Write-Host "[*] Installing OpenClaw from GitHub ($repoUrl)..." -ForegroundColor Yellow + + Assert-GitCheckoutHasCommit -RepoDir $RepoDir + if (-not (Test-Path $RepoDir) -or @(Get-ChildItem -LiteralPath $RepoDir -Force).Count -eq 0) { + New-TransactionalGitCheckout -RepoUrl $repoUrl -RepoDir $RepoDir + } + + if (-not $SkipUpdate) { + # PowerShell 7+ surfaces native-command stderr as terminating errors when + # $ErrorActionPreference=Stop, so git's normal "From <url>" progress line + # would abort the script. Swallow failures here — pull is best-effort. + $dirty = $null + try { $dirty = git -C $RepoDir status --porcelain 2>$null } catch {} + if (-not $dirty) { + try { git -C $RepoDir pull --rebase 2>$null } catch {} + } else { + Write-Host "[!] Repo is dirty; skipping git pull" -ForegroundColor Yellow + } + } else { + Write-Host "[!] Git update disabled; skipping git pull" -ForegroundColor Yellow + } + Remove-LegacySubmodule -RepoDir $RepoDir + + $prevPnpmChildConcurrency = $env:PNPM_CONFIG_CHILD_CONCURRENCY + $prevPnpmNetworkConcurrency = $env:PNPM_CONFIG_NETWORK_CONCURRENCY + $prevPnpmWorkspaceConcurrency = $env:PNPM_CONFIG_WORKSPACE_CONCURRENCY + $prevPnpmVerifyDepsBeforeRun = $env:PNPM_CONFIG_VERIFY_DEPS_BEFORE_RUN + $prevPnpmSideEffectsCache = $env:PNPM_CONFIG_SIDE_EFFECTS_CACHE + $prevNodeOptions = $env:NODE_OPTIONS + $pnpmInstallDirectory = Join-Path $script:InstallerTempDirectory ("openclaw-pnpm-" + [Guid]::NewGuid().ToString("N")) + $previousPnpmContext = foreach ($name in @("PATH", "COREPACK_ENABLE_DOWNLOAD_PROMPT", "NPM_CONFIG_WORKSPACE_DIR", "npm_config_workspace_dir", "PNPM_CONFIG_LOCKFILE_DIR", "pnpm_config_lockfile_dir")) { + [PSCustomObject]@{ Name = $name; Value = [Environment]::GetEnvironmentVariable($name, "Process") } + } + $env:PNPM_CONFIG_CHILD_CONCURRENCY = "1" + $env:PNPM_CONFIG_NETWORK_CONCURRENCY = "4" + $env:PNPM_CONFIG_WORKSPACE_CONCURRENCY = "1" + $env:PNPM_CONFIG_VERIFY_DEPS_BEFORE_RUN = "false" + $env:PNPM_CONFIG_SIDE_EFFECTS_CACHE = "false" + $pushedRepoLocation = $false + try { + Push-Location -LiteralPath $RepoDir + $pushedRepoLocation = $true + New-Item -ItemType Directory -Path $pnpmInstallDirectory | Out-Null + # Both explicit and nested commands belong to this checkout. Restore the + # caller's environment even when bootstrap, install, or build fails. + $env:PATH = "$pnpmInstallDirectory;$env:PATH" + # Corepack must not await terminal input in the hidden version probe. + $env:COREPACK_ENABLE_DOWNLOAD_PROMPT = "0" + $env:NPM_CONFIG_WORKSPACE_DIR = $env:npm_config_workspace_dir = $RepoDir + $env:PNPM_CONFIG_LOCKFILE_DIR = $env:pnpm_config_lockfile_dir = $RepoDir + $pnpmCommand = Ensure-Pnpm -RepoDir $RepoDir -InstallDirectory $pnpmInstallDirectory + $env:PATH = "$(Split-Path -Parent $pnpmCommand);$env:PATH" + $sourceInstallArgs = @("install") + if (Test-ShouldPreferOfflinePnpmInstall -ProjectDir $RepoDir -PnpmCommand $pnpmCommand) { + $sourceInstallArgs += "--prefer-offline" + } + $sourceInstallArgs += @( + "--config.node-linker=hoisted", + "--config.engine-strict=false", + "--config.enable-pre-post-scripts=true", + "--config.side-effects-cache=false", + "--no-frozen-lockfile", + "--config.child-concurrency=$env:PNPM_CONFIG_CHILD_CONCURRENCY", + "--config.network-concurrency=$env:PNPM_CONFIG_NETWORK_CONCURRENCY", + "--config.workspace-concurrency=$env:PNPM_CONFIG_WORKSPACE_CONCURRENCY" + ) + & $pnpmCommand @sourceInstallArgs + $installSucceeded = ($LASTEXITCODE -eq 0) + if (-not $installSucceeded) { + Write-Host "[!] pnpm install failed for the Git checkout; clearing node_modules and retrying once" -ForegroundColor Yellow + Remove-Item -Recurse -Force node_modules -ErrorAction SilentlyContinue + & $pnpmCommand @sourceInstallArgs + $installSucceeded = ($LASTEXITCODE -eq 0) + } + if (-not $installSucceeded) { + Write-Host "[!] pnpm install failed for the Git checkout" -ForegroundColor Red + return $false + } + & $pnpmCommand ui:build + if ($LASTEXITCODE -ne 0) { + Write-Host "[!] UI build failed; continuing (CLI may still work)" -ForegroundColor Yellow + } + $env:NODE_OPTIONS = Resolve-NodeOptionsWithMinOldSpace -NodeOptions $prevNodeOptions -MinOldSpaceMb 8192 + & $pnpmCommand build + if ($LASTEXITCODE -ne 0) { + Write-Host "[!] pnpm build failed for the Git checkout" -ForegroundColor Red + return $false + } + } finally { + if ($pushedRepoLocation) { + Pop-Location + } + $env:PNPM_CONFIG_CHILD_CONCURRENCY = $prevPnpmChildConcurrency + $env:PNPM_CONFIG_NETWORK_CONCURRENCY = $prevPnpmNetworkConcurrency + $env:PNPM_CONFIG_WORKSPACE_CONCURRENCY = $prevPnpmWorkspaceConcurrency + $env:PNPM_CONFIG_VERIFY_DEPS_BEFORE_RUN = $prevPnpmVerifyDepsBeforeRun + $env:PNPM_CONFIG_SIDE_EFFECTS_CACHE = $prevPnpmSideEffectsCache + $env:NODE_OPTIONS = $prevNodeOptions + foreach ($entry in $previousPnpmContext) { + # The Env provider preserves absence; .NET string binding turns + # $null into an empty variable on PowerShell 7.5+. + Set-Item -LiteralPath "Env:$($entry.Name)" -Value $entry.Value + } + if (Test-Path -LiteralPath $pnpmInstallDirectory) { + Remove-Item -LiteralPath $pnpmInstallDirectory -Recurse -Force + } + } + + $entryPath = Join-Path $RepoDir "dist\\entry.js" + if (-not (Test-Path $entryPath)) { + Write-Host "[!] OpenClaw build did not produce $entryPath" -ForegroundColor Red + return $false + } + + $binDir = Join-Path $env:USERPROFILE ".local\\bin" + if (-not (Test-Path $binDir)) { + New-Item -ItemType Directory -Force -Path $binDir | Out-Null + } + node $entryPath --version 2>$null | Out-Null + if ($LASTEXITCODE -ne 0) { + Write-Host "[!] Git replacement failed CLI verification" -ForegroundColor Red + return $false + } + $cmdPath = Join-Path $binDir "openclaw.cmd" + $cmdContents = Format-OpenClawGitWrapper -EntryPath $entryPath + Publish-TextFileAtomically -Path $cmdPath -Contents $cmdContents + + if (Add-ToUserPath $binDir) { + Write-Host "[!] Added $binDir to user PATH (restart terminal if command not found)" -ForegroundColor Yellow + } + + Write-Host "[OK] OpenClaw wrapper installed to $cmdPath" -ForegroundColor Green + Write-Host "[i] Manual builds need the checkout-pinned pnpm launcher; installer bootstrap is temporary: https://docs.openclaw.ai/install/installer#source-build-toolchain" -ForegroundColor Gray + return $true +} + +# Run doctor for migrations (safe, non-interactive) +function Run-Doctor { + Write-Host "[*] Running doctor to migrate settings..." -ForegroundColor Yellow + try { + Invoke-OpenClawCommand doctor --fix --non-interactive + Write-Host "[OK] Migration complete" -ForegroundColor Green + return $true + } catch { + Write-Host "[!] Migration failed: $($_.Exception.Message)" -ForegroundColor Red + return $false + } +} + +function Test-GatewayServiceLoaded { + try { + $statusJson = (Invoke-OpenClawCommand daemon status --json 2>$null) + if ([string]::IsNullOrWhiteSpace($statusJson)) { + return $false + } + $parsed = $statusJson | ConvertFrom-Json + if ($parsed -and $parsed.service -and $parsed.service.loaded) { + return $true + } + } catch { + return $false + } + return $false +} + +function Refresh-GatewayServiceIfLoaded { + if (-not (Get-OpenClawCommandPath)) { + return + } + if (-not (Test-GatewayServiceLoaded)) { + return + } + + Write-Host "[*] Refreshing loaded gateway service..." -ForegroundColor Yellow + try { + Invoke-OpenClawCommand gateway install --force | Out-Null + } catch { + Write-Host "[!] Gateway service refresh failed; continuing." -ForegroundColor Yellow + return + } + + try { + Invoke-OpenClawCommand gateway restart | Out-Null + Invoke-OpenClawCommand gateway status --json | Out-Null + Write-Host "[OK] Gateway service refreshed" -ForegroundColor Green + } catch { + Write-Host "[!] Gateway service restart failed; continuing. Run: openclaw gateway restart" -ForegroundColor Yellow + } +} + +function Get-LegacyRepoDir { + if (-not [string]::IsNullOrWhiteSpace($env:OPENCLAW_GIT_DIR)) { + return $env:OPENCLAW_GIT_DIR + } + $userHome = [Environment]::GetFolderPath("UserProfile") + return (Join-Path $userHome "openclaw") +} + +function Remove-LegacySubmodule { + param( + [string]$RepoDir + ) + if ([string]::IsNullOrWhiteSpace($RepoDir)) { + $RepoDir = Get-LegacyRepoDir + } + $legacyDir = Join-Path $RepoDir "Peekaboo" + if (Test-Path $legacyDir) { + Write-Host "[!] Removing legacy submodule checkout: $legacyDir" -ForegroundColor Yellow + Remove-Item -Recurse -Force $legacyDir + } +} + +function Test-PreviousGitWrapper { + $wrapper = Join-Path (Join-Path $env:USERPROFILE ".local\bin") "openclaw.cmd" + if (-not (Test-Path -LiteralPath $wrapper -PathType Leaf)) { return $false } + return ([System.IO.File]::ReadAllText($wrapper) -match '^@echo off\r?\nnode ".+[\\/]dist[\\/]entry\.js" %\*\r?\n?$') +} + +function Remove-PreviousGitWrapper { + if (Test-PreviousGitWrapper) { + $wrapper = Join-Path (Join-Path $env:USERPROFILE ".local\bin") "openclaw.cmd" + Remove-Item -LiteralPath $wrapper -Force + Write-Host "[OK] Previous git wrapper retired" -ForegroundColor Green + } +} + +function Remove-PreviousNpmOwner { + param([string]$GitWrapper) + $npmCommand = Get-NpmCommandPath + $rootOutput = @(Invoke-NpmCommand -CommandPath $npmCommand -Arguments @("root", "-g") 2>$null) + if ($LASTEXITCODE -ne 0 -or $rootOutput.Count -eq 0) { throw "Could not resolve the previous npm owner." } + $packageRoot = Join-Path $rootOutput[-1].ToString().Trim() "openclaw" + $packageJson = Join-Path $packageRoot "package.json" + if (-not (Test-Path -LiteralPath $packageJson)) { return } + $package = Get-Content -LiteralPath $packageJson -Raw | ConvertFrom-Json + if ($package.name -ne "openclaw") { throw "Refusing to retire a package whose identity is not openclaw." } + $prefixOutput = @(Invoke-NpmCommand -CommandPath $npmCommand -Arguments @("config", "get", "prefix") 2>$null) + $npmShim = if ($prefixOutput.Count -gt 0) { Join-Path $prefixOutput[-1].ToString().Trim() "openclaw.cmd" } else { $null } + if ($npmShim -and [System.IO.Path]::GetFullPath($npmShim) -eq [System.IO.Path]::GetFullPath($GitWrapper)) { + Remove-Item -LiteralPath $packageRoot -Recurse -Force + } else { + Invoke-NpmCommand -CommandPath $npmCommand -Arguments @("uninstall", "-g", "openclaw") | Out-Null + if ($LASTEXITCODE -ne 0) { throw "npm could not retire the previous OpenClaw package." } + } + Write-Host "[OK] Previous npm install retired" -ForegroundColor Green +} + +function Start-NpmShimBackup { + param( + [string]$Path, + [string]$ExpectedLauncher + ) + $backupPath = Join-Path (Split-Path -Parent $Path) (".openclaw-shim-backup-" + [guid]::NewGuid().ToString("N")) + [System.IO.File]::Move($Path, $backupPath) + return [pscustomobject]@{ Path = $Path; BackupPath = $backupPath; ExpectedLauncher = $ExpectedLauncher } +} + +function Test-NpmOpenClawCmdShim { + param( + [string]$Path, + [string]$ExpectedLauncher + ) + if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) { return $false } + $item = Get-Item -LiteralPath $Path -Force + if (($item.Attributes -band [System.IO.FileAttributes]::ReparsePoint) -or $item.Length -gt 16384) { return $false } + $contents = [System.IO.File]::ReadAllText($Path) + if (-not $contents.StartsWith("@ECHO off`r`nGOTO start`r`n", [System.StringComparison]::Ordinal)) { return $false } + $targetMatch = [regex]::Match($contents, '"%(?:~dp0|dp0%)\\(?<target>[^"]+?)"\s+%\*') + if (-not $targetMatch.Success) { return $false } + $resolvedTarget = [System.IO.Path]::GetFullPath((Join-Path (Split-Path -Parent $Path) $targetMatch.Groups["target"].Value)) + return [string]::Equals($resolvedTarget, [System.IO.Path]::GetFullPath($ExpectedLauncher), [System.StringComparison]::OrdinalIgnoreCase) +} + +function Restore-NpmShimBackup { + param([object]$Backup) + if (-not $Backup -or -not (Test-Path -LiteralPath $Backup.BackupPath -PathType Leaf)) { return } + if (Test-Path -LiteralPath $Backup.Path) { + if (-not (Test-NpmOpenClawCmdShim -Path $Backup.Path -ExpectedLauncher $Backup.ExpectedLauncher)) { + throw "Refusing to replace an unrelated file while restoring $($Backup.Path)." + } + Remove-Item -LiteralPath $Backup.Path -Force + } + [System.IO.File]::Move($Backup.BackupPath, $Backup.Path) +} + +function Complete-NpmShimBackup { + param([object]$Backup) + if ($Backup -and (Test-Path -LiteralPath $Backup.BackupPath -PathType Leaf)) { + Remove-Item -LiteralPath $Backup.BackupPath -Force + } +} + +# Main installation flow +function Main { + if ($NodeOnly) { + $prefixRoot = if (-not [string]::IsNullOrWhiteSpace($NodePrefix)) { [System.IO.Path]::GetPathRoot($NodePrefix) } else { "" } + if ( + [string]::IsNullOrWhiteSpace($prefixRoot) -or + $prefixRoot.EndsWith(":") -or + ($prefixRoot -eq "\") -or + [string]::Equals([System.IO.Path]::GetFullPath($NodePrefix).TrimEnd('\', '/'), $prefixRoot.TrimEnd('\', '/'), [System.StringComparison]::OrdinalIgnoreCase) + ) { + Write-Host "Error: -NodeOnly requires -NodePrefix with an absolute private directory, not a filesystem root." -ForegroundColor Red + Fail-Install -Code 2 + return + } + if ($DryRun) { + Write-Host "[OK] Would install private Node.js to $NodePrefix (PATH unchanged)." -ForegroundColor Green + return $true + } + try { + Install-PrivateNode -Prefix ([System.IO.Path]::GetFullPath($NodePrefix)) + } catch { + Write-Host "Error: Node.js update failed: $($_.Exception.Message)" -ForegroundColor Red + Fail-Install + } + return + } + if (-not [string]::IsNullOrWhiteSpace($NodePrefix)) { + Write-Host "Error: -NodePrefix requires -NodeOnly." -ForegroundColor Red + Fail-Install -Code 2 + return + } + + if ($InstallMethod -ne "npm" -and $InstallMethod -ne "git") { + Write-Host "Error: invalid -InstallMethod (use npm or git)." -ForegroundColor Red + Fail-Install -Code 2 + return + } + + if ($DryRun) { + Write-Host "[OK] Dry run" -ForegroundColor Green + Write-Host "[OK] Install method: $InstallMethod" -ForegroundColor Green + if ($InstallMethod -eq "git") { + Write-Host "[OK] Git dir: $GitDir" -ForegroundColor Green + if ($NoGitUpdate) { + Write-Host "[OK] Git update: disabled" -ForegroundColor Green + } else { + Write-Host "[OK] Git update: enabled" -ForegroundColor Green + } + } + if ($NoOnboard) { + Write-Host "[OK] Onboard: skipped" -ForegroundColor Green + } + return $true + } + + # Check for existing installation + $isUpgrade = Check-ExistingOpenClaw + + # Step 1: Node.js + if (-not (Check-Node)) { + if (-not (Install-Node)) { + Fail-Install + return + } + + # Verify installation + if (-not (Check-Node)) { + Write-Host "" + Write-Host "Error: Node.js installation may require a terminal restart" -ForegroundColor Red + Write-Host "Please close this terminal, open a new one, and run this installer again." -ForegroundColor Yellow + Fail-Install + return + } + } + + $finalGitDir = $null + + # Step 2: OpenClaw + if ($InstallMethod -eq "git") { + $hadNpmOwner = $false + try { + $npmCommand = Get-NpmCommandPath + if ($npmCommand) { + Invoke-NpmCommand -CommandPath $npmCommand -Arguments @("list", "-g", "openclaw") 2>$null | Out-Null + $hadNpmOwner = ($LASTEXITCODE -eq 0) + } + } catch { } + $finalGitDir = $GitDir + $gitInstallResults = @(Install-OpenClawFromGit -RepoDir $GitDir -SkipUpdate:$NoGitUpdate) + if (-not (Test-BooleanSuccessResult -Results $gitInstallResults)) { + Fail-Install + return + } + if ($hadNpmOwner) { + Remove-PreviousNpmOwner -GitWrapper (Join-Path (Join-Path $env:USERPROFILE ".local\bin") "openclaw.cmd") + } + } else { + $hadGitWrapper = Test-PreviousGitWrapper + $npmShimBackup = $null + try { + $npmCommand = Get-NpmCommandPath + $npmCwd = Get-WindowsCommandSafeDirectory + $prefixOutput = @(Invoke-NpmCommand -CommandPath $npmCommand -WorkingDirectory $npmCwd -Arguments @("config", "get", "prefix") 2>$null) + $npmPrefix = if ($prefixOutput.Count -gt 0) { $prefixOutput[-1].ToString().Trim() } else { $null } + $previousGitWrapper = Join-Path (Join-Path $env:USERPROFILE ".local\bin") "openclaw.cmd" + if ($hadGitWrapper) { + foreach ($npmBin in (Get-NpmGlobalBinCandidates -NpmPrefix $npmPrefix)) { + $candidate = Join-Path $npmBin "openclaw.cmd" + if ([string]::Equals([System.IO.Path]::GetFullPath($candidate), [System.IO.Path]::GetFullPath($previousGitWrapper), [System.StringComparison]::OrdinalIgnoreCase)) { + $rootOutput = @(Invoke-NpmCommand -CommandPath $npmCommand -WorkingDirectory $npmCwd -Arguments @("root", "-g") 2>$null) + if ($LASTEXITCODE -ne 0 -or $rootOutput.Count -eq 0) { + Fail-Install + return + } + $expectedNpmLauncher = Join-Path $rootOutput[-1].ToString().Trim() "openclaw\openclaw.mjs" + $npmShimBackup = Start-NpmShimBackup -Path $previousGitWrapper -ExpectedLauncher $expectedNpmLauncher + break + } + } + } + + $npmInstallResults = @(Install-OpenClaw) + if (-not (Test-BooleanSuccessResult -Results $npmInstallResults)) { + Fail-Install + return + } + if ($hadGitWrapper) { + $npmCandidate = if ($npmShimBackup) { $npmShimBackup.Path } else { + $candidatePath = $null + foreach ($npmBin in (Get-NpmGlobalBinCandidates -NpmPrefix $npmPrefix)) { + $candidate = Join-Path $npmBin "openclaw.cmd" + if (Test-Path -LiteralPath $candidate -PathType Leaf) { $candidatePath = $candidate; break } + } + $candidatePath + } + if (-not $npmCandidate -or -not (Test-Path -LiteralPath $npmCandidate -PathType Leaf)) { + Fail-Install + return + } + & $npmCandidate --version 2>$null | Out-Null + if ($LASTEXITCODE -ne 0) { + Fail-Install + return + } + Complete-NpmShimBackup -Backup $npmShimBackup + $npmShimBackup = $null + Remove-PreviousGitWrapper + } + } finally { + if ($npmShimBackup) { + Restore-NpmShimBackup -Backup $npmShimBackup + } + } + } + + if (-not (Ensure-OpenClawOnPath)) { + Write-Host "Install completed, but OpenClaw is not on PATH yet." -ForegroundColor Yellow + Write-Host "Open a new terminal, then run: openclaw doctor" -ForegroundColor Cyan + return + } + + Refresh-GatewayServiceIfLoaded + + # Step 3: Run doctor for migrations if upgrading or git install + if ($isUpgrade -or $InstallMethod -eq "git") { + $doctorResults = @(Run-Doctor) + if (-not (Test-BooleanSuccessResult -Results $doctorResults)) { + Fail-Install + return + } + } + + $installedVersion = $null + try { + $installedVersion = (Invoke-OpenClawCommand --version 2>$null).Trim() + } catch { + $installedVersion = $null + } + if (-not $installedVersion) { + try { + $npmList = Invoke-NpmCommand -Arguments @("list", "-g", "--depth", "0", "--json") 2>$null | ConvertFrom-Json + if ($npmList -and $npmList.dependencies -and $npmList.dependencies.openclaw -and $npmList.dependencies.openclaw.version) { + $installedVersion = $npmList.dependencies.openclaw.version + } + } catch { + $installedVersion = $null + } + } + + Write-Host "" + if ($installedVersion) { + Write-Host "OpenClaw installed successfully ($installedVersion)!" -ForegroundColor Green + } else { + Write-Host "OpenClaw installed successfully!" -ForegroundColor Green + } + Write-Host "" + if ($isUpgrade) { + $updateMessages = @( + "Leveled up! New skills unlocked. You're welcome.", + "Fresh code, same lobster. Miss me?", + "Back and better. Did you even notice I was gone?", + "Update complete. I learned some new tricks while I was out.", + "Upgraded! Now with 23% more sass.", + "I've evolved. Try to keep up.", + "New version, who dis? Oh right, still me but shinier.", + "Patched, polished, and ready to pinch. Let's go.", + "The lobster has molted. Harder shell, sharper claws.", + "Update done! Check the changelog or just trust me, it's good.", + "Reborn from the boiling waters of npm. Stronger now.", + "I went away and came back smarter. You should try it sometime.", + "Update complete. The bugs feared me, so they left.", + "New version installed. Old version sends its regards.", + "Firmware fresh. Brain wrinkles: increased.", + "I've seen things you wouldn't believe. Anyway, I'm updated.", + "Back online. The changelog is long but our friendship is longer.", + "Upgraded! Peter fixed stuff. Blame him if it breaks.", + "Molting complete. Please don't look at my soft shell phase.", + "Version bump! Same chaos energy, fewer crashes (probably)." + ) + Write-Host (Get-Random -InputObject $updateMessages) -ForegroundColor Gray + Write-Host "" + } else { + $completionMessages = @( + "Ahh nice, I like it here. Got any snacks? ", + "Home sweet home. Don't worry, I won't rearrange the furniture.", + "I'm in. Let's cause some responsible chaos.", + "Installation complete. Your productivity is about to get weird.", + "Settled in. Time to automate your life whether you're ready or not.", + "Cozy. I've already read your calendar. We need to talk.", + "Finally unpacked. Now point me at your problems.", + "cracks claws Alright, what are we building?", + "The lobster has landed. Your terminal will never be the same.", + "All done! I promise to only judge your code a little bit." + ) + Write-Host (Get-Random -InputObject $completionMessages) -ForegroundColor Gray + Write-Host "" + } + + if ($InstallMethod -eq "git") { + Write-Host "Source checkout: $finalGitDir" -ForegroundColor Cyan + Write-Host "Wrapper: $env:USERPROFILE\\.local\\bin\\openclaw.cmd" -ForegroundColor Cyan + Write-Host "" + } + + if ($isUpgrade) { + Write-Host "Upgrade complete." -ForegroundColor Green + } else { + if ($NoOnboard) { + Write-Host "Skipping onboard (requested). Run " -NoNewline + Write-Host "openclaw onboard" -ForegroundColor Cyan -NoNewline + Write-Host " later." + } else { + Write-Host "Starting setup..." -ForegroundColor Cyan + Write-Host "" + Invoke-InteractiveOpenClawCommand onboard + } + } + + return $true +} + +$null = Main +Complete-Install diff --git a/scripts/install.sh b/scripts/install.sh new file mode 100644 index 0000000000000000000000000000000000000000..aaf459cfc9d398c9ca17fb3d6f193be1498c2037 --- /dev/null +++ b/scripts/install.sh @@ -0,0 +1,4224 @@ +#!/bin/bash + +# Bash 5.3+ can deadlock writing heredoc pipes on macOS before the reader starts. +if [[ ${OSTYPE:-} == darwin* && $BASH != /bin/bash ]] && ((BASH_VERSINFO[0] > 5 || (BASH_VERSINFO[0] == 5 && BASH_VERSINFO[1] >= 3))); then + if (return 0 2>/dev/null); then + printf '%s\n' 'Run this installer with /bin/bash on macOS instead of sourcing it.' >&2 + return 1 + fi + case "${BASH_SOURCE[0]:-}" in + ""|bash|-bash|/dev/stdin) + # Bash reads piped scripts unbuffered; stdin now starts after this guard. + OPENCLAW_INSTALLER_REEXEC_FILE="$(mktemp "${TMPDIR:-/tmp}/openclaw-installer.XXXXXX")" || exit 1 + export OPENCLAW_INSTALLER_REEXEC_FILE + trap 'rm -f -- "$OPENCLAW_INSTALLER_REEXEC_FILE"' EXIT + { printf '#!/bin/bash\n'; cat; } > "$OPENCLAW_INSTALLER_REEXEC_FILE" || exit 1 + exec /bin/bash "$OPENCLAW_INSTALLER_REEXEC_FILE" "$@" + ;; + *) exec /bin/bash "$0" "$@" ;; + esac +fi + +set -euo pipefail + +# BEGIN GENERATED UPDATE NETWORK BUDGET +# Source: src/infra/update-network-budget.ts; regenerate: node scripts/generate-update-network-budget.mjs +UPDATE_NETWORK_TIMEOUT_SECONDS=300 +# END GENERATED UPDATE NETWORK BUDGET + +# The re-executed shell has the script open, so unlink its private copy now. +if [[ -n "${OPENCLAW_INSTALLER_REEXEC_FILE:-}" && "${BASH_SOURCE[0]:-}" == "$OPENCLAW_INSTALLER_REEXEC_FILE" ]]; then + rm -f -- "$OPENCLAW_INSTALLER_REEXEC_FILE" +fi +unset OPENCLAW_INSTALLER_REEXEC_FILE + +# OpenClaw Installer for macOS and Linux +# Usage: curl -fsSL --proto '=https' --tlsv1.2 https://openclaw.ai/install.sh | bash + +BOLD='\033[1m' +ACCENT='\033[38;2;255;77;77m' # coral-bright #ff4d4d +# shellcheck disable=SC2034 +ACCENT_BRIGHT='\033[38;2;255;110;110m' # lighter coral +INFO='\033[38;2;136;146;176m' # text-secondary #8892b0 +SUCCESS='\033[38;2;0;229;204m' # cyan-bright #00e5cc +WARN='\033[38;2;255;176;32m' # amber (no site equiv, keep warm) +ERROR='\033[38;2;230;57;70m' # coral-mid #e63946 +MUTED='\033[38;2;90;100;128m' # text-muted #5a6480 +NC='\033[0m' # No Color + +DEFAULT_TAGLINE="All your chats, one OpenClaw." +NODE_DEFAULT_MAJOR=26 +# Homebrew ships the current Node line as plain "node" (no versioned node@26 +# formula exists); versioned formulas only cover LTS lines like node@24. +NODE_BREW_FORMULA="node" +# Linux package repositories can publish builds ahead of the Node release line. +# Provision the supported LTS line there so a fresh install never receives a prerelease runtime. +NODE_LINUX_DEFAULT_MAJOR=24 +NODE_24_MIN_MINOR=16 +NODE_24_MIN_PATCH=0 +NODE_26_MIN_MINOR=1 +NODE_26_MIN_PATCH=0 +NODE_SUPPORTED_VERSION_LABEL="24.16.0+ or 26.1.0+" + +ORIGINAL_PATH="${PATH:-}" + +TMPFILES=() +OPENCLAW_BIN_BACKUP_TARGET="" +OPENCLAW_BIN_BACKUP_PATH="" +OPENCLAW_BIN_BACKUP_CANDIDATE="" +OPENCLAW_BIN_BACKUP_DISCARD=0 +cleanup_tmpfiles() { + if [[ "$(type -t restore_openclaw_bin_backup 2>/dev/null || true)" == "function" ]]; then + restore_openclaw_bin_backup || true + fi + local f + for f in "${TMPFILES[@]:-}"; do + rm -rf "$f" 2>/dev/null || true + done +} +trap cleanup_tmpfiles EXIT + +abort_install_int() { + cleanup_tmpfiles + echo "" + ui_warn "Installation interrupted" + exit 130 +} +abort_install_term() { + cleanup_tmpfiles + echo "" + ui_warn "Installation terminated" + exit 143 +} +trap abort_install_int INT +trap abort_install_term TERM + +mktempfile() { + local output_var="${1:?output variable required}" f + f="$(mktemp)" + # Assign into caller scope; command substitution would lose this cleanup + # registration in its subshell. + TMPFILES+=("$f") + printf -v "$output_var" '%s' "$f" +} + +resolve_openclaw_effective_home() { + local openclaw_home="${OPENCLAW_HOME:-}" + if [[ -z "$openclaw_home" ]]; then + echo "$HOME" + return + fi + if [[ "$openclaw_home" == "~" ]]; then + echo "$HOME" + return + fi + if [[ "$openclaw_home" == \~/* ]]; then + echo "${HOME}${openclaw_home:1}" + return + fi + echo "$openclaw_home" +} + +resolve_openclaw_user_path() { + local input="$1" + local effective_home + effective_home="$(resolve_openclaw_effective_home)" + if [[ "$input" == "~" ]]; then + echo "$effective_home" + elif [[ "$input" == \~/* ]]; then + echo "${effective_home}${input:1}" + elif [[ "$input" == /* ]]; then + echo "$input" + else + echo "$PWD/$input" + fi +} + +DOWNLOADER="" +detect_downloader() { + if command -v curl &> /dev/null; then + DOWNLOADER="curl" + return 0 + fi + if command -v wget &> /dev/null; then + DOWNLOADER="wget" + return 0 + fi + ui_error "Missing downloader (curl or wget required)" + exit 1 +} + +download_file() { + local url="$1" + local output="$2" + local redirect_mode="${3:-follow}" + if [[ -z "$DOWNLOADER" ]]; then + detect_downloader + fi + if [[ "$DOWNLOADER" == "curl" ]]; then + if [[ "$redirect_mode" == "deny" ]]; then + curl -fsSL --max-redirs 0 --proto '=https' --tlsv1.2 \ + --speed-limit 1 --speed-time "$UPDATE_NETWORK_TIMEOUT_SECONDS" \ + --retry 3 --retry-delay 1 --retry-connrefused \ + -o "$output" "$url" + return + fi + # Bound post-connect stalls without imposing a total download duration. + curl -fsSL --proto '=https' --tlsv1.2 \ + --speed-limit 1 --speed-time "$UPDATE_NETWORK_TIMEOUT_SECONDS" \ + --retry 3 --retry-delay 1 --retry-connrefused \ + -o "$output" "$url" + return + fi + if [[ "$redirect_mode" == "deny" ]]; then + wget -q --max-redirect=0 --https-only --secure-protocol=TLSv1_2 --tries=3 --timeout="$UPDATE_NETWORK_TIMEOUT_SECONDS" -O "$output" "$url" + return + fi + wget -q --https-only --secure-protocol=TLSv1_2 --tries=3 --timeout="$UPDATE_NETWORK_TIMEOUT_SECONDS" -O "$output" "$url" +} + +# Managed setup endpoints must return a non-empty script with a raw shebang. +# This is a response-shape check, not an authenticity or completeness check. +validate_downloaded_script() { + local file="$1" url="$2" + if [[ ! -s "$file" ]]; then + ui_error "Downloaded script is empty: ${url}" + return 1 + fi + # Check the first two raw bytes are '#!' (0x23 0x21) BEFORE command + # substitution, which strips NUL/control bytes and could false-accept + # a file whose raw content does not actually start with a shebang. + local raw_magic + raw_magic="$(od -An -tx1 -N2 "$file" | tr -d ' ')" + if [[ "$raw_magic" != "2321" ]]; then + ui_error "Downloaded file does not look like a shell script (no shebang): ${url}" + return 1 + fi +} + +download_validated_script() { + local url="$1" output="$2" + # These fixed executable-script endpoints must not redirect: Wget's + # --https-only only filters recursive traversal, not ordinary redirects. + download_file "$url" "$output" deny || return 1 + validate_downloaded_script "$output" "$url" +} + +run_remote_bash() { + local url="$1" + local tmp + mktempfile tmp + download_validated_script "$url" "$tmp" || return 1 + /bin/bash "$tmp" +} + +GUM_VERSION="${OPENCLAW_GUM_VERSION:-2.0.0}" +GUM="" +GUM_STATUS="skipped" +GUM_REASON="" +LAST_NPM_INSTALL_CMD="" + +is_non_interactive_shell() { + if [[ "${NO_PROMPT:-0}" == "1" ]]; then + return 0 + fi + if [[ ! -t 0 || ! -t 1 ]]; then + return 0 + fi + return 1 +} + +# Returns true when stdin should be isolated from the script stream. +# Checks stdin directly (not stdout) and respects NO_PROMPT so that +# stdout redirection (e.g. install.sh > log.txt) does not suppress +# interactive prompts. +needs_stdin_isolation() { + [[ ! -t 0 ]] || [[ "${NO_PROMPT:-0}" == "1" ]] +} + +has_controlling_tty() { + if [[ ! -r /dev/tty || ! -w /dev/tty ]]; then + return 1 + fi + if ! { : </dev/tty; } 2>/dev/null; then + return 1 + fi + return 0 +} + +has_visible_prompt_output() { + [[ -t 1 ]] +} + +resolve_subprocess_stdin_path() { + local prompt_output_visible="${1:-0}" + if [[ "${NO_PROMPT:-0}" == "1" ]]; then + echo "/dev/null" + return 0 + fi + if ! needs_stdin_isolation; then + return 1 + fi + if has_controlling_tty && [[ "$prompt_output_visible" == "1" ]]; then + echo "/dev/tty" + else + echo "/dev/null" + fi +} + +run_with_safe_stdin() { + local stdin_path="" + local prompt_output_visible=0 + if has_visible_prompt_output; then + prompt_output_visible=1 + fi + if stdin_path="$(resolve_subprocess_stdin_path "$prompt_output_visible")"; then + "$@" < "$stdin_path" + else + "$@" + fi +} + +gum_is_tty() { + if [[ -n "${NO_COLOR:-}" ]]; then + return 1 + fi + if [[ "${TERM:-dumb}" == "dumb" ]]; then + return 1 + fi + if [[ -t 2 || -t 1 ]]; then + return 0 + fi + if has_controlling_tty; then + return 0 + fi + return 1 +} + +gum_detect_os() { + case "$(uname -s 2>/dev/null || true)" in + Darwin) echo "Darwin" ;; + Linux) echo "Linux" ;; + *) echo "unsupported" ;; + esac +} + +gum_detect_arch() { + case "$(uname -m 2>/dev/null || true)" in + x86_64|amd64) echo "x86_64" ;; + arm64|aarch64) echo "arm64" ;; + i386|i686) echo "i386" ;; + armv7l|armv7) echo "armv7" ;; + armv6l|armv6) echo "armv6" ;; + *) echo "unknown" ;; + esac +} + +verify_sha256sum_file() { + local checksums="$1" + if command -v sha256sum >/dev/null 2>&1; then + sha256sum --ignore-missing -c "$checksums" >/dev/null 2>&1 + return $? + fi + if command -v shasum >/dev/null 2>&1; then + shasum -a 256 --ignore-missing -c "$checksums" >/dev/null 2>&1 + return $? + fi + return 1 +} + +bootstrap_gum_temp() { + GUM="" + GUM_STATUS="skipped" + GUM_REASON="" + + if is_non_interactive_shell; then + GUM_REASON="non-interactive shell (auto-disabled)" + return 1 + fi + + if ! gum_is_tty; then + GUM_REASON="terminal does not support gum UI" + return 1 + fi + + if command -v gum >/dev/null 2>&1; then + GUM="gum" + GUM_STATUS="found" + GUM_REASON="already installed" + return 0 + fi + + if ! command -v tar >/dev/null 2>&1; then + GUM_REASON="tar not found" + return 1 + fi + + local os arch asset base gum_tmpdir gum_path + os="$(gum_detect_os)" + arch="$(gum_detect_arch)" + if [[ "$os" == "unsupported" || "$arch" == "unknown" ]]; then + GUM_REASON="unsupported os/arch ($os/$arch)" + return 1 + fi + + asset="gum_${GUM_VERSION}_${os}_${arch}.tar.gz" + base="https://github.com/charmbracelet/gum/releases/download/v${GUM_VERSION}" + + gum_tmpdir="$(mktemp -d)" + TMPFILES+=("$gum_tmpdir") + + ui_info "Preparing spinner support" + if ! download_file "${base}/${asset}" "$gum_tmpdir/$asset"; then + GUM_REASON="download failed" + return 1 + fi + + ui_info "Verifying spinner support download" + if ! download_file "${base}/checksums.txt" "$gum_tmpdir/checksums.txt"; then + GUM_REASON="checksum unavailable or failed" + return 1 + fi + + if ! (cd "$gum_tmpdir" && verify_sha256sum_file "checksums.txt"); then + GUM_REASON="checksum unavailable or failed" + return 1 + fi + + if ! tar -xzf "$gum_tmpdir/$asset" -C "$gum_tmpdir" >/dev/null 2>&1; then + GUM_REASON="extract failed" + return 1 + fi + + gum_path="$(find "$gum_tmpdir" -type f -name gum 2>/dev/null | head -n1 || true)" + if [[ -z "$gum_path" ]]; then + GUM_REASON="gum binary missing after extract" + return 1 + fi + + chmod +x "$gum_path" >/dev/null 2>&1 || true + if [[ ! -x "$gum_path" ]]; then + GUM_REASON="gum binary is not executable" + return 1 + fi + + GUM="$gum_path" + GUM_STATUS="installed" + GUM_REASON="temp, verified" + return 0 +} + +print_gum_status() { + case "$GUM_STATUS" in + found) + ui_success "gum available (${GUM_REASON})" + ;; + installed) + ui_success "gum bootstrapped (${GUM_REASON}, v${GUM_VERSION})" + ;; + *) + if [[ -n "$GUM_REASON" && "$GUM_REASON" != "non-interactive shell (auto-disabled)" ]]; then + ui_info "gum skipped (${GUM_REASON})" + fi + ;; + esac +} + +print_installer_banner() { + if [[ -n "$GUM" ]]; then + local title tagline hint card + title="$("$GUM" style --foreground "#ff4d4d" --bold "🦞 OpenClaw Installer")" + tagline="$("$GUM" style --foreground "#8892b0" "$TAGLINE")" + hint="$("$GUM" style --foreground "#5a6480" "modern installer mode")" + card="$(printf '%s\n%s\n%s' "$title" "$tagline" "$hint")" + "$GUM" style --border rounded --border-foreground "#ff4d4d" --padding "1 2" "$card" + echo "" + return + fi + + echo -e "${ACCENT}${BOLD}" + echo " 🦞 OpenClaw Installer" + echo -e "${NC}${INFO} ${TAGLINE}${NC}" + echo "" +} + +detect_os_or_die() { + OS="unknown" + if [[ "$OSTYPE" == "darwin"* ]]; then + OS="macos" + elif [[ "$OSTYPE" == "linux"* ]] || [[ -n "${WSL_DISTRO_NAME:-}" ]]; then + OS="linux" + fi + + if [[ "$OS" == "unknown" ]]; then + ui_error "Unsupported operating system" + echo "This installer supports macOS and Linux (including WSL)." + echo "For Windows, use: iwr -useb https://openclaw.ai/install.ps1 | iex" + exit 1 + fi + + ui_success "Detected: $OS" +} + +ui_info() { + local msg="$*" + if [[ -n "$GUM" ]]; then + "$GUM" log --level info "$msg" + else + echo -e "${MUTED}·${NC} ${msg}" + fi +} + +ui_warn() { + local msg="$*" + if [[ -n "$GUM" ]]; then + "$GUM" log --level warn "$msg" + else + echo -e "${WARN}!${NC} ${msg}" + fi +} + +ui_success() { + local msg="$*" + if [[ -n "$GUM" ]]; then + local mark + mark="$("$GUM" style --foreground "#00e5cc" --bold "✓")" + echo "${mark} ${msg}" + else + echo -e "${SUCCESS}✓${NC} ${msg}" + fi +} + +ui_error() { + local msg="$*" + if [[ -n "$GUM" ]]; then + "$GUM" log --level error "$msg" + else + echo -e "${ERROR}✗${NC} ${msg}" + fi +} + +INSTALL_STAGE_TOTAL=3 +INSTALL_STAGE_CURRENT=0 + +configure_install_stage_total() { + INSTALL_STAGE_TOTAL=3 + INSTALL_STAGE_CURRENT=0 + if [[ "${VERIFY_INSTALL:-0}" == "1" ]]; then + INSTALL_STAGE_TOTAL=4 + fi +} + +ui_section() { + local title="$1" + if [[ -n "$GUM" ]]; then + "$GUM" style --bold --foreground "#ff4d4d" --padding "1 0" "$title" + else + echo "" + echo -e "${ACCENT}${BOLD}${title}${NC}" + fi +} + +ui_stage() { + local title="$1" + INSTALL_STAGE_CURRENT=$((INSTALL_STAGE_CURRENT + 1)) + ui_section "[${INSTALL_STAGE_CURRENT}/${INSTALL_STAGE_TOTAL}] ${title}" +} + +ui_kv() { + local key="$1" + local value="$2" + if [[ -n "$GUM" ]]; then + local key_part value_part + key_part="$("$GUM" style --foreground "#5a6480" --width 20 "$key")" + value_part="$("$GUM" style --bold "$value")" + "$GUM" join --horizontal "$key_part" "$value_part" + else + echo -e "${MUTED}${key}:${NC} ${value}" + fi +} + +ui_panel() { + local content="$1" + if [[ -n "$GUM" ]]; then + "$GUM" style --border rounded --border-foreground "#5a6480" --padding "0 1" "$content" + else + echo "$content" + fi +} + +show_install_plan() { + local detected_checkout="$1" + + ui_section "Install plan" + ui_kv "OS" "$OS" + ui_kv "Install method" "$INSTALL_METHOD" + ui_kv "Requested version" "$OPENCLAW_VERSION" + if [[ "$USE_BETA" == "1" ]]; then + ui_kv "Beta channel" "enabled" + fi + if [[ "$INSTALL_METHOD" == "git" ]]; then + ui_kv "Git directory" "$GIT_DIR" + ui_kv "Git update" "$GIT_UPDATE" + fi + if [[ -n "$detected_checkout" ]]; then + ui_kv "Detected checkout" "$detected_checkout" + fi + if [[ "$DRY_RUN" == "1" ]]; then + ui_kv "Dry run" "yes" + fi + if [[ "$NO_ONBOARD" == "1" ]]; then + ui_kv "Onboarding" "skipped" + fi +} + +show_footer_links() { + local faq_url="https://docs.openclaw.ai/start/faq" + if [[ -n "$GUM" ]]; then + local content + content="$(printf '%s\n%s' "Need help?" "FAQ: ${faq_url}")" + ui_panel "$content" + else + echo "" + echo -e "FAQ: ${INFO}${faq_url}${NC}" + fi +} + +ui_celebrate() { + local msg="$1" + if [[ -n "$GUM" ]]; then + "$GUM" style --bold --foreground "#00e5cc" "$msg" + else + echo -e "${SUCCESS}${BOLD}${msg}${NC}" + fi +} + +is_shell_function() { + local name="${1:-}" + [[ -n "$name" ]] && declare -F "$name" >/dev/null 2>&1 +} + +is_gum_raw_mode_failure() { + local err_log="$1" + [[ -s "$err_log" ]] || return 1 + grep -Eiq 'setrawmode|inappropriate ioctl' "$err_log" +} + +run_with_spinner() { + local title="$1" + shift + + if [[ -n "$GUM" ]] && gum_is_tty && ! is_shell_function "${1:-}"; then + local gum_err gum_out + mktempfile gum_err + mktempfile gum_out + local gum_status=0 + if needs_stdin_isolation; then + "$GUM" spin --spinner dot --title "$title" -- "$@" < /dev/null >"$gum_out" 2>"$gum_err" || gum_status=$? + else + "$GUM" spin --spinner dot --title "$title" -- "$@" >"$gum_out" 2>"$gum_err" || gum_status=$? + fi + if [[ "$gum_status" -eq 0 ]]; then + if is_gum_raw_mode_failure "$gum_out" || is_gum_raw_mode_failure "$gum_err"; then + GUM="" + GUM_STATUS="skipped" + GUM_REASON="gum raw mode unavailable" + ui_warn "Spinner unavailable in this terminal; continuing without spinner" + if needs_stdin_isolation; then + "$@" < /dev/null + else + "$@" + fi + return $? + fi + if [[ -s "$gum_out" ]]; then + cat "$gum_out" + fi + return 0 + fi + if is_gum_raw_mode_failure "$gum_err" || is_gum_raw_mode_failure "$gum_out"; then + GUM="" + GUM_STATUS="skipped" + GUM_REASON="gum raw mode unavailable" + ui_warn "Spinner unavailable in this terminal; continuing without spinner" + if needs_stdin_isolation; then + "$@" < /dev/null + else + "$@" + fi + return $? + fi + if [[ -s "$gum_err" ]]; then + cat "$gum_err" >&2 + fi + return "$gum_status" + fi + + if needs_stdin_isolation; then + "$@" < /dev/null + else + "$@" + fi +} + +run_quiet_step() { + local title="$1" + shift + + if [[ "$VERBOSE" == "1" ]]; then + run_with_spinner "$title" "$@" + return $? + fi + + local log + mktempfile log + local showed_progress=false + + local cmd_exit=0 + + if [[ -n "$GUM" ]] && gum_is_tty && ! is_shell_function "${1:-}"; then + local cmd_quoted="" + local log_quoted="" + printf -v cmd_quoted '%q ' "$@" + printf -v log_quoted '%q' "$log" + run_with_spinner "$title" bash -c "${cmd_quoted}>${log_quoted} 2>&1" || cmd_exit=$? + if (( cmd_exit == 0 )); then + return 0 + fi + showed_progress=true + else + # Keep users informed even when gum spinner cannot run (for example shell functions). + ui_info "${title}" + showed_progress=true + if needs_stdin_isolation; then + "$@" < /dev/null >"$log" 2>&1 || cmd_exit=$? + else + "$@" >"$log" 2>&1 || cmd_exit=$? + fi + if (( cmd_exit == 0 )); then + return 0 + fi + fi + + if [[ "$showed_progress" == "false" ]]; then + ui_info "${title}" + fi + + ui_error "${title} failed — re-run with --verbose for details" + if [[ -s "$log" ]]; then + tail -n 80 "$log" >&2 || true + fi + # Preserve signal exit codes (130=SIGINT, 143=SIGTERM) so callers + # like run_doctor can distinguish user cancellation from normal errors. + # Return 1 for all other failures to keep existing caller semantics. + if (( cmd_exit > 128 )); then + return "$cmd_exit" + fi + return 1 +} + +run_required_step() { + local title="$1" + shift + if run_quiet_step "$title" "$@"; then + return 0 + fi + exit 1 +} + +cleanup_legacy_submodules() { + local repo_dir="$1" + local legacy_dir="$repo_dir/Peekaboo" + if [[ -d "$legacy_dir" ]]; then + ui_info "Removing legacy submodule checkout: ${legacy_dir}" + rm -rf "$legacy_dir" + fi +} + +begin_openclaw_bin_backup() { + local target="$1" candidate="$2" discard="${3:-0}" backup="" + [[ -z "$OPENCLAW_BIN_BACKUP_PATH" ]] || return 0 + [[ -e "$target" || -L "$target" ]] || return 0 + backup="$(mktemp "${target}.openclaw-backup.XXXXXX")" || return 1 + rm -f "$backup" || return 1 + OPENCLAW_BIN_BACKUP_TARGET="$target" + OPENCLAW_BIN_BACKUP_PATH="$backup" + OPENCLAW_BIN_BACKUP_CANDIDATE="$candidate" + OPENCLAW_BIN_BACKUP_DISCARD="$discard" + if ! mv "$target" "$backup"; then + OPENCLAW_BIN_BACKUP_TARGET="" + OPENCLAW_BIN_BACKUP_PATH="" + OPENCLAW_BIN_BACKUP_CANDIDATE="" + OPENCLAW_BIN_BACKUP_DISCARD=0 + return 1 + fi +} + +is_npm_openclaw_shim() { + local target="$1" launcher="$2" + if [[ -L "$target" ]]; then + local link_target="" + link_target="$(readlink "$target" 2>/dev/null || true)" + [[ "$link_target" == "$launcher" || "$link_target" == *"/node_modules/openclaw/openclaw.mjs" ]] + return + fi + [[ -f "$target" ]] && grep -Fq "/node_modules/openclaw/openclaw.mjs" "$target" +} + +restore_openclaw_bin_backup() { + local target="$OPENCLAW_BIN_BACKUP_TARGET" backup="$OPENCLAW_BIN_BACKUP_PATH" + [[ -n "$backup" && ( -e "$backup" || -L "$backup" ) ]] || return 0 + if [[ -e "$target" || -L "$target" ]]; then + is_npm_openclaw_shim "$target" "$OPENCLAW_BIN_BACKUP_CANDIDATE" || return 1 + rm -f "$target" || return 1 + fi + mv "$backup" "$target" || return 1 + OPENCLAW_BIN_BACKUP_TARGET="" + OPENCLAW_BIN_BACKUP_PATH="" + OPENCLAW_BIN_BACKUP_CANDIDATE="" + OPENCLAW_BIN_BACKUP_DISCARD=0 +} + +commit_openclaw_bin_backup() { + local backup="$OPENCLAW_BIN_BACKUP_PATH" + [[ -n "$backup" ]] || return 0 + if [[ "$OPENCLAW_BIN_BACKUP_DISCARD" == "1" ]]; then + rm -f "$backup" || return 1 + else + ui_info "Preserved previous openclaw command at ${backup}" + fi + OPENCLAW_BIN_BACKUP_TARGET="" + OPENCLAW_BIN_BACKUP_PATH="" + OPENCLAW_BIN_BACKUP_CANDIDATE="" + OPENCLAW_BIN_BACKUP_DISCARD=0 +} + +extract_openclaw_conflict_path() { + local log="$1" + local path="" + path="$(sed -n 's/.*File exists: //p' "$log" | head -n1)" + if [[ -z "$path" ]]; then + path="$(sed -n 's/.*EEXIST: file already exists, //p' "$log" | head -n1)" + fi + if [[ -n "$path" ]]; then + echo "$path" + return 0 + fi + return 1 +} + +cleanup_openclaw_bin_conflict() { + local bin_path="$1" + if [[ -z "$bin_path" || ( ! -e "$bin_path" && ! -L "$bin_path" ) ]]; then + return 1 + fi + local npm_bin="" + npm_bin="$(npm_global_bin_dir 2>/dev/null || true)" + if [[ -n "$npm_bin" && "$bin_path" != "$npm_bin/openclaw" ]]; then + case "$bin_path" in + "/opt/homebrew/bin/openclaw"|"/usr/local/bin/openclaw") + ;; + *) + return 1 + ;; + esac + fi + local npm_root="" + npm_root="$(npm root -g 2>/dev/null || true)" + [[ -n "$npm_root" ]] || return 1 + begin_openclaw_bin_backup "$bin_path" "${npm_root%/}/openclaw/openclaw.mjs" 0 || return 1 + ui_info "Moved existing openclaw command aside for npm retry" +} + +cleanup_npm_stale_rename_dirs() { + local npm_root="" stale="" found=0 + npm_root="$(npm root -g 2>/dev/null || true)" + [[ -n "$npm_root" && "$npm_root" == *node_modules* ]] || return 1 + for stale in "$npm_root"/.openclaw-*; do + [[ -d "$stale" && ! -L "$stale" ]] || continue + found=1 + rm -rf "$stale" || return 1 + done + (( found == 0 )) || ui_info "Removed interrupted npm rename directories" +} + +npm_log_indicates_missing_build_tools() { + local log="$1" + if [[ -z "$log" || ! -f "$log" ]]; then + return 1 + fi + + grep -Eiq "(not found: make|make: command not found|cmake: command not found|CMAKE_MAKE_PROGRAM is not set|Could not find CMAKE|gyp ERR! find Python|no developer tools were found|is not able to compile a simple test program|Failed to build llama\\.cpp|It seems that \"make\" is not installed in your system|It seems that the used \"cmake\" doesn't work properly)" "$log" +} + +# Detect Arch-based distributions (Arch Linux, Manjaro, EndeavourOS, etc.) +is_arch_linux() { + if [[ -f /etc/os-release ]]; then + local os_id + os_id="$(grep -E '^ID=' /etc/os-release 2>/dev/null | cut -d'=' -f2 | tr -d '"' || true)" + case "$os_id" in + arch|manjaro|endeavouros|arcolinux|garuda|archarm|cachyos|archcraft) + return 0 + ;; + esac + # Also check ID_LIKE for Arch derivatives + local os_id_like + os_id_like="$(grep -E '^ID_LIKE=' /etc/os-release 2>/dev/null | cut -d'=' -f2 | tr -d '"' || true)" + if [[ "$os_id_like" == *arch* ]]; then + return 0 + fi + fi + return 1 +} + +is_alpine_linux() { + if [[ -f /etc/alpine-release ]]; then + return 0 + fi + if [[ -f /etc/os-release ]]; then + local os_id os_id_like + os_id="$(grep -E '^ID=' /etc/os-release 2>/dev/null | cut -d'=' -f2 | tr -d '"' || true)" + os_id_like="$(grep -E '^ID_LIKE=' /etc/os-release 2>/dev/null | cut -d'=' -f2 | tr -d '"' || true)" + if [[ "$os_id" == "alpine" || "$os_id_like" == *alpine* ]]; then + return 0 + fi + fi + return 1 +} + +apt_get() { + if is_root; then + env DEBIAN_FRONTEND="${DEBIAN_FRONTEND:-noninteractive}" NEEDRESTART_MODE="${NEEDRESTART_MODE:-a}" apt-get "$@" + else + sudo env DEBIAN_FRONTEND="${DEBIAN_FRONTEND:-noninteractive}" NEEDRESTART_MODE="${NEEDRESTART_MODE:-a}" apt-get "$@" + fi +} + +apt_get_update() { + apt_get update -qq +} + +apt_get_install() { + apt_get install -y -qq \ + -o Dpkg::Options::=--force-confdef \ + -o Dpkg::Options::=--force-confold \ + "$@" +} + +install_build_tools_linux() { + require_sudo + + # apt_get already escalates privileges itself, so it stays separate from the + # sudo-prefixed command list below. + if command -v apt-get &> /dev/null; then + run_quiet_step "Updating package index" apt_get_update + run_quiet_step "Installing build tools" apt_get_install build-essential python3 make g++ cmake + return + fi + + local -a build_tools_cmd=() + if command -v pacman &> /dev/null && is_arch_linux; then + build_tools_cmd=(pacman -Sy --noconfirm base-devel python make cmake gcc) + elif command -v dnf &> /dev/null; then + build_tools_cmd=(dnf install -y -q gcc gcc-c++ make cmake python3) + elif command -v yum &> /dev/null; then + build_tools_cmd=(yum install -y -q gcc gcc-c++ make cmake python3) + elif command -v apk &> /dev/null && is_alpine_linux; then + build_tools_cmd=(apk add --no-cache build-base python3 cmake) + else + ui_warn "Could not detect package manager for auto-installing build tools" + return 1 + fi + + is_root || build_tools_cmd=(sudo "${build_tools_cmd[@]}") + # Return the package manager's status: callers choose between "Build tools + # installed" and the continue-without-build-tools warning based on it, so + # swallowing a failure here makes the installer claim success after an error. + run_quiet_step "Installing build tools" "${build_tools_cmd[@]}" +} + +install_build_tools_macos() { + local ok=true + + if ! xcode-select -p >/dev/null 2>&1; then + ui_info "Installing Xcode Command Line Tools (required for make/clang)" + xcode-select --install >/dev/null 2>&1 || true + if ! xcode-select -p >/dev/null 2>&1; then + ui_warn "Xcode Command Line Tools are not ready yet" + ui_info "Complete the installer dialog, then re-run this installer" + ok=false + fi + fi + + if ! command -v cmake >/dev/null 2>&1; then + if command -v brew >/dev/null 2>&1; then + run_quiet_step "Installing cmake" brew install cmake + else + ui_warn "Homebrew not available; cannot auto-install cmake" + ok=false + fi + fi + + if ! command -v make >/dev/null 2>&1; then + ui_warn "make is still unavailable" + ok=false + fi + if ! command -v cmake >/dev/null 2>&1; then + ui_warn "cmake is still unavailable" + ok=false + fi + + [[ "$ok" == "true" ]] +} + +auto_install_build_tools_for_npm_failure() { + local log="$1" + if ! npm_log_indicates_missing_build_tools "$log"; then + return 1 + fi + + ui_warn "Detected missing native build tools; attempting automatic setup" + if [[ "$OS" == "linux" ]]; then + install_build_tools_linux || return 1 + elif [[ "$OS" == "macos" ]]; then + install_build_tools_macos || return 1 + else + return 1 + fi + ui_success "Build tools setup complete" + return 0 +} + +resolve_npm_config_path() { + local raw="$1" + if [[ -z "$raw" || "$raw" == "null" || "$raw" == "undefined" ]]; then + return 1 + fi + if [[ "$raw" == \~/* && -n "${HOME:-}" ]]; then + printf '%s\n' "${HOME}/${raw#"~/"}" + return 0 + fi + if [[ "$raw" == "\${HOME}/"* && -n "${HOME:-}" ]]; then + printf '%s\n' "${HOME}/${raw#"\${HOME}/"}" + return 0 + fi + printf '%s\n' "$raw" +} + +npm_config_file_has_key() { + local file="$1" + local key="$2" + [[ -f "$file" ]] || return 1 + grep -Eiq "^[[:space:]]*${key}[[:space:]]*=" "$file" +} + +npm_command_path() { + local npm_cmd="$1" + local npm_path="$npm_cmd" + if [[ "$npm_path" != */* ]]; then + npm_path="$(command -v "$npm_cmd" 2>/dev/null)" || return 1 + fi + if command -v node >/dev/null 2>&1; then + node -e 'const fs = require("node:fs"); console.log(fs.realpathSync(process.argv[1]));' "$npm_path" 2>/dev/null && return 0 + fi + printf '%s\n' "$npm_path" +} + +npm_builtin_config_path() { + local npm_cmd="$1" + local npm_path + npm_path="$(npm_command_path "$npm_cmd")" || return 1 + local npm_root + npm_root="$(cd "$(dirname "$npm_path")/.." >/dev/null 2>&1 && pwd -P)" || return 1 + printf '%s\n' "${npm_root}/npmrc" +} + +npm_config_has_raw_key() { + local npm_cmd="$1" + local key="$2" + local project_dir="${3:-}" + local raw="" + local file="" + local -a files=() + + if [[ -n "$project_dir" ]]; then + files+=("${project_dir}/.npmrc") + fi + + raw="${NPM_CONFIG_USERCONFIG:-${npm_config_userconfig:-}}" + if [[ -n "$raw" ]]; then + file="$(resolve_npm_config_path "$raw" 2>/dev/null || true)" + [[ -n "$file" ]] && files+=("$file") + elif [[ -n "${HOME:-}" ]]; then + files+=("${HOME}/.npmrc") + fi + + raw="${NPM_CONFIG_GLOBALCONFIG:-${npm_config_globalconfig:-}}" + if [[ -n "$raw" ]]; then + file="$(resolve_npm_config_path "$raw" 2>/dev/null || true)" + [[ -n "$file" ]] && files+=("$file") + fi + + raw="$(env -u NPM_CONFIG_BEFORE -u npm_config_before -u NPM_CONFIG_MIN_RELEASE_AGE -u npm_config_min_release_age -u npm_config_min-release-age "$npm_cmd" config get globalconfig --global 2>/dev/null || true)" + file="$(resolve_npm_config_path "$raw" 2>/dev/null || true)" + [[ -n "$file" ]] && files+=("$file") + + file="$(npm_builtin_config_path "$npm_cmd" 2>/dev/null || true)" + [[ -n "$file" ]] && files+=("$file") + + for file in "${files[@]}"; do + if npm_config_file_has_key "$file" "$key"; then + return 0 + fi + done + return 1 +} + +npm_lifecycle_allow_arg() { + local npm_cmd="$1" spec="$2" npm_cwd="${3:-$PWD}" exact_identity="${4:-}" version="" output="" + if ! version="$("$npm_cmd" --version 2>/dev/null)"; then + echo "Unable to determine npm version from ${npm_cmd}; no package changes were made." >&2 + return 1 + fi + output="$(node - "$version" "$spec" "$npm_cwd" "$exact_identity" <<'NODE' +const path = require("node:path"); +const [versionOutput, spec, cwd, exactIdentity] = process.argv.slice(2); +const version = versionOutput.trim().split(/\r?\n/).at(-1) ?? ""; +const parsed = version.match(/^[vV]?(\d+)\.(\d+)\.(\d+)(?:[-+][0-9A-Za-z.-]+)?$/); +const fail = (message) => { process.stderr.write(`${message}\n`); process.exit(1); }; +if (!parsed) fail("Unable to determine npm version; no package changes were made."); +if (+parsed[1] < 12 && (+parsed[1] !== 11 || +parsed[2] < 16)) process.exit(0); +const normalized = spec.trim(); +const unaliased = normalized.toLowerCase().startsWith("openclaw@") ? normalized.slice(9).trim() : normalized; +const explicit = (value) => /\.(?:tgz|tar\.gz)$/i.test(value) || value.includes("://") || value.includes("#") || /^(?:file|github|git\+(?:ssh|https|http|file)|npm):/i.test(value); +let identity = !normalized || explicit(normalized) || explicit(unaliased) || /^\.{1,2}(?:[\\/]|$)/.test(unaliased) || path.isAbsolute(normalized) || path.isAbsolute(unaliased) ? unaliased : "openclaw"; +const alias = /^npm:/i.test(identity); +if (alias) identity = /^npm:(@[^/]+\/[^@]+|[^@]+?)(?:@.*)?$/i.exec(identity)?.[1] ?? ""; +const filePrefix = /^file:/i.test(identity) ? "file:" : ""; +const archivePath = identity.slice(filePrefix.length); +const gitShorthand = !/^~[\\/]/.test(identity) && /^[^./@\s:#][^/\s:@#]*\/[^/\s:@#]+(?:#[\s\S]*)?$/.test(identity); +const localArchive = !alias && !gitShorthand && /\.(?:tgz|tar\.gz|tar)$/i.test(archivePath) && (filePrefix || path.isAbsolute(archivePath) || !/^[a-z][a-z0-9+.-]*:/i.test(archivePath)); +let absoluteArchive = ""; +if (localArchive) { + const npmPath = process.platform === "win32" ? archivePath.replaceAll("\\", "/") : archivePath; + // Escape raw paths before URL normalization so literal %, #, and ? retain their identity. + let fileUrl = `file:${encodeURI(npmPath).replace(/[?#]/g, encodeURIComponent)}`; + fileUrl = fileUrl.replace(/^file:\/\/(?=[^/])/, "file:/").replace(/^file:\/{1,3}(?=\.\.?(?:\/|$))/, "file:"); + const specPath = decodeURIComponent(new URL(fileUrl).pathname); + let resolvedPath = decodeURIComponent(new URL(fileUrl, `${require("node:url").pathToFileURL(path.resolve(cwd || process.cwd())).href}/`).pathname); + if (process.platform === "win32") resolvedPath = resolvedPath.replace(/^\/+([a-z]:\/)/i, "$1"); + absoluteArchive = /^\/~(?:\/|$)/.test(specPath) ? path.resolve(require("node:os").homedir(), specPath.slice(3)) : path.resolve(cwd || process.cwd(), resolvedPath); +} +// Tarballs match the absolute npm resolved identity; directory links accept relative paths. +// Keep the npm 11 comma-path identity: its advisory/strict decision stays npm-owned. +if (absoluteArchive && (+parsed[1] >= 12 || !absoluteArchive.includes(","))) identity = `${filePrefix}${absoluteArchive}`; +else { + const relative = cwd && path.isAbsolute(identity) ? path.relative(cwd, identity) || "." : ""; + if (relative) identity = path.isAbsolute(relative) || relative === "." || relative === ".." || relative.startsWith(`..${path.sep}`) ? relative : `.${path.sep}${relative}`; +} +if (exactIdentity) identity = exactIdentity; +if (!identity || identity.includes(",")) fail(`npm cannot allow lifecycle scripts for install target '${spec}'; use a package URL or local path without commas.`); +process.stdout.write(`--allow-scripts=${identity}\n`); +NODE +)" || return 1 + printf '%s' "$output" +} + +verify_npm_lifecycle_completed() { + local npm_cmd="$1" npm_root="" + npm_root="$("$npm_cmd" root -g 2>/dev/null | awk 'NF { value = $0 } END { print value }')" || true + [[ -n "$npm_root" ]] || { echo "Unable to resolve npm global root after install." >&2; return 1; } + [[ ! -e "${npm_root%/}/openclaw/.openclaw-lifecycle-pending" && ! -e "${npm_root%/}/openclaw/dist/openclaw-install-guard" ]] || { + echo "OpenClaw lifecycle scripts did not complete; refusing installer success." >&2 + return 1 + } +} + +run_npm_global_install() { + local spec="$1" + local log="$2" + local npm_cmd="" lifecycle_arg="" + npm_cmd="$(npm_command_path npm)" || { echo "npm not found on PATH; no package changes were made." >&2; return 1; } + local npm_cwd="$PWD" + lifecycle_arg="$(npm_lifecycle_allow_arg "$npm_cmd" "$spec" "$npm_cwd")" || return 1 + + local freshness_flag="--min-release-age=0" + local min_release_age="" + min_release_age="$(env -u NPM_CONFIG_BEFORE -u npm_config_before "$npm_cmd" config get min-release-age --global 2>/dev/null || true)" + if npm_config_has_raw_key "$npm_cmd" "min-release-age"; then + freshness_flag="--min-release-age=0" + elif [[ -z "$min_release_age" || "$min_release_age" == "null" || "$min_release_age" == "undefined" ]]; then + local before_value="" + before_value="$(env -u NPM_CONFIG_MIN_RELEASE_AGE -u npm_config_min_release_age -u npm_config_min-release-age "$npm_cmd" config get before --global 2>/dev/null || true)" + if [[ -n "$before_value" && "$before_value" != "null" && "$before_value" != "undefined" ]]; then + freshness_flag="--before=$(date -u '+%Y-%m-%dT%H:%M:%S.000Z')" + fi + fi + + local -a cmd + cmd=(env -u NPM_CONFIG_BEFORE -u npm_config_before -u NPM_CONFIG_MIN_RELEASE_AGE -u npm_config_min_release_age -u npm_config_min-release-age "$npm_cmd" --loglevel "$NPM_LOGLEVEL") + cmd+=(--no-fund --no-audit "$freshness_flag" install -g) + [[ -z "$lifecycle_arg" ]] || cmd+=("$lifecycle_arg") + cmd+=("$spec") + local cmd_display="" + printf -v cmd_display '%q ' "${cmd[@]}" + LAST_NPM_INSTALL_CMD="${cmd_display% }" + + local install_status=0 + if [[ "$VERBOSE" == "1" ]]; then + "${cmd[@]}" < /dev/null 2>&1 | tee "$log" || install_status=$? + elif [[ -n "$GUM" ]] && gum_is_tty; then + local cmd_quoted="" + local log_quoted="" + printf -v cmd_quoted '%q ' "${cmd[@]}" + printf -v log_quoted '%q' "$log" + run_with_spinner "Installing OpenClaw package" bash -c "${cmd_quoted}>${log_quoted} 2>&1" || install_status=$? + else + ui_info "Installing OpenClaw package" + "${cmd[@]}" < /dev/null >"$log" 2>&1 || install_status=$? + fi + (( install_status == 0 )) || return "$install_status" +} + +run_verified_npm_global_install() { + local npm_cmd="" + npm_cmd="$(npm_command_path npm)" || return 1 + run_npm_global_install "$1" "$2" && verify_npm_lifecycle_completed "$npm_cmd" +} + +extract_npm_debug_log_path() { + local log="$1" + local path="" + path="$(sed -n -E 's/.*A complete log of this run can be found in:[[:space:]]*//p' "$log" | tail -n1)" + if [[ -n "$path" ]]; then + echo "$path" + return 0 + fi + + path="$(grep -Eo '/[^[:space:]]+_logs/[^[:space:]]+debug[^[:space:]]*\.log' "$log" | tail -n1 || true)" + if [[ -n "$path" ]]; then + echo "$path" + return 0 + fi + + return 1 +} + +extract_first_npm_error_line() { + local log="$1" + grep -E 'npm (ERR!|error)|ERR!' "$log" | head -n1 || true +} + +extract_npm_error_code() { + local log="$1" + sed -n -E 's/^npm (ERR!|error) code[[:space:]]+([^[:space:]]+).*$/\2/p' "$log" | head -n1 +} + +extract_npm_error_syscall() { + local log="$1" + sed -n -E 's/^npm (ERR!|error) syscall[[:space:]]+(.+)$/\2/p' "$log" | head -n1 +} + +extract_npm_error_errno() { + local log="$1" + sed -n -E 's/^npm (ERR!|error) errno[[:space:]]+(.+)$/\2/p' "$log" | head -n1 +} + +print_npm_failure_diagnostics() { + local spec="$1" + local log="$2" + local debug_log="" + local first_error="" + local error_code="" + local error_syscall="" + local error_errno="" + + ui_warn "npm install failed for ${spec}" + if [[ -n "${LAST_NPM_INSTALL_CMD}" ]]; then + echo " Command: ${LAST_NPM_INSTALL_CMD}" + fi + # EXIT cleanup removes this capture; expose its contents and npm-owned log instead. + + error_code="$(extract_npm_error_code "$log")" + if [[ -n "$error_code" ]]; then + echo " npm code: ${error_code}" + fi + + error_syscall="$(extract_npm_error_syscall "$log")" + if [[ -n "$error_syscall" ]]; then + echo " npm syscall: ${error_syscall}" + fi + + error_errno="$(extract_npm_error_errno "$log")" + if [[ -n "$error_errno" ]]; then + echo " npm errno: ${error_errno}" + fi + + debug_log="$(extract_npm_debug_log_path "$log" || true)" + if [[ -n "$debug_log" ]]; then + echo " npm debug log: ${debug_log}" + fi + + first_error="$(extract_first_npm_error_line "$log")" + if [[ -n "$first_error" ]]; then + echo " First npm error: ${first_error}" + fi +} + +install_openclaw_npm() { + local spec="$1" + local log + mktempfile log + if ! run_verified_npm_global_install "$spec" "$log"; then + local attempted_build_tool_fix=false + if auto_install_build_tools_for_npm_failure "$log"; then + attempted_build_tool_fix=true + ui_info "Retrying npm install after build tools setup" + if run_verified_npm_global_install "$spec" "$log"; then + ui_success "OpenClaw npm package installed" + return 0 + fi + fi + + print_npm_failure_diagnostics "$spec" "$log" + + if [[ "$VERBOSE" != "1" ]]; then + if [[ "$attempted_build_tool_fix" == "true" ]]; then + ui_warn "npm install still failed after build tools setup; showing last log lines" + else + ui_warn "npm install failed; showing last log lines" + fi + tail -n 80 "$log" >&2 || true + fi + + if grep -q "ENOTEMPTY: directory not empty, rename .*openclaw" "$log"; then + ui_warn "npm left stale directory; cleaning and retrying" + cleanup_npm_stale_rename_dirs || return 1 + if run_verified_npm_global_install "$spec" "$log"; then + ui_success "OpenClaw npm package installed" + return 0 + fi + return 1 + fi + if grep -q "EEXIST" "$log"; then + local conflict="" + conflict="$(extract_openclaw_conflict_path "$log" || true)" + if [[ -n "$conflict" ]] && cleanup_openclaw_bin_conflict "$conflict"; then + if run_verified_npm_global_install "$spec" "$log"; then + ui_success "OpenClaw npm package installed" + return 0 + fi + return 1 + fi + ui_error "npm failed because an openclaw binary already exists" + if [[ -n "$conflict" ]]; then + ui_info "Remove or move ${conflict}, then retry" + fi + ui_info "Or rerun with: npm install -g --force ${spec}" + fi + return 1 + fi + ui_success "OpenClaw npm package installed" + return 0 +} + +TAGLINES=() +TAGLINES+=("Your terminal just grew claws—type something and let the bot pinch the busywork.") +TAGLINES+=("Welcome to the command line: where dreams compile and confidence segfaults.") +TAGLINES+=("I run on caffeine, JSON5, and the audacity of \"it worked on my machine.\"") +TAGLINES+=("Gateway online—please keep hands, feet, and appendages inside the shell at all times.") +TAGLINES+=("I speak fluent bash, mild sarcasm, and aggressive tab-completion energy.") +TAGLINES+=("One CLI to rule them all, and one more restart because you changed the port.") +TAGLINES+=("Your .env is showing; don't worry, I'll pretend I didn't see it.") +TAGLINES+=("I'll do the boring stuff while you dramatically stare at the logs like it's cinema.") +TAGLINES+=("I'm not saying your workflow is chaotic... I'm just bringing a linter and a helmet.") +TAGLINES+=("Type the command with confidence—nature will provide the stack trace if needed.") +TAGLINES+=("I don't judge, but your missing API keys are absolutely judging you.") +TAGLINES+=("I can grep it, git blame it, and gently roast it—pick your coping mechanism.") +TAGLINES+=("Hot reload for config, cold sweat for deploys.") +TAGLINES+=("I'm the assistant your terminal demanded, not the one your sleep schedule requested.") +TAGLINES+=("I keep secrets like a vault... unless you print them in debug logs again.") +TAGLINES+=("Automation with claws: minimal fuss, maximal pinch.") +TAGLINES+=("If you're lost, run doctor; if you're brave, run prod; if you're wise, run tests.") +TAGLINES+=("Your task has been queued; your dignity has been deprecated.") +TAGLINES+=("I'm not magic—I'm just extremely persistent with retries and coping strategies.") +TAGLINES+=("It's not \"failing,\" it's \"discovering new ways to configure the same thing wrong.\"") +TAGLINES+=("I read logs so you can keep pretending you don't have to.") +TAGLINES+=("If something's on fire, I can't extinguish it—but I can write a beautiful postmortem.") +TAGLINES+=("I'll refactor your busywork like it owes me money.") +TAGLINES+=("Say \"stop\" and I'll stop—say \"ship\" and we'll both learn a lesson.") +TAGLINES+=("I'm the reason your shell history looks like a hacker-movie montage.") +TAGLINES+=("I'm like tmux: confusing at first, then suddenly you can't live without me.") +TAGLINES+=("I can run local, remote, or purely on vibes—results may vary with DNS.") +TAGLINES+=("If you can describe it, I can probably automate it—or at least make it funnier.") +TAGLINES+=("Your config is valid, your assumptions are not.") +TAGLINES+=("Claws out, commit in—let's ship something mildly responsible.") +TAGLINES+=("I'll butter your workflow like a lobster roll: messy, delicious, effective.") +TAGLINES+=("Shell yeah—I'm here to pinch the toil and leave you the glory.") +TAGLINES+=("If it's repetitive, I'll automate it; if it's hard, I'll bring jokes and a rollback plan.") +TAGLINES+=("WhatsApp, but make it ✨engineering✨.") +TAGLINES+=("Turning \"I'll reply later\" into \"my bot replied instantly\".") +TAGLINES+=("The only crab in your contacts you actually want to hear from. 🦞") +TAGLINES+=("Chat automation for people who peaked at IRC.") +TAGLINES+=("Because Siri wasn't answering at 3AM.") +TAGLINES+=("IPC, but it's your phone.") +TAGLINES+=("The UNIX philosophy meets your DMs.") +TAGLINES+=("curl for conversations.") +TAGLINES+=("WhatsApp Business, but without the business.") +TAGLINES+=("Meta wishes they shipped this fast.") +TAGLINES+=("End-to-end encrypted, Zuck-to-Zuck excluded.") +TAGLINES+=("The only bot Mark can't train on your DMs.") +TAGLINES+=("WhatsApp automation without the \"please accept our new privacy policy\".") +TAGLINES+=("Chat APIs that don't require a Senate hearing.") +TAGLINES+=("Because Threads wasn't the answer either.") +TAGLINES+=("Your messages, your servers, Meta's tears.") +TAGLINES+=("Siri's competent cousin.") +TAGLINES+=("Works on Android. Crazy concept, we know.") +TAGLINES+=("No \$999 stand required.") +TAGLINES+=("We ship features faster than Apple ships calculator updates.") +TAGLINES+=("Your AI assistant, now without the \$3,499 headset.") +TAGLINES+=("Think different. Actually think.") +TAGLINES+=("Ah, the fruit tree company! 🍎") + +HOLIDAY_NEW_YEAR="New Year's Day: New year, new config—same old EADDRINUSE, but this time we resolve it like grown-ups." +HOLIDAY_LUNAR_NEW_YEAR="Lunar New Year: May your builds be lucky, your branches prosperous, and your merge conflicts chased away with fireworks." +HOLIDAY_CHRISTMAS="Christmas: Ho ho ho—Santa's little claw-sistant is here to ship joy, roll back chaos, and stash the keys safely." +HOLIDAY_EID="Eid al-Fitr: Celebration mode: queues cleared, tasks completed, and good vibes committed to main with clean history." +HOLIDAY_DIWALI="Diwali: Let the logs sparkle and the bugs flee—today we light up the terminal and ship with pride." +HOLIDAY_EASTER="Easter: I found your missing environment variable—consider it a tiny CLI egg hunt with fewer jellybeans." +HOLIDAY_HANUKKAH="Hanukkah: Eight nights, eight retries, zero shame—may your gateway stay lit and your deployments stay peaceful." +HOLIDAY_HALLOWEEN="Halloween: Spooky season: beware haunted dependencies, cursed caches, and the ghost of node_modules past." +HOLIDAY_THANKSGIVING="Thanksgiving: Grateful for stable ports, working DNS, and a bot that reads the logs so nobody has to." +HOLIDAY_VALENTINES="Valentine's Day: Roses are typed, violets are piped—I'll automate the chores so you can spend time with humans." + +append_holiday_taglines() { + local today + local month_day + today="$(date -u +%Y-%m-%d 2>/dev/null || date +%Y-%m-%d)" + month_day="$(date -u +%m-%d 2>/dev/null || date +%m-%d)" + + case "$month_day" in + "01-01") TAGLINES+=("$HOLIDAY_NEW_YEAR") ;; + "02-14") TAGLINES+=("$HOLIDAY_VALENTINES") ;; + "10-31") TAGLINES+=("$HOLIDAY_HALLOWEEN") ;; + "12-25") TAGLINES+=("$HOLIDAY_CHRISTMAS") ;; + esac + + case "$today" in + "2025-01-29"|"2026-02-17"|"2027-02-06") TAGLINES+=("$HOLIDAY_LUNAR_NEW_YEAR") ;; + "2025-03-30"|"2025-03-31"|"2026-03-20"|"2027-03-10") TAGLINES+=("$HOLIDAY_EID") ;; + "2025-10-20"|"2026-11-08"|"2027-10-28") TAGLINES+=("$HOLIDAY_DIWALI") ;; + "2025-04-20"|"2026-04-05"|"2027-03-28") TAGLINES+=("$HOLIDAY_EASTER") ;; + "2025-11-27"|"2026-11-26"|"2027-11-25") TAGLINES+=("$HOLIDAY_THANKSGIVING") ;; + "2025-12-15"|"2025-12-16"|"2025-12-17"|"2025-12-18"|"2025-12-19"|"2025-12-20"|"2025-12-21"|"2025-12-22"|"2026-12-05"|"2026-12-06"|"2026-12-07"|"2026-12-08"|"2026-12-09"|"2026-12-10"|"2026-12-11"|"2026-12-12"|"2027-12-25"|"2027-12-26"|"2027-12-27"|"2027-12-28"|"2027-12-29"|"2027-12-30"|"2027-12-31"|"2028-01-01") TAGLINES+=("$HOLIDAY_HANUKKAH") ;; + esac +} + +pick_tagline() { + append_holiday_taglines + local count=${#TAGLINES[@]} + if [[ "$count" -eq 0 ]]; then + echo "$DEFAULT_TAGLINE" + return + fi + if [[ -n "${OPENCLAW_TAGLINE_INDEX:-}" ]]; then + if [[ "${OPENCLAW_TAGLINE_INDEX}" =~ ^[0-9]+$ ]]; then + local idx=$((OPENCLAW_TAGLINE_INDEX % count)) + echo "${TAGLINES[$idx]}" + return + fi + fi + local idx=$((RANDOM % count)) + echo "${TAGLINES[$idx]}" +} + +TAGLINE=$(pick_tagline) + +NO_ONBOARD=${OPENCLAW_NO_ONBOARD:-0} +NO_PROMPT=${OPENCLAW_NO_PROMPT:-0} +DRY_RUN=${OPENCLAW_DRY_RUN:-0} +INSTALL_METHOD=${OPENCLAW_INSTALL_METHOD:-} +OPENCLAW_VERSION=${OPENCLAW_VERSION:-latest} +USE_BETA=${OPENCLAW_BETA:-0} +GIT_DIR=${OPENCLAW_GIT_DIR:-"$(resolve_openclaw_effective_home)/openclaw"} +GIT_DIR_EXPLICIT=${OPENCLAW_GIT_DIR:+1} +GIT_UPDATE=${OPENCLAW_GIT_UPDATE:-1} +NPM_LOGLEVEL="${OPENCLAW_NPM_LOGLEVEL:-error}" +VERBOSE="${OPENCLAW_VERBOSE:-0}" +VERIFY_INSTALL="${OPENCLAW_VERIFY_INSTALL:-0}" +OPENCLAW_BIN="" +PNPM_CMD=() +GIT_REF_KIND="" +HELP=0 + +print_usage() { + cat <<EOF +OpenClaw installer (macOS + Linux) + +Usage: + curl -fsSL --proto '=https' --tlsv1.2 https://openclaw.ai/install.sh | bash -s -- [options] + +Options: + --install-method, --method npm|git Install via npm (default) or from a git checkout + --npm Shortcut for --install-method npm + --git, --github Shortcut for --install-method git + --version <version|dist-tag|spec> npm install target (default: latest) + --beta Use beta if available, else latest + --git-dir, --dir <path> Checkout directory (default: ~/openclaw) + --no-git-update Skip git pull for existing checkout + --no-onboard Skip onboarding (non-interactive) + --no-prompt Disable prompts (required in CI/automation) + --verify Run a post-install smoke verify + --dry-run Print what would happen (no changes) + --verbose Print debug output (set -x, npm verbose) + --help, -h Show this help + +Environment variables: + OPENCLAW_INSTALL_METHOD=git|npm + OPENCLAW_VERSION=latest|next|<semver>|<spec> + OPENCLAW_BETA=0|1 + OPENCLAW_GIT_DIR=... + OPENCLAW_GIT_UPDATE=0|1 + OPENCLAW_NO_PROMPT=1 + OPENCLAW_VERIFY_INSTALL=1 + OPENCLAW_DRY_RUN=1 + OPENCLAW_NO_ONBOARD=1 + OPENCLAW_VERBOSE=1 + OPENCLAW_NPM_LOGLEVEL=error|warn|notice Default: error (hide npm deprecation noise) +Examples: + curl -fsSL --proto '=https' --tlsv1.2 https://openclaw.ai/install.sh | bash + curl -fsSL --proto '=https' --tlsv1.2 https://openclaw.ai/install.sh | bash -s -- --no-onboard + curl -fsSL --proto '=https' --tlsv1.2 https://openclaw.ai/install.sh | bash -s -- --no-onboard --verify + curl -fsSL --proto '=https' --tlsv1.2 https://openclaw.ai/install.sh | bash -s -- --install-method git --version main + curl -fsSL --proto '=https' --tlsv1.2 https://openclaw.ai/install.sh | bash -s -- --install-method git --no-onboard +EOF +} + +parse_args() { + while [[ $# -gt 0 ]]; do + case "$1" in + --no-onboard) + NO_ONBOARD=1 + shift + ;; + --onboard) + NO_ONBOARD=0 + shift + ;; + --dry-run) + DRY_RUN=1 + shift + ;; + --verbose) + VERBOSE=1 + shift + ;; + --verify) + VERIFY_INSTALL=1 + shift + ;; + --no-prompt) + NO_PROMPT=1 + shift + ;; + --help|-h) + HELP=1 + shift + ;; + --install-method|--method) + if [[ $# -lt 2 || "${2:-}" == --* ]]; then + ui_error "Missing value for $1" + return 2 + fi + INSTALL_METHOD="$2" + shift 2 + ;; + --version) + if [[ $# -lt 2 || "${2:-}" == --* ]]; then + ui_error "Missing value for $1" + return 2 + fi + OPENCLAW_VERSION="$2" + shift 2 + ;; + --beta) + USE_BETA=1 + shift + ;; + --npm) + INSTALL_METHOD="npm" + shift + ;; + --git|--github) + INSTALL_METHOD="git" + shift + ;; + --git-dir|--dir) + if [[ $# -lt 2 || "${2:-}" == --* ]]; then + ui_error "Missing value for $1" + return 2 + fi + GIT_DIR="$2" + GIT_DIR_EXPLICIT=${2:+1} + shift 2 + ;; + --no-git-update) + GIT_UPDATE=0 + shift + ;; + *) + ui_error "Unknown option: $1" + return 2 + ;; + esac + done +} + +configure_verbose() { + if [[ "$VERBOSE" != "1" ]]; then + return 0 + fi + if [[ "$NPM_LOGLEVEL" == "error" ]]; then + NPM_LOGLEVEL="notice" + fi + set -x +} + +is_promptable() { + if [[ "$NO_PROMPT" == "1" ]]; then + return 1 + fi + if has_controlling_tty; then + return 0 + fi + return 1 +} + +prompt_choice() { + local prompt="$1" + local answer="" + if ! is_promptable; then + return 1 + fi + echo -e "$prompt" > /dev/tty + read -r answer < /dev/tty || true + echo "$answer" +} + +choose_install_method_interactive() { + local detected_checkout="$1" + + if ! is_promptable; then + return 1 + fi + + if [[ -n "$GUM" ]] && gum_is_tty; then + local header selection + header="Detected OpenClaw checkout in: ${detected_checkout} +Choose install method" + selection="$("$GUM" choose \ + --header "$header" \ + --cursor-prefix "❯ " \ + "git · update this checkout and use it" \ + "npm · install globally via npm" < /dev/tty || true)" + + case "$selection" in + git*) + echo "git" + return 0 + ;; + npm*) + echo "npm" + return 0 + ;; + esac + return 1 + fi + + local choice="" + choice="$(prompt_choice "$(cat <<EOF +${WARN}→${NC} Detected a OpenClaw source checkout in: ${INFO}${detected_checkout}${NC} +Choose install method: + 1) Update this checkout (git) and use it + 2) Install global via npm (migrate away from git) +Enter 1 or 2: +EOF +)" || true)" + + case "$choice" in + 1) + echo "git" + return 0 + ;; + 2) + echo "npm" + return 0 + ;; + esac + + return 1 +} + +detect_openclaw_checkout() { + local dir="$1" + if [[ ! -f "$dir/package.json" ]]; then + return 1 + fi + if [[ ! -f "$dir/pnpm-workspace.yaml" ]]; then + return 1 + fi + if ! grep -q '"name"[[:space:]]*:[[:space:]]*"openclaw"' "$dir/package.json" 2>/dev/null; then + return 1 + fi + echo "$dir" + return 0 +} + +# Check for Homebrew on macOS +is_macos_admin_user() { + if [[ "$OS" != "macos" ]]; then + return 0 + fi + if is_root; then + return 0 + fi + id -Gn "$(id -un)" 2>/dev/null | grep -qw "admin" +} + +print_homebrew_admin_fix() { + local current_user + current_user="$(id -un 2>/dev/null || echo "${USER:-current user}")" + ui_error "Homebrew installation requires a macOS Administrator account" + echo "Current user (${current_user}) is not in the admin group." + echo "Fix options:" + echo " 1) Use an Administrator account and re-run the installer." + echo " 2) Ask an Administrator to grant admin rights, then sign out/in:" + echo " sudo dseditgroup -o edit -a ${current_user} -t user admin" + echo "Then retry:" + echo " curl -fsSL https://openclaw.ai/install.sh | bash" +} + +install_homebrew() { + if [[ "$OS" == "macos" ]]; then + if ! command -v brew &> /dev/null; then + if ! is_macos_admin_user; then + print_homebrew_admin_fix + exit 1 + fi + ui_info "Homebrew not found, installing" + run_quiet_step "Installing Homebrew" run_remote_bash "https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh" + + # Add Homebrew to PATH for this session + if [[ -f "/opt/homebrew/bin/brew" ]]; then + eval "$(/opt/homebrew/bin/brew shellenv)" + elif [[ -f "/usr/local/bin/brew" ]]; then + eval "$(/usr/local/bin/brew shellenv)" + fi + ui_success "Homebrew installed" + else + ui_success "Homebrew already installed" + fi + fi +} + +# Check Node.js version +parse_node_version_components_for_binary() { + local node_bin="${1:-node}" + if ! command -v "$node_bin" &> /dev/null && [[ ! -x "$node_bin" ]]; then + return 1 + fi + local version major minor patch + version="$("$node_bin" -v 2>/dev/null || true)" + version="${version#"${version%%[![:space:]]*}"}" + version="${version%"${version##*[![:space:]]}"}" + + # This standalone installer runs before OpenClaw exists on disk. Mirror the + # release grammar in node-version.mjs; parity cases guard this boundary. + if [[ ! "$version" =~ ^v?(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(\+[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$ ]]; then + return 1 + fi + major="${BASH_REMATCH[1]}" + minor="${BASH_REMATCH[2]}" + patch="${BASH_REMATCH[3]}" + + local component + for component in "$major" "$minor" "$patch"; do + if ((${#component} > 16)) || + ((${#component} == 16 && 10#$component > 9007199254740991)); then + return 1 + fi + done + echo "${major} ${minor} ${patch}" + return 0 +} + +parse_node_version_components() { + if ! command -v node &> /dev/null; then + return 1 + fi + parse_node_version_components_for_binary node +} + +node_major_version() { + local version_components major minor patch + version_components="$(parse_node_version_components || true)" + read -r major minor patch <<< "$version_components" + if [[ "$major" =~ ^[0-9]+$ && "$minor" =~ ^[0-9]+$ ]]; then + echo "$major" + return 0 + fi + return 1 +} + +node_version_components_are_supported() { + local major="$1" + local minor="$2" + local patch="$3" + + case "$major" in + 24) + ((minor > NODE_24_MIN_MINOR)) || + ((minor == NODE_24_MIN_MINOR && patch >= NODE_24_MIN_PATCH)) + ;; + 26) + ((minor > NODE_26_MIN_MINOR)) || + ((minor == NODE_26_MIN_MINOR && patch >= NODE_26_MIN_PATCH)) + ;; + *) + ((major > 26)) + ;; + esac +} + +node_binary_has_safe_sqlite() { + local node_bin="$1" + "$node_bin" -e ' + const { DatabaseSync } = require("node:sqlite"); + const db = new DatabaseSync(":memory:"); + try { + const value = db.prepare("SELECT sqlite_version() AS version").get()?.version; + const match = typeof value === "string" ? /^(\d+)\.(\d+)\.(\d+)$/.exec(value) : null; + const major = Number(match?.[1]); + const minor = Number(match?.[2]); + const patch = Number(match?.[3]); + const safe = + major > 3 || + (major === 3 && + (minor > 51 || + (minor === 51 && patch >= 3) || + (minor === 50 && patch >= 7) || + (minor === 44 && patch >= 6))); + const text = "a\u0000b\u0000"; + const bytes = Buffer.from(text, "utf8"); + const json = JSON.stringify({ value: text }); + db.exec("CREATE TABLE probe (text_value TEXT, blob_value BLOB, json_value TEXT)"); + db.prepare("INSERT INTO probe VALUES (?, ?, ?)").run(text, bytes, json); + const row = db.prepare("SELECT text_value, blob_value, json_value FROM probe").get(); + const textSafe = typeof row?.text_value === "string" && row.text_value.length === text.length && Buffer.from(row.text_value, "utf8").equals(bytes); + const blobSafe = row?.blob_value instanceof Uint8Array && Buffer.from(row.blob_value).equals(bytes); + const jsonSafe = row?.json_value === json && JSON.parse(row.json_value).value === text; + if (!textSafe) { + console.error("Node " + process.versions.node + ": node:sqlite truncates TEXT at embedded NUL (nodejs/node#61954); use 24.16+/26.1+ or a build with the fix"); + } else if (!blobSafe || !jsonSafe) { + console.error("Node " + process.versions.node + ": node:sqlite NUL round-trip capability probe failed; use 24.16+/26.1+ or a build with the fix"); + } else if (!safe) { + console.error("Node " + process.versions.node + ": SQLite " + value + " is not WAL-reset-safe"); + } + if (!safe || !textSafe || !blobSafe || !jsonSafe) process.exitCode = 1; + } finally { + db.close(); + } + ' --no-warnings >/dev/null +} + +node_binary_sqlite_version() { + local node_bin="$1" + local version + version="$("$node_bin" -e ' + const { DatabaseSync } = require("node:sqlite"); + const db = new DatabaseSync(":memory:"); + try { + process.stdout.write(String(db.prepare("SELECT sqlite_version() AS version").get()?.version ?? "unknown")); + } finally { + db.close(); + } + ' 2>/dev/null || true)" + printf '%s\n' "${version:-unavailable}" +} + +node_version_is_supported() { + local version_components major minor patch + version_components="$(parse_node_version_components || true)" + read -r major minor patch <<< "$version_components" + if [[ ! "$major" =~ ^[0-9]+$ || ! "$minor" =~ ^[0-9]+$ || ! "$patch" =~ ^[0-9]+$ ]]; then + return 1 + fi + node_version_components_are_supported "$major" "$minor" "$patch" +} + +node_is_supported() { + node_binary_is_supported node +} + +node_binary_is_supported() { + local node_bin="$1" + local version_components major minor patch + version_components="$(parse_node_version_components_for_binary "$node_bin" || true)" + read -r major minor patch <<< "$version_components" + if [[ ! "$major" =~ ^[0-9]+$ || ! "$minor" =~ ^[0-9]+$ || ! "$patch" =~ ^[0-9]+$ ]]; then + return 1 + fi + node_version_components_are_supported "$major" "$minor" "$patch" && + node_binary_has_safe_sqlite "$node_bin" +} + +prepend_path_dir() { + local dir="${1%/}" + if [[ -z "$dir" || ! -d "$dir" ]]; then + return 1 + fi + local current=":${PATH:-}:" + current="${current//:${dir}:/:}" + current="${current#:}" + current="${current%:}" + if [[ -n "$current" ]]; then + export PATH="${dir}:${current}" + else + export PATH="${dir}" + fi + refresh_shell_command_cache +} + +persist_shell_path_prepend() { + local dir="${1%/}" + if [[ -z "$dir" ]]; then + return 1 + fi + + local path_expr="${2:-$dir}" + local shell_name="${SHELL:-}" + shell_name="${shell_name##*/}" + local bash_login_rc="$HOME/.profile" + if [[ -r "$HOME/.bash_profile" ]]; then + bash_login_rc="$HOME/.bash_profile" + elif [[ -r "$HOME/.bash_login" ]]; then + bash_login_rc="$HOME/.bash_login" + fi + + local targets=() + local fish_rc="$HOME/.config/fish/conf.d/openclaw.fish" + case "$shell_name" in + bash) + targets+=("bash:$HOME/.bashrc" "bash:$bash_login_rc") + [[ -e "$HOME/.zshrc" || -L "$HOME/.zshrc" ]] && targets+=("zsh:$HOME/.zshrc") + [[ -e "$HOME/.zprofile" || -L "$HOME/.zprofile" ]] && targets+=("zsh:$HOME/.zprofile") + [[ -e "$fish_rc" || -L "$fish_rc" ]] && targets+=("fish:$fish_rc") + ;; + zsh) + targets+=("zsh:$HOME/.zshrc" "zsh:$HOME/.zprofile") + [[ -e "$HOME/.bashrc" || -L "$HOME/.bashrc" ]] && targets+=("bash:$HOME/.bashrc") + [[ -e "$bash_login_rc" || -L "$bash_login_rc" ]] && targets+=("bash:$bash_login_rc") + [[ -e "$fish_rc" || -L "$fish_rc" ]] && targets+=("fish:$fish_rc") + ;; + fish) + targets+=("fish:$fish_rc") + [[ -e "$HOME/.bashrc" || -L "$HOME/.bashrc" ]] && targets+=("bash:$HOME/.bashrc") + [[ -e "$bash_login_rc" || -L "$bash_login_rc" ]] && targets+=("bash:$bash_login_rc") + [[ -e "$HOME/.zshrc" || -L "$HOME/.zshrc" ]] && targets+=("zsh:$HOME/.zshrc") + [[ -e "$HOME/.zprofile" || -L "$HOME/.zprofile" ]] && targets+=("zsh:$HOME/.zprofile") + ;; + *) + echo "" + ui_warn "Could not identify your shell from SHELL=${SHELL:-unset}; PATH was not persisted" + echo " Add this directory to PATH in your shell startup file: ${dir}" + echo " Bash/zsh: export PATH=\"${path_expr}:\$PATH\"" + echo " Fish: fish_add_path -- \"${path_expr}\"" + return 0 + ;; + esac + + local target contract rc path_line failed=0 + for target in "${targets[@]}"; do + contract="${target%%:*}" + rc="${target#*:}" + if [[ "$contract" == "fish" ]]; then + path_line="fish_add_path -- \"${path_expr}\"" + else + path_line="export PATH=\"${path_expr}:\$PATH\"" + fi + if ! persist_path_line_to_profile "$rc" "$path_line"; then + failed=1 + fi + done + return "$failed" +} + +resolve_safe_profile_target() { + local profile="$1" current="$1" link parent resolved hops=0 + local home_real + home_real="$(cd "$HOME" 2>/dev/null && pwd -P)" || return 1 + while [[ -L "$current" ]]; do + ((hops += 1)) + if (( hops > 40 )); then + ui_warn "Refusing to update profile symlink loop: ${profile}" >&2 + return 1 + fi + link="$(readlink "$current")" || return 1 + parent="$(dirname "$current")" + if [[ "$link" == /* ]]; then + current="$link" + else + current="$parent/$link" + fi + parent="$(cd "$(dirname "$current")" 2>/dev/null && pwd -P)" || { + ui_warn "Refusing profile symlink with missing parent: ${profile}" >&2 + return 1 + } + current="$parent/$(basename "$current")" + done + parent="$(cd "$(dirname "$current")" 2>/dev/null && pwd -P)" || return 1 + resolved="$parent/$(basename "$current")" + case "$resolved" in + "$home_real"/*) ;; + *) + ui_warn "Refusing profile symlink outside your home: ${profile}" >&2 + return 1 + ;; + esac + if [[ ! -f "$resolved" || -L "$resolved" ]]; then + ui_warn "Refusing non-regular profile target: ${profile}" >&2 + return 1 + fi + local owner current_uid + current_uid="$(id -u)" + owner="$(stat -c '%u' "$resolved" 2>/dev/null || stat -f '%u' "$resolved" 2>/dev/null || true)" + if [[ "$owner" != "$current_uid" ]]; then + ui_warn "Refusing profile target not owned by the current user: ${profile}" >&2 + return 1 + fi + printf '%s\n' "$resolved" +} + +prepare_safe_profile_parent() { + local profile="$1" parent ancestor home_real ancestor_real parent_real + parent="$(dirname "$profile")" + ancestor="$parent" + home_real="$(cd "$HOME" 2>/dev/null && pwd -P)" || return 1 + while [[ ! -d "$ancestor" ]]; do + if [[ -e "$ancestor" || -L "$ancestor" ]]; then + ui_warn "Refusing non-directory shell profile parent: ${profile}" + return 1 + fi + ancestor="$(dirname "$ancestor")" + done + ancestor_real="$(cd "$ancestor" 2>/dev/null && pwd -P)" || return 1 + case "$ancestor_real" in + "$home_real"|"$home_real"/*) ;; + *) + ui_warn "Refusing shell profile parent outside your home: ${profile}" + return 1 + ;; + esac + mkdir -p "$parent" || return 1 + parent_real="$(cd "$parent" 2>/dev/null && pwd -P)" || return 1 + case "$parent_real" in + "$home_real"|"$home_real"/*) ;; + *) + ui_warn "Refusing shell profile parent outside your home: ${profile}" + return 1 + ;; + esac +} + +persist_path_line_to_profile() { + local profile="$1" path_line="$2" rc tmp_rc original_mode + rc="$profile" + if [[ -L "$profile" ]]; then + rc="$(resolve_safe_profile_target "$profile")" || return 1 + elif [[ -e "$profile" && ! -f "$profile" ]]; then + ui_warn "Refusing non-regular shell profile: ${profile}" + return 1 + fi + + prepare_safe_profile_parent "$rc" || return 1 + if [[ "$(sed -n '1p' "$rc" 2>/dev/null || true)" == "$path_line" ]]; then + return 0 + fi + tmp_rc="$(mktemp "${rc}.openclaw-tmp.XXXXXX")" + TMPFILES+=("$tmp_rc") + if [[ -f "$rc" ]]; then + if ! cp -p "$rc" "$tmp_rc"; then + ui_warn "Failed to copy shell profile: ${profile}" + return 1 + fi + original_mode="$(stat -c '%a' "$rc" 2>/dev/null || stat -f '%Lp' "$rc" 2>/dev/null)" || return 1 + chmod u+w "$tmp_rc" || return 1 + fi + if ! { + printf '%s\n' "$path_line" + if [[ -f "$rc" ]]; then + grep -Fvx "$path_line" "$rc" || true + fi + } > "$tmp_rc"; then + ui_warn "Failed to write shell profile: ${profile}" + return 1 + fi + if [[ -n "${original_mode:-}" ]]; then + chmod "$original_mode" "$tmp_rc" || return 1 + fi + mv "$tmp_rc" "$rc" || return 1 +} + +promote_supported_node_binary() { + local candidates=() + local candidate dir seen_dirs=":" + + while IFS= read -r candidate; do + candidates+=("$candidate") + done < <(type -P -a node 2>/dev/null || true) + + candidates+=( + "/usr/bin/node" + "/usr/local/bin/node" + "/opt/homebrew/bin/node" + "/opt/homebrew/opt/${NODE_BREW_FORMULA}/bin/node" + "/usr/local/opt/${NODE_BREW_FORMULA}/bin/node" + # Keep-alive for installs provisioned when node@24 was the default. + "/opt/homebrew/opt/node@24/bin/node" + "/usr/local/opt/node@24/bin/node" + ) + + for candidate in "${candidates[@]}"; do + if [[ -z "$candidate" || ! -x "$candidate" ]]; then + continue + fi + if dir="$(cd "$(dirname "$candidate")" && pwd 2>/dev/null)"; then + : + else + dir="" + fi + if [[ -z "$dir" || "$seen_dirs" == *":$dir:"* ]]; then + continue + fi + seen_dirs="${seen_dirs}${dir}:" + if node_binary_is_supported "$candidate"; then + prepend_path_dir "$dir" || continue + if [[ "$OS" == "linux" && "${NVM_DETECTED:-0}" != "1" ]]; then + persist_shell_path_prepend "$dir" || true + fi + ui_info "Using Node.js runtime at ${candidate}" + return 0 + fi + done + + return 1 +} + +activate_supported_node_on_path() { + promote_supported_node_binary +} + +print_active_node_paths() { + if ! command -v node &> /dev/null; then + return 1 + fi + local node_path node_version npm_path npm_version + node_path="$(command -v node 2>/dev/null || true)" + node_version="$(node -v 2>/dev/null || true)" + ui_info "Active Node.js: ${node_version:-unknown} (${node_path:-unknown})" + + if command -v npm &> /dev/null; then + npm_path="$(command -v npm 2>/dev/null || true)" + npm_version="$(npm -v 2>/dev/null || true)" + ui_info "Active npm: ${npm_version:-unknown} (${npm_path:-unknown})" + fi + return 0 +} + +ensure_macos_default_node_active() { + if [[ "$OS" != "macos" ]]; then + return 0 + fi + + local brew_node_prefix="" + if command -v brew &> /dev/null; then + brew_node_prefix="$(brew --prefix "${NODE_BREW_FORMULA}" 2>/dev/null || true)" + if [[ -n "$brew_node_prefix" && -x "${brew_node_prefix}/bin/node" ]]; then + export PATH="${brew_node_prefix}/bin:$PATH" + refresh_shell_command_cache + fi + fi + + if node_is_supported; then + return 0 + fi + + local active_path active_version + active_path="$(command -v node 2>/dev/null || echo "not found")" + active_version="$(node -v 2>/dev/null || echo "missing")" + + if [[ -z "$brew_node_prefix" || ! -x "${brew_node_prefix}/bin/node" ]]; then + ui_error "Homebrew ${NODE_BREW_FORMULA} is not installed on disk" + echo "The previous 'brew install' step appears to have failed." + echo "Re-run 'brew install ${NODE_BREW_FORMULA}' directly or rerun the installer with --verbose to see the underlying error." + return 1 + fi + + ui_error "Node.js v${NODE_DEFAULT_MAJOR} was installed but this shell is using ${active_version} (${active_path})" + echo "Add this to your shell profile and restart shell:" + echo " export PATH=\"${brew_node_prefix}/bin:\$PATH\"" + return 1 +} + +ensure_default_node_active_shell() { + if node_is_supported; then + return 0 + fi + + local active_path active_version + active_path="$(command -v node 2>/dev/null || echo "not found")" + active_version="$(node -v 2>/dev/null || echo "missing")" + + ui_error "Active Node.js must be ${NODE_SUPPORTED_VERSION_LABEL} but this shell is using ${active_version} (${active_path})" + print_active_node_paths || true + + echo "Install/select Node.js ${NODE_DEFAULT_MAJOR} and ensure it is first on PATH, then rerun installer." + return 1 +} + +load_nvm_for_node_detection() { + NVM_DETECTED=0 + local nvm_dir="${NVM_DIR:-}" profile + if [[ -n "$nvm_dir" || -d "$HOME/.nvm" ]] || command -v nvm >/dev/null 2>&1; then + NVM_DETECTED=1 + fi + # Detect custom/lazy hooks without executing arbitrary shell startup files. + for profile in "$HOME/.bashrc" "$HOME/.bash_profile" "$HOME/.bash_login" "$HOME/.profile" \ + "${ZDOTDIR:-$HOME}/.zshrc" "${ZDOTDIR:-$HOME}/.zprofile"; do + if [[ -r "$profile" ]] && grep -Eq '^[[:space:]]*([^#[:space:]].*)?(NVM_DIR|nvm[.]sh)' "$profile"; then + NVM_DETECTED=1 + fi + done + if [[ ! -s "$nvm_dir/nvm.sh" && -s "$HOME/.nvm/nvm.sh" ]]; then + nvm_dir="$HOME/.nvm" + fi + if [[ -n "$nvm_dir" && -s "$nvm_dir/nvm.sh" ]]; then + NVM_DETECTED=1 + export NVM_DIR="$nvm_dir" + # --no-use preserves the caller's selected system or managed runtime. + # shellcheck disable=SC1090,SC1091 + if ! . "$NVM_DIR/nvm.sh" --no-use; then + ui_error "Could not load existing nvm at ${NVM_DIR}; load it in your shell and rerun the installer" + return 1 + fi + fi + refresh_shell_command_cache +} + +use_supported_nvm_node() { + command -v nvm >/dev/null 2>&1 || return 1 + local candidate version + for candidate in "${NVM_DIR:-$HOME/.nvm}"/versions/node/*/bin/node; do + [[ -x "$candidate" ]] || continue + node_binary_is_supported "$candidate" || continue + version="${candidate%/bin/node}" + version="${version##*/}" + nvm use --silent "$version" || return 1 + refresh_shell_command_cache + node_is_supported || return 1 + ui_info "Using existing nvm Node.js ${version} for this installation (${NVM_DIR})" + echo " Shell profiles and the nvm default are unchanged. For later commands, run: nvm use ${version}" + return 0 + done + return 1 +} + +install_node_with_existing_nvm() { + local reason="${1:-no compatible Node.js runtime is available}" + local default_version="" default_note="Your existing default alias setting will be preserved." + if command -v nvm >/dev/null 2>&1; then + default_version="$(nvm version default 2>/dev/null || true)" + fi + case "$default_version" in + v*|system) default_note="Keep current default ${default_version}; if nvm refreshes aliases, pin default to ${default_version}." ;; + *) + if [[ -n "${NVM_DIR:-}" && ! -e "$NVM_DIR/alias/default" ]]; then + default_note="nvm will also create its currently unset default alias." + fi + ;; + esac + ui_warn "Existing nvm detected; ${reason}" + echo "Load your existing nvm in your shell, then run:" + echo " nvm install ${NODE_DEFAULT_MAJOR}" + echo " nvm use ${NODE_DEFAULT_MAJOR}" + echo "nvm install can refresh LTS aliases; check your default with: nvm version default" + echo "Then rerun the installer. Shell profiles will not be changed." + + local answer="" + if command -v nvm >/dev/null 2>&1 && is_promptable; then + answer="$(prompt_choice "Install Node.js ${NODE_DEFAULT_MAJOR} in your existing nvm (${NVM_DIR}) for this session? ${default_note} [y/N]" || true)" + fi + case "$answer" in + y|Y|yes|YES) + ui_info "Installing Node.js ${NODE_DEFAULT_MAJOR} in existing nvm (${NVM_DIR}). ${default_note}" + local install_result=0 + nvm install "$NODE_DEFAULT_MAJOR" || install_result=$? + # Remote LTS metadata can move an existing default even on download failure. + case "$default_version" in + v*|system) + if [[ "$(nvm version default 2>/dev/null || true)" != "$default_version" ]]; then + ui_info "Preserving the previous default: nvm alias default ${default_version} (approved)" + nvm alias default "$default_version" || return 1 + fi + ;; + esac + if [[ "$install_result" -ne 0 ]]; then + ui_error "nvm install failed; any downloaded files remain in ${NVM_DIR}. Fix the reported error and rerun the command above." + return 1 + fi + nvm use --silent "$NODE_DEFAULT_MAJOR" || return 1 + refresh_shell_command_cache + ensure_default_node_active_shell || return 1 + ui_info "nvm default now resolves to: $(nvm version default 2>/dev/null || true)" + ui_success "Using nvm Node.js $(node -v) for this installation; shell profiles unchanged" + ;; + *) + ui_error "Installation stopped without changing Node.js; run the nvm commands above to continue" + return 1 + ;; + esac +} + +check_node() { + if command -v node &> /dev/null; then + NODE_VERSION="$(node_major_version || true)" + if node_is_supported; then + ui_success "Node.js v$(node -v | cut -d'v' -f2) found" + print_active_node_paths || true + return 0 + else + if [[ -n "$NODE_VERSION" ]]; then + ui_info "Node.js $(node -v) found, upgrading to a supported version" + else + ui_info "Node.js found but version could not be parsed; reinstalling a supported version" + fi + return 1 + fi + else + ui_info "Node.js not found, installing it now" + return 1 + fi +} + +finish_linux_node_install() { + if ! node_is_supported; then + activate_supported_node_on_path || true + fi + if ! node_is_supported; then + local active_path active_version + active_path="$(command -v node 2>/dev/null || echo "not found")" + active_version="$(node -v 2>/dev/null || echo "missing")" + ui_error "Installed Node.js must be ${NODE_SUPPORTED_VERSION_LABEL} but this shell is using ${active_version} (${active_path})" + echo "Upgrade the system Node.js package or install Node.js ${NODE_DEFAULT_MAJOR} manually, then rerun the installer." + exit 1 + fi + + ui_success "Node.js v$(node -v | cut -d'v' -f2) installed" + print_active_node_paths || true +} + +install_node_with_apk() { + ui_info "Installing Node.js via apk (Alpine Linux detected)" + if is_root; then + run_required_step "Installing Node.js" apk add --no-cache nodejs npm + else + run_required_step "Installing Node.js" sudo apk add --no-cache nodejs npm + fi + + activate_supported_node_on_path || true + if node_is_supported; then + finish_linux_node_install + return 0 + fi + + local apk_node_version + apk_node_version="$(node -v 2>/dev/null || echo "missing")" + ui_warn "Alpine nodejs package installed ${apk_node_version}, which does not meet the Node and SQLite runtime contract" + ui_info "Trying Alpine nodejs-current package" + if is_root; then + run_required_step "Installing nodejs-current" apk add --no-cache nodejs-current npm + else + run_required_step "Installing nodejs-current" sudo apk add --no-cache nodejs-current npm + fi + + activate_supported_node_on_path || true + if node_is_supported; then + finish_linux_node_install + return 0 + fi + + local active_path active_version sqlite_version + active_path="$(command -v node 2>/dev/null || echo "not found")" + active_version="$(node -v 2>/dev/null || echo "missing")" + sqlite_version="$(node_binary_sqlite_version node)" + ui_error "Alpine apk repositories did not provide Node.js with WAL-reset-safe SQLite; found ${active_version} with SQLite ${sqlite_version} (${active_path})" + echo "Use an official node:${NODE_DEFAULT_MAJOR}-alpine container or a glibc-based host until Alpine ships patched SQLite, then rerun the installer." + exit 1 +} + +install_node_with_user_prefix() { + local cli_installer prefix node_bin_dir + prefix="${HOME}/.openclaw" + node_bin_dir="${prefix}/tools/node/bin" + mktempfile cli_installer + + ui_info "Using a user-space Node.js runtime because the system Node.js links unsafe SQLite" + run_required_step "Downloading user-space Node.js installer" \ + download_validated_script "https://openclaw.ai/install-cli.sh" "$cli_installer" + # The child Bash expands this script's positional arguments, not this shell. + # shellcheck disable=SC2016 + run_required_step "Installing user-space Node.js" \ + env OPENCLAW_INSTALL_CLI_SH_NO_RUN=1 OPENCLAW_PREFIX="$prefix" \ + bash -c ' + set -euo pipefail + source "$1" + install_node "$(os_detect)" "$(arch_detect)" + ' openclaw-install-node "$cli_installer" + + prepend_path_dir "$node_bin_dir" + persist_shell_path_prepend "$node_bin_dir" "\$HOME/.openclaw/tools/node/bin" || true + finish_linux_node_install +} + +# Install Node.js +install_node() { + if [[ "$OS" == "macos" ]]; then + ui_info "Installing Node.js via Homebrew" + if ! run_quiet_step "Installing ${NODE_BREW_FORMULA}" brew install "${NODE_BREW_FORMULA}"; then + echo "Re-run with --verbose or run 'brew install ${NODE_BREW_FORMULA}' directly, then rerun the installer." + exit 1 + fi + brew link "${NODE_BREW_FORMULA}" --overwrite --force 2>/dev/null || true + if ! ensure_macos_default_node_active; then + exit 1 + fi + ui_success "Node.js installed" + print_active_node_paths || true + elif [[ "$OS" == "linux" ]]; then + require_sudo + + ui_info "Installing Linux build tools (make/g++/cmake/python3)" + if install_build_tools_linux; then + ui_success "Build tools installed" + else + ui_warn "Continuing without auto-installing build tools" + fi + + # RPM distributions can link a supported Node release to a vulnerable + # system SQLite. Preserve distro packages and use the managed runtime. + if { command -v dnf &> /dev/null || command -v yum &> /dev/null; } && + node_version_is_supported && ! node_binary_has_safe_sqlite node; then + install_node_with_user_prefix + return 0 + fi + + # Arch-based distros: use pacman with official repos + if command -v pacman &> /dev/null && is_arch_linux; then + ui_info "Installing Node.js via pacman (Arch-based distribution detected)" + if is_root; then + run_required_step "Installing Node.js" pacman -Sy --noconfirm nodejs npm + else + run_required_step "Installing Node.js" sudo pacman -Sy --noconfirm nodejs npm + fi + finish_linux_node_install + return 0 + fi + + if command -v apk &> /dev/null && is_alpine_linux; then + install_node_with_apk + return 0 + fi + + ui_info "Installing Node.js via NodeSource" + if command -v apt-get &> /dev/null; then + local tmp setup_url + setup_url="https://deb.nodesource.com/setup_${NODE_LINUX_DEFAULT_MAJOR}.x" + mktempfile tmp + run_required_step "Downloading NodeSource setup script" download_validated_script "$setup_url" "$tmp" + if is_root; then + run_required_step "Configuring NodeSource repository" bash "$tmp" + run_required_step "Installing Node.js" apt_get_install nodejs + else + run_required_step "Configuring NodeSource repository" sudo -E bash "$tmp" + run_required_step "Installing Node.js" apt_get_install nodejs + fi + elif command -v dnf &> /dev/null; then + local tmp setup_url + setup_url="https://rpm.nodesource.com/setup_${NODE_LINUX_DEFAULT_MAJOR}.x" + mktempfile tmp + run_required_step "Downloading NodeSource setup script" download_validated_script "$setup_url" "$tmp" + if is_root; then + run_required_step "Configuring NodeSource repository" bash "$tmp" + run_required_step "Installing Node.js" dnf install -y -q --disablerepo='*' --enablerepo=nodesource-nodejs nodejs + else + run_required_step "Configuring NodeSource repository" sudo bash "$tmp" + run_required_step "Installing Node.js" sudo dnf install -y -q --disablerepo='*' --enablerepo=nodesource-nodejs nodejs + fi + elif command -v yum &> /dev/null; then + local tmp setup_url + setup_url="https://rpm.nodesource.com/setup_${NODE_LINUX_DEFAULT_MAJOR}.x" + mktempfile tmp + run_required_step "Downloading NodeSource setup script" download_validated_script "$setup_url" "$tmp" + if is_root; then + run_required_step "Configuring NodeSource repository" bash "$tmp" + run_required_step "Installing Node.js" yum install -y -q --disablerepo='*' --enablerepo=nodesource-nodejs nodejs + else + run_required_step "Configuring NodeSource repository" sudo bash "$tmp" + run_required_step "Installing Node.js" sudo yum install -y -q --disablerepo='*' --enablerepo=nodesource-nodejs nodejs + fi + else + ui_error "Could not detect package manager" + echo "Please install Node.js ${NODE_DEFAULT_MAJOR} manually: https://nodejs.org" + exit 1 + fi + + finish_linux_node_install + fi +} + +# Check Git +check_git() { + if command -v git &> /dev/null; then + ui_success "Git already installed" + return 0 + fi + ui_info "Git not found, installing it now" + return 1 +} + +is_root() { + [[ "$(id -u)" -eq 0 ]] +} + +require_sudo() { + if [[ "$OS" != "linux" ]]; then + return 0 + fi + if is_root; then + return 0 + fi + if command -v sudo &> /dev/null; then + if ! sudo -n true >/dev/null 2>&1; then + ui_info "Administrator privileges required; enter your password" + sudo -v + fi + return 0 + fi + ui_error "sudo is required for system installs on Linux" + echo " Install sudo or re-run as root." + exit 1 +} + +install_git() { + if [[ "$OS" == "macos" ]]; then + install_homebrew + run_quiet_step "Installing Git" brew install git + elif [[ "$OS" == "linux" ]]; then + require_sudo + if command -v apk &> /dev/null && is_alpine_linux; then + if is_root; then + run_quiet_step "Installing Git" apk add --no-cache git + else + run_quiet_step "Installing Git" sudo apk add --no-cache git + fi + elif command -v apt-get &> /dev/null; then + run_quiet_step "Updating package index" apt_get_update + run_quiet_step "Installing Git" apt_get_install git + elif command -v pacman &> /dev/null && is_arch_linux; then + if is_root; then + run_quiet_step "Installing Git" pacman -Sy --noconfirm git + else + run_quiet_step "Installing Git" sudo pacman -Sy --noconfirm git + fi + elif command -v dnf &> /dev/null; then + if is_root; then + run_quiet_step "Installing Git" dnf install -y -q git + else + run_quiet_step "Installing Git" sudo dnf install -y -q git + fi + elif command -v yum &> /dev/null; then + if is_root; then + run_quiet_step "Installing Git" yum install -y -q git + else + run_quiet_step "Installing Git" sudo yum install -y -q git + fi + else + ui_error "Could not detect package manager for Git" + exit 1 + fi + fi + ui_success "Git installed" +} + +# Fix npm permissions for global installs (Linux) +fix_npm_permissions() { + if [[ "$OS" != "linux" ]]; then + return 0 + fi + + local npm_prefix + npm_prefix="$(npm config get prefix 2>/dev/null || true)" + if [[ -z "$npm_prefix" ]]; then + return 0 + fi + + if [[ -w "$npm_prefix" || -w "$npm_prefix/lib" ]]; then + return 0 + fi + + if [[ "${NVM_DETECTED:-0}" == "1" ]]; then + # npm's persistent prefix setting makes subsequent nvm use commands fail. + ui_warn "npm global prefix is not writable: ${npm_prefix}; preserving nvm-compatible npm settings" + use_supported_nvm_node || install_node_with_existing_nvm "the active npm prefix is not writable" || return 1 + npm_prefix="$(npm config get prefix 2>/dev/null || true)" + if [[ -n "$npm_prefix" && ( -w "$npm_prefix" || -w "$npm_prefix/lib" ) ]]; then + return 0 + fi + ui_error "The selected nvm runtime still has an unwritable npm prefix (${npm_prefix}); check your npm config before rerunning" + return 1 + fi + + ui_warn "npm global prefix is not writable: ${npm_prefix}" + ui_warn "The installer will switch npm's user prefix to ${HOME}/.npm-global; npm normally writes that setting to ~/.npmrc." + ui_info "Configuring npm for user-local installs" + mkdir -p "$HOME/.npm-global" + npm config set prefix "$HOME/.npm-global" < /dev/null + ui_warn "Avoid sudo npm i -g for future OpenClaw updates; use npm i -g openclaw@latest so npm keeps using this user prefix instead of a different global prefix." + + persist_shell_path_prepend "$HOME/.npm-global/bin" "\$HOME/.npm-global/bin" || true + + export PATH="$HOME/.npm-global/bin:$PATH" + ui_success "npm configured for user installs" +} + +ensure_openclaw_bin_link() { + local npm_root="" + npm_root="$(npm root -g 2>/dev/null || true)" + local launcher="${npm_root}/openclaw/openclaw.mjs" + if [[ -z "$npm_root" || ! -x "$launcher" ]] || ! "$launcher" --version >/dev/null 2>&1; then + return 1 + fi + local npm_bin="" + npm_bin="$(npm_global_bin_dir || true)" + if [[ -z "$npm_bin" ]]; then + return 1 + fi + mkdir -p "$npm_bin" || return 1 + local target="${npm_bin}/openclaw" temp="" + if [[ -e "$target" || -L "$target" ]]; then + is_npm_openclaw_shim "$target" "$launcher" || return 1 + fi + temp="$(mktemp "${npm_bin}/.openclaw-link.XXXXXX")" || return 1 + TMPFILES+=("$temp") + rm -f "$temp" || return 1 + ln -s "$launcher" "$temp" || return 1 + mv -f "$temp" "$target" || return 1 + ui_info "Published openclaw bin link at ${target}" + "$target" --version >/dev/null 2>&1 +} + +# Check for existing OpenClaw installation +check_existing_openclaw() { + if [[ -n "$(type -P openclaw 2>/dev/null || true)" ]]; then + ui_info "Existing OpenClaw installation detected, upgrading" + return 0 + fi + return 1 +} + +set_pnpm_cmd() { + PNPM_CMD=("$@") +} + +pnpm_cmd_pretty() { + if [[ ${#PNPM_CMD[@]} -eq 0 ]]; then + echo "" + return 1 + fi + printf '%s' "${PNPM_CMD[*]}" + return 0 +} + +ensure_pnpm() { + local repo_dir="${1:-$PWD}" + local spec version pnpm_dir corepack_cmd="" npm_cmd lifecycle_arg selected_version + spec="$(repo_pnpm_spec "$repo_dir" || true)" + [[ "$spec" == pnpm@* ]] || spec="pnpm@12.3.4" + version="${spec#pnpm@}" + version="${version%%+*}" + pnpm_dir="$(mktemp -d "${TMPDIR:-/tmp}/openclaw-pnpm.XXXXXX")" || return 1 + TMPFILES+=("$pnpm_dir") + corepack_cmd="$(command -v corepack || true)" + if [[ -n "$corepack_cmd" ]]; then + ui_info "Selecting repo pnpm ${version} via Corepack" + set_pnpm_cmd "$pnpm_dir/pnpm" + if "$corepack_cmd" enable --install-directory "$pnpm_dir" pnpm && + selected_version="$(run_pnpm -C "$repo_dir" --version 2>/dev/null)" && + [[ "$selected_version" == "$version" ]]; then + ui_success "pnpm ready ($(pnpm_cmd_pretty))" + return 0 + fi + ui_warn "Corepack could not provision pnpm; falling back to npm" + fi + + ui_info "Installing pnpm ${version} via npm" + npm_cmd="$(command -v npm)" + lifecycle_arg="$(npm_lifecycle_allow_arg "$npm_cmd" "pnpm@${version}" "$repo_dir" "pnpm@${version}")" || return 1 + # The explicit npm prefix owns this executable; never rediscover ambient pnpm. + "$npm_cmd" install -g --prefix "$pnpm_dir/npm" "pnpm@${version}" ${lifecycle_arg:+"$lifecycle_arg"} || return 1 + set_pnpm_cmd "$pnpm_dir/npm/bin/pnpm" + if [[ ! -x "${PNPM_CMD[0]}" ]] || ! selected_version="$(run_pnpm -C "$repo_dir" --version 2>/dev/null)" || [[ "$selected_version" != "$version" ]]; then + ui_error "Could not provision pnpm ${version} for ${repo_dir}" + return 1 + fi + ui_success "pnpm ready ($(pnpm_cmd_pretty))" +} + +run_pnpm() ( + local repo_dir="$PWD" + if [[ "${1:-}" == "-C" ]]; then + repo_dir="$2" + shift 2 + fi + cd "$repo_dir" || return 1 + # Pin nested commands and inherited roots only for this child. Corepack's + # cold-cache prompt would otherwise wait invisibly in the version probe. + env COREPACK_ENABLE_DOWNLOAD_PROMPT=0 PATH="${PNPM_CMD[0]%/*}:$PATH" \ + NPM_CONFIG_WORKSPACE_DIR="$PWD" npm_config_workspace_dir="$PWD" \ + PNPM_CONFIG_LOCKFILE_DIR="$PWD" pnpm_config_lockfile_dir="$PWD" \ + "${PNPM_CMD[@]}" "$@" +) + +should_prefer_offline_pnpm_install() { + local project_dir="${1:-$PWD}" + [[ -z "${PNPM_CONFIG_PREFER_OFFLINE+x}" && -z "${pnpm_config_prefer_offline+x}" ]] || return 1 + local configured="" + configured="$(run_pnpm -C "$project_dir" config get prefer-offline 2>/dev/null)" || return 1 + [[ -z "$configured" || "$configured" == "undefined" || "$configured" == "null" ]] +} + +resolve_git_openclaw_ref() { + local requested="${OPENCLAW_VERSION:-latest}" + local resolved_version="" + + case "$requested" in + ""|latest) + resolved_version="$(npm view "openclaw" "dist-tags.${requested:-latest}" 2>/dev/null || true)" + if [[ -n "$resolved_version" ]]; then + echo "v${resolved_version}" + return 0 + fi + echo "main" + return 0 + ;; + next|beta) + resolved_version="$(npm view "openclaw" "dist-tags.${requested:-latest}" 2>/dev/null || true)" + if [[ -n "$resolved_version" ]]; then + echo "v${resolved_version}" + return 0 + fi + echo "$requested" + return 0 + ;; + main) + echo "main" + return 0 + ;; + v[0-9]*) + echo "$requested" + return 0 + ;; + [0-9]*.[0-9]*.[0-9]*) + echo "v${requested}" + return 0 + ;; + *) + echo "$requested" + return 0 + ;; + esac +} + +verify_git_rebase_recovery() { + local repo_dir="$1" + local expected_head="$2" + local expected_status="$3" + local git_dir + + git_dir="$(git -C "$repo_dir" rev-parse --absolute-git-dir)" || return 1 + if [[ -d "$git_dir/rebase-merge" || -d "$git_dir/rebase-apply" ]]; then + git -C "$repo_dir" rebase --abort >/dev/null 2>&1 || return 1 + fi + + [[ "$(git -C "$repo_dir" rev-parse --verify HEAD 2>/dev/null)" == "$expected_head" ]] && + [[ "$(git -C "$repo_dir" status --porcelain=v1 --untracked-files=all 2>/dev/null)" == "$expected_status" ]] && + [[ ! -d "$git_dir/rebase-merge" && ! -d "$git_dir/rebase-apply" ]] +} + +checkout_git_openclaw_ref() { + local repo_dir="$1" + local ref="$2" + local original_head="" + local original_status="" + local namespaces=(heads tags) + + GIT_REF_KIND="" + + if [[ -z "$ref" ]]; then + return 0 + fi + + # Full commit IDs pin source bytes, even when a remote ref has the same name. + # Bundled/existing checkouts already have the object and need no remote lookup. + if [[ "$ref" =~ ^[[:xdigit:]]{40}$ ]]; then + if ! git -C "$repo_dir" cat-file -e "$ref" 2>/dev/null; then + if ! run_quiet_step "Fetching requested commit" git -C "$repo_dir" fetch --no-tags origin "$ref"; then + ui_error "Could not fetch requested git commit: ${ref}" + return 1 + fi + fi + if ! git -C "$repo_dir" rev-parse --verify --quiet "${ref}^{commit}" >/dev/null; then + ui_error "Requested git version is not a commit: ${ref}" + return 1 + fi + run_quiet_step "Checking out ${ref}" git -C "$repo_dir" checkout --detach "$ref" + GIT_REF_KIND="immutable" + return 0 + fi + + if [[ "$ref" == "main" ]]; then + run_quiet_step "Fetching requested version" git -C "$repo_dir" fetch --no-tags origin "refs/heads/main:refs/remotes/origin/main" + run_quiet_step "Checking out main" git -C "$repo_dir" checkout main + if [[ "$GIT_UPDATE" == "1" ]]; then + if ! original_head="$(git -C "$repo_dir" rev-parse --verify HEAD 2>/dev/null)"; then + ui_error "Could not record repository state before updating from origin/main" + return 1 + fi + if ! original_status="$(git -C "$repo_dir" status --porcelain=v1 --untracked-files=all 2>/dev/null)"; then + ui_error "Could not record repository state before updating from origin/main" + return 1 + fi + if ! run_quiet_step "Updating repository" git -C "$repo_dir" rebase origin/main; then + if verify_git_rebase_recovery "$repo_dir" "$original_head" "$original_status"; then + ui_error "Could not update repository from origin/main; the checkout was restored to its pre-update state" + else + ui_error "Could not update repository from origin/main; checkout recovery was not verified. Run git -C \"$repo_dir\" rebase --abort and inspect the checkout before retrying" + fi + return 1 + fi + fi + GIT_REF_KIND="moving" + return 0 + fi + + # Normalized release selectors prefer immutable tags. A same-name branch + # remains a fallback for operator-supplied v-prefixed branch names. + if [[ "$ref" == v[0-9]* ]]; then + namespaces=(tags heads) + fi + + local namespace="" + local probe_status=0 + for namespace in "${namespaces[@]}"; do + if git -C "$repo_dir" ls-remote --exit-code origin "refs/${namespace}/${ref}" >/dev/null 2>&1; then + if [[ "$namespace" == "heads" ]]; then + run_quiet_step "Fetching requested version" git -C "$repo_dir" fetch --no-tags origin "refs/heads/${ref}:refs/remotes/origin/${ref}" + run_quiet_step "Checking out ${ref}" git -C "$repo_dir" checkout -B "$ref" "origin/$ref" + GIT_REF_KIND="moving" + else + run_quiet_step "Fetching requested version" git -C "$repo_dir" fetch --no-tags origin "refs/tags/${ref}:refs/tags/${ref}" + if ! git -C "$repo_dir" rev-parse --verify --quiet "refs/tags/${ref}^{commit}" >/dev/null; then + ui_error "Requested git version is not a commit: ${ref}" + return 1 + fi + run_quiet_step "Checking out ${ref}" git -C "$repo_dir" checkout --detach "refs/tags/${ref}" + GIT_REF_KIND="immutable" + fi + return 0 + else + probe_status=$? + fi + if (( probe_status != 2 )); then + ui_error "Could not resolve requested git ref: ${ref}" + return 1 + fi + done + + ui_error "Requested git version not found: ${ref}" + return 1 +} + +git_install_lockfile_flag() { + if [[ "$1" == "moving" ]]; then + echo "--no-frozen-lockfile" + else + echo "--frozen-lockfile" + fi +} + +validate_git_checkout_head() { + local repo_dir="$1" + + if [[ ! -d "$repo_dir/.git" ]]; then + return 0 + fi + if git --git-dir="$repo_dir/.git" --work-tree="$repo_dir" rev-parse --verify --quiet 'HEAD^{commit}' >/dev/null 2>&1; then + return 0 + fi + + ui_error "Git checkout has no commit: ${repo_dir}" + ui_info "Move or remove this incomplete checkout, then retry the installer." + return 1 +} + +clone_git_checkout_transactionally() { + local repo_url="$1" + local repo_dir="$2" + shift 2 + + local parent_dir staging_dir clone_status=0 preserve_repo_dir=0 + parent_dir="$(dirname "$repo_dir")" + mkdir -p "$parent_dir" + parent_dir="$(cd "$parent_dir" && pwd -P)" + if [[ -d "$repo_dir" && -z "$(ls -A "$repo_dir" 2>/dev/null || true)" ]]; then + preserve_repo_dir=1 + repo_dir="$(cd "$repo_dir" && pwd -P)" + staging_dir="$(mktemp -d "${repo_dir}/.openclaw-clone.XXXXXX")" + else + repo_dir="${parent_dir}/$(basename "$repo_dir")" + staging_dir="$(mktemp -d "${parent_dir}/.openclaw-clone.XXXXXX")" + fi + TMPFILES+=("$staging_dir") + + run_quiet_step "Cloning OpenClaw" git clone "$@" "$repo_url" "$staging_dir" || clone_status=$? + if (( clone_status != 0 )); then + return "$clone_status" + fi + + if ! node - "$staging_dir" "$repo_dir" "$preserve_repo_dir" <<'NODE' +const fs = require("node:fs"); +const [source, target, preserveTarget] = process.argv.slice(2); +if (preserveTarget === "0") { + try { + fs.lstatSync(target); + } catch (error) { + if (error?.code !== "ENOENT") throw error; + fs.renameSync(source, target); + process.exit(0); + } + throw new Error(`Git install dir appeared while cloning: ${target}`); +} +const expected = preserveTarget === "1" ? [source.slice(source.lastIndexOf("/") + 1)] : []; +if (!fs.statSync(target).isDirectory() || fs.readdirSync(target).sort().join("\0") !== expected.sort().join("\0")) { + throw new Error(`Git install dir appeared while cloning: ${target}`); +} +const entries = fs.readdirSync(source).sort((a, b) => (a === ".git" ? 1 : b === ".git" ? -1 : 0)); +const moved = []; +try { + for (const entry of entries) { + fs.renameSync(`${source}/${entry}`, `${target}/${entry}`); + moved.push(entry); + } + fs.rmdirSync(source); +} catch (error) { + const rollbackErrors = []; + for (const entry of moved.reverse()) { + try { + fs.renameSync(`${target}/${entry}`, `${source}/${entry}`); + } catch (rollbackError) { + rollbackErrors.push(rollbackError); + } + } + if (rollbackErrors.length > 0) { + let recovery = source; + try { + recovery = `${source}.recovery`; + fs.renameSync(source, recovery); + } catch (recoveryError) { + rollbackErrors.push(recoveryError); + recovery = source; + } + throw new AggregateError( + [error, ...rollbackErrors], + `Could not publish or fully roll back the cloned checkout at ${target}; recovery files remain at ${recovery}`, + ); + } + throw error; +} +NODE + then + ui_error "Could not publish the cloned checkout: ${repo_dir}" + ui_info "Inspect the destination for partial files, move it or choose another --git-dir, then retry." + return 1 + fi +} + +repo_pnpm_spec() { + local repo_dir="$1" + local package_json="${repo_dir}/package.json" + + if [[ ! -f "$package_json" ]]; then + return 1 + fi + + node -e 'const fs = require("node:fs"); const pkg = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); if (typeof pkg.packageManager === "string") process.stdout.write(pkg.packageManager);' "$package_json" +} + + +ensure_user_local_bin_on_path() { + local target="$HOME/.local/bin" + mkdir -p "$target" + + prepend_path_dir "$target" + persist_shell_path_prepend "$target" "\$HOME/.local/bin" || true +} + +npm_global_bin_dir() { + local npm_cmd="${1:-npm}" prefix="" + prefix="$(bounded_probe_output "npm prefix -g" "$npm_cmd" prefix -g || true)" + if [[ -n "$prefix" ]]; then + if [[ "$prefix" == /* ]]; then + echo "${prefix%/}/bin" + return 0 + fi + fi + + prefix="$(bounded_probe_output "npm config get prefix" "$npm_cmd" config get prefix || true)" + if [[ -n "$prefix" && "$prefix" != "undefined" && "$prefix" != "null" ]]; then + if [[ "$prefix" == /* ]]; then + echo "${prefix%/}/bin" + return 0 + fi + fi + + echo "" + return 1 +} + +canonicalize_dir() { + local dir="$1" + if [[ -z "$dir" || ! -d "$dir" ]]; then + return 1 + fi + (cd "$dir" 2>/dev/null && pwd -P) || return 1 +} + +openclaw_package_version() { + local package_json="$1" + if [[ ! -f "$package_json" ]]; then + echo "unknown" + return 0 + fi + + local version="" + if command -v node >/dev/null 2>&1; then + version="$(node -e 'const fs = require("fs"); const pkg = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(String(pkg.version || "unknown"));' "$package_json" 2>/dev/null || true)" + fi + if [[ -z "$version" ]]; then + version="$(sed -n -E 's/^[[:space:]]*"version"[[:space:]]*:[[:space:]]*"([^"]+)".*/\1/p' "$package_json" | head -n1)" + fi + echo "${version:-unknown}" +} + +emit_npm_root_candidate() { + local root="${1%/}" + if [[ -n "$root" && "$root" == /* ]]; then + echo "$root" + fi +} + +collect_openclaw_npm_root_candidates() { + local root="" + root="$(npm root -g 2>/dev/null || true)" + emit_npm_root_candidate "$root" + + local npm_cmd="" + while IFS= read -r npm_cmd; do + [[ -n "$npm_cmd" ]] || continue + root="$("$npm_cmd" root -g 2>/dev/null || true)" + emit_npm_root_candidate "$root" + done < <(type -aP npm 2>/dev/null | awk '!seen[$0]++' || true) + + local extra_root="" + local old_ifs="$IFS" + IFS=":" + for extra_root in ${OPENCLAW_INSTALL_EXTRA_NPM_ROOTS:-}; do + emit_npm_root_candidate "$extra_root" + done + IFS="$old_ifs" + + emit_npm_root_candidate "/opt/homebrew/lib/node_modules" + emit_npm_root_candidate "/usr/local/lib/node_modules" + emit_npm_root_candidate "/usr/lib/node_modules" + + local manager_dir="" + local candidate="" + for manager_dir in "${NVM_DIR:-}" "$HOME/.nvm"; do + [[ -n "$manager_dir" && -d "$manager_dir" ]] || continue + for candidate in "$manager_dir"/versions/node/*/lib/node_modules; do + [[ -d "$candidate" ]] && emit_npm_root_candidate "$candidate" + done + done + + for manager_dir in "${FNM_DIR:-}" "$HOME/.fnm" "$HOME/.local/share/fnm"; do + [[ -n "$manager_dir" && -d "$manager_dir" ]] || continue + for candidate in "$manager_dir"/node-versions/*/installation/lib/node_modules; do + [[ -d "$candidate" ]] && emit_npm_root_candidate "$candidate" + done + done + + for manager_dir in "${VOLTA_HOME:-}" "$HOME/.volta"; do + [[ -n "$manager_dir" && -d "$manager_dir" ]] || continue + for candidate in "$manager_dir"/tools/image/node/*/lib/node_modules; do + [[ -d "$candidate" ]] && emit_npm_root_candidate "$candidate" + done + done +} + +find_openclaw_global_installs() { + local seen="|" + local npm_root="" + while IFS= read -r npm_root; do + [[ -n "$npm_root" ]] || continue + local package_dir="${npm_root%/}/openclaw" + local package_json="${package_dir}/package.json" + [[ -f "$package_json" ]] || continue + + local real_package_dir="" + real_package_dir="$(canonicalize_dir "$package_dir" || true)" + [[ -n "$real_package_dir" ]] || real_package_dir="$package_dir" + case "$seen" in + *"|${real_package_dir}|"*) continue ;; + esac + seen="${seen}${real_package_dir}|" + + local version="" + version="$(openclaw_package_version "$package_json")" + printf '%s\t%s\t%s\n' "$version" "$real_package_dir" "$npm_root" + done < <(collect_openclaw_npm_root_candidates) +} + +warn_duplicate_openclaw_global_installs() { + local installs=() + local line="" + while IFS= read -r line; do + [[ -n "$line" ]] && installs+=("$line") + done < <(find_openclaw_global_installs) + + if [[ "${#installs[@]}" -le 1 ]]; then + return 0 + fi + + ui_warn "Multiple OpenClaw global installs detected" + echo " Different Node/npm environments can run different OpenClaw versions." + + local active_node active_npm active_openclaw + active_node="$(command -v node 2>/dev/null || true)" + active_npm="$(command -v npm 2>/dev/null || true)" + active_openclaw="${OPENCLAW_BIN:-}" + if [[ -z "$active_openclaw" ]]; then + active_openclaw="$(type -P openclaw 2>/dev/null || true)" + fi + echo -e " Active node: ${INFO}${active_node:-none}${NC}" + echo -e " Active npm: ${INFO}${active_npm:-none}${NC}" + echo -e " Active openclaw: ${INFO}${active_openclaw:-none}${NC}" + echo "" + echo " Found installs:" + + local install version package_dir npm_root + for install in "${installs[@]}"; do + IFS=$'\t' read -r version package_dir npm_root <<< "$install" + echo -e " - ${INFO}${version:-unknown}${NC} ${package_dir}" + echo -e " npm root: ${MUTED}${npm_root}${NC}" + done + + echo "" + echo " Keep one install source, then remove stale installs with that environment's npm:" + echo " npm uninstall -g openclaw" +} + +refresh_shell_command_cache() { + hash -r 2>/dev/null || true +} + +path_has_dir() { + local path="$1" + local dir="${2%/}" + if [[ -z "$dir" ]]; then + return 1 + fi + case ":${path}:" in + *":${dir}:"*) return 0 ;; + *) return 1 ;; + esac +} + +warn_shell_path_missing_dir() { + local dir="${1%/}" + local label="$2" + if [[ -z "$dir" ]]; then + return 0 + fi + if path_has_dir "$ORIGINAL_PATH" "$dir"; then + return 0 + fi + + if [[ -n "${NVM_DIR:-}" && "$dir" == "$NVM_DIR"/versions/node/*/bin ]]; then + local version="${dir%/bin}" + version="${version##*/}" + ui_info "OpenClaw was installed under nvm Node.js ${version}" + echo " For this shell and future shells, run: nvm use ${version}" + echo " Shell profiles were not changed." + return 0 + fi + + # persist_shell_path_prepend may already have written the export line; in + # that case new shells are fine and the user only needs to reload this one. + # RC lines may spell the home dir as $HOME instead of the expanded path. + local dir_home_form="\$HOME${dir#"$HOME"}" + local managed_node_bin="$HOME/.openclaw/tools/node/bin" + local managed_node_home_form="\$HOME/.openclaw/tools/node/bin" + if [[ ! -d "$managed_node_bin" || ! -d "$dir" || + "$(canonicalize_dir "$managed_node_bin" || true)" != "$(canonicalize_dir "$dir" || true)" ]]; then + managed_node_bin="" + managed_node_home_form="" + fi + for rc in "$HOME/.bashrc" "$HOME/.bash_profile" "$HOME/.bash_login" "$HOME/.profile" "$HOME/.zshrc" "$HOME/.zprofile" "$HOME/.config/fish/conf.d/openclaw.fish"; do + if [[ -f "$rc" ]] && { + grep -Fq "$dir" "$rc" || grep -Fq "$dir_home_form" "$rc" || + { [[ -n "$managed_node_bin" ]] && { grep -Fq "$managed_node_bin" "$rc" || grep -Fq "$managed_node_home_form" "$rc"; }; } + }; then + echo "" + ui_info "PATH updated in ${rc}: added ${label} (${dir})" + echo " New terminals pick this up automatically." + if [[ "$rc" == *.fish ]]; then + echo " For this shell, run: source ${rc}" + else + echo " For this shell, run: source ${rc}; hash -r" + fi + return 0 + fi + done + + echo "" + ui_warn "PATH missing ${label}: ${dir}" + echo " This can make openclaw show as \"command not found\" in new terminals." + if [[ "${SHELL:-}" == */fish ]]; then + echo " Fix (Fish: ~/.config/fish/conf.d/openclaw.fish):" + echo " fish_add_path -- \"${dir}\"" + else + echo " Fix (zsh: ~/.zshrc, bash: ~/.bashrc):" + echo " export PATH=\"${dir}:\$PATH\"" + fi +} + +openclaw_command_for_user() { + local claw="${1:-}" + if [[ -z "$claw" ]]; then + echo "openclaw" + return 0 + fi + + local original_claw="" + original_claw="$(PATH="$ORIGINAL_PATH" type -P openclaw 2>/dev/null || true)" + if [[ "$original_claw" == "$claw" ]]; then + echo "openclaw" + return 0 + fi + + local quoted_claw="" + printf -v quoted_claw '%q' "$claw" + echo "$quoted_claw" +} + +ensure_npm_global_bin_on_path() { + local bin_dir="" + bin_dir="$(npm_global_bin_dir || true)" + if [[ -n "$bin_dir" ]]; then + export PATH="${bin_dir}:$PATH" + fi +} + +maybe_nodenv_rehash() { + if command -v nodenv &> /dev/null; then + nodenv rehash >/dev/null 2>&1 || true + fi +} + +bounded_probe_output() { + local label="$1" + shift + local timeout_seconds="${OPENCLAW_INSTALL_PROBE_TIMEOUT_SECONDS:-5}" + local output_file status_file timeout_file pid watchdog status + output_file="$(mktemp)" + status_file="$(mktemp)" + timeout_file="$(mktemp)" + TMPFILES+=("$output_file" "$status_file" "$timeout_file") + + ( + "$@" >"$output_file" 2>/dev/null + printf '%s' "$?" >"$status_file" + ) & + pid="$!" + + ( + local sleeper + # Builtin wait lets TERM interrupt the watchdog; a foreground sleep + # would outlive it and hold the caller's command-substitution pipe open. + trap 'exit' TERM + trap ' + for sleeper in $(jobs -p); do + kill "$sleeper" 2>/dev/null || true + wait "$sleeper" 2>/dev/null || true + done + ' EXIT + sleep "$timeout_seconds" & + wait "$!" + if kill -0 "$pid" 2>/dev/null; then + printf '1' >"$timeout_file" + kill "$pid" 2>/dev/null || true + sleep 0.1 & + wait "$!" + kill -9 "$pid" 2>/dev/null || true + printf 'timeout' >"$status_file" + fi + ) & + watchdog="$!" + + wait "$pid" 2>/dev/null || true + kill "$watchdog" 2>/dev/null || true + wait "$watchdog" 2>/dev/null || true + + status="$(cat "$status_file" 2>/dev/null || true)" + if [[ -s "$timeout_file" || "$status" == "timeout" ]]; then + echo "Warning: timed out during installer finalization probe: ${label}" >&2 + return 124 + fi + + cat "$output_file" 2>/dev/null || true + if [[ -n "$status" && "$status" =~ ^[0-9]+$ ]]; then + return "$status" + fi + return 1 +} + +warn_openclaw_not_found() { + ui_warn "Installed, but openclaw is not discoverable on PATH in this shell" + echo " Try: hash -r (bash) or rehash (zsh), then retry." + local t="" + t="$(type -t openclaw 2>/dev/null || true)" + if [[ "$t" == "alias" || "$t" == "function" ]]; then + ui_warn "Found a shell ${t} named openclaw; it may shadow the real binary" + fi + if command -v nodenv &> /dev/null; then + echo -e "Using nodenv? Run: ${INFO}nodenv rehash${NC}" + fi + + local npm_prefix="" + npm_prefix="$(bounded_probe_output "npm prefix -g" npm prefix -g || true)" + local npm_bin="" + npm_bin="$(npm_global_bin_dir 2>/dev/null || true)" + if [[ -n "$npm_prefix" ]]; then + echo -e "npm prefix -g: ${INFO}${npm_prefix}${NC}" + fi + if [[ -n "$npm_bin" ]]; then + echo -e "npm bin -g: ${INFO}${npm_bin}${NC}" + echo -e "If needed: ${INFO}export PATH=\"${npm_bin}:\\$PATH\"${NC}" + fi +} + +resolve_openclaw_bin() { + refresh_shell_command_cache + local resolved="" + resolved="$(type -P openclaw 2>/dev/null || true)" + if [[ -n "$resolved" && -x "$resolved" ]]; then + echo "$resolved" + return 0 + fi + + ensure_npm_global_bin_on_path + refresh_shell_command_cache + resolved="$(type -P openclaw 2>/dev/null || true)" + if [[ -n "$resolved" && -x "$resolved" ]]; then + echo "$resolved" + return 0 + fi + + local npm_bin="" + npm_bin="$(npm_global_bin_dir || true)" + if [[ -n "$npm_bin" && -x "${npm_bin}/openclaw" ]]; then + echo "${npm_bin}/openclaw" + return 0 + fi + + maybe_nodenv_rehash + refresh_shell_command_cache + resolved="$(type -P openclaw 2>/dev/null || true)" + if [[ -n "$resolved" && -x "$resolved" ]]; then + echo "$resolved" + return 0 + fi + + if [[ -n "$npm_bin" && -x "${npm_bin}/openclaw" ]]; then + echo "${npm_bin}/openclaw" + return 0 + fi + + echo "" + return 1 +} + +resolve_installed_openclaw_bin() { + local installed_bin="" + if [[ "$INSTALL_METHOD" == "git" ]]; then + installed_bin="$HOME/.local/bin/openclaw" + elif [[ "$INSTALL_METHOD" == "npm" ]]; then + local npm_bin="" + npm_bin="$(npm_global_bin_dir || true)" + if [[ -n "$npm_bin" ]]; then + installed_bin="${npm_bin}/openclaw" + fi + fi + + if [[ -n "$installed_bin" && -x "$installed_bin" ]]; then + echo "$installed_bin" + return 0 + fi + resolve_openclaw_bin +} + +publish_executable_wrapper() { + local target="$1" target_dir="" temp="" + target_dir="${target%/*}" + mkdir -p "$target_dir" + temp="$(mktemp "${target_dir}/.openclaw-wrapper.XXXXXX")" || return 1 + TMPFILES+=("$temp") + cat > "$temp" + chmod +x "$temp" + mv -f "$temp" "$target" +} + +install_openclaw_from_git() { + local repo_dir="$1" + local repo_url="https://github.com/openclaw/openclaw.git" + + mkdir -p "$(dirname "$repo_dir")" + if [[ -d "$repo_dir" ]]; then + repo_dir="$(cd "$repo_dir" && pwd -P)" + else + repo_dir="$(cd "$(dirname "$repo_dir")" && pwd -P)/$(basename "$repo_dir")" + fi + + if [[ -d "$repo_dir/.git" ]]; then + ui_info "Installing OpenClaw from git checkout: ${repo_dir}" + else + ui_info "Installing OpenClaw from GitHub (${repo_url})" + fi + + if ! check_git; then + install_git + fi + + validate_git_checkout_head "$repo_dir" || return 1 + if [[ ! -d "$repo_dir" || -z "$(ls -A "$repo_dir" 2>/dev/null || true)" ]]; then + # Blobless clone: the installer checks out one release tag, so full blob + # history is downloaded and then discarded. blob:none keeps ref metadata + # (unlike --depth 1) so ref switching and later updates still work, and + # git warns and falls back to a full clone if the server cannot filter. + clone_git_checkout_transactionally "$repo_url" "$repo_dir" --filter=blob:none + fi + + local git_ref + git_ref="$(resolve_git_openclaw_ref)" + if [[ -z "$(git -C "$repo_dir" status --porcelain 2>/dev/null || true)" ]]; then + ui_info "Using git ref: ${git_ref}" + checkout_git_openclaw_ref "$repo_dir" "$git_ref" + else + ui_info "Repo has local changes; skipping git checkout/update" + if git -C "$repo_dir" symbolic-ref --quiet HEAD >/dev/null; then + GIT_REF_KIND="moving" + else + GIT_REF_KIND="immutable" + fi + fi + + cleanup_legacy_submodules "$repo_dir" + ensure_pnpm "$repo_dir" + + local install_lockfile_flag + install_lockfile_flag="$(git_install_lockfile_flag "$GIT_REF_KIND")" + local -a pnpm_prefer_offline_args=() + if should_prefer_offline_pnpm_install "$repo_dir"; then + pnpm_prefer_offline_args=(--prefer-offline) + fi + CI="${CI:-true}" run_quiet_step "Installing dependencies" run_pnpm -C "$repo_dir" install ${pnpm_prefer_offline_args[@]+"${pnpm_prefer_offline_args[@]}"} "$install_lockfile_flag" + + if ! run_quiet_step "Building UI" run_pnpm -C "$repo_dir" ui:build; then + ui_warn "UI build failed; continuing (CLI may still work)" + fi + run_quiet_step "Building OpenClaw" run_pnpm -C "$repo_dir" build + + ensure_user_local_bin_on_path + + local node_bin="" node_bin_quoted="" entry_path_quoted="" + node_bin="$(type -P node 2>/dev/null || true)" + if [[ -n "$node_bin" && "$node_bin" != /* ]]; then + local node_dir="" + node_dir="$(cd "$(dirname "$node_bin")" && pwd -P 2>/dev/null)" || node_dir="" + if [[ -n "$node_dir" ]]; then + node_bin="${node_dir}/$(basename "$node_bin")" + fi + fi + if [[ -z "$node_bin" || ! -x "$node_bin" ]]; then + ui_error "Node.js runtime not found after build" + return 1 + fi + if ! "$node_bin" "${repo_dir}/dist/entry.js" --version >/dev/null 2>&1; then + ui_error "Git replacement failed CLI verification" + return 1 + fi + printf -v node_bin_quoted "%q" "$node_bin" + printf -v entry_path_quoted "%q" "${repo_dir}/dist/entry.js" + + publish_executable_wrapper "$HOME/.local/bin/openclaw" <<EOF +#!/usr/bin/env bash +set -euo pipefail +exec ${node_bin_quoted} ${entry_path_quoted} "\$@" +EOF + ui_success "OpenClaw wrapper installed to \$HOME/.local/bin/openclaw" + ui_info "Manual builds need the checkout-pinned pnpm launcher; installer bootstrap is temporary: https://docs.openclaw.ai/install/installer#source-build-toolchain" +} + +# Install OpenClaw +resolve_beta_version() { + local beta="" + beta="$(npm view openclaw dist-tags.beta 2>/dev/null || true)" + if [[ -z "$beta" || "$beta" == "undefined" || "$beta" == "null" ]]; then + return 1 + fi + echo "$beta" +} + +to_lowercase_ascii() { + # macOS still ships Bash 3.2, so avoid `${value,,}` here. + printf '%s' "${1:-}" | tr '[:upper:]' '[:lower:]' +} + +is_explicit_package_install_spec() { + local value="${1:-}" + [[ "$value" == *"://"* || "$value" == *"#"* || "$value" == /* || "$value" == ./* || "$value" == ../* || "$value" =~ \.(tgz|tar\.gz)$ || "$value" =~ ^(file|github|git\+ssh|git\+https|git\+http|git\+file|npm): ]] +} + +is_openclaw_source_package_install_spec() { + local value="${1:-}" + local normalized_value="" + normalized_value="$(to_lowercase_ascii "$value")" + normalized_value="${normalized_value#openclaw@}" + + [[ "$normalized_value" == "main" ]] && return 0 + [[ "$normalized_value" =~ ^github:openclaw/openclaw($|[#/]) ]] && return 0 + + normalized_value="${normalized_value#git+}" + [[ "$normalized_value" =~ ^https?://github\.com/openclaw/openclaw(\.git)?($|[?#]) ]] && return 0 + [[ "$normalized_value" =~ ^ssh://git@github\.com[:/]openclaw/openclaw(\.git)?($|[?#]) ]] && return 0 + [[ "$normalized_value" =~ ^git://github\.com/openclaw/openclaw(\.git)?($|[?#]) ]] && return 0 + [[ "$normalized_value" =~ ^git@github\.com:openclaw/openclaw(\.git)?($|[?#]) ]] && return 0 + return 1 +} + +can_resolve_registry_package_version() { + local value="${1:-}" + local normalized_value="" + normalized_value="$(to_lowercase_ascii "$value")" + if [[ -z "$value" ]]; then + return 0 + fi + if [[ "$normalized_value" == "main" ]]; then + return 1 + fi + if is_explicit_package_install_spec "$value"; then + return 1 + fi + return 0 +} + +resolve_package_install_spec() { + local package_name="$1" + local value="$2" + local normalized_value="" + normalized_value="$(to_lowercase_ascii "$value")" + if [[ "$normalized_value" == "main" ]]; then + echo "github:openclaw/openclaw#main" + return 0 + fi + if is_explicit_package_install_spec "$value"; then + echo "$value" + return 0 + fi + if [[ "$value" == "latest" ]]; then + echo "${package_name}@latest" + return 0 + fi + echo "${package_name}@${value}" +} + +install_openclaw() { + local package_name="openclaw" + if [[ "$USE_BETA" == "1" ]]; then + local beta_version="" + beta_version="$(resolve_beta_version || true)" + if [[ -n "$beta_version" ]]; then + OPENCLAW_VERSION="$beta_version" + ui_info "Beta tag detected (${beta_version})" + package_name="openclaw" + else + OPENCLAW_VERSION="latest" + ui_info "No beta tag found; using latest" + fi + fi + + if [[ -z "${OPENCLAW_VERSION}" ]]; then + OPENCLAW_VERSION="latest" + fi + + if is_openclaw_source_package_install_spec "${OPENCLAW_VERSION}"; then + ui_error "npm installs do not support OpenClaw GitHub source targets like '${OPENCLAW_VERSION}'." + ui_info "Use --install-method git --version main for the moving main checkout, or use latest, beta, an exact version, or a built .tgz package." + return 1 + fi + + local resolved_version="" + if can_resolve_registry_package_version "${OPENCLAW_VERSION}"; then + resolved_version="$(npm view "${package_name}@${OPENCLAW_VERSION}" version 2>/dev/null || true)" + fi + if [[ -n "$resolved_version" ]]; then + ui_info "Installing OpenClaw v${resolved_version}" + else + ui_info "Installing OpenClaw (${OPENCLAW_VERSION})" + fi + local install_spec="" + install_spec="$(resolve_package_install_spec "${package_name}" "${OPENCLAW_VERSION}")" + + if ! install_openclaw_npm "${install_spec}" || ! ensure_openclaw_bin_link; then + ui_warn "npm install did not produce a usable OpenClaw package; retrying" + if ! install_openclaw_npm "${install_spec}" || ! ensure_openclaw_bin_link; then + ui_error "npm install did not produce a usable OpenClaw package" + restore_openclaw_bin_backup || ui_error "Could not restore the previous openclaw command" + return 1 + fi + fi + +} + +# Run doctor for migrations (safe, non-interactive) +run_doctor() { + ui_info "Running doctor to migrate settings" + local claw="${OPENCLAW_BIN:-}" + if [[ -z "$claw" ]]; then + claw="$(resolve_openclaw_bin || true)" + fi + if [[ -z "$claw" ]]; then + ui_info "Skipping doctor (openclaw not on PATH yet)" + warn_openclaw_not_found + return 0 + fi + local doctor_exit=0 + run_quiet_step "Running doctor" "$claw" doctor --fix --non-interactive || doctor_exit=$? + if (( doctor_exit == 130 )); then + abort_install_int + fi + if (( doctor_exit != 0 )); then + return "$doctor_exit" + fi + ui_success "Doctor complete" +} + +maybe_open_dashboard() { + local claw="${OPENCLAW_BIN:-}" + if [[ -z "$claw" ]]; then + claw="$(resolve_openclaw_bin || true)" + fi + if [[ -z "$claw" ]]; then + return 0 + fi + if ! "$claw" dashboard --help >/dev/null 2>&1; then + return 0 + fi + run_with_safe_stdin "$claw" dashboard || true +} + +has_openclaw_config() { + local effective_home + effective_home="$(resolve_openclaw_effective_home)" + if [[ -n "${OPENCLAW_CONFIG_PATH:-}" ]]; then + local config_path + config_path="$(resolve_openclaw_user_path "$OPENCLAW_CONFIG_PATH")" + [[ -f "$config_path" ]] + return + fi + + if [[ -n "${OPENCLAW_STATE_DIR:-}" ]]; then + local state_dir + state_dir="$(resolve_openclaw_user_path "$OPENCLAW_STATE_DIR")" + if [[ -f "$state_dir/openclaw.json" || -f "$state_dir/clawdbot.json" ]]; then + return 0 + fi + return 1 + fi + + if [[ -f "$effective_home/.openclaw/openclaw.json" || + -f "$effective_home/.openclaw/clawdbot.json" || + -f "$effective_home/.clawdbot/openclaw.json" || + -f "$effective_home/.clawdbot/clawdbot.json" ]]; then + return 0 + fi + return 1 +} + +load_install_version_helpers() { + local source_path="${BASH_SOURCE[0]-}" + local script_dir="" + local helper_path="" + if [[ -z "$source_path" || ! -f "$source_path" ]]; then + return 0 + fi + if script_dir="$(cd "$(dirname "$source_path")" && pwd 2>/dev/null)"; then + : + else + script_dir="" + fi + helper_path="${script_dir}/docker/install-sh-common/version-parse.sh" + if [[ -n "$script_dir" && -r "$helper_path" ]]; then + # shellcheck source=docker/install-sh-common/version-parse.sh + # shellcheck disable=SC1091 + source "$helper_path" + fi +} + +load_install_version_helpers + +if ! declare -F extract_openclaw_semver >/dev/null 2>&1; then +# Inline fallback when version-parse.sh could not be sourced (for example, stdin install). +extract_openclaw_semver() { + local raw="${1:-}" + raw="${raw//$'\r'/}" + if [[ "$raw" =~ v?([0-9]+\.[0-9]+\.[0-9]+([.-][0-9A-Za-z]+(\.[0-9A-Za-z]+)*)?(\+[0-9A-Za-z.-]+)?) ]]; then + printf '%s' "${BASH_REMATCH[1]}" + fi +} +fi + +resolve_openclaw_version() { + local version="" + local raw_version_output="" + local claw="${OPENCLAW_BIN:-}" + if [[ -z "$claw" ]] && command -v openclaw &> /dev/null; then + claw="$(command -v openclaw)" + fi + if [[ -n "$claw" ]]; then + raw_version_output=$("$claw" --version 2>/dev/null || true) + raw_version_output="${raw_version_output%%$'\n'*}" + raw_version_output="${raw_version_output//$'\r'/}" + version="$(extract_openclaw_semver "$raw_version_output")" + if [[ -z "$version" ]]; then + version="$raw_version_output" + fi + fi + if [[ -z "$version" ]]; then + local npm_root="" + npm_root=$(npm root -g 2>/dev/null || true) + if [[ -n "$npm_root" && -f "$npm_root/openclaw/package.json" ]]; then + version=$(node -e "console.log(require('${npm_root}/openclaw/package.json').version)" 2>/dev/null || true) + fi + fi + echo "$version" +} + +is_gateway_daemon_loaded() { + local claw="$1" + if [[ -z "$claw" ]]; then + return 1 + fi + + local status_json="" + status_json="$(bounded_probe_output "openclaw daemon status --json" "$claw" daemon status --json || true)" + if [[ -z "$status_json" ]]; then + return 1 + fi + + printf '%s' "$status_json" | node -e ' +const fs = require("fs"); +const raw = fs.readFileSync(0, "utf8").trim(); +if (!raw) process.exit(1); +try { + const data = JSON.parse(raw); + process.exit(data?.service?.loaded ? 0 : 1); +} catch { + process.exit(1); +} +' >/dev/null 2>&1 +} + +refresh_gateway_service_if_loaded() { + local claw="${OPENCLAW_BIN:-}" refresh_output + if [[ -z "$claw" ]]; then + claw="$(resolve_openclaw_bin || true)" + fi + if [[ -z "$claw" ]]; then + return 0 + fi + + if ! is_gateway_daemon_loaded "$claw"; then + return 0 + fi + + ui_info "Refreshing loaded gateway service" + if ! refresh_output="$({ set +x; "$claw" gateway install --force; } 2>&1 | sed -n -e 's/^Replacing unsupported Gateway service Node .*; refreshing the install\.$/node-runtime-replaced/p' -e 's/^Replacing missing Gateway service Node .*; refreshing the install\.$/node-runtime-replaced/p' -e 's/.*SERVICE_DEFINITION_SEALED:.*/ask the privileged deployment owner to manually repair it/p' -e 's/.*SERVICE_DEFINITION_UNKNOWN:.*/inspect service-definition access and manually repair it/p')"; then + refresh_output="$(printf '%s\n' "$refresh_output" | sed '/^node-runtime-replaced$/d')" + if [[ -n "$refresh_output" ]]; then + ui_warn "Code installed; gateway service definition left unchanged; ${refresh_output}" + ui_info "Run openclaw gateway status --deep, verify the installation owner, and restart it manually if needed." + return 0 + else + ui_warn "Gateway service refresh failed; continuing" + return 0 + fi + else + if [[ "$refresh_output" == *node-runtime-replaced* ]]; then + ui_success "Gateway service Node runtime replaced" + fi + ui_success "Gateway service metadata refreshed" + fi + + # `gateway install --force` activates the replacement service. Keep the + # explicit lifecycle restart in the finalization phase so doctor/plugin + # changes can still be applied without restarting twice here. + run_quiet_step "Probing gateway service" "$claw" gateway status --deep || true +} + +verify_installation() { + if [[ "${VERIFY_INSTALL}" != "1" ]]; then + return 0 + fi + local verify_gateway="${1:-true}" + + ui_stage "Verifying installation" + local claw="${OPENCLAW_BIN:-}" + if [[ -z "$claw" ]]; then + claw="$(resolve_openclaw_bin || true)" + fi + if [[ -z "$claw" ]]; then + ui_error "Install verify failed: openclaw not on PATH yet" + warn_openclaw_not_found + return 1 + fi + + run_quiet_step "Checking OpenClaw version" "$claw" --version || return 1 + + if [[ "$verify_gateway" != "true" ]]; then + ui_info "Setup not complete; skipping gateway service check" + elif is_gateway_daemon_loaded "$claw"; then + run_quiet_step "Checking gateway service" "$claw" gateway status --deep || { + local user_claw + user_claw="$(openclaw_command_for_user "$claw")" + ui_error "Install verify failed: gateway service unhealthy" + ui_info "Run: ${user_claw} gateway status --deep" + return 1 + } + else + ui_info "Gateway service not loaded; skipping gateway deep probe" + fi + + ui_success "Install verify complete" +} + +retire_npm_owner_after_git_install() { + local wrapper="$HOME/.local/bin/openclaw" npm_cmd="" npm_root="" npm_bin="" package_root="" package_name="" + if ! npm_cmd="$(npm_command_path npm)"; then + ui_error "Could not retire the previous npm install: npm not found on PATH" + return 1 + fi + npm_root="$("$npm_cmd" root -g 2>/dev/null | awk 'NF { value = $0 } END { print value }')" || true + package_root="${npm_root%/}/openclaw" + [[ -n "$npm_root" && -f "$package_root/package.json" ]] || return 0 + package_name="$(node -e 'const p=require(process.argv[1]); process.stdout.write(String(p.name || ""))' "$package_root/package.json" 2>/dev/null || true)" + if [[ "$package_name" != "openclaw" ]]; then + ui_error "Could not retire the previous npm install: ${package_root} contains package '${package_name:-unknown}', not openclaw" + return 1 + fi + npm_bin="$(npm_global_bin_dir "$npm_cmd" || true)" + if [[ "${npm_bin%/}/openclaw" == "$wrapper" ]]; then + if ! rm -rf "$package_root"; then + ui_error "Could not retire the previous npm install: failed to remove ${package_root}" + return 1 + fi + else + if ! "$npm_cmd" uninstall -g openclaw >/dev/null 2>&1; then + ui_error "Could not retire the previous npm install: npm uninstall -g openclaw failed" + return 1 + fi + fi + ui_success "Previous npm install retired" +} + +is_installer_git_wrapper() { + local wrapper="${1:-$HOME/.local/bin/openclaw}" first="" second="" third="" fourth="" + [[ -f "$wrapper" && ! -L "$wrapper" ]] || return 1 + IFS= read -r first < "$wrapper" || return 1 + second="$(sed -n '2p' "$wrapper")"; third="$(sed -n '3p' "$wrapper")"; fourth="$(sed -n '4p' "$wrapper")" + [[ "$first" == "#!/usr/bin/env bash" && "$second" == "set -euo pipefail" && -z "$fourth" ]] || return 1 + case "$third" in "exec "*"/dist/entry.js \"\$@\"") return 0 ;; *) return 1 ;; esac +} + +prepare_git_wrapper_backup_for_npm() { + local npm_cmd="" npm_root="" npm_bin="" target="" launcher="" + # Without a resolvable npm there is nothing to back up; let the npm + # install step report the missing npm with its own remediation text + # instead of silently exiting here (Arch splits node and npm packages). + npm_cmd="$(npm_command_path npm)" || return 0 + npm_root="$("$npm_cmd" root -g 2>/dev/null || true)" + npm_bin="$(npm_global_bin_dir "$npm_cmd" || true)" + [[ -n "$npm_root" && -n "$npm_bin" ]] || return 0 + target="${npm_bin%/}/openclaw" + is_installer_git_wrapper "$target" || return 0 + launcher="${npm_root%/}/openclaw/openclaw.mjs" + begin_openclaw_bin_backup "$target" "$launcher" 1 +} + +retire_git_wrapper_after_npm_install() { + local wrapper="$HOME/.local/bin/openclaw" + is_installer_git_wrapper "$wrapper" || return 0 + if ! rm -f "$wrapper"; then + ui_error "Could not retire the previous git wrapper: failed to remove ${wrapper}" + return 1 + fi + ui_success "Previous git wrapper retired" +} + +# Main installation flow +main() { + if [[ "$HELP" == "1" ]]; then + print_usage + return 0 + fi + + # A dry run must stay side-effect free; gum bootstrap may download binaries. + if [[ "$DRY_RUN" != "1" ]]; then + echo -e "${INFO}Preparing installer interface...${NC}" + bootstrap_gum_temp || true + fi + print_installer_banner + if [[ "$DRY_RUN" != "1" ]]; then + print_gum_status + fi + detect_os_or_die + + if [[ "$OS" == "linux" ]]; then + export DEBIAN_FRONTEND="${DEBIAN_FRONTEND:-noninteractive}" + export NEEDRESTART_MODE="${NEEDRESTART_MODE:-a}" + fi + + local detected_checkout="" + detected_checkout="$(detect_openclaw_checkout "$PWD" || true)" + + if [[ -z "$INSTALL_METHOD" && -n "$detected_checkout" ]]; then + if ! is_promptable; then + ui_info "Found OpenClaw checkout but no TTY; defaulting to npm install" + INSTALL_METHOD="npm" + else + local selected_method="" + selected_method="$(choose_install_method_interactive "$detected_checkout" || true)" + case "$selected_method" in + git|npm) + INSTALL_METHOD="$selected_method" + ;; + *) + ui_error "no install method selected" + echo "Re-run with: --install-method git|npm (or set OPENCLAW_INSTALL_METHOD)." + exit 2 + ;; + esac + fi + fi + + if [[ -z "$INSTALL_METHOD" ]]; then + INSTALL_METHOD="npm" + fi + + if [[ "$INSTALL_METHOD" != "npm" && "$INSTALL_METHOD" != "git" ]]; then + ui_error "invalid --install-method: ${INSTALL_METHOD}" + echo "Use: --install-method npm|git" + exit 2 + fi + + show_install_plan "$detected_checkout" + + if [[ "$DRY_RUN" == "1" ]]; then + ui_success "Dry run complete (no changes made)" + return 0 + fi + + # Check for existing installation + local is_upgrade=false + if check_existing_openclaw; then + is_upgrade=true + VERIFY_INSTALL=1 + fi + configure_install_stage_total + local should_open_dashboard=false + + ui_stage "Preparing environment" + + # Step 1: Node.js. macOS package-manager branches install Homebrew lazily + # only when they are about to call brew. + load_nvm_for_node_detection || exit 1 + if ! node_is_supported; then + use_supported_nvm_node || activate_supported_node_on_path || true + fi + if ! check_node; then + if [[ "${NVM_DETECTED:-0}" == "1" ]]; then + install_node_with_existing_nvm || exit 1 + else + install_homebrew + install_node + fi + fi + if ! ensure_default_node_active_shell; then + exit 1 + fi + + ui_stage "Installing OpenClaw" + + local final_git_dir="" + if [[ "$INSTALL_METHOD" == "git" ]]; then + local had_npm_owner=false + if npm list -g openclaw &>/dev/null; then + had_npm_owner=true + fi + + final_git_dir="$GIT_DIR" + if [[ -z "$GIT_DIR_EXPLICIT" && -n "$detected_checkout" ]]; then + final_git_dir="$detected_checkout" + fi + install_openclaw_from_git "$final_git_dir" + if [[ "$had_npm_owner" == "true" ]]; then + retire_npm_owner_after_git_install || return $? + fi + else + # Step 3: Git (required for npm installs that may fetch from git or apply patches) + if ! check_git; then + install_git + fi + + # Step 4: npm permissions (Linux) + fix_npm_permissions || exit 1 + + # Step 5: OpenClaw + prepare_git_wrapper_backup_for_npm || return $? + install_openclaw + local npm_candidate="" + npm_candidate="$(resolve_installed_openclaw_bin || true)" + if [[ -z "$npm_candidate" ]] || ! "$npm_candidate" --version >/dev/null 2>&1; then + ui_error "npm replacement failed verification" + restore_openclaw_bin_backup || ui_error "Could not restore the previous openclaw command" + return 1 + fi + if ! commit_openclaw_bin_backup; then + restore_openclaw_bin_backup || ui_error "Could not restore the previous openclaw command" + return 1 + fi + ui_success "OpenClaw installed" + retire_git_wrapper_after_npm_install || return $? + fi + + ui_stage "Finalizing setup" + + OPENCLAW_BIN="$(resolve_installed_openclaw_bin || true)" + warn_duplicate_openclaw_global_installs || true + + # PATH warning: installs can succeed while the user's login shell still lacks npm's global bin dir. + local npm_bin="" + npm_bin="$(npm_global_bin_dir || true)" + if [[ "$INSTALL_METHOD" == "npm" ]]; then + warn_shell_path_missing_dir "$npm_bin" "npm global bin dir" + fi + if [[ "$INSTALL_METHOD" == "git" ]]; then + if [[ -x "$HOME/.local/bin/openclaw" ]]; then + warn_shell_path_missing_dir "$HOME/.local/bin" "user-local bin dir (~/.local/bin)" + fi + fi + + local config_present=false defer_success=false + if has_openclaw_config; then + config_present=true + refresh_gateway_service_if_loaded + fi + + if [[ "$is_upgrade" == "true" || "$config_present" == "true" || "$VERIFY_INSTALL" == "1" ]]; then + defer_success=true + fi + + if [[ "$config_present" == "true" && "$is_upgrade" == "true" ]]; then + if has_controlling_tty || [[ "$NO_ONBOARD" == "1" || "$NO_PROMPT" == "1" ]]; then + local claw="${OPENCLAW_BIN:-}" + if [[ -z "$claw" ]]; then + claw="$(resolve_installed_openclaw_bin || true)" + fi + if [[ -z "$claw" ]]; then + ui_info "Skipping doctor (openclaw not on PATH yet)" + warn_openclaw_not_found + return 0 + fi + local -a doctor_args=("--fix") + if [[ "$NO_ONBOARD" == "1" || "$NO_PROMPT" == "1" ]]; then + doctor_args+=("--non-interactive") + fi + ui_info "Running openclaw doctor" + local doctor_exit=0 + if [[ "$NO_ONBOARD" == "1" || "$NO_PROMPT" == "1" ]]; then + OPENCLAW_UPDATE_IN_PROGRESS=1 "$claw" doctor "${doctor_args[@]}" </dev/null || doctor_exit=$? + else + OPENCLAW_UPDATE_IN_PROGRESS=1 "$claw" doctor "${doctor_args[@]}" </dev/tty || doctor_exit=$? + fi + if (( doctor_exit == 130 )); then + abort_install_int + fi + if (( doctor_exit != 0 )); then + ui_warn "Doctor failed; skipping plugin updates" + return "$doctor_exit" + fi + should_open_dashboard=true + ui_info "Updating plugins" + OPENCLAW_UPDATE_IN_PROGRESS=1 run_with_safe_stdin "$claw" plugins update --all || true + else + run_doctor || return $? + should_open_dashboard=true + local user_claw + user_claw="$(openclaw_command_for_user "${OPENCLAW_BIN:-}")" + ui_info "No TTY; run ${user_claw} plugins update --all manually" + fi + elif [[ "$config_present" == "true" ]]; then + ui_info "Config already present; running doctor" + run_doctor || return $? + should_open_dashboard=true + ui_info "Config already present; skipping onboarding" + fi + + if [[ "$config_present" == "true" ]]; then + local claw="${OPENCLAW_BIN:-}" + if [[ -z "$claw" ]]; then + claw="$(resolve_installed_openclaw_bin || true)" + fi + if [[ -n "$claw" ]] && is_gateway_daemon_loaded "$claw"; then + local user_claw + user_claw="$(openclaw_command_for_user "$claw")" + if [[ "$DRY_RUN" == "1" ]]; then + ui_info "Gateway daemon detected; would restart (${user_claw} daemon restart)" + else + ui_info "Gateway daemon detected; restarting" + if OPENCLAW_UPDATE_IN_PROGRESS=1 "$claw" daemon restart < /dev/null >/dev/null 2>&1; then + ui_success "Gateway restarted" + else + ui_warn "Gateway restart failed; try: ${user_claw} daemon restart" + fi + fi + fi + fi + + if [[ "$defer_success" == "true" ]] && ! verify_installation "$config_present"; then + if [[ "$config_present" != "true" && "$NO_ONBOARD" != "1" ]] && ! is_promptable; then + local user_claw + user_claw="$(openclaw_command_for_user "${OPENCLAW_BIN:-}")" + ui_info "No TTY; run ${user_claw} onboard to finish setup" + fi + return 1 + fi + + local installed_version="" + installed_version="$(resolve_openclaw_version)" + echo "" + if [[ -n "$installed_version" ]]; then + ui_celebrate "🦞 OpenClaw installed successfully (${installed_version})!" + else + ui_celebrate "🦞 OpenClaw installed successfully!" + fi + if [[ "$is_upgrade" == "true" ]]; then + ui_info "Upgrade complete" + else + local completion_messages=( + "Ahh nice, I like it here. Got any snacks? " + "Home sweet home. Don't worry, I won't rearrange the furniture." + "I'm in. Let's cause some responsible chaos." + "Installation complete. Your productivity is about to get weird." + "Settled in. Time to automate your life whether you're ready or not." + "Cozy. I've already read your calendar. We need to talk." + "Finally unpacked. Now point me at your problems." + "cracks claws Alright, what are we building?" + "The lobster has landed. Your terminal will never be the same." + "All done! I promise to only judge your code a little bit." + ) + local completion_message + completion_message="${completion_messages[RANDOM % ${#completion_messages[@]}]}" + echo -e "${MUTED}${completion_message}${NC}" + echo "" + fi + + if [[ "$INSTALL_METHOD" == "git" && -n "$final_git_dir" ]]; then + local user_claw + user_claw="$(openclaw_command_for_user "${OPENCLAW_BIN:-}")" + ui_section "Source install details" + ui_kv "Checkout" "$final_git_dir" + ui_kv "Wrapper" "$HOME/.local/bin/openclaw" + ui_kv "Update command" "${user_claw} update" + ui_kv "Switch to npm" "curl -fsSL --proto '=https' --tlsv1.2 https://openclaw.ai/install.sh | bash -s -- --install-method npm" + fi + + if [[ "$config_present" != "true" ]]; then + if [[ "$NO_ONBOARD" == "1" ]]; then + local user_claw + user_claw="$(openclaw_command_for_user "${OPENCLAW_BIN:-}")" + ui_info "Skipping onboard (requested); run ${user_claw} onboard later" + else + ui_info "Starting setup" + echo "" + if is_promptable; then + local claw="${OPENCLAW_BIN:-}" + if [[ -z "$claw" ]]; then + claw="$(resolve_installed_openclaw_bin || true)" + fi + if [[ -z "$claw" ]]; then + ui_info "Skipping onboarding (openclaw not on PATH yet)" + warn_openclaw_not_found + return 0 + fi + exec </dev/tty + exec "$claw" onboard + fi + local user_claw + user_claw="$(openclaw_command_for_user "${OPENCLAW_BIN:-}")" + ui_info "No TTY; run ${user_claw} onboard to finish setup" + fi + fi + + if [[ "$should_open_dashboard" == "true" ]]; then + maybe_open_dashboard + fi + + show_footer_links +} + +if [[ "${OPENCLAW_INSTALL_SH_NO_RUN:-0}" != "1" ]]; then + parse_args "$@" + configure_verbose + main +fi diff --git a/scripts/ios-app-review-notes-pdf.swift b/scripts/ios-app-review-notes-pdf.swift new file mode 100644 index 0000000000000000000000000000000000000000..3d809cee892589fc8657765a9c2140d2b6dc1294 --- /dev/null +++ b/scripts/ios-app-review-notes-pdf.swift @@ -0,0 +1,347 @@ +#!/usr/bin/env swift + +import AppKit +import Foundation + +func fail(_ message: String) -> Never { + FileHandle.standardError.write(Data((message + "\n").utf8)) + exit(1) +} + +func htmlEscaped(_ value: String) -> String { + var escaped = "" + escaped.reserveCapacity(value.count) + for character in value { + switch character { + case "&": + escaped += "&" + case "<": + escaped += "<" + case ">": + escaped += ">" + case "\"": + escaped += """ + default: + escaped.append(character) + } + } + return escaped +} + +func renderInlineMarkdown(_ value: String) -> String { + let escaped = htmlEscaped(value) + let pieces = escaped.split(separator: "`", omittingEmptySubsequences: false) + guard pieces.count > 1 else { + return escaped + } + + return pieces.enumerated().map { index, piece in + index.isMultiple(of: 2) ? String(piece) : "<code>\(piece)</code>" + }.joined() +} + +func absoluteFileURL(_ path: String) -> URL { + if path.hasPrefix("/") { + return URL(fileURLWithPath: path).standardizedFileURL + } + + return URL(fileURLWithPath: FileManager.default.currentDirectoryPath) + .appendingPathComponent(path) + .standardizedFileURL +} + +func markdownToHTML(_ markdown: String) -> String { + var body: [String] = [] + var paragraph: [String] = [] + var inCodeBlock = false + var codeLines: [String] = [] + var previousListItem = false + + func flushParagraph() { + guard !paragraph.isEmpty else { + return + } + body.append("<p>\(paragraph.map(renderInlineMarkdown).joined(separator: " "))</p>") + paragraph.removeAll() + previousListItem = false + } + + func flushCodeBlock() { + guard !codeLines.isEmpty else { + return + } + body.append("<pre><code>\(htmlEscaped(codeLines.joined(separator: "\n")))</code></pre>") + codeLines.removeAll() + previousListItem = false + } + + let headingPattern = try! NSRegularExpression(pattern: "^(#{1,6})\\s+(.+)$") + let orderedListPattern = try! NSRegularExpression(pattern: "^\\s*(\\d+)\\.\\s+(.+)$") + let unorderedListPattern = try! NSRegularExpression(pattern: "^\\s*-\\s+(.+)$") + + for rawLine in markdown.components(separatedBy: .newlines) { + let line = rawLine.trimmingCharacters(in: .whitespaces) + + if line.hasPrefix("```") { + if inCodeBlock { + flushCodeBlock() + inCodeBlock = false + } else { + flushParagraph() + inCodeBlock = true + previousListItem = false + } + continue + } + + if inCodeBlock { + codeLines.append(rawLine) + continue + } + + if line.isEmpty { + flushParagraph() + previousListItem = false + continue + } + + let lineRange = NSRange(line.startIndex..<line.endIndex, in: line) + if let match = headingPattern.firstMatch(in: line, range: lineRange), + let levelRange = Range(match.range(at: 1), in: line), + let textRange = Range(match.range(at: 2), in: line) { + flushParagraph() + let level = line[levelRange].count + if level == 2 { + body.append("<p class=\"section-spacer\"> </p>") + body.append("<p class=\"section-spacer\"> </p>") + } + body.append("<h\(level)>\(renderInlineMarkdown(String(line[textRange])))</h\(level)>") + if level <= 2 { + body.append("<p class=\"heading-rule\">\(String(repeating: "─", count: 85))</p>") + } + previousListItem = false + continue + } + + if let match = orderedListPattern.firstMatch(in: line, range: lineRange), + let numberRange = Range(match.range(at: 1), in: line), + let itemRange = Range(match.range(at: 2), in: line) { + flushParagraph() + let marker = htmlEscaped(String(line[numberRange])) + "." + body.append( + "<p class=\"list-item\"><span class=\"marker\">\(marker)</span> \(renderInlineMarkdown(String(line[itemRange])))</p>" + ) + previousListItem = true + continue + } + + if let match = unorderedListPattern.firstMatch(in: line, range: lineRange), + let itemRange = Range(match.range(at: 1), in: line) { + flushParagraph() + body.append("<p class=\"list-item\"><span class=\"marker\">•</span> \(renderInlineMarkdown(String(line[itemRange])))</p>") + previousListItem = true + continue + } + + if previousListItem && rawLine.hasPrefix(" ") { + body.append("<p class=\"list-continuation\">\(renderInlineMarkdown(line))</p>") + } else { + paragraph.append(line) + previousListItem = false + } + } + + if inCodeBlock { + flushCodeBlock() + } + flushParagraph() + + return """ + <!doctype html> + <html> + <head> + <meta charset="utf-8"> + <style> + body { + background: #ffffff; + color: #111111; + font-family: "Helvetica Neue", Helvetica, Arial, sans-serif; + font-size: 10pt; + line-height: 1.42; + } + h1 { + color: #111111; + font-size: 25pt; + margin: 0 0 5pt 0; + } + h2 { + color: #111111; + font-size: 17pt; + margin: 0 0 5pt 0; + } + .section-spacer { + color: #ffffff; + font-size: 10pt; + margin: 0; + } + .heading-rule { + color: #d8dee4; + font-family: Menlo, Monaco, monospace; + font-size: 6pt; + line-height: 0.55; + margin: 0 0 7pt 0; + } + p { + margin: 0 0 9pt 0; + } + .list-item { + margin: 0 0 5pt 0; + padding-left: 24pt; + text-indent: -24pt; + } + .marker { + display: inline-block; + min-width: 20pt; + } + .list-continuation { + margin-left: 24pt; + } + code { + background: #eef2f6; + border-radius: 4pt; + color: #24292f; + font-family: Menlo, Monaco, monospace; + font-size: 9.5pt; + padding: 1.5pt 3pt; + } + pre { + background: #f6f8fa; + border: 1px solid #d0d7de; + border-radius: 5pt; + color: #24292f; + margin: 0 0 10pt 0; + padding: 8pt; + } + pre code { + background: transparent; + border-radius: 0; + padding: 0; + } + </style> + </head> + <body> + \(body.joined(separator: "\n")) + </body> + </html> + """ +} + +let arguments = CommandLine.arguments +guard arguments.count == 3 else { + fail("Usage: scripts/ios-app-review-notes-pdf.swift <APP-REVIEW-NOTES.md> <output.pdf>") +} + +let sourceURL = absoluteFileURL(arguments[1]) +let outputURL = absoluteFileURL(arguments[2]) + +guard FileManager.default.fileExists(atPath: sourceURL.path) else { + fail("Missing App Review notes Markdown: \(sourceURL.path)") +} + +let markdown: String +do { + markdown = try String(contentsOf: sourceURL, encoding: .utf8) +} catch { + fail("Failed to read \(sourceURL.path): \(error)") +} + +let htmlData = Data(markdownToHTML(markdown).utf8) +let attributed: NSAttributedString +do { + attributed = try NSAttributedString( + data: htmlData, + options: [ + .documentType: NSAttributedString.DocumentType.html, + .characterEncoding: String.Encoding.utf8.rawValue + ], + documentAttributes: nil + ) +} catch { + fail("Failed to render App Review notes Markdown as HTML: \(error)") +} + +let fileManager = FileManager.default +do { + try fileManager.createDirectory( + at: outputURL.deletingLastPathComponent(), + withIntermediateDirectories: true + ) +} catch { + fail("Failed to create PDF output directory: \(error)") +} + +let pageSize = NSSize(width: 612, height: 792) +let margin: CGFloat = 42 +let textWidth = pageSize.width - (margin * 2) + +let textStorage = NSTextStorage(attributedString: attributed) +let layoutManager = NSLayoutManager() +layoutManager.usesFontLeading = true +textStorage.addLayoutManager(layoutManager) + +let contentSize = NSSize(width: textWidth, height: pageSize.height - (margin * 2)) +var pageGlyphRanges: [NSRange] = [] + +while pageGlyphRanges.last.map({ NSMaxRange($0) < layoutManager.numberOfGlyphs }) ?? true { + let textContainer = NSTextContainer(containerSize: contentSize) + textContainer.lineFragmentPadding = 0 + layoutManager.addTextContainer(textContainer) + layoutManager.ensureLayout(for: textContainer) + + let glyphRange = layoutManager.glyphRange(for: textContainer) + guard glyphRange.length > 0 else { + break + } + + pageGlyphRanges.append(glyphRange) + if pageGlyphRanges.count > 100 { + fail("Refusing to render more than 100 App Review notes PDF pages.") + } +} + +let pdfData = NSMutableData() +var mediaBox = CGRect(origin: .zero, size: pageSize) +guard let dataConsumer = CGDataConsumer(data: pdfData), + let pdfContext = CGContext(consumer: dataConsumer, mediaBox: &mediaBox, nil) else { + fail("Failed to create PDF context: \(outputURL.path)") +} + +let pageBackground = CGColor(gray: 1, alpha: 1) + +for glyphRange in pageGlyphRanges { + pdfContext.beginPDFPage(nil) + pdfContext.setFillColor(pageBackground) + pdfContext.fill(mediaBox) + + pdfContext.saveGState() + pdfContext.translateBy(x: 0, y: pageSize.height) + pdfContext.scaleBy(x: 1, y: -1) + NSGraphicsContext.saveGraphicsState() + NSGraphicsContext.current = NSGraphicsContext(cgContext: pdfContext, flipped: true) + layoutManager.drawBackground(forGlyphRange: glyphRange, at: NSPoint(x: margin, y: margin)) + layoutManager.drawGlyphs(forGlyphRange: glyphRange, at: NSPoint(x: margin, y: margin)) + NSGraphicsContext.restoreGraphicsState() + pdfContext.restoreGState() + + pdfContext.endPDFPage() +} + +pdfContext.closePDF() + +do { + try pdfData.write(to: outputURL, options: .atomic) +} catch { + fail("Failed to write App Review notes PDF: \(error)") +} + +print(outputURL.path) diff --git a/scripts/ios-configure-signing.sh b/scripts/ios-configure-signing.sh new file mode 100644 index 0000000000000000000000000000000000000000..c1d773c6271469d3d5ea434eff7a39255154bb8e --- /dev/null +++ b/scripts/ios-configure-signing.sh @@ -0,0 +1,126 @@ +#!/bin/bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +IOS_DIR="${ROOT_DIR}/apps/ios" +TEAM_ID_SCRIPT="${ROOT_DIR}/scripts/ios-team-id.sh" +LOCAL_SIGNING_FILE="${IOS_DIR}/.local-signing.xcconfig" +CANONICAL_TEAM_ID="FWJYW4S8P8" + +sanitize_identifier_segment() { + local raw="${1:-}" + raw="$(printf '%s' "$raw" | tr '[:upper:]' '[:lower:]')" + raw="$(printf '%s' "$raw" | sed -E 's/[^a-z0-9]+/-/g; s/^-+//; s/-+$//; s/-+/-/g')" + if [[ -z "$raw" ]]; then + raw="local" + fi + printf '%s\n' "$raw" +} + +normalize_bundle_id() { + local raw="${1:-}" + raw="$(printf '%s' "$raw" | tr '[:upper:]' '[:lower:]')" + raw="$(printf '%s' "$raw" | sed -E 's/[^a-z0-9.-]+/-/g; s/\.+/./g; s/^-+//; s/[.-]+$//')" + if [[ -z "$raw" ]]; then + raw="ai.openclawfoundation.app.test.local" + fi + printf '%s\n' "$raw" +} + +if [[ ! -x "${TEAM_ID_SCRIPT}" ]]; then + echo "ERROR: Missing team detection helper: ${TEAM_ID_SCRIPT}" >&2 + exit 1 +fi + +team_id="" +if team_id="$("${TEAM_ID_SCRIPT}" 2>/dev/null)"; then + : +else + if [[ "${IOS_SIGNING_REQUIRED:-0}" == "1" ]]; then + "${TEAM_ID_SCRIPT}" + exit 1 + fi + + echo "WARN: Unable to detect an Apple Team ID; keeping existing iOS signing override (if any)." >&2 + exit 0 +fi + +if [[ -n "${OPENCLAW_IOS_BUNDLE_SUFFIX:-}" ]]; then + identity_source="${OPENCLAW_IOS_BUNDLE_SUFFIX}" +else + identity_source="${USER:-}" + if [[ -z "${identity_source}" ]]; then + identity_source="$(id -un 2>/dev/null || true)" + fi + team_segment="$(sanitize_identifier_segment "${team_id}")" + identity_source="${identity_source}-${team_segment}" +fi +bundle_suffix="$(sanitize_identifier_segment "${identity_source}")" + +bundle_base="${OPENCLAW_IOS_APP_BUNDLE_ID:-${OPENCLAW_IOS_BUNDLE_ID_BASE:-}}" +if [[ -z "${bundle_base}" ]]; then + if [[ "${team_id}" == "${CANONICAL_TEAM_ID}" ]]; then + bundle_base="ai.openclawfoundation.app" + else + bundle_base="ai.openclawfoundation.app.test.${bundle_suffix}" + fi +fi +bundle_base="$(normalize_bundle_id "${bundle_base}")" + +share_bundle_id="${OPENCLAW_IOS_SHARE_BUNDLE_ID:-${bundle_base}.share}" +app_group_id="${OPENCLAW_IOS_APP_GROUP_ID:-group.${bundle_base}.shared}" +activity_widget_bundle_id="${OPENCLAW_IOS_ACTIVITY_WIDGET_BUNDLE_ID:-${bundle_base}.activitywidget}" +watch_app_bundle_id="${OPENCLAW_IOS_WATCH_APP_BUNDLE_ID:-${bundle_base}.watchkitapp}" + +code_sign_style="${OPENCLAW_IOS_CODE_SIGN_STYLE:-Automatic}" +code_sign_identity="${OPENCLAW_IOS_CODE_SIGN_IDENTITY:-Apple Development}" +apns_entitlement_environment="${OPENCLAW_IOS_APNS_ENTITLEMENT_ENVIRONMENT:-development}" +app_profile="${OPENCLAW_IOS_APP_PROFILE:-}" +share_profile="${OPENCLAW_IOS_SHARE_PROFILE:-}" +activity_widget_profile="${OPENCLAW_IOS_ACTIVITY_WIDGET_PROFILE:-}" +watch_app_profile="${OPENCLAW_IOS_WATCH_APP_PROFILE:-}" + +tmp_file="$(mktemp "${TMPDIR:-/tmp}/openclaw-ios-configure-signing.XXXXXX")" +cat >"${tmp_file}" <<EOF +// Auto-generated by scripts/ios-configure-signing.sh. +// This file is local-only and should not be committed. +// Override values with env vars if needed: +// IOS_DEVELOPMENT_TEAM / IOS_PREFERRED_TEAM_ID +// OPENCLAW_IOS_APP_BUNDLE_ID / OPENCLAW_IOS_BUNDLE_ID_BASE +// OPENCLAW_IOS_SHARE_BUNDLE_ID / OPENCLAW_IOS_APP_GROUP_ID +// OPENCLAW_IOS_ACTIVITY_WIDGET_BUNDLE_ID +// OPENCLAW_IOS_WATCH_APP_BUNDLE_ID +// OPENCLAW_IOS_CODE_SIGN_STYLE / OPENCLAW_IOS_CODE_SIGN_IDENTITY +// OPENCLAW_IOS_APNS_ENTITLEMENT_ENVIRONMENT / OPENCLAW_IOS_APP_PROFILE +// OPENCLAW_IOS_SHARE_PROFILE / OPENCLAW_IOS_ACTIVITY_WIDGET_PROFILE +// OPENCLAW_IOS_WATCH_APP_PROFILE +OPENCLAW_CODE_SIGN_STYLE = ${code_sign_style} +OPENCLAW_CODE_SIGN_IDENTITY = ${code_sign_identity} +OPENCLAW_CODE_SIGN_ENTITLEMENTS = Sources/OpenClaw.entitlements +OPENCLAW_DEVELOPMENT_TEAM = ${team_id} +// Keep legacy key for compatibility with older signing config paths. +OPENCLAW_IOS_SELECTED_TEAM = ${team_id} +OPENCLAW_APP_BUNDLE_ID = ${bundle_base} +OPENCLAW_SHARE_BUNDLE_ID = ${share_bundle_id} +OPENCLAW_APP_GROUP_ID = ${app_group_id} +OPENCLAW_ACTIVITY_WIDGET_BUNDLE_ID = ${activity_widget_bundle_id} +OPENCLAW_WATCH_APP_BUNDLE_ID = ${watch_app_bundle_id} +OPENCLAW_APNS_ENTITLEMENT_ENVIRONMENT = ${apns_entitlement_environment} +OPENCLAW_APP_PROFILE = ${app_profile} +OPENCLAW_SHARE_PROFILE = ${share_profile} +OPENCLAW_ACTIVITY_WIDGET_PROFILE = ${activity_widget_profile} +OPENCLAW_WATCH_APP_PROFILE = ${watch_app_profile} +EOF + +if [[ -f "${LOCAL_SIGNING_FILE}" ]] && cmp -s "${tmp_file}" "${LOCAL_SIGNING_FILE}"; then + rm -f "${tmp_file}" + echo "iOS signing config already up to date: team=${team_id} app=${bundle_base}" + exit 0 +fi + +mv "${tmp_file}" "${LOCAL_SIGNING_FILE}" +if [[ "${team_id}" == "${CANONICAL_TEAM_ID}" ]]; then + echo "Configured iOS signing: team=${team_id} app=${bundle_base}" +else + echo "Configured iOS local signing: canonical_team=${CANONICAL_TEAM_ID} local_team=${team_id} app=${bundle_base}" +fi diff --git a/scripts/ios-release-cut.sh b/scripts/ios-release-cut.sh new file mode 100644 index 0000000000000000000000000000000000000000..6a59788ea59723886e7370802c8464861d49d3a9 --- /dev/null +++ b/scripts/ios-release-cut.sh @@ -0,0 +1,19 @@ +#!/bin/bash +set -euo pipefail + +usage() { + cat <<'EOF' +Standalone iOS release cutting is retired. Use scripts/mobile-release-version.ts +--prepare, capture pnpm ios:release:plan -- --json, then use --finalize. +EOF +} + +for argument in "$@"; do + if [[ "${argument}" == "-h" || "${argument}" == "--help" ]]; then + usage + exit 0 + fi +done + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +exec node --import tsx "${ROOT_DIR}/scripts/ios-release-cut.ts" "$@" diff --git a/scripts/ios-release-plan.sh b/scripts/ios-release-plan.sh new file mode 100644 index 0000000000000000000000000000000000000000..cb3cbc153a5b0ef1b45888a5101abf86172a3764 --- /dev/null +++ b/scripts/ios-release-plan.sh @@ -0,0 +1,36 @@ +#!/bin/bash +set -euo pipefail + +usage() { + cat <<'EOF' +Usage: + scripts/ios-release-plan.sh [--json] [--version 2026.7.2] [--revision 1] [--build-number 3] + +Reads App Store Connect state and prints the deterministic iOS release plan. +This command does not mutate App Store Connect or repository files. +EOF +} + +BUILD_NUMBER="" +APP_STORE_REVISION="" +RELEASE_VERSION="" +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +source "${ROOT_DIR}/scripts/lib/ios-fastlane.sh" + +parse_ios_release_args plan "$@" + +PLAN_FILE="$(mktemp "${TMPDIR:-/tmp}/openclaw-ios-release-plan.XXXXXX")" +trap 'rm -f "${PLAN_FILE}"' EXIT +FASTLANE_ARGS=(ios release_plan "output_path:${PLAN_FILE}") +[[ -n "${RELEASE_VERSION}" ]] && FASTLANE_ARGS+=("release_version:${RELEASE_VERSION}") +[[ -n "${APP_STORE_REVISION}" ]] && FASTLANE_ARGS+=("app_store_revision:${APP_STORE_REVISION}") +[[ -n "${BUILD_NUMBER}" ]] && FASTLANE_ARGS+=("build_number:${BUILD_NUMBER}") + +if ! ( + cd "${ROOT_DIR}/apps/ios" + run_ios_fastlane "${FASTLANE_ARGS[@]}" 1>&2 +); then + echo "Failed to resolve the iOS release plan." >&2 + exit 1 +fi +cat "${PLAN_FILE}" diff --git a/scripts/ios-screenshot-evidence.d.mts b/scripts/ios-screenshot-evidence.d.mts new file mode 100644 index 0000000000000000000000000000000000000000..bbc2d9c30dd77e6db5bbb75ae5fbdf9c1bb0e877 --- /dev/null +++ b/scripts/ios-screenshot-evidence.d.mts @@ -0,0 +1,33 @@ +export type IosScreenshotTooling = { + xcode: string; + fastlane: string; + node: string; +}; + +export type IosScreenshotProvenance = { + targetSha: string; + workflowSha: string; + runId: string; + runAttempt: string | number; + tooling: IosScreenshotTooling; +}; + +export type XcresultSummary = { + testResult: string; + failedTests: number; +}; + +export function collectIosScreenshotEvidence(options: { + family: string; + screenshotDirectory: string; + xcresultDirectory: string; + outputDirectory: string; + provenance: IosScreenshotProvenance; + readXcresultSummary?: (resultPath: string) => XcresultSummary; +}): Record<string, unknown>; + +export function reduceIosScreenshotEvidence(options: { + inputDirectory: string; + outputRoot: string; + expectedProvenance: IosScreenshotProvenance; +}): Record<string, unknown>; diff --git a/scripts/ios-screenshot-evidence.mjs b/scripts/ios-screenshot-evidence.mjs new file mode 100644 index 0000000000000000000000000000000000000000..7bd11c93d4fbfe539386243657ddc07c69f8627f --- /dev/null +++ b/scripts/ios-screenshot-evidence.mjs @@ -0,0 +1,784 @@ +#!/usr/bin/env node + +import { spawnSync } from "node:child_process"; +import crypto from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import { isDirectRunUrl } from "./lib/direct-run.mjs"; + +const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); +const CAPTURE_ATTEMPTS_FILENAME = "capture-attempts.json"; +const SCREENSHOT_NAMES = [ + "01-control-connected", + "02-chat-connected", + "03-agent-connected", + "04-settings-connected", +]; +const FAMILY_SPECS = { + iphone: { + devicePattern: /^iPhone /u, + screenshotNames: SCREENSHOT_NAMES, + captureAttempts: true, + }, + "ipad-13": { + devicePattern: /^iPad (?:Air|Pro) 13-inch/u, + screenshotNames: SCREENSHOT_NAMES, + captureAttempts: true, + }, + watch: { + devicePattern: /^Apple Watch/u, + screenshotNames: ["01-now-face"], + captureAttempts: false, + }, +}; +const EXPECTED_FAMILIES = Object.keys(FAMILY_SPECS).toSorted(); +// Model only whole OpenClaw Fastlane invocations. Fastlane's internal launch +// retries remain workflow-log evidence. +const ATTEMPT_MODEL = Object.freeze({ + owner: "openclaw", + unit: "capture_ios_screenshots invocation", + maxAttempts: 2, + fastlaneInternalRetries: "workflow-log", +}); + +function fail(message) { + throw new Error(message); +} + +function requireString(value, label) { + if (typeof value !== "string" || value.trim() === "") { + fail(`${label} must be a nonempty string`); + } + return value.trim(); +} + +function requireSha(value, label) { + const sha = requireString(value, label); + if (!/^[0-9a-f]{40}$/u.test(sha)) { + fail(`${label} must be a full lowercase commit SHA`); + } + return sha; +} + +function requirePositiveInteger(value, label) { + const number = Number(value); + if (!Number.isInteger(number) || number < 1) { + fail(`${label} must be a positive integer`); + } + return number; +} + +function listEntries(directory) { + if (!fs.existsSync(directory)) { + fail(`missing evidence directory: ${directory}`); + } + return fs.readdirSync(directory, { withFileTypes: true }); +} + +function sha256File(filePath) { + return crypto.createHash("sha256").update(fs.readFileSync(filePath)).digest("hex"); +} + +function listFilesRecursive(directory, prefix = "") { + return fs + .readdirSync(directory, { withFileTypes: true }) + .flatMap((entry) => { + const relativePath = path.posix.join(prefix, entry.name); + const absolutePath = path.join(directory, entry.name); + return entry.isDirectory() + ? listFilesRecursive(absolutePath, relativePath) + : [{ absolutePath, relativePath }]; + }) + .toSorted((left, right) => left.relativePath.localeCompare(right.relativePath)); +} + +function sha256Directory(directory) { + const hash = crypto.createHash("sha256"); + for (const file of listFilesRecursive(directory)) { + hash.update(file.relativePath); + hash.update("\0"); + hash.update(fs.readFileSync(file.absolutePath)); + hash.update("\0"); + } + return hash.digest("hex"); +} + +function verifyPng(filePath) { + const stat = fs.statSync(filePath); + if (!stat.isFile() || stat.size <= PNG_SIGNATURE.length) { + fail(`invalid empty PNG evidence: ${filePath}`); + } + const header = Buffer.alloc(PNG_SIGNATURE.length); + const descriptor = fs.openSync(filePath, "r"); + try { + fs.readSync(descriptor, header, 0, header.length, 0); + } finally { + fs.closeSync(descriptor); + } + if (!header.equals(PNG_SIGNATURE)) { + fail(`invalid PNG signature: ${filePath}`); + } + return { bytes: stat.size, sha256: sha256File(filePath) }; +} + +function defaultReadXcresultSummary(resultPath) { + const result = spawnSync( + "xcrun", + ["xcresulttool", "get", "test-results", "summary", "--path", resultPath, "--compact"], + { encoding: "utf8" }, + ); + if (result.status !== 0) { + fail( + `failed to inspect ${resultPath}: ${result.stderr.trim() || result.stdout.trim() || `exit ${result.status}`}`, + ); + } + try { + const summary = JSON.parse(result.stdout); + return { + testResult: requireString(summary.result, `${resultPath} test result`), + failedTests: Number(summary.failedTests), + }; + } catch (error) { + return fail(`invalid xcresult summary for ${resultPath}: ${String(error)}`); + } +} + +function copyEntry(source, destination) { + fs.mkdirSync(path.dirname(destination), { recursive: true }); + fs.cpSync(source, destination, { recursive: true }); +} + +function screenshotDeviceName(filename, screenshotName) { + const suffix = `-${screenshotName}.png`; + return filename.endsWith(suffix) ? filename.slice(0, -suffix.length) : undefined; +} + +function collectScreenshots({ family, screenshotDirectory, familyDirectory, spec }) { + const pngNames = listEntries(screenshotDirectory) + .filter((entry) => entry.isFile() && entry.name.endsWith(".png")) + .map((entry) => entry.name); + const matches = spec.screenshotNames.map((screenshotName) => { + const candidates = pngNames.filter((filename) => { + const deviceName = screenshotDeviceName(filename, screenshotName); + return deviceName && spec.devicePattern.test(deviceName); + }); + if (candidates.length !== 1) { + fail( + `${family} expected one ${screenshotName} PNG, found ${candidates.length}: ${candidates.join(", ") || "none"}`, + ); + } + return { filename: candidates[0], screenshotName }; + }); + const deviceNames = new Set( + matches.map(({ filename, screenshotName }) => screenshotDeviceName(filename, screenshotName)), + ); + if (deviceNames.size !== 1) { + fail(`${family} screenshots span multiple devices: ${[...deviceNames].join(", ")}`); + } + const deviceName = [...deviceNames][0]; + const familyPngs = pngNames.filter((filename) => spec.devicePattern.test(filename)); + const expectedPngs = matches.map(({ filename }) => filename).toSorted(); + if (familyPngs.toSorted().join("\n") !== expectedPngs.join("\n")) { + fail( + `${family} PNG union mismatch; expected ${expectedPngs.join(", ")}, found ${familyPngs.toSorted().join(", ")}`, + ); + } + + const screenshots = matches.map(({ filename, screenshotName }) => { + const source = path.join(screenshotDirectory, filename); + const artifactPath = path.posix.join("screenshots", filename); + const metadata = verifyPng(source); + copyEntry(source, path.join(familyDirectory, artifactPath)); + return Object.assign( + { + name: screenshotName, + deviceName, + artifactPath, + canonicalPath: path.posix.join("apps/ios/fastlane/screenshots/en-US", filename), + }, + metadata, + ); + }); + return { deviceName, screenshots }; +} + +function readCaptureAttemptLedger(xcresultDirectory) { + const ledgerPath = path.join(xcresultDirectory, CAPTURE_ATTEMPTS_FILENAME); + if (!fs.existsSync(ledgerPath) || !fs.statSync(ledgerPath).isFile()) { + fail(`missing OpenClaw capture attempt ledger: ${ledgerPath}`); + } + let ledger; + try { + ledger = JSON.parse(fs.readFileSync(ledgerPath, "utf8")); + } catch (error) { + return fail(`invalid OpenClaw capture attempt ledger: ${String(error)}`); + } + const expectedKeys = ["attempts", "schemaVersion"]; + const actualKeys = + ledger && typeof ledger === "object" && !Array.isArray(ledger) + ? Object.keys(ledger).toSorted((left, right) => left.localeCompare(right)) + : []; + if (actualKeys.join("\n") !== expectedKeys.join("\n")) { + fail("OpenClaw capture attempt ledger has an unexpected shape"); + } + if (ledger.schemaVersion !== 1 || !Array.isArray(ledger.attempts)) { + fail("OpenClaw capture attempt ledger has an unsupported schema"); + } + return ledger.attempts; +} + +function collectCaptureAttempts({ + deviceName, + familyDirectory, + screenshotNames, + xcresultDirectory, + readXcresultSummary, +}) { + const directoryEntries = listEntries(xcresultDirectory); + const unexpectedEntries = directoryEntries.filter( + (entry) => + !( + (entry.isFile() && entry.name === CAPTURE_ATTEMPTS_FILENAME) || + (entry.isDirectory() && entry.name.endsWith(".xcresult")) + ), + ); + if (unexpectedEntries.length > 0) { + fail( + `OpenClaw capture evidence contains unexpected entries: ${unexpectedEntries.map((entry) => entry.name).join(", ")}`, + ); + } + const xcresultNames = directoryEntries + .filter((entry) => entry.isDirectory() && entry.name.endsWith(".xcresult")) + .map((entry) => entry.name); + const ledgerEntries = readCaptureAttemptLedger(xcresultDirectory); + const results = []; + for (const screenshotName of screenshotNames) { + const attempts = ledgerEntries + .filter((entry) => entry.deviceName === deviceName && entry.screenshotName === screenshotName) + .toSorted((left, right) => left.attempt - right.attempt); + const attemptNumbers = attempts.map(({ attempt }) => attempt).join(","); + if (attemptNumbers !== "1" && attemptNumbers !== "1,2") { + fail( + `${deviceName} ${screenshotName} expected OpenClaw attempt 1 and optional retry 2; found ${attemptNumbers || "none"}`, + ); + } + const summaries = attempts.map((entry, index) => { + const expectedKeys = ["attempt", "captureOutcome", "deviceName", "screenshotName"]; + const actualKeys = + entry && typeof entry === "object" && !Array.isArray(entry) + ? Object.keys(entry).toSorted((left, right) => left.localeCompare(right)) + : []; + if (actualKeys.join("\n") !== expectedKeys.join("\n")) { + fail(`${deviceName} ${screenshotName} has an invalid capture attempt record`); + } + const { attempt, captureOutcome } = entry; + const expectedOutcome = index === attempts.length - 1 ? "succeeded" : "failed"; + if (captureOutcome !== expectedOutcome) { + fail(`${deviceName} ${screenshotName} has an unexpected capture outcome sequence`); + } + const name = `${deviceName}-${screenshotName}-attempt-${attempt}.xcresult`; + const source = path.join(xcresultDirectory, name); + if (!fs.existsSync(source)) { + if (captureOutcome === "succeeded") { + fail(`${name} is missing for the successful final capture attempt`); + } + return { + screenshotName, + attempt, + captureOutcome, + artifactPath: null, + canonicalPath: null, + testResult: null, + failedTests: null, + sha256: null, + }; + } + const summary = readXcresultSummary(source); + if (!Number.isInteger(summary.failedTests) || summary.failedTests < 0) { + fail(`${name} has invalid failedTests`); + } + const artifactPath = path.posix.join("xcresults", name); + copyEntry(source, path.join(familyDirectory, artifactPath)); + return { + screenshotName, + attempt, + artifactPath, + canonicalPath: path.posix.join("apps/ios/build/SnapshotTestResults", name), + captureOutcome, + testResult: summary.testResult, + failedTests: summary.failedTests, + sha256: sha256Directory(source), + }; + }); + const final = summaries.at(-1); + if ( + final.captureOutcome !== "succeeded" || + final.artifactPath === null || + final.testResult !== "Passed" || + final.failedTests !== 0 + ) { + fail(`${deviceName} ${screenshotName} does not have a passing final capture attempt`); + } + results.push(...summaries); + } + if (ledgerEntries.length !== results.length) { + fail(`${deviceName} capture attempt ledger contains unexpected evidence`); + } + const expectedNames = results + .filter(({ artifactPath }) => artifactPath !== null) + .map(({ artifactPath }) => path.posix.basename(artifactPath)); + const familyEntries = xcresultNames.filter((name) => name.startsWith(`${deviceName}-`)); + if ( + familyEntries.length !== xcresultNames.length || + familyEntries.toSorted((left, right) => left.localeCompare(right)).join("\n") !== + expectedNames.toSorted((left, right) => left.localeCompare(right)).join("\n") + ) { + fail(`${deviceName} xcresult union contains unexpected evidence`); + } + return results; +} + +export function collectIosScreenshotEvidence({ + family, + screenshotDirectory, + xcresultDirectory, + outputDirectory, + provenance, + readXcresultSummary = defaultReadXcresultSummary, +}) { + const spec = FAMILY_SPECS[family]; + if (!spec) { + fail(`unsupported screenshot family: ${family}`); + } + const normalizedProvenance = { + targetSha: requireSha(provenance.targetSha, "target SHA"), + workflowSha: requireSha(provenance.workflowSha, "workflow SHA"), + runId: requireString(provenance.runId, "workflow run id"), + runAttempt: requirePositiveInteger(provenance.runAttempt, "workflow run attempt"), + tooling: { + xcode: requireString(provenance.tooling?.xcode, "Xcode version"), + fastlane: requireString(provenance.tooling?.fastlane, "Fastlane version"), + node: requireString(provenance.tooling?.node, "Node version"), + }, + }; + const familyDirectory = path.join(outputDirectory, family); + fs.rmSync(familyDirectory, { recursive: true, force: true }); + fs.mkdirSync(familyDirectory, { recursive: true }); + const { deviceName, screenshots } = collectScreenshots({ + family, + screenshotDirectory, + familyDirectory, + spec, + }); + const captureAttempts = spec.captureAttempts + ? collectCaptureAttempts({ + deviceName, + familyDirectory, + screenshotNames: spec.screenshotNames, + xcresultDirectory, + readXcresultSummary, + }) + : []; + const manifest = { + schemaVersion: 1, + attemptModel: { ...ATTEMPT_MODEL }, + family, + deviceName, + ...normalizedProvenance, + screenshots, + captureAttempts, + }; + fs.writeFileSync( + path.join(familyDirectory, "manifest.json"), + `${JSON.stringify(manifest, null, 2)}\n`, + ); + return manifest; +} + +function loadExpectedManifests(inputDirectory, targetSha) { + const expectedContainers = { + [`ios-release-screenshot-shard-iphone-${targetSha}`]: ["iphone"], + [`ios-release-screenshot-shard-ipad-13-${targetSha}`]: ["ipad-13", "watch"], + }; + const actualContainers = listEntries(inputDirectory); + if (actualContainers.some((entry) => !entry.isDirectory())) { + fail("screenshot evidence input contains a non-container entry"); + } + const actualContainerNames = actualContainers.map((entry) => entry.name).toSorted(); + const expectedContainerNames = Object.keys(expectedContainers).toSorted(); + if (actualContainerNames.join("\n") !== expectedContainerNames.join("\n")) { + fail( + `screenshot artifact container topology mismatch; expected ${expectedContainerNames.join(", ")}, found ${actualContainerNames.join(", ") || "none"}`, + ); + } + + return Object.entries(expectedContainers).flatMap(([containerName, expectedFamilies]) => { + const containerDirectory = path.join(inputDirectory, containerName); + const containerEntries = listEntries(containerDirectory); + if (containerEntries.some((entry) => !entry.isDirectory())) { + fail(`${containerName} contains a non-family entry`); + } + const actualFamilies = containerEntries.map((entry) => entry.name).toSorted(); + if (actualFamilies.join("\n") !== expectedFamilies.toSorted().join("\n")) { + fail( + `${containerName} family topology mismatch; expected ${expectedFamilies.join(", ")}, found ${actualFamilies.join(", ") || "none"}`, + ); + } + return expectedFamilies.map((family) => { + const manifestPath = path.join(containerDirectory, family, "manifest.json"); + if (!fs.existsSync(manifestPath)) { + fail(`${containerName} is missing ${family}/manifest.json`); + } + const manifest = JSON.parse(fs.readFileSync(manifestPath, "utf8")); + if (manifest.family !== family) { + fail(`${containerName}/${family} contains ${manifest.family ?? "unknown"} family evidence`); + } + return { containerName, manifestPath, manifest }; + }); + }); +} + +function verifyManifestEntry(manifestPath, entry, kind) { + const familyDirectory = path.dirname(manifestPath); + const artifactPath = requireString(entry.artifactPath, `${kind} artifact path`); + const canonicalPath = requireString(entry.canonicalPath, `${kind} canonical path`); + const expectedRoot = + kind === "screenshot" + ? "apps/ios/fastlane/screenshots/en-US/" + : "apps/ios/build/SnapshotTestResults/"; + const expectedArtifactRoot = kind === "screenshot" ? "screenshots/" : "xcresults/"; + const filename = path.posix.basename(canonicalPath); + if (canonicalPath !== `${expectedRoot}${filename}`) { + fail(`unexpected ${kind} canonical path: ${canonicalPath}`); + } + if (artifactPath !== `${expectedArtifactRoot}${filename}`) { + fail(`unexpected ${kind} artifact path: ${artifactPath}`); + } + const source = path.join(familyDirectory, artifactPath); + const actualDigest = kind === "screenshot" ? verifyPng(source).sha256 : sha256Directory(source); + if (actualDigest !== entry.sha256) { + fail(`${kind} digest mismatch for ${canonicalPath}`); + } + return { canonicalPath, source }; +} + +function verifyFamilyArtifactUnion(manifestPath, manifest) { + const familyDirectory = path.dirname(manifestPath); + if (path.basename(familyDirectory) !== manifest.family) { + fail(`${manifest.family} manifest is stored under an unexpected directory`); + } + const xcresults = manifest.captureAttempts.filter((entry) => entry.artifactPath !== null); + const expectedTopLevel = ["manifest.json", "screenshots"]; + if (xcresults.length > 0) { + expectedTopLevel.push("xcresults"); + } + const actualTopLevel = listEntries(familyDirectory) + .map((entry) => entry.name) + .toSorted(); + if (actualTopLevel.join("\n") !== expectedTopLevel.toSorted().join("\n")) { + fail(`${manifest.family} shard contains unexpected evidence`); + } + + const expectedScreenshots = manifest.screenshots + .map((entry) => path.posix.basename(entry.artifactPath)) + .toSorted(); + const actualScreenshots = listEntries(path.join(familyDirectory, "screenshots")) + .map((entry) => entry.name) + .toSorted(); + if (actualScreenshots.join("\n") !== expectedScreenshots.join("\n")) { + fail(`${manifest.family} screenshot artifact union mismatch`); + } + + if (xcresults.length > 0) { + const expectedXcresults = xcresults + .map((entry) => path.posix.basename(entry.artifactPath)) + .toSorted(); + const actualXcresults = listEntries(path.join(familyDirectory, "xcresults")) + .map((entry) => entry.name) + .toSorted(); + if (actualXcresults.join("\n") !== expectedXcresults.join("\n")) { + fail(`${manifest.family} xcresult artifact union mismatch`); + } + } +} + +function verifyAttemptModel(manifest) { + const model = manifest.attemptModel; + const expectedKeys = Object.keys(ATTEMPT_MODEL).toSorted(); + const actualKeys = + model && typeof model === "object" && !Array.isArray(model) + ? Object.keys(model).toSorted() + : []; + if ( + actualKeys.join("\n") !== expectedKeys.join("\n") || + expectedKeys.some((key) => model[key] !== ATTEMPT_MODEL[key]) + ) { + fail(`${manifest.family} has an unexpected attempt model`); + } +} + +function verifyManifestFamily(manifestPath, manifest) { + const spec = FAMILY_SPECS[manifest.family]; + verifyAttemptModel(manifest); + const deviceName = requireString(manifest.deviceName, `${manifest.family} device name`); + if (!spec.devicePattern.test(deviceName)) { + fail(`${manifest.family} has unexpected device name: ${deviceName}`); + } + requireString(manifest.runId, `${manifest.family} workflow run id`); + requirePositiveInteger(manifest.runAttempt, `${manifest.family} workflow run attempt`); + requireString(manifest.tooling?.xcode, `${manifest.family} Xcode version`); + requireString(manifest.tooling?.fastlane, `${manifest.family} Fastlane version`); + requireString(manifest.tooling?.node, `${manifest.family} Node version`); + + const screenshotNames = manifest.screenshots?.map((entry) => entry.name).toSorted(); + if ( + screenshotNames?.join("\n") !== + spec.screenshotNames.toSorted((left, right) => left.localeCompare(right)).join("\n") + ) { + fail(`${manifest.family} screenshot name union mismatch`); + } + for (const screenshot of manifest.screenshots) { + const expectedFilename = `${deviceName}-${screenshot.name}.png`; + const expectedCanonicalPath = `apps/ios/fastlane/screenshots/en-US/${expectedFilename}`; + if (screenshot.canonicalPath !== expectedCanonicalPath) { + fail(`${manifest.family} has unexpected screenshot path: ${screenshot.canonicalPath}`); + } + if (screenshot.artifactPath !== `screenshots/${expectedFilename}`) { + fail( + `${manifest.family} has unexpected screenshot artifact path: ${screenshot.artifactPath}`, + ); + } + } + + if (!Array.isArray(manifest.captureAttempts)) { + fail(`${manifest.family} capture attempts must be an array`); + } + if (!spec.captureAttempts) { + if (manifest.captureAttempts?.length !== 0) { + fail(`${manifest.family} must not contain capture attempt evidence`); + } + verifyFamilyArtifactUnion(manifestPath, manifest); + return; + } + const knownScreenshotNames = new Set(spec.screenshotNames); + if (manifest.captureAttempts.some((entry) => !knownScreenshotNames.has(entry.screenshotName))) { + fail(`${manifest.family} capture attempt union contains an unexpected screenshot`); + } + for (const screenshotName of spec.screenshotNames) { + const attempts = manifest.captureAttempts + ?.filter((entry) => entry.screenshotName === screenshotName) + .toSorted((left, right) => left.attempt - right.attempt); + const attemptNumbers = attempts?.map((entry) => entry.attempt).join(","); + if (attemptNumbers !== "1" && attemptNumbers !== "1,2") { + fail(`${manifest.family} ${screenshotName} capture attempt union mismatch`); + } + const final = attempts.at(-1); + if ( + final.captureOutcome !== "succeeded" || + final.artifactPath === null || + final.testResult !== "Passed" || + final.failedTests !== 0 + ) { + fail(`${manifest.family} ${screenshotName} final xcresult is not passing`); + } + if ( + attempts.some( + (entry, index) => + entry.captureOutcome !== (index === attempts.length - 1 ? "succeeded" : "failed"), + ) + ) { + fail(`${manifest.family} ${screenshotName} has an unexpected capture outcome sequence`); + } + for (const attempt of attempts) { + if (attempt.artifactPath === null) { + if ( + attempt.captureOutcome !== "failed" || + attempt.canonicalPath !== null || + attempt.testResult !== null || + attempt.failedTests !== null || + attempt.sha256 !== null + ) { + fail(`${manifest.family} ${screenshotName} has invalid missing xcresult evidence`); + } + continue; + } + requireString(attempt.testResult, `${manifest.family} ${screenshotName} test result`); + if (!Number.isInteger(attempt.failedTests) || attempt.failedTests < 0) { + fail(`${manifest.family} ${screenshotName} has invalid failedTests`); + } + const expectedFilename = `${deviceName}-${screenshotName}-attempt-${attempt.attempt}.xcresult`; + const expectedCanonicalPath = `apps/ios/build/SnapshotTestResults/${expectedFilename}`; + if (attempt.canonicalPath !== expectedCanonicalPath) { + fail(`${manifest.family} has unexpected xcresult path: ${attempt.canonicalPath}`); + } + if (attempt.artifactPath !== `xcresults/${expectedFilename}`) { + fail(`${manifest.family} has unexpected xcresult artifact path: ${attempt.artifactPath}`); + } + } + } + verifyFamilyArtifactUnion(manifestPath, manifest); +} + +export function reduceIosScreenshotEvidence({ inputDirectory, outputRoot, expectedProvenance }) { + const expected = { + targetSha: requireSha(expectedProvenance.targetSha, "expected target SHA"), + workflowSha: requireSha(expectedProvenance.workflowSha, "expected workflow SHA"), + runId: requireString(expectedProvenance.runId, "expected workflow run id"), + runAttempt: requirePositiveInteger( + expectedProvenance.runAttempt, + "expected workflow run attempt", + ), + tooling: { + xcode: requireString(expectedProvenance.tooling?.xcode, "expected Xcode version"), + fastlane: requireString(expectedProvenance.tooling?.fastlane, "expected Fastlane version"), + node: requireString(expectedProvenance.tooling?.node, "expected Node version"), + }, + }; + const manifests = loadExpectedManifests(inputDirectory, expected.targetSha); + const families = manifests + .map(({ manifest }) => manifest.family) + .toSorted((left, right) => left.localeCompare(right)); + if (families.join("\n") !== EXPECTED_FAMILIES.join("\n")) { + fail( + `screenshot family union mismatch; expected ${EXPECTED_FAMILIES.join(", ")}, found ${families.join(", ") || "none"}`, + ); + } + const canonicalEntries = []; + for (const { manifestPath, manifest } of manifests) { + if (manifest.schemaVersion !== 1) { + fail(`unsupported screenshot manifest schema in ${manifestPath}`); + } + if (manifest.targetSha !== expected.targetSha) { + fail( + `cross-SHA screenshot evidence in ${manifestPath}: expected ${expected.targetSha}, found ${manifest.targetSha}`, + ); + } + if (manifest.workflowSha !== expected.workflowSha) { + fail(`${manifest.family} workflow SHA does not match the reducer context`); + } + if (manifest.runId !== expected.runId) { + fail(`${manifest.family} workflow run id does not match the reducer context`); + } + if (manifest.runAttempt !== expected.runAttempt) { + fail(`${manifest.family} workflow run attempt does not match the reducer context`); + } + for (const tool of ["xcode", "fastlane", "node"]) { + if (manifest.tooling?.[tool] !== expected.tooling[tool]) { + fail(`${manifest.family} ${tool} version does not match the reducer context`); + } + } + verifyManifestFamily(manifestPath, manifest); + for (const screenshot of manifest.screenshots) { + canonicalEntries.push({ + ...verifyManifestEntry(manifestPath, screenshot, "screenshot"), + family: manifest.family, + }); + } + for (const attempt of manifest.captureAttempts) { + if (attempt.artifactPath === null) { + continue; + } + canonicalEntries.push({ + ...verifyManifestEntry(manifestPath, attempt, "xcresult"), + family: manifest.family, + }); + } + } + const canonicalPaths = canonicalEntries.map(({ canonicalPath }) => canonicalPath); + if (new Set(canonicalPaths).size !== canonicalPaths.length) { + fail("duplicate canonical screenshot evidence paths"); + } + + const screenshotOutput = path.join(outputRoot, "apps/ios/fastlane/screenshots/en-US"); + const xcresultOutput = path.join(outputRoot, "apps/ios/build/SnapshotTestResults"); + const manifestOutput = path.join(outputRoot, "apps/ios/build/ScreenshotEvidence"); + for (const directory of [screenshotOutput, xcresultOutput, manifestOutput]) { + fs.rmSync(directory, { recursive: true, force: true }); + fs.mkdirSync(directory, { recursive: true }); + } + for (const entry of canonicalEntries) { + copyEntry(entry.source, path.join(outputRoot, entry.canonicalPath)); + } + const combinedManifest = { + schemaVersion: 1, + attemptModel: { ...ATTEMPT_MODEL }, + ...expected, + families: manifests + .map(({ manifest }) => manifest) + .toSorted((left, right) => left.family.localeCompare(right.family)), + }; + fs.writeFileSync( + path.join(manifestOutput, "manifest.json"), + `${JSON.stringify(combinedManifest, null, 2)}\n`, + ); + return combinedManifest; +} + +function parseIosScreenshotEvidenceArgs(argv) { + const [command, ...rest] = argv; + const options = {}; + for (let index = 0; index < rest.length; index += 2) { + const key = rest[index]; + const value = rest[index + 1]; + if (!key?.startsWith("--") || value === undefined) { + fail(`invalid argument list near ${key ?? "end of input"}`); + } + options[key.slice(2)] = value; + } + return { command, options }; +} + +function main(argv) { + const { command, options } = parseIosScreenshotEvidenceArgs(argv); + if (command === "collect") { + const manifest = collectIosScreenshotEvidence({ + family: options.family, + screenshotDirectory: options.screenshots, + xcresultDirectory: options.xcresults, + outputDirectory: options.output, + provenance: { + targetSha: options["target-sha"], + workflowSha: options["workflow-sha"], + runId: options["run-id"], + runAttempt: options["run-attempt"], + tooling: { + xcode: options["xcode-version"], + fastlane: options["fastlane-version"], + node: options["node-version"], + }, + }, + }); + console.log(`collected ${manifest.family} screenshot evidence for ${manifest.targetSha}`); + return; + } + if (command === "reduce") { + const manifest = reduceIosScreenshotEvidence({ + inputDirectory: options.input, + outputRoot: options.output, + expectedProvenance: { + targetSha: options["target-sha"], + workflowSha: options["workflow-sha"], + runId: options["run-id"], + runAttempt: options["run-attempt"], + tooling: { + xcode: options["xcode-version"], + fastlane: options["fastlane-version"], + node: options["node-version"], + }, + }, + }); + console.log(`reduced iOS screenshot evidence for ${manifest.targetSha}`); + return; + } + fail(`expected command collect or reduce, got ${command ?? "none"}`); +} + +if (isDirectRunUrl(process.argv[1], import.meta.url)) { + try { + main(process.argv.slice(2)); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + console.error("[ios-screenshot-evidence] FAILED (exit 1)"); + process.exitCode = 1; + } +} diff --git a/scripts/ios-screenshots.sh b/scripts/ios-screenshots.sh new file mode 100644 index 0000000000000000000000000000000000000000..4cb814945ec29388ec679ff0edd875986f894f31 --- /dev/null +++ b/scripts/ios-screenshots.sh @@ -0,0 +1,10 @@ +#!/bin/bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +source "${ROOT_DIR}/scripts/lib/ios-fastlane.sh" + +( + cd "${ROOT_DIR}/apps/ios" + run_ios_fastlane ios screenshots +) diff --git a/scripts/ios-sync-versioning.ts b/scripts/ios-sync-versioning.ts new file mode 100644 index 0000000000000000000000000000000000000000..870ce132bbd43cfc56254ad8fb5a45c6c5a57f6a --- /dev/null +++ b/scripts/ios-sync-versioning.ts @@ -0,0 +1,49 @@ +// Ios Sync Versioning script supports OpenClaw repository automation. +import path from "node:path"; +import { syncIosVersioning } from "./lib/ios-version.ts"; +import { parseVersionSyncArgs } from "./lib/version-script-args.ts"; + +export function parseArgs(argv: string[]) { + return parseVersionSyncArgs(argv, { allowAppStoreRevision: true }); +} + +function printUsage(): void { + process.stdout.write( + "Usage: node --import tsx scripts/ios-sync-versioning.ts [--write|--check] [--version YYYY.M.D] [--revision 0-9] [--root dir]\n\nValidates that iOS versioning inputs can produce generated local artifacts.\n", + ); +} + +function main(argv = process.argv.slice(2)): number { + const options = parseArgs(argv); + if (options.help) { + printUsage(); + return 0; + } + + const result = syncIosVersioning({ + appStoreRevision: options.appStoreRevision, + mode: options.mode, + releaseVersion: options.releaseVersion, + rootDir: options.rootDir, + }); + + if (options.mode === "check") { + process.stdout.write("iOS versioning inputs are valid.\n"); + } else if (result.updatedPaths.length === 0) { + process.stdout.write( + "iOS versioning inputs are valid; local artifacts are generated by iOS prep commands.\n", + ); + } else { + process.stdout.write( + `Updated iOS versioning artifacts:\n- ${result.updatedPaths.map((filePath) => path.relative(process.cwd(), filePath)).join("\n- ")}\n`, + ); + } + return 0; +} + +try { + process.exitCode = main(); +} catch (error) { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + process.exitCode = 1; +} diff --git a/scripts/ios-team-id.sh b/scripts/ios-team-id.sh new file mode 100644 index 0000000000000000000000000000000000000000..c0d557388dc1d176e7bb81bffaa3ed7f39e2ff30 --- /dev/null +++ b/scripts/ios-team-id.sh @@ -0,0 +1,278 @@ +#!/bin/bash +set -euo pipefail + +usage() { + cat <<'EOF' +Usage: + scripts/ios-team-id.sh [--require-canonical] + +Prints an Apple Developer Team ID for iOS signing. + +Default behavior: +- return IOS_DEVELOPMENT_TEAM when set +- prefer the canonical OpenClaw iOS team when available in Xcode +- otherwise fall back to a local Xcode team for local development builds + +Options: + --require-canonical fail unless the resolved team is FWJYW4S8P8 +EOF +} + +canonical_team="FWJYW4S8P8" +require_canonical="0" +while [[ $# -gt 0 ]]; do + case "$1" in + --require-canonical) + require_canonical="1" + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unknown argument: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +canonical_team="${canonical_team//$'\r'/}" + +if [[ -n "${IOS_DEVELOPMENT_TEAM:-}" ]]; then + explicit_team="${IOS_DEVELOPMENT_TEAM//$'\r'/}" + if [[ "$require_canonical" == "1" && "$explicit_team" != "$canonical_team" ]]; then + echo "Resolved iOS Team ID '${explicit_team}' is not the canonical OpenClaw iOS team '${canonical_team}'." >&2 + exit 1 + fi + printf '%s\n' "$explicit_team" + exit 0 +fi + +preferred_team="${IOS_PREFERRED_TEAM_ID:-}" +preferred_team_name="${IOS_PREFERRED_TEAM_NAME:-}" +allow_keychain_fallback="${IOS_ALLOW_KEYCHAIN_TEAM_FALLBACK:-0}" +prefer_non_free_team="${IOS_PREFER_NON_FREE_TEAM:-1}" +preferred_team="${preferred_team//$'\r'/}" +preferred_team_name="${preferred_team_name//$'\r'/}" + +declare -a team_ids=() +declare -a team_is_free=() +declare -a team_names=() +python_cmd="" + +detect_python() { + local candidate + for candidate in "${IOS_PYTHON_BIN:-}" python3 python /usr/bin/python3; do + [[ -n "$candidate" ]] || continue + if command -v "$candidate" >/dev/null 2>&1; then + printf '%s\n' "$candidate" + return 0 + fi + done + return 1 +} + +python_cmd="$(detect_python || true)" + +append_team() { + local candidate_id="$1" + local candidate_is_free="$2" + local candidate_name="$3" + candidate_id="${candidate_id//$'\r'/}" + candidate_is_free="${candidate_is_free//$'\r'/}" + candidate_name="${candidate_name//$'\r'/}" + [[ -z "$candidate_id" ]] && return + + local i + for i in "${!team_ids[@]}"; do + if [[ "${team_ids[$i]}" == "$candidate_id" ]]; then + return + fi + done + + team_ids+=("$candidate_id") + team_is_free+=("$candidate_is_free") + team_names+=("$candidate_name") +} + +load_teams_from_xcode_team_key() { + local key="$1" + local plist_path="${HOME}/Library/Preferences/com.apple.dt.Xcode.plist" + [[ -f "$plist_path" ]] || return 0 + [[ -n "$python_cmd" ]] || return 0 + + while IFS=$'\t' read -r team_id is_free team_name; do + [[ -z "$team_id" ]] && continue + append_team "$team_id" "${is_free:-0}" "${team_name:-}" + done < <( + plutil -extract "$key" json -o - "$plist_path" 2>/dev/null \ + | "$python_cmd" -c ' +import json +import sys + +try: + data = json.load(sys.stdin) +except Exception: + raise SystemExit(0) + +if not isinstance(data, dict): + raise SystemExit(0) + +for teams in data.values(): + if not isinstance(teams, list): + continue + for team in teams: + if not isinstance(team, dict): + continue + team_id = str(team.get("teamID", "")).strip() + if not team_id: + continue + is_free = "1" if bool(team.get("isFreeProvisioningTeam", False)) else "0" + team_name = str(team.get("teamName", "")).replace("\t", " ").strip() + print(f"{team_id}\t{is_free}\t{team_name}") +' + ) +} + +load_teams_from_xcode_preferences() { + load_teams_from_xcode_team_key IDEProvisioningTeamByIdentifier + load_teams_from_xcode_team_key IDEProvisioningTeams +} + +load_teams_from_legacy_defaults_key() { + while IFS= read -r team; do + [[ -z "$team" ]] && continue + append_team "$team" "0" "" + done < <( + defaults read com.apple.dt.Xcode IDEProvisioningTeamIdentifiers 2>/dev/null \ + | grep -Eo '[A-Z0-9]{10}' || true + ) +} + +load_teams_from_xcode_managed_profiles() { + local profiles_dir="${HOME}/Library/MobileDevice/Provisioning Profiles" + [[ -d "$profiles_dir" ]] || return 0 + [[ -n "$python_cmd" ]] || return 0 + + while IFS= read -r team; do + [[ -z "$team" ]] && continue + append_team "$team" "0" "" + done < <( + for p in "${profiles_dir}"/*.mobileprovision; do + [[ -f "$p" ]] || continue + security cms -D -i "$p" 2>/dev/null \ + | "$python_cmd" -c ' +import plistlib, sys +try: + raw = sys.stdin.buffer.read() + if not raw: + raise SystemExit(0) + d = plistlib.loads(raw) + for tid in d.get("TeamIdentifier", []): + print(tid) +except Exception: + pass +' 2>/dev/null + done | sort -u + ) +} + +has_xcode_account() { + local plist_path="${HOME}/Library/Preferences/com.apple.dt.Xcode.plist" + [[ -f "$plist_path" ]] || return 1 + local accts + accts="$(defaults read com.apple.dt.Xcode DVTDeveloperAccountManagerAppleIDLists 2>/dev/null || true)" + [[ -n "$accts" ]] && [[ "$accts" != *"does not exist"* ]] && grep -q 'identifier' <<< "$accts" +} + +load_teams_from_xcode_preferences +load_teams_from_legacy_defaults_key + +if [[ ${#team_ids[@]} -eq 0 ]]; then + load_teams_from_xcode_managed_profiles +fi + +if [[ ${#team_ids[@]} -eq 0 && "$allow_keychain_fallback" == "1" ]]; then + while IFS= read -r team; do + [[ -z "$team" ]] && continue + append_team "$team" "0" "" + done < <( + security find-identity -p codesigning -v 2>/dev/null \ + | grep -Eo '\([A-Z0-9]{10}\)' \ + | tr -d '()' || true + ) +fi + +if [[ ${#team_ids[@]} -eq 0 ]]; then + if [[ "$require_canonical" == "1" ]]; then + echo "Canonical OpenClaw iOS Team ID '${canonical_team}' is not available in Xcode on this machine." >&2 + echo "Sign into the Apple Developer account that owns the canonical team, or set IOS_DEVELOPMENT_TEAM=${canonical_team}." >&2 + exit 1 + fi + + if has_xcode_account; then + echo "An Apple account is signed in to Xcode, but no Team ID could be resolved." >&2 + echo "" >&2 + echo "On Xcode 16+, team data is not written until you build a project." >&2 + echo "To fix this, do ONE of the following:" >&2 + echo "" >&2 + echo " 1. Open the iOS project in Xcode, select your Team in Signing &" >&2 + echo " Capabilities, and build once. Then re-run this script." >&2 + echo "" >&2 + echo " 2. Set your Team ID directly:" >&2 + echo " export IOS_DEVELOPMENT_TEAM=<your-10-char-team-id>" >&2 + echo " Find your Team ID at: https://developer.apple.com/account#MembershipDetailsCard" >&2 + elif [[ "$allow_keychain_fallback" == "1" ]]; then + echo "No Apple Team ID found. Open Xcode or install signing certificates first." >&2 + else + echo "No Apple Team ID found in Xcode accounts. Open Xcode → Settings → Accounts and sign in, then retry." >&2 + echo "(Set IOS_ALLOW_KEYCHAIN_TEAM_FALLBACK=1 to allow keychain-only team detection.)" >&2 + fi + exit 1 +fi + +for i in "${!team_ids[@]}"; do + if [[ "${team_ids[$i]}" == "$canonical_team" ]]; then + printf '%s\n' "${team_ids[$i]}" + exit 0 + fi +done + +if [[ "$require_canonical" == "1" ]]; then + echo "Canonical OpenClaw iOS Team ID '${canonical_team}' is not available in Xcode on this machine." >&2 + echo "Sign into the Apple Developer account that owns the canonical team, or set IOS_DEVELOPMENT_TEAM=${canonical_team}." >&2 + exit 1 +fi + +if [[ -n "$preferred_team" ]]; then + for i in "${!team_ids[@]}"; do + if [[ "${team_ids[$i]}" == "$preferred_team" ]]; then + printf '%s\n' "${team_ids[$i]}" + exit 0 + fi + done +fi + +if [[ -n "$preferred_team_name" ]]; then + preferred_team_name_lc="$(printf '%s' "$preferred_team_name" | tr '[:upper:]' '[:lower:]')" + for i in "${!team_ids[@]}"; do + team_name_lc="$(printf '%s' "${team_names[$i]}" | tr '[:upper:]' '[:lower:]')" + if [[ "$team_name_lc" == "$preferred_team_name_lc" ]]; then + printf '%s\n' "${team_ids[$i]}" + exit 0 + fi + done +fi + +if [[ "$prefer_non_free_team" == "1" ]]; then + for i in "${!team_ids[@]}"; do + if [[ "${team_is_free[$i]}" == "0" ]]; then + printf '%s\n' "${team_ids[$i]}" + exit 0 + fi + done +fi + +printf '%s\n' "${team_ids[0]}" diff --git a/scripts/ios-validate-app-store-ipa.sh b/scripts/ios-validate-app-store-ipa.sh new file mode 100644 index 0000000000000000000000000000000000000000..691f6e6d01c4e6ee8462da6265af7f7caa5d54df --- /dev/null +++ b/scripts/ios-validate-app-store-ipa.sh @@ -0,0 +1,278 @@ +#!/bin/bash +set -euo pipefail + +usage() { + cat <<'EOF' +Usage: + scripts/ios-validate-app-store-ipa.sh --ipa apps/ios/build/app-store/OpenClaw-<version>.ipa \ + [--expected-commit <full-sha>] [--expected-build-timestamp <utc-iso>] + +Validates the exported iOS App Store IPA before App Store Connect upload. +EOF +} + +IPA_PATH="" +EXPECTED_GIT_COMMIT="" +EXPECTED_BUILD_TIMESTAMP="" +EXPECTED_TEAM_ID="FWJYW4S8P8" +EXPECTED_BUNDLE_ID="ai.openclawfoundation.app" +EXPECTED_PROFILE_NAME="OpenClaw App Store ai.openclawfoundation.app" +EXPECTED_APP_GROUP="group.ai.openclawfoundation.app.shared" +EXPECTED_PUSH_MODE="appStore" + +PLIST_BUDDY_BIN="${IOS_VALIDATE_PLIST_BUDDY_BIN:-/usr/libexec/PlistBuddy}" +PLUTIL_BIN="${IOS_VALIDATE_PLUTIL_BIN:-plutil}" +CODESIGN_BIN="${IOS_VALIDATE_CODESIGN_BIN:-codesign}" +SECURITY_BIN="${IOS_VALIDATE_SECURITY_BIN:-security}" +UNZIP_BIN="${IOS_VALIDATE_UNZIP_BIN:-unzip}" + +require_option_value() { + local option="$1" + local value="${2-}" + + if [[ -z "${value}" || "${value}" == --* ]]; then + echo "Missing value for ${option}." >&2 + usage >&2 + exit 1 + fi +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --ipa) + require_option_value "$1" "${2-}" + IPA_PATH="$2" + shift 2 + ;; + --expected-commit) + require_option_value "$1" "${2-}" + EXPECTED_GIT_COMMIT="$(printf '%s' "$2" | tr '[:upper:]' '[:lower:]')" + shift 2 + ;; + --expected-build-timestamp) + require_option_value "$1" "${2-}" + EXPECTED_BUILD_TIMESTAMP="$2" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unknown argument: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +if [[ -z "${IPA_PATH}" ]]; then + echo "Missing required --ipa." >&2 + usage >&2 + exit 1 +fi + +if [[ ! -f "${IPA_PATH}" ]]; then + echo "IPA not found: ${IPA_PATH}" >&2 + exit 1 +fi + +tmp_dir="$(mktemp -d -t openclaw-ios-ipa.XXXXXX)" +trap 'rm -rf "${tmp_dir}"' EXIT + +"${UNZIP_BIN}" -q "${IPA_PATH}" -d "${tmp_dir}" + +payload_dir="${tmp_dir}/Payload" +if [[ ! -d "${payload_dir}" ]]; then + echo "Invalid IPA: missing Payload directory." >&2 + exit 1 +fi + +app_paths=() +while IFS= read -r app_bundle; do + app_paths+=("${app_bundle}") +done < <(find "${payload_dir}" -maxdepth 1 -type d -name "*.app" | sort) +if [[ "${#app_paths[@]}" -ne 1 ]]; then + echo "Invalid IPA: expected exactly one app bundle in Payload, found ${#app_paths[@]}." >&2 + exit 1 +fi + +app_path="${app_paths[0]}" +info_plist="${app_path}/Info.plist" +embedded_profile="${app_path}/embedded.mobileprovision" +entitlements_plist="${tmp_dir}/entitlements.plist" +profile_plist="${tmp_dir}/profile.plist" + +if [[ ! -f "${info_plist}" ]]; then + echo "Invalid IPA: missing app Info.plist." >&2 + exit 1 +fi +if [[ ! -f "${embedded_profile}" ]]; then + echo "Invalid IPA: missing embedded.mobileprovision." >&2 + exit 1 +fi + +plist_value() { + local plist="$1" + local key_path="$2" + "${PLIST_BUDDY_BIN}" -c "Print:${key_path}" "${plist}" 2>/dev/null || true +} + +plist_has_key() { + local plist="$1" + local key_path="$2" + "${PLIST_BUDDY_BIN}" -c "Print:${key_path}" "${plist}" >/dev/null 2>&1 +} + +assert_plist_string() { + local plist="$1" + local key_path="$2" + local expected="$3" + local label="$4" + local actual + actual="$(plist_value "${plist}" "${key_path}")" + if [[ "${actual}" != "${expected}" ]]; then + echo "Invalid IPA: ${label}; expected ${expected}, got ${actual:-missing}." >&2 + exit 1 + fi +} + +assert_plist_nonempty_string() { + local plist="$1" + local key_path="$2" + local label="$3" + local actual + actual="$("${PLUTIL_BIN}" -extract "${key_path}" raw -expect string -o - "${plist}" 2>/dev/null || true)" + if [[ -z "${actual}" ]]; then + echo "Invalid IPA: ${label} must be a non-empty string." >&2 + exit 1 + fi +} + +assert_plist_true() { + local plist="$1" + local key_path="$2" + local label="$3" + local actual + actual="$(plist_value "${plist}" "${key_path}")" + if [[ "${actual}" != "true" ]]; then + echo "Invalid IPA: ${label}; expected true, got ${actual:-missing}." >&2 + exit 1 + fi +} + +assert_plist_key_absent() { + local plist="$1" + local key_path="$2" + local label="$3" + if plist_has_key "${plist}" "${key_path}"; then + echo "Invalid IPA: ${label} must not be present in App Store builds." >&2 + exit 1 + fi +} + +assert_plist_array_contains() { + local plist="$1" + local key_path="$2" + local expected="$3" + local label="$4" + local raw + raw="$(plist_value "${plist}" "${key_path}")" + if ! printf '%s\n' "${raw}" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//' | grep -Fxq "${expected}"; then + echo "Invalid IPA: ${label}; expected ${expected} in ${key_path}." >&2 + exit 1 + fi +} + +assert_plist_empty_or_absent() { + local plist="$1" + local key_path="$2" + local label="$3" + local actual + actual="$(plist_value "${plist}" "${key_path}")" + if [[ -n "${actual}" ]]; then + echo "Invalid IPA: ${label} must be empty for App Store builds; got ${actual}." >&2 + exit 1 + fi +} + +assert_build_provenance() { + local commit + local timestamp + commit="$(plist_value "${info_plist}" "OpenClawGitCommit")" + timestamp="$(plist_value "${info_plist}" "OpenClawBuildTimestamp")" + if [[ ! "${commit}" =~ ^[0-9a-f]{40}$ ]]; then + echo "Invalid IPA: OpenClawGitCommit must be a full lowercase commit SHA; got ${commit:-missing}." >&2 + exit 1 + fi + if [[ ! "${timestamp}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]{3})?Z$ ]]; then + echo "Invalid IPA: OpenClawBuildTimestamp must be a canonical UTC timestamp; got ${timestamp:-missing}." >&2 + exit 1 + fi + if [[ -n "${EXPECTED_GIT_COMMIT}" && "${commit}" != "${EXPECTED_GIT_COMMIT}" ]]; then + echo "Invalid IPA: embedded Git commit mismatch; expected ${EXPECTED_GIT_COMMIT}, got ${commit}." >&2 + exit 1 + fi + if [[ -n "${EXPECTED_BUILD_TIMESTAMP}" && "${timestamp}" != "${EXPECTED_BUILD_TIMESTAMP}" ]]; then + echo "Invalid IPA: embedded build timestamp mismatch; expected ${EXPECTED_BUILD_TIMESTAMP}, got ${timestamp}." >&2 + exit 1 + fi +} + +assert_localized_plists_resolve_build_settings() { + local localized_plist + local rendered + while IFS= read -r -d '' localized_plist; do + if ! rendered="$("${PLUTIL_BIN}" -convert xml1 -o - "${localized_plist}" 2>/dev/null)"; then + echo "Invalid IPA: could not read localized plist ${localized_plist#"${app_path}/"}." >&2 + exit 1 + fi + if grep -Eq '\$\([A-Za-z0-9_.-]+\)|\$\{[A-Za-z0-9_.-]+\}' <<<"${rendered}"; then + echo "Invalid IPA: unresolved build setting in localized plist ${localized_plist#"${app_path}/"}." >&2 + exit 1 + fi + done < <(find "${app_path}" -type f -path "*.lproj/InfoPlist.strings" -print0) +} + +assert_plist_string "${info_plist}" "CFBundleIdentifier" "${EXPECTED_BUNDLE_ID}" "bundle identifier mismatch" +assert_plist_string "${info_plist}" "CFBundleDisplayName" "OpenClaw" "display name mismatch" +assert_plist_string "${info_plist}" "OpenClawPushMode" "${EXPECTED_PUSH_MODE}" "push mode mismatch" +assert_plist_nonempty_string "${info_plist}" "NSHealthShareUsageDescription" "Health share usage description" +assert_plist_nonempty_string "${info_plist}" "NSHealthUpdateUsageDescription" "Health update usage description" +assert_build_provenance +assert_localized_plists_resolve_build_settings +assert_plist_empty_or_absent "${info_plist}" "OpenClawPushRelayBaseURL" "push relay URL override" +assert_plist_key_absent "${info_plist}" "OpenClawPushTransport" "legacy push transport" +assert_plist_key_absent "${info_plist}" "OpenClawPushDistribution" "legacy push distribution" +assert_plist_key_absent "${info_plist}" "OpenClawPushAPNsEnvironment" "legacy APNs environment" +assert_plist_key_absent "${info_plist}" "OpenClawPushRelayProfile" "legacy relay profile" +assert_plist_key_absent "${info_plist}" "OpenClawPushProofPolicy" "legacy proof policy" + +if ! "${CODESIGN_BIN}" -d --entitlements :- "${app_path}" >"${entitlements_plist}" 2>"${tmp_dir}/codesign.err"; then + detail="$(<"${tmp_dir}/codesign.err")" + echo "Invalid IPA: failed to read signed entitlements${detail:+: ${detail}}" >&2 + exit 1 +fi + +assert_plist_string "${entitlements_plist}" "application-identifier" "${EXPECTED_TEAM_ID}.${EXPECTED_BUNDLE_ID}" "signed application identifier mismatch" +assert_plist_string "${entitlements_plist}" "com.apple.developer.team-identifier" "${EXPECTED_TEAM_ID}" "signed team identifier mismatch" +assert_plist_string "${entitlements_plist}" "aps-environment" "production" "signed APNs entitlement mismatch" +assert_plist_string "${entitlements_plist}" "com.apple.developer.devicecheck.appattest-environment" "production" "signed App Attest entitlement mismatch" +assert_plist_true "${entitlements_plist}" "com.apple.developer.healthkit" "signed HealthKit entitlement mismatch" +assert_plist_array_contains "${entitlements_plist}" "com.apple.security.application-groups" "${EXPECTED_APP_GROUP}" "signed App Group entitlement mismatch" + +if ! "${SECURITY_BIN}" cms -D -i "${embedded_profile}" >"${profile_plist}" 2>"${tmp_dir}/security.err"; then + detail="$(<"${tmp_dir}/security.err")" + echo "Invalid IPA: failed to decode embedded provisioning profile${detail:+: ${detail}}" >&2 + exit 1 +fi + +assert_plist_string "${profile_plist}" "Name" "${EXPECTED_PROFILE_NAME}" "embedded profile name mismatch" +assert_plist_array_contains "${profile_plist}" "TeamIdentifier" "${EXPECTED_TEAM_ID}" "embedded profile team mismatch" +assert_plist_string "${profile_plist}" "Entitlements:application-identifier" "${EXPECTED_TEAM_ID}.${EXPECTED_BUNDLE_ID}" "embedded profile application identifier mismatch" +assert_plist_string "${profile_plist}" "Entitlements:aps-environment" "production" "embedded profile APNs entitlement mismatch" +assert_plist_array_contains "${profile_plist}" "Entitlements:com.apple.developer.devicecheck.appattest-environment" "production" "embedded profile App Attest entitlement mismatch" +assert_plist_true "${profile_plist}" "Entitlements:com.apple.developer.healthkit" "embedded profile HealthKit entitlement mismatch" +assert_plist_array_contains "${profile_plist}" "Entitlements:com.apple.security.application-groups" "${EXPECTED_APP_GROUP}" "embedded profile App Group entitlement mismatch" + +echo "Validated iOS App Store IPA: ${IPA_PATH}" diff --git a/scripts/ios-write-swift-filelist.mts b/scripts/ios-write-swift-filelist.mts new file mode 100644 index 0000000000000000000000000000000000000000..5ad1ef46033dfa76c28bce3a4694c5d4b44b624e --- /dev/null +++ b/scripts/ios-write-swift-filelist.mts @@ -0,0 +1,68 @@ +#!/usr/bin/env node +import { existsSync, lstatSync, mkdirSync, readdirSync, writeFileSync } from "node:fs"; +import path from "node:path"; +import { resolveRepoRoot } from "./lib/repo-root.mjs"; +const repoRoot = resolveRepoRoot(import.meta.url); +const iosRoot = path.join(repoRoot, "apps", "ios"); +const outputPath = path.join(iosRoot, "SwiftSources.input.xcfilelist"); + +const iosSourceRoots = [ + "Sources", + "ShareExtension", + "ActivityWidget", + path.join("WatchApp", "Sources"), +]; + +const sharedSourceRoots = [ + path.join("..", "shared", "OpenClawKit", "Sources", "OpenClawChatUI"), + path.join("..", "shared", "OpenClawKit", "Sources", "OpenClawKit"), + path.join("..", "shared", "OpenClawKit", "Sources", "OpenClawNativeState"), + path.join("..", "shared", "OpenClawKit", "Sources", "OpenClawProtocol"), + path.join("..", "swabble", "Sources", "SwabbleKit"), +]; + +const excludedSwiftFiles = new Set([ + "../shared/OpenClawKit/Sources/OpenClawProtocol/GatewayModels.swift", +]); + +function normalizeFileListPath(filePath: string): string { + return filePath.split(path.sep).join("/"); +} + +function collectSwiftFiles(rootRelativePath: string): string[] { + const root = path.join(iosRoot, rootRelativePath); + if (!existsSync(root)) { + throw new Error(`Missing Swift source root: ${rootRelativePath}`); + } + + const entries: string[] = []; + const visit = (dir: string): void => { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const fullPath = path.join(dir, entry.name); + if (entry.isDirectory()) { + visit(fullPath); + } else if (entry.isFile() && entry.name.endsWith(".swift")) { + entries.push(normalizeFileListPath(path.relative(iosRoot, fullPath))); + } + } + }; + visit(root); + return entries; +} + +function writeGeneratedFile(filePath: string, contents: string): void { + if (existsSync(filePath) && lstatSync(filePath).isSymbolicLink()) { + throw new Error(`Refusing to overwrite symlinked file: ${filePath}`); + } + mkdirSync(path.dirname(filePath), { recursive: true }); + writeFileSync(filePath, contents, "utf8"); +} + +const iosFiles = iosSourceRoots.flatMap(collectSwiftFiles); +const sharedFiles = sharedSourceRoots.flatMap(collectSwiftFiles); +const fileList = [...new Set([...iosFiles, ...sharedFiles])] + .filter((filePath) => !excludedSwiftFiles.has(filePath)) + .toSorted((left, right) => left.localeCompare(right)); + +writeGeneratedFile(outputPath, `${fileList.join("\n")}\n`); +process.stdout.write(`Prepared iOS Swift file list: ${path.relative(repoRoot, outputPath)}\n`); diff --git a/scripts/kova-ci-summary.mts b/scripts/kova-ci-summary.mts new file mode 100644 index 0000000000000000000000000000000000000000..1a58a312e2ac12bf828c03c2a4e018b6cb89f616 --- /dev/null +++ b/scripts/kova-ci-summary.mts @@ -0,0 +1,360 @@ +#!/usr/bin/env node +// Summarizes Kova CI run metadata for diagnostics. +import { readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { isRecord } from "./lib/record-shared.mjs"; + +const knownArgKeys = new Set(["report", "output", "lane", "reporturl", "artifacturl"]); +const rawArgs = process.argv.slice(2); +if (shouldPrintHelp(rawArgs)) { + usage("", 0); +} + +const args = parseArgs(rawArgs); +if (!args.report) { + usage("missing --report"); +} + +const keyMetricIds = [ + "timeToHealthReadyMs", + "timeToListeningMs", + "healthP95Ms", + "peakRssMb", + "resourcePeakGatewayRssMb", + "cpuPercentMax", + "openclawEventLoopMaxMs", + "agentTurnP95Ms", + "coldAgentTurnMs", + "warmAgentTurnMs", + "agentPreProviderP95Ms", + "agentProviderFinalP95Ms", + "agentCleanupP95Ms", + "runtimeDepsStagingMs", +]; +const rssMetricIds = ["peakRssMb", "resourcePeakGatewayRssMb"]; +const cpuMetricIds = ["cpuPercentMax"]; +const reportPath = path.resolve(args.report); +type Metric = Partial<Record<"count" | "title" | "median" | "p95" | "max" | "unit", unknown>>; +type PerformanceGroup = Partial<Record<"scenario" | "state", unknown>> & { + metrics?: Record<string, Metric | undefined>; +}; +type KovaRecord = Partial<Record<"scenario" | "state" | "status" | "failureReason", unknown>> & { + error?: { message?: unknown }; + violations?: Array<Record<string, unknown>>; +}; +type KovaReport = Partial<Record<"runId" | "generatedAt" | "target", unknown>> & { + summary?: { statuses?: Record<string, unknown> }; + performance?: { + repeat?: unknown; + groups?: PerformanceGroup[]; + resourceCollectionSkippedReason?: unknown; + }; + records?: KovaRecord[]; +}; +type SummaryOptions = { lane: string; reportUrl: string; artifactUrl: string }; + +const report: KovaReport = JSON.parse(await readFile(reportPath, "utf8")); +const invalidReport = validateKovaSummaryReport(report); +if (invalidReport) { + console.error(`error: invalid Kova report: ${invalidReport}`); + process.exit(1); +} +const markdown = renderSummary(report, { + lane: args.lane || "kova", + reportUrl: args.reportUrl || "", + artifactUrl: args.artifactUrl || "", +}); + +if (args.output) { + await writeFile(path.resolve(args.output), markdown, "utf8"); +} else { + process.stdout.write(markdown); +} + +function renderSummary(reportLocal: KovaReport, options: SummaryOptions) { + const lines = []; + const statuses = reportLocal.summary?.statuses || {}; + const statusText = + Object.entries(statuses) + .map(([status, count]) => `${status}: ${value(count)}`) + .join(", ") || "unknown"; + + lines.push(`# OpenClaw Performance Report`); + lines.push(""); + lines.push(`- Lane: ${options.lane}`); + lines.push(`- Run: ${value(reportLocal.runId)}`); + lines.push(`- Generated: ${value(reportLocal.generatedAt)}`); + lines.push(`- Target: ${value(reportLocal.target)}`); + lines.push(`- Statuses: ${statusText}`); + lines.push(`- Repeat: ${value(reportLocal.performance?.repeat)}`); + if (options.reportUrl) { + lines.push(`- Published report: ${options.reportUrl}`); + } + if (options.artifactUrl) { + lines.push(`- GitHub artifact: ${options.artifactUrl}`); + } + lines.push(""); + + const groups = Array.isArray(reportLocal.performance?.groups) + ? reportLocal.performance.groups + : []; + const metricRows = []; + for (const group of groups) { + for (const metricId of keyMetricIds) { + const metric = group.metrics?.[metricId]; + if (!metric || !hasPositiveSampleCount(metric)) { + continue; + } + metricRows.push( + [ + value(group.scenario), + value(group.state), + value(metric.title || metricId), + formatMetric(metric.median, metric.unit), + formatMetric(metric.p95, metric.unit), + formatMetric(metric.max, metric.unit), + ] + .join(" | ") + .replace(/^/, "| ") + .replace(/$/, " |"), + ); + } + } + if (metricRows.length > 0) { + lines.push("## Key metrics"); + lines.push(""); + lines.push("| Scenario | State | Metric | Median | p95 | Max |"); + lines.push("| --- | --- | --- | ---: | ---: | ---: |"); + lines.push(...metricRows); + lines.push(""); + } else if (groups.length > 0) { + lines.push("## Key metrics"); + lines.push(""); + lines.push("- No sampled key metrics were available; inspect the blocking records below."); + lines.push(""); + } + + const violations = collectViolations(reportLocal.records); + if (violations.length > 0) { + lines.push("## Threshold violations"); + lines.push(""); + lines.push("| Scenario | State | Metric | Actual | Threshold |"); + lines.push("| --- | --- | --- | ---: | ---: |"); + for (const item of violations.slice(0, 20)) { + lines.push( + [ + item.scenario, + item.state, + item.metric, + formatMetric(item.actual, item.unit), + formatMetric(item.threshold, item.unit), + ] + .join(" | ") + .replace(/^/, "| ") + .replace(/$/, " |"), + ); + } + if (violations.length > 20) { + lines.push(""); + lines.push(`_Only first 20 of ${violations.length} violations shown._`); + } + lines.push(""); + } + + const records = Array.isArray(reportLocal.records) ? reportLocal.records : []; + if (records.length > 0) { + lines.push("## Records"); + lines.push(""); + lines.push("| Scenario | State | Status | Failure |"); + lines.push("| --- | --- | --- | --- |"); + for (const record of records.slice(0, 30)) { + lines.push( + [ + value(record.scenario), + value(stateValue(record.state)), + value(record.status), + value(record.failureReason || record.error?.message || ""), + ] + .join(" | ") + .replace(/^/, "| ") + .replace(/$/, " |"), + ); + } + lines.push(""); + } + + return `${lines.join("\n").trimEnd()}\n`; +} + +function validateKovaSummaryReport(reportLocal: KovaReport) { + if (!isRecord(reportLocal)) { + return "report must be an object"; + } + const statuses = reportLocal.summary?.statuses; + if ( + !statuses || + typeof statuses !== "object" || + Array.isArray(statuses) || + Object.keys(statuses).length === 0 + ) { + return "missing summary.statuses"; + } + const records = Array.isArray(reportLocal.records) ? reportLocal.records : []; + const groups = Array.isArray(reportLocal.performance?.groups) + ? reportLocal.performance.groups + : []; + if (records.length === 0 && groups.length === 0) { + return "missing records or performance groups"; + } + if ( + groups.length > 0 && + reportHasOnlyPassingStatuses(statuses) && + !hasExplicitResourceCollectionSkip(reportLocal) + ) { + if (!groups.some((group) => hasSampledMetric(group, rssMetricIds))) { + return "missing sampled RSS metric in performance groups"; + } + if (!groups.some((group) => hasSampledMetric(group, cpuMetricIds))) { + return "missing sampled CPU metric in performance groups"; + } + } + return null; +} + +function reportHasOnlyPassingStatuses(statuses: Record<string, unknown>) { + const populated = Object.entries(statuses).filter(([, count]) => Number(count) > 0); + return ( + populated.length > 0 && populated.every(([status]) => status.trim().toUpperCase() === "PASS") + ); +} + +function hasExplicitResourceCollectionSkip(reportLocal: KovaReport) { + const reason = reportLocal.performance?.resourceCollectionSkippedReason; + return typeof reason === "string" && reason.trim().length > 0; +} + +function hasSampledMetric(group: PerformanceGroup, metricIds: string[]) { + return metricIds.some((metricId) => hasPositiveSampleCount(group?.metrics?.[metricId])); +} + +function hasPositiveSampleCount(metric: Metric | undefined) { + return ( + typeof metric?.count === "number" && Number.isSafeInteger(metric.count) && metric.count > 0 + ); +} + +function stateValue(state: unknown): unknown { + return isRecord(state) ? state.id : state; +} + +function collectViolations(records: KovaRecord[] | undefined) { + if (!Array.isArray(records)) { + return []; + } + return records.flatMap((record) => { + if (!Array.isArray(record.violations)) { + return []; + } + return record.violations.map((violation) => ({ + scenario: value(record.scenario), + state: value(stateValue(record.state)), + metric: value(violation.metric || violation.id || violation.name), + actual: violation.actual ?? violation.value, + threshold: violation.threshold ?? violation.max ?? violation.expected, + unit: violation.unit, + })); + }); +} + +function formatMetric(valueToFormat: unknown, unit: unknown) { + if ( + valueToFormat === null || + valueToFormat === undefined || + (typeof valueToFormat === "number" && Number.isNaN(valueToFormat)) + ) { + return ""; + } + const numeric = Number(valueToFormat); + const rendered = Number.isFinite(numeric) + ? numeric.toLocaleString("en-US", { maximumFractionDigits: numeric >= 100 ? 0 : 1 }) + : value(valueToFormat); + const unitText = value(unit); + return unitText ? `${rendered} ${unitText}` : rendered; +} + +function value(input: unknown) { + if (input === null || input === undefined) { + return ""; + } + const rendered = + typeof input === "string" || + typeof input === "number" || + typeof input === "boolean" || + typeof input === "bigint" + ? String(input) + : (JSON.stringify(input) ?? ""); + return rendered.replaceAll("|", "\\|").replaceAll("\n", " "); +} + +function parseArgs(argv: string[]) { + const parsed: Record<string, string> = {}; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (!arg?.startsWith("--")) { + usage(`unexpected argument: ${arg}`); + } + const key = arg.slice(2).replaceAll("-", ""); + if (!knownArgKeys.has(key)) { + usage(`unknown argument: ${arg}`); + } + const valueLocal = argv[index + 1]; + if (!valueLocal || valueLocal.startsWith("-")) { + usage(`${arg} requires a value`); + } + parsed[key] = valueLocal; + index += 1; + } + return { + report: parsed.report, + output: parsed.output, + lane: parsed.lane, + reportUrl: parsed.reporturl, + artifactUrl: parsed.artifacturl, + }; +} + +function shouldPrintHelp(argv: string[]) { + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === "--help" || arg === "-h") { + return true; + } + if (!arg?.startsWith("--")) { + return false; + } + const key = arg.slice(2).replaceAll("-", ""); + if (!knownArgKeys.has(key)) { + return false; + } + const optionValue = argv[index + 1]; + if (!optionValue || optionValue.startsWith("-")) { + return false; + } + index += 1; + } + return false; +} + +function usage(message: string, status = 2): never { + const text = + "usage: node --import tsx scripts/kova-ci-summary.mts --report <report.json> [--output <summary.md>] [--lane <name>] [--report-url <url>] [--artifact-url <url>]\n"; + if (message) { + console.error(`error: ${message}`); + } + if (status === 0 && !message) { + process.stdout.write(text); + } else { + process.stderr.write(text); + } + process.exit(status); +} diff --git a/scripts/lint-swift.sh b/scripts/lint-swift.sh new file mode 100644 index 0000000000000000000000000000000000000000..ad909bda92d8127259a8e50197ad49563b392a5b --- /dev/null +++ b/scripts/lint-swift.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +set -euo pipefail + +repo_root="$(cd "$(dirname "$0")/.." && pwd)" +cd "$repo_root" + +scope="${1:-all}" +if [[ "$scope" != "all" && "$scope" != "ios" && "$scope" != "macos" ]]; then + echo "usage: $0 [ios|macos]" >&2 + exit 2 +fi + +./scripts/check-swift-tools.sh swiftlint + +if [[ "$scope" != "ios" ]]; then + swiftlint lint --strict --config config/swiftlint.yml + ( + cd apps/swabble + swiftlint lint --strict --config .swiftlint.yml + ) +fi + +if [[ "$scope" == "macos" ]]; then + exit 0 +fi + +( + cd apps/ios + swiftlint lint --strict --config .swiftlint.yml +) diff --git a/scripts/linux-app-channel.mjs b/scripts/linux-app-channel.mjs new file mode 100644 index 0000000000000000000000000000000000000000..c58d1039fcf44a94dd06135a7c5a03ab56879688 --- /dev/null +++ b/scripts/linux-app-channel.mjs @@ -0,0 +1,1339 @@ +#!/usr/bin/env node + +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { + closeSync, + copyFileSync, + lstatSync, + mkdtempSync, + openSync, + readFileSync, + readSync, + readdirSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { pathToFileURL } from "node:url"; +import { parseArgs } from "node:util"; +import { + classifyReleaseTrain, + compareReleaseVersions, + parseReleaseVersion, +} from "./lib/release-version.mjs"; +import { + resolveReleaseToolingIdentity, + verifyReleaseToolingIdentity, + verifyReleaseWorkflowRun, +} from "./release-tooling-identity.mjs"; + +const REPOSITORY = "openclaw/openclaw"; +const CHANNEL = "linux-stable"; +const INITIAL_BODY = "OpenClaw Linux update channel. Publication is not complete."; +const METADATA_LIMIT = 1024 * 1024; +const BUNDLE_LIMIT = 2 * 1024 * 1024 * 1024; +const SHA = /^[0-9a-f]{40}$/u; +const DIGEST = /^[0-9a-f]{64}$/u; + +function digest(bytes) { + return createHash("sha256").update(bytes).digest("hex"); +} + +function fileDigest(path) { + const hash = createHash("sha256"); + const fd = openSync(path, "r"); + const buffer = Buffer.alloc(1024 * 1024); + try { + for (let count; (count = readSync(fd, buffer, 0, buffer.length, null)) !== 0;) { + hash.update(buffer.subarray(0, count)); + } + return hash.digest("hex"); + } finally { + closeSync(fd); + } +} + +function command(binary, args, timeout = 60_000) { + return execFileSync(binary, args, { + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + timeout, + killSignal: "SIGKILL", + maxBuffer: 2 * METADATA_LIMIT, + env: { + ...process.env, + GH_PROMPT_DISABLED: "1", + TAURI_SIGNING_PRIVATE_KEY: "", + TAURI_SIGNING_PRIVATE_KEY_PASSWORD: "", + }, + }); +} + +function authorizeWrite(mode, options) { + const alphaBranch = + mode === "finalize-core" && options["workflow-ref"].startsWith("tideclaw/alpha/"); + if (alphaBranch) { + assert( + options.latest === "false" && parseReleaseVersion(options.tag.slice(1))?.channel === "alpha", + "Tideclaw branch finalization requires an alpha tag and explicit non-latest intent", + ); + // Reuse the release owner's exact direct-branch grammar before allowing + // the validator's live branch/SHA and publisher-attempt checks. + resolveReleaseToolingIdentity({ + workflowContract: "2", + workflowRef: options["workflow-ref"], + workflowFullRef: options["workflow-full-ref"], + workflowSha: options["tooling-sha"], + }); + } + verifyReleaseToolingIdentity({ + allowPrevalidatedRef: alphaBranch, + repository: REPOSITORY, + workflowRef: options["workflow-ref"], + workflowFullRef: options["workflow-full-ref"], + workflowSha: options["tooling-sha"], + ...(mode === "publish" + ? {} + : { + releasePublishRunId: options["release-publish-run-id"], + releasePublishRunAttempt: options["release-publish-run-attempt"], + releasePublishRef: options["release-publish-ref"], + releasePublishFullRef: options["release-publish-full-ref"], + releasePublishParentStatePolicy: "active-or-success", + }), + }); + if (mode === "publish") { + // Native builds are admitted by their successful request, not a core publisher. + const request = JSON.parse( + command("gh", ["api", `repos/${REPOSITORY}/actions/runs/${options["request-run-id"]}`]), + ); + const [path, fullRef] = String(request.path ?? "").split("@", 2); + assert(!fullRef || fullRef === "refs/heads/main", "Linux request workflow ref changed"); + assert.deepEqual( + { + id: request.id, + attempt: request.run_attempt, + repository: request.repository?.full_name, + event: request.event, + path, + branch: request.head_branch, + sha: request.head_sha, + title: request.display_title, + status: request.status, + conclusion: request.conclusion, + }, + { + id: Number(options["request-run-id"]), + attempt: Number(options["request-run-attempt"]), + repository: REPOSITORY, + event: "workflow_dispatch", + path: ".github/workflows/linux-app-release-request.yml", + branch: "main", + sha: options["tooling-sha"], + title: `Linux App Release Request [${options.tag}] desktop=${options["desktop-test"]}`, + status: "completed", + conclusion: "success", + }, + "Linux release request authority changed", + ); + } + verifyReleaseWorkflowRun({ + repository: REPOSITORY, + workflowRef: options["workflow-ref"], + workflowFullRef: options["workflow-full-ref"], + workflowSha: options["tooling-sha"], + runId: process.env.GITHUB_RUN_ID, + runAttempt: process.env.GITHUB_RUN_ATTEMPT, + workflowPath: + mode === "finalize-core" + ? ".github/workflows/openclaw-release-publish.yml" + : ".github/workflows/linux-app-release.yml", + workflowEvent: mode === "publish" ? "workflow_run" : "workflow_dispatch", + runStatePolicy: "active", + }); +} + +function regularVersion(version) { + const parsed = parseReleaseVersion(version); + assert( + parsed && parsed.version === version && classifyReleaseTrain(parsed) === "stable", + `Not a canonical regular Linux release: ${version}`, + ); + return version; +} + +function releaseVersion(tag) { + assert(typeof tag === "string" && tag.startsWith("v"), "Expected a versioned release tag"); + return regularVersion(tag.slice(1)); +} + +function assetUrl(tag, name) { + return `https://github.com/${REPOSITORY}/releases/download/${tag}/${name}`; +} + +function manifestName(version) { + return `OpenClaw-${version}-linux.json`; +} + +function bundleNames(version) { + return { + appimage: `OpenClaw-${version}-amd64.AppImage`, + deb: `OpenClaw-${version}-amd64.deb`, + desktop: [ + `OpenClaw-${version}-darwin-aarch64.dmg`, + `OpenClaw-${version}-darwin-aarch64.app.tar.gz`, + `OpenClaw-${version}-windows-x86_64.exe`, + ], + checksums: "SHA256SUMS.linux-app.txt", + }; +} + +function asset(release, name) { + const matches = release.assets.filter((entry) => entry.name === name); + assert(matches.length <= 1, `Duplicate asset ${name} on ${release.tag_name}`); + return matches[0]; +} + +function identity(release) { + return [release.id, release.tag_name, release.draft, release.prerelease]; +} + +function assetIdentity(entry) { + return ( + entry && [ + entry.id, + entry.name, + entry.size, + entry.state, + entry.digest, + entry.browser_download_url, + ] + ); +} + +class GitHub { + constructor(directory, mode, options) { + this.directory = directory; + this.sequence = 0; + this.mode = mode; + this.options = options; + } + + authorize() { + authorizeWrite(this.mode, this.options); + } + + temp(name) { + return join(this.directory, `${++this.sequence}-${name}`); + } + + api(endpoint, missing = false) { + try { + return JSON.parse(command("gh", ["api", `repos/${REPOSITORY}/${endpoint}`])); + } catch (error) { + if (missing && /\bHTTP 404\b/u.test(String(error.stderr))) { + return null; + } + throw error; + } + } + + release(tag, missing = false) { + const release = this.api(`releases/tags/${encodeURIComponent(tag)}`, missing); + if (!release) { + return null; + } + assert( + Number.isSafeInteger(release.id) && + release.id > 0 && + release.tag_name === tag && + typeof release.draft === "boolean" && + typeof release.prerelease === "boolean", + "Release identity is invalid", + ); + const assets = []; + for (let page = 1; page <= 5; page++) { + const rows = this.api(`releases/${release.id}/assets?per_page=100&page=${page}`); + assert(Array.isArray(rows) && rows.length <= 100, "Invalid asset inventory"); + assets.push(...rows); + if (rows.length < 100) { + return { ...release, assets }; + } + } + throw new Error("Release asset inventory exceeds the publication bound"); + } + + latest() { + const latest = this.api("releases/latest", true); + if (!latest) { + return null; + } + const release = this.release(latest.tag_name); + assert.equal(release.id, latest.id, "Latest release changed during observation"); + return release; + } + + source(tag, missing = false) { + const ref = this.api(`git/ref/tags/${encodeURIComponent(tag)}`, missing); + if (!ref) { + return null; + } + let object = ref.object; + for (let depth = 0; depth < 5; depth++) { + assert(object && SHA.test(object.sha), "Invalid release tag object"); + if (object.type === "commit") { + return object.sha; + } + assert.equal(object.type, "tag", "Release tag must resolve to a commit"); + object = this.api(`git/tags/${object.sha}`).object; + } + throw new Error("Release tag nesting exceeds the publication bound"); + } + + download(url, limit) { + const path = this.temp("download"); + command( + "curl", + [ + "--disable", + "--fail", + "--location", + "--silent", + "--show-error", + "--proto", + "=https", + "--proto-redir", + "=https", + "--connect-timeout", + "10", + "--max-time", + "90", + "--max-filesize", + String(limit), + "--output", + path, + url, + ], + 95_000, + ); + const stat = lstatSync(path); + assert(stat.isFile() && stat.size > 0 && stat.size <= limit, "Invalid public download"); + return path; + } + + publicAsset(release, entry, limit = METADATA_LIMIT) { + assert( + Number.isSafeInteger(entry.id) && + entry.id > 0 && + Number.isSafeInteger(entry.size) && + entry.size > 0 && + entry.size <= limit && + entry.state === "uploaded", + `Invalid asset identity: ${entry.name}`, + ); + const url = assetUrl(release.tag_name, entry.name); + assert.equal(entry.browser_download_url, url, "Unexpected public asset URL"); + const path = this.download(url, limit); + assert.equal(lstatSync(path).size, entry.size, "Public asset size mismatch"); + if (entry.digest != null) { + assert.equal(entry.digest, `sha256:${fileDigest(path)}`, "Public asset digest mismatch"); + } + return path; + } + + metadata(release, name) { + const entry = asset(release, name); + return entry ? readFileSync(this.publicAsset(release, entry)) : null; + } + + unchanged(release, source) { + const fresh = this.release(release.tag_name); + assert.deepEqual(identity(fresh), identity(release), "Release identity changed before write"); + assert.equal(this.source(release.tag_name), source, "Release source changed before write"); + return fresh; + } + + upload(release, source, name, path, mutable = false, beforeWrite = () => {}) { + const bytesHash = fileDigest(path); + const previous = asset(release, name); + if (previous) { + const old = this.publicAsset(release, previous, BUNDLE_LIMIT); + if (fileDigest(old) === bytesHash) { + return; + } + assert(mutable, `Immutable asset conflict: ${name}`); + } + const uploadDirectory = mkdtempSync(join(this.directory, "upload-")); + const uploadPath = join(uploadDirectory, name); + copyFileSync(path, uploadPath); + const fresh = this.unchanged(release, source); + assert.deepEqual( + assetIdentity(asset(fresh, name)), + assetIdentity(previous), + `Asset changed before write: ${name}`, + ); + beforeWrite(); + this.authorize(); + if (previous) { + command("gh", [ + "api", + "--method", + "DELETE", + `repos/${REPOSITORY}/releases/assets/${previous.id}`, + ]); + beforeWrite(); + this.authorize(); + } + // No clobber/retry: uncertain replacement must be reconciled from a fresh inventory. + command( + "gh", + ["release", "upload", release.tag_name, "--repo", REPOSITORY, uploadPath], + 120_000, + ); + const published = this.unchanged(release, source); + const uploaded = asset(published, name); + assert(uploaded, `Upload missing from release: ${name}`); + assert.equal( + fileDigest(this.publicAsset(published, uploaded, BUNDLE_LIMIT)), + bytesHash, + `Public upload readback failed: ${name}`, + ); + } +} + +function parseManifest(bytes, publicKey) { + assert(bytes.length <= METADATA_LIMIT, "Linux manifest is too large"); + const manifest = JSON.parse(bytes); + const version = regularVersion(manifest.version); + const names = bundleNames(version); + assert.deepEqual( + Object.keys(manifest.platforms), + ["linux-x86_64"], + "Linux stable currently requires exactly the admitted AMD64 platform", + ); + const platform = manifest.platforms["linux-x86_64"]; + assert.equal( + platform.url, + assetUrl(`v${version}`, names.appimage), + "Manifest bundle URL mismatch", + ); + assert( + typeof platform.signature === "string" && + platform.signature.length > 0 && + platform.signature.length <= 16_384, + "Invalid manifest signature", + ); + const proof = manifest.linuxPublication; + assert( + proof?.schemaVersion === 1 && + SHA.test(proof.sourceSha) && + SHA.test(proof.toolingSha) && + SHA.test(proof.channelSha) && + Number.isSafeInteger(proof.releaseId) && + proof.releaseId > 0, + "Missing Linux publisher identity", + ); + assert.equal( + proof.publicKeySha256, + digest(Buffer.from(publicKey, "base64")), + "Linux publication trust root changed", + ); + assert( + Array.isArray(proof.assets) && [3, 6].includes(proof.assets.length), + "Incomplete publication asset identity", + ); + const expected = [ + names.appimage, + names.deb, + names.checksums, + ...(proof.assets.length === 6 ? names.desktop : []), + ].toSorted(); + assert.deepEqual( + proof.assets.map((entry) => entry.name).toSorted(), + expected, + "Unexpected publication assets", + ); + for (const entry of proof.assets) { + assert( + Number.isSafeInteger(entry.id) && + entry.id > 0 && + Number.isSafeInteger(entry.size) && + entry.size > 0 && + entry.size <= BUNDLE_LIMIT && + DIGEST.test(entry.sha256), + "Invalid publication asset identity", + ); + } + return manifest; +} + +function parseLegacyManifest(bytes) { + const manifest = JSON.parse(bytes); + assert.deepEqual( + Object.keys(manifest).toSorted(), + ["notes", "platforms", "pub_date", "version"], + "Unrecognized legacy Linux manifest", + ); + const version = regularVersion(manifest.version); + assert( + typeof manifest.notes === "string" && + typeof manifest.pub_date === "string" && + Number.isFinite(Date.parse(manifest.pub_date)), + "Invalid legacy manifest fields", + ); + assert.deepEqual( + Object.keys(manifest.platforms), + ["linux-x86_64"], + "Unrecognized legacy Linux platforms", + ); + const platform = manifest.platforms["linux-x86_64"]; + assert.deepEqual( + Object.keys(platform).toSorted(), + ["signature", "url"], + "Unrecognized legacy Linux platform fields", + ); + assert.equal( + platform.url, + assetUrl(`v${version}`, bundleNames(version).appimage), + "Legacy Linux bundle URL mismatch", + ); + assert( + typeof platform.signature === "string" && + platform.signature.length > 0 && + platform.signature.length <= 16_384 && + /^[A-Za-z0-9+/]+={0,2}$/u.test(platform.signature), + "Invalid legacy Linux signature encoding", + ); + return manifest; +} + +function publishedManifest(github, bytes, publicKey) { + const manifest = parseManifest(bytes, publicKey); + const proof = manifest.linuxPublication; + const release = github.release(`v${manifest.version}`); + assert( + !release.draft && !release.prerelease && release.id === proof.releaseId, + "Linux bundles must belong to the identified public stable release", + ); + assert.equal(github.source(release.tag_name), proof.sourceSha, "Published Linux source changed"); + assert.deepEqual( + github.metadata(release, manifestName(manifest.version)), + bytes, + "Canonical bytes differ from the immutable Linux publication", + ); + // Core consumes the publisher's verified immutable asset IDs, not a claimed signature field. + // GitHub replacement creates a new ID. Check its digest too when the API supplies one. + for (const entry of proof.assets) { + const current = asset(release, entry.name); + assert( + current && + current.id === entry.id && + current.size === entry.size && + current.state === "uploaded" && + current.browser_download_url === assetUrl(release.tag_name, entry.name) && + (current.digest == null || current.digest === `sha256:${entry.sha256}`), + `Published Linux asset changed: ${entry.name}`, + ); + } + return manifest; +} + +function channelBody(version) { + const names = bundleNames(version); + return ( + `Latest published Linux companion: v${version}\n\n` + + `- [AppImage](${assetUrl(`v${version}`, names.appimage)})\n` + + `- [Debian package](${assetUrl(`v${version}`, names.deb)})\n` + ); +} + +function updateChannelBody(github, release, source, version) { + const body = channelBody(version); + if (release.body === body) { + return; + } + const path = github.temp("channel-notes.md"); + writeFileSync(path, body, { flag: "wx" }); + github.unchanged(release, source); + github.authorize(); + command("gh", [ + "release", + "edit", + CHANNEL, + "--repo", + REPOSITORY, + "--prerelease", + "--latest=false", + "--notes-file", + path, + ]); + assert.equal(github.release(CHANNEL).body, body, "Linux download-page readback failed"); +} + +function canonical(github, publicKey) { + const release = github.release(CHANNEL); + assert(!release.draft && release.prerelease, "Linux channel must be public and prerelease"); + const bytes = github.metadata(release, "latest.json"); + assert( + bytes, + "Linux channel manifest is missing; reconcile the interrupted publication from retained promotion evidence", + ); + const manifest = publishedManifest(github, bytes, publicKey); + assert.equal( + github.source(CHANNEL), + manifest.linuxPublication.channelSha, + "Linux channel tag moved", + ); + return { release, bytes, manifest }; +} + +function mirror(github, publicKey, target) { + const latest = github.latest(); + assert(latest, "No public core latest release exists"); + const latestVersion = releaseVersion(latest.tag_name); + assert(!latest.draft && !latest.prerelease, "Core latest is not a public regular release"); + const source = github.source(latest.tag_name); + if (target && (target.tag !== latest.tag_name || target.source !== source)) { + // A resume of an older release must never promote or modify it. + const expected = github.release(target.tag); + assert.equal(github.source(target.tag), target.source, "Selected core tag moved"); + assert(!expected.draft && !expected.prerelease, "Selected core release is not public"); + return { state: "skipped-not-latest", tag: target.tag }; + } + const verifyLatest = () => { + assert.deepEqual( + identity(github.latest()), + identity(latest), + "Core latest changed before mirror", + ); + assert.equal( + github.source(latest.tag_name), + source, + "Core latest source changed before mirror", + ); + }; + const selected = canonical(github, publicKey); + assert( + compareReleaseVersions(selected.manifest.version, latestVersion) <= 0, + "Canonical Linux version is newer than core latest", + ); + const previousBytes = github.metadata(latest, "latest.json"); + if (previousBytes && !previousBytes.equals(selected.bytes)) { + const previous = JSON.parse(previousBytes); + const legacy = !Object.hasOwn(previous, "linuxPublication"); + const owned = legacy + ? parseLegacyManifest(previousBytes) + : publishedManifest(github, previousBytes, publicKey); + const comparison = compareReleaseVersions(selected.manifest.version, owned.version); + assert(comparison >= 0, "Legacy endpoint already carries a newer Linux manifest"); + if (legacy) { + const origin = github.release(`v${owned.version}`); + assert(!origin.draft && !origin.prerelease, "Legacy manifest origin is not public stable"); + assert.deepEqual( + github.metadata(origin, "latest.json"), + previousBytes, + "Legacy manifest differs from its public versioned origin", + ); + const bundle = asset(origin, bundleNames(owned.version).appimage); + assert( + bundle && + Number.isSafeInteger(bundle.id) && + bundle.id > 0 && + bundle.size > 0 && + bundle.state === "uploaded" && + bundle.browser_download_url === owned.platforms["linux-x86_64"].url, + "Legacy manifest has no matching public bundle", + ); + if (comparison === 0) { + const originalFields = { ...selected.manifest }; + delete originalFields.linuxPublication; + assert.deepEqual( + owned, + originalFields, + "Same-version legacy bootstrap changed original manifest fields", + ); + } + } else { + assert.equal( + owned.linuxPublication.channelSha, + selected.manifest.linuxPublication.channelSha, + "Legacy endpoint belongs to a different Linux channel identity", + ); + assert(comparison > 0, "Same-version canonical manifest conflict at legacy endpoint"); + } + } + const beforeWrite = () => { + assert.deepEqual( + canonical(github, publicKey).bytes, + selected.bytes, + "Canonical Linux manifest changed before mirror", + ); + verifyLatest(); + }; + beforeWrite(); + const path = github.temp("latest.json"); + writeFileSync(path, selected.bytes, { flag: "wx" }); + github.upload(latest, source, "latest.json", path, true, beforeWrite); + assert.deepEqual(identity(github.latest()), identity(latest), "Core latest changed after mirror"); + assert.deepEqual( + canonical(github, publicKey).bytes, + selected.bytes, + "Canonical Linux manifest changed after mirror", + ); + updateChannelBody( + github, + selected.release, + selected.manifest.linuxPublication.channelSha, + selected.manifest.version, + ); + const publicPath = github.download( + `https://github.com/${REPOSITORY}/releases/latest/download/latest.json`, + METADATA_LIMIT, + ); + assert.deepEqual(readFileSync(publicPath), selected.bytes, "Legacy endpoint readback failed"); + assert.deepEqual( + identity(github.latest()), + identity(latest), + "Core latest changed during readback", + ); + assert.equal( + github.source(latest.tag_name), + source, + "Core latest source changed during readback", + ); + return { + state: "mirrored", + tag: latest.tag_name, + releaseId: latest.id, + sourceSha: source, + version: selected.manifest.version, + manifestSha256: digest(selected.bytes), + }; +} + +function finalizeCore(github, options) { + const version = options.tag.slice(1); + const parsed = parseReleaseVersion(version); + assert( + parsed && + parsed.version === version && + ["stable", "alpha", "beta"].includes(classifyReleaseTrain(parsed)), + "Unsupported core GitHub release train", + ); + assert(["true", "false"].includes(options.latest), "Expected explicit core latest intent"); + const prerelease = parsed.channel !== "stable"; + assert(!prerelease || options.latest === "false", "Prereleases cannot become core latest"); + github.authorize(); + // gh release view also discovers drafts by their pending tag; REST's tag route does not. + const discovered = JSON.parse( + command("gh", [ + "release", + "view", + options.tag, + "--repo", + REPOSITORY, + "--json", + "databaseId,tagName,isDraft,isPrerelease", + ]), + ); + const release = { + id: discovered.databaseId, + tag_name: discovered.tagName, + draft: discovered.isDraft, + prerelease: discovered.isPrerelease, + }; + assert( + Number.isSafeInteger(release.id) && + release.id > 0 && + release.tag_name === options.tag && + typeof release.draft === "boolean" && + typeof release.prerelease === "boolean", + "Selected core release identity is invalid", + ); + assert.equal(release.prerelease, prerelease, "Selected core prerelease classification changed"); + assert.equal(github.source(options.tag), options["source-sha"], "Selected core tag moved"); + const latest = github.latest(); + const latestSource = latest ? github.source(latest.tag_name) : null; + let makeLatest = options.latest === "true"; + if (latest) { + assert(!latest.draft && !latest.prerelease, "Core latest is not a public regular release"); + const latestVersion = releaseVersion(latest.tag_name); + if (makeLatest && compareReleaseVersions(version, latestVersion) < 0) { + makeLatest = false; + } + assert( + makeLatest || latest.id !== release.id, + "Non-latest finalization must not demote the current latest release", + ); + } + const verifyLatest = () => { + const fresh = github.latest(); + assert.deepEqual( + fresh ? identity(fresh) : null, + latest ? identity(latest) : null, + "Core latest changed before finalization", + ); + if (latest) { + assert.equal(github.source(latest.tag_name), latestSource, "Core latest tag moved"); + } + }; + assert.deepEqual( + identity(github.api(`releases/${release.id}`)), + identity(release), + "Release identity changed before write", + ); + assert.equal(github.source(options.tag), options["source-sha"], "Selected core tag moved"); + verifyLatest(); + github.authorize(); + // Address the admitted release ID, never a re-resolved replacement with the same tag. + command("gh", [ + "api", + "--method", + "PATCH", + `repos/${REPOSITORY}/releases/${release.id}`, + "--field", + "draft=false", + "--raw-field", + `make_latest=${makeLatest}`, + ]); + const published = github.release(options.tag); + assert.deepEqual( + identity(published), + [release.id, options.tag, false, prerelease], + "Finalized core release identity mismatch", + ); + assert.equal(github.source(options.tag), options["source-sha"], "Finalized core tag moved"); + const actualLatest = github.latest(); + const expectedLatest = makeLatest ? published : latest; + assert.deepEqual( + actualLatest ? identity(actualLatest) : null, + expectedLatest ? identity(expectedLatest) : null, + "Core latest readback mismatch", + ); + if (actualLatest) { + assert.equal( + github.source(actualLatest.tag_name), + makeLatest ? options["source-sha"] : latestSource, + "Core latest source readback mismatch", + ); + } + return { + state: "finalized", + tag: options.tag, + releaseId: release.id, + sourceSha: options["source-sha"], + madeLatest: makeLatest, + latestReleaseId: actualLatest?.id ?? null, + latestTag: actualLatest?.tag_name ?? null, + }; +} + +function publishDesktopChannel(github, options, directory) { + const tag = "desktop-test"; + const name = "latest-desktop-test.json"; + const version = releaseVersion(options.tag); + let channel = github.release(tag, true); + const creating = channel === null; + if (!channel) { + const existingTag = github.source(tag, true); + assert( + existingTag === null || existingTag === options["source-sha"], + "Existing desktop channel tag has a different source; reconcile before creation", + ); + github.authorize(); + command("gh", [ + "release", + "create", + tag, + "--repo", + REPOSITORY, + "--target", + options["source-sha"], + "--prerelease", + "--latest=false", + "--title", + "OpenClaw desktop test update channel", + "--notes", + "Opt-in updater manifest for unsigned macOS and Windows Tauri test builds.", + ]); + channel = github.release(tag); + } + assert(!channel.draft && channel.prerelease, "Desktop channel must be public and prerelease"); + // Existing channels keep their historical tag; an update never retags it. + const source = github.source(tag); + if (creating) { + assert.equal(source, options["source-sha"], "Created desktop channel source mismatch"); + } + const previous = github.metadata(channel, name); + if (previous) { + const current = regularVersion(JSON.parse(previous).version); + if (compareReleaseVersions(version, current) < 0) { + return { state: "kept-newer-channel", version: current }; + } + } + github.upload(channel, source, name, join(directory, name), true, () => + github.unchanged(channel, source), + ); + return { state: "published", version }; +} + +function publicInputs(github, release, publicKey, desktop) { + const version = releaseVersion(release.tag_name); + const names = bundleNames(version); + const immutable = github.metadata(release, manifestName(version)); + const bytes = immutable ?? github.metadata(release, "latest.json"); + assert(bytes, "Complete public Linux assets require their published signature manifest"); + const manifest = Object.hasOwn(JSON.parse(bytes), "linuxPublication") + ? publishedManifest(github, bytes, publicKey) + : parseLegacyManifest(bytes); + assert.equal(manifest.version, version, "Public Linux inputs belong to a different version"); + const checksumAsset = asset(release, names.checksums); + assert(checksumAsset, "Public Linux checksum asset is missing"); + const checksumPath = github.publicAsset(release, checksumAsset); + const rows = readFileSync(checksumPath, "utf8").trimEnd().split("\n"); + const hasDesktop = rows.length === 5; + const expected = [names.appimage, names.deb, ...(hasDesktop ? names.desktop : [])].toSorted(); + assert.deepEqual( + rows + .map((row) => { + const match = /^([0-9a-f]{64}) {2}\.\/([^/]+)$/u.exec(row); + assert(match, "Invalid public Linux checksum entry"); + return match[2]; + }) + .toSorted(), + expected, + "Public Linux checksum inventory mismatch", + ); + assert(!desktop || hasDesktop, "Opt-in desktop publication requires its complete public bundles"); + const directory = mkdtempSync(join(github.directory, "public-inputs-")); + copyFileSync(checksumPath, join(directory, names.checksums)); + for (const name of expected) { + const entry = asset(release, name); + assert(entry, `Public Linux input is missing: ${name}`); + copyFileSync(github.publicAsset(release, entry, BUNDLE_LIMIT), join(directory, name)); + } + if (desktop) { + const metadata = github.metadata(release, "latest-desktop-test.json"); + assert(metadata, "Public desktop test manifest is missing"); + writeFileSync(join(directory, "latest-desktop-test.json"), metadata, { flag: "wx" }); + } + return { directory, signature: manifest.platforms["linux-x86_64"].signature, hasDesktop }; +} + +function publish(github, options, publicKey) { + const version = releaseVersion(options.tag); + const names = bundleNames(version); + const release = github.release(options.tag); + assert( + !release.draft && !release.prerelease, + "Publish Linux bundles only to a public stable release", + ); + assert.equal(github.source(options.tag), options["source-sha"], "Linux release source mismatch"); + const desktop = options["desktop-test"] === "true"; + // Idempotent native requests can finish channel publication from the exact + // public bundles. They do not require or rebuild discarded runner artifacts. + const retainedInputs = options.assets ? null : publicInputs(github, release, publicKey, desktop); + const directory = retainedInputs?.directory ?? resolve(options.assets); + const expected = [ + names.appimage, + names.deb, + names.checksums, + ...(retainedInputs?.hasDesktop || desktop ? names.desktop : []), + ].toSorted(); + assert.deepEqual( + readdirSync(directory).toSorted(), + [...expected, ...(desktop ? ["latest-desktop-test.json"] : [])].toSorted((a, b) => + a < b ? -1 : a > b ? 1 : 0, + ), + "Release directory does not match the admitted bundle inventory", + ); + const inputs = expected.map((name) => { + const path = join(directory, name); + const stat = lstatSync(path); + assert( + stat.isFile() && stat.size > 0 && stat.size <= BUNDLE_LIMIT, + `Invalid release input: ${name}`, + ); + return { name, path, size: stat.size, sha256: fileDigest(path) }; + }); + const checksums = inputs + .filter((entry) => entry.name !== names.checksums) + .map((entry) => `${entry.sha256} ./${entry.name}`) + .toSorted() + .join("\n"); + assert.equal( + readFileSync(join(directory, names.checksums), "utf8") + .trimEnd() + .split("\n") + .toSorted() + .join("\n"), + checksums, + "Bundle checksum inventory mismatch", + ); + const signature = retainedInputs?.signature ?? readFileSync(options.signature, "utf8").trim(); + assert(signature.length > 0 && signature.length <= 16_384, "Invalid AppImage signature input"); + const signaturePath = github.temp("appimage.minisig"); + const publicKeyPath = github.temp("updater.pub"); + writeFileSync(signaturePath, Buffer.from(signature, "base64"), { flag: "wx" }); + writeFileSync(publicKeyPath, Buffer.from(publicKey, "base64"), { flag: "wx" }); + command("minisign", [ + "-Vm", + join(directory, names.appimage), + "-x", + signaturePath, + "-p", + publicKeyPath, + ]); + + // Detect every immutable conflict before uploading any missing file. + for (const entry of inputs) { + const previous = asset(release, entry.name); + if (previous) { + assert.equal( + fileDigest(github.publicAsset(release, previous, BUNDLE_LIMIT)), + entry.sha256, + `Immutable asset conflict: ${entry.name}`, + ); + } + } + for (const entry of inputs) { + github.upload( + github.unchanged(release, options["source-sha"]), + options["source-sha"], + entry.name, + entry.path, + ); + } + const published = github.unchanged(release, options["source-sha"]); + const assets = inputs.map((entry) => { + const current = asset(published, entry.name); + assert(current, `Missing public bundle: ${entry.name}`); + assert.equal( + fileDigest(github.publicAsset(published, current, BUNDLE_LIMIT)), + entry.sha256, + `Public bundle differs from signed input: ${entry.name}`, + ); + return { id: current.id, name: entry.name, size: entry.size, sha256: entry.sha256 }; + }); + let channel = github.release(CHANNEL, true); + const creatingChannel = channel === null; + if (!channel) { + const retainedTag = github.source(CHANNEL, true); + assert( + retainedTag === null || retainedTag === options["source-sha"], + "Existing Linux channel tag has a different source; reconcile before creating its release", + ); + github.authorize(); + command("gh", [ + "release", + "create", + CHANNEL, + "--repo", + REPOSITORY, + "--target", + options["source-sha"], + "--prerelease", + "--latest=false", + "--title", + "OpenClaw Linux update channel", + "--notes", + INITIAL_BODY, + ]); + channel = github.release(CHANNEL); + } + assert(!channel.draft && channel.prerelease, "Linux channel must be public and prerelease"); + const channelSha = github.source(CHANNEL); + const currentBytes = github.metadata(channel, "latest.json"); + let current; + if (currentBytes) { + current = publishedManifest(github, currentBytes, publicKey); + assert.equal(current.linuxPublication.channelSha, channelSha, "Linux channel tag moved"); + } else { + assert( + creatingChannel && channel.body === INITIAL_BODY && channelSha === options["source-sha"], + "Missing channel metadata is not a new channel; reconcile before publishing", + ); + } + const existing = github.metadata(published, manifestName(version)); + let bytes = existing; + if (existing) { + const retained = publishedManifest(github, existing, publicKey); + assert.equal(retained.version, version, "Versioned manifest identity mismatch"); + assert.equal( + retained.linuxPublication.channelSha, + channelSha, + "Channel identity changed on replay", + ); + assert.deepEqual(retained.linuxPublication.assets, assets, "Bundle identity changed on replay"); + // Re-signing identical bytes can change the trusted timestamp comment. Verify + // and retain the original signature instead of regenerating immutable metadata. + writeFileSync( + signaturePath, + Buffer.from(retained.platforms["linux-x86_64"].signature, "base64"), + ); + command("minisign", [ + "-Vm", + join(directory, names.appimage), + "-x", + signaturePath, + "-p", + publicKeyPath, + ]); + } else { + assert( + typeof published.published_at === "string" && + Number.isFinite(Date.parse(published.published_at)), + "Public release publication date is required", + ); + let originalFields; + const legacyBytes = github.metadata(published, "latest.json"); + if (legacyBytes) { + if (Object.hasOwn(JSON.parse(legacyBytes), "linuxPublication")) { + publishedManifest(github, legacyBytes, publicKey); + } else { + const legacy = parseLegacyManifest(legacyBytes); + if (legacy.version === version) { + originalFields = legacy; + writeFileSync( + signaturePath, + Buffer.from(legacy.platforms["linux-x86_64"].signature, "base64"), + ); + command("minisign", [ + "-Vm", + join(directory, names.appimage), + "-x", + signaturePath, + "-p", + publicKeyPath, + ]); + } + } + } + bytes = Buffer.from( + `${JSON.stringify( + { + ...(originalFields ?? { + version, + notes: Array.from(published.body ?? "") + .slice(0, 2000) + .join(""), + pub_date: published.published_at, + platforms: { + "linux-x86_64": { + signature, + url: assetUrl(options.tag, names.appimage), + }, + }, + }), + linuxPublication: { + schemaVersion: 1, + sourceSha: options["source-sha"], + toolingSha: options["tooling-sha"], + channelSha, + releaseId: published.id, + publicKeySha256: digest(Buffer.from(publicKey, "base64")), + assets, + }, + }, + null, + 2, + )}\n`, + ); + const path = github.temp("versioned-linux.json"); + writeFileSync(path, bytes, { flag: "wx" }); + github.upload(published, options["source-sha"], manifestName(version), path); + } + const comparison = current ? compareReleaseVersions(version, current.version) : 1; + if (comparison === 0) { + assert.deepEqual(bytes, currentBytes, "Same-version canonical manifest conflict"); + } + if (comparison >= 0) { + const path = github.temp("canonical-linux.json"); + writeFileSync(path, bytes, { flag: "wx" }); + console.error( + JSON.stringify({ + state: "canonical-publication-intent", + version, + manifestSha256: digest(bytes), + previousVersion: current?.version ?? null, + previousManifestSha256: currentBytes ? digest(currentBytes) : null, + }), + ); + github.upload(channel, channelSha, "latest.json", path, true); + updateChannelBody(github, channel, channelSha, version); + assert.deepEqual(canonical(github, publicKey).bytes, bytes, "Linux channel readback failed"); + } + if (desktop) { + const desktopName = "latest-desktop-test.json"; + const desktopPath = join(directory, desktopName); + const candidate = JSON.parse(readFileSync(desktopPath, "utf8")); + assert.equal(candidate.version, version, "Desktop test manifest version mismatch"); + assert.deepEqual( + Object.keys(candidate.platforms).toSorted(), + ["darwin-aarch64", "windows-x86_64"], + "Unexpected desktop test platforms", + ); + const previous = github.metadata(github.unchanged(release, options["source-sha"]), desktopName); + if (previous) { + const retained = JSON.parse(previous); + assert.equal(retained.version, candidate.version, "Desktop test manifest version changed"); + assert.deepEqual( + Object.keys(retained.platforms).toSorted(), + Object.keys(candidate.platforms).toSorted(), + "Desktop test platforms changed", + ); + for (const [platform, name] of [ + ["darwin-aarch64", names.desktop[1]], + ["windows-x86_64", names.desktop[2]], + ]) { + assert.equal( + retained.platforms[platform].url, + candidate.platforms[platform].url, + "Desktop test bundle URL changed", + ); + const previousSignature = retained.platforms[platform].signature; + assert( + typeof previousSignature === "string" && + previousSignature.length > 0 && + previousSignature.length <= 16_384, + "Invalid retained desktop signature", + ); + writeFileSync(signaturePath, Buffer.from(previousSignature, "base64")); + command("minisign", [ + "-Vm", + join(directory, name), + "-x", + signaturePath, + "-p", + publicKeyPath, + ]); + } + // Keep the previously published date/notes on an identical bundle replay. + writeFileSync(desktopPath, previous); + } + github.upload( + github.unchanged(release, options["source-sha"]), + options["source-sha"], + desktopName, + desktopPath, + ); + } + const legacy = mirror(github, publicKey); + const desktopChannel = desktop ? publishDesktopChannel(github, options, directory) : undefined; + return { + state: comparison < 0 ? "kept-newer-channel" : "published", + version, + manifestSha256: digest(bytes), + legacy, + ...(desktop ? { desktop: desktopChannel } : {}), + }; +} + +function main() { + const { values, positionals } = parseArgs({ + allowPositionals: true, + options: Object.fromEntries( + [ + "tag", + "source-sha", + "tooling-sha", + "workflow-ref", + "workflow-full-ref", + "release-publish-run-id", + "release-publish-run-attempt", + "release-publish-ref", + "release-publish-full-ref", + "request-run-id", + "request-run-attempt", + "assets", + "signature", + "public-key-config", + "desktop-test", + "latest", + ].map((name) => [name, { type: "string" }]), + ), + }); + const [mode] = positionals; + assert( + positionals.length === 1 && ["publish", "mirror", "finalize-core"].includes(mode), + "Usage: linux-app-channel.mjs publish|mirror|finalize-core --tag TAG --source-sha SHA [mode options]", + ); + assert(SHA.test(values["source-sha"]), "Expected the approved source SHA"); + assert( + SHA.test(values["tooling-sha"]) && values["workflow-ref"] && values["workflow-full-ref"], + "Missing publication tooling identity", + ); + const identityKeys = + mode === "publish" + ? ["request-run-id", "request-run-attempt"] + : ["release-publish-run-id", "release-publish-run-attempt"]; + for (const key of identityKeys) { + assert( + /^[1-9][0-9]*$/u.test(values[key]) && Number.isSafeInteger(Number(values[key])), + `Missing or invalid ${key}`, + ); + } + if (mode === "publish") { + assert( + values["workflow-ref"] === "main" && values["workflow-full-ref"] === "refs/heads/main", + "Native publication requires the admitted main request route", + ); + } else { + assert( + values["release-publish-ref"] && values["release-publish-full-ref"], + "Missing core publisher identity", + ); + } + assert(typeof values.tag === "string" && values.tag.startsWith("v"), "Expected a release tag"); + if (mode !== "finalize-core") { + releaseVersion(values.tag); + } + if (mode === "publish") { + assert( + SHA.test(values["tooling-sha"]) && + Boolean(values.assets) === Boolean(values.signature) && + ["true", "false"].includes(values["desktop-test"]), + "Missing Linux publication inputs", + ); + } + let publicKey; + if (mode !== "finalize-core") { + const config = JSON.parse(readFileSync(values["public-key-config"], "utf8")); + publicKey = config.plugins?.updater?.pubkey; + assert( + typeof publicKey === "string" && + publicKey.length > 0 && + publicKey.length <= 4096 && + /^[A-Za-z0-9+/]+={0,2}$/u.test(publicKey), + "Expected the trusted updater public key", + ); + } + const directory = mkdtempSync(join(tmpdir(), "openclaw-linux-channel-")); + try { + const github = new GitHub(directory, mode, values); + if (mode !== "finalize-core") { + github.authorize(); + } + const result = + mode === "finalize-core" + ? finalizeCore(github, values) + : mode === "publish" + ? publish(github, values, publicKey) + : mirror(github, publicKey, { tag: values.tag, source: values["source-sha"] }); + process.stdout.write(`${JSON.stringify(result, null, 2)}\n`); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { + try { + main(); + } catch (error) { + console.error(`Release publication incomplete; reconcile before retry: ${error.message}`); + process.exitCode = 1; + } +} diff --git a/scripts/linux-updater-manifest.mjs b/scripts/linux-updater-manifest.mjs new file mode 100644 index 0000000000000000000000000000000000000000..6d7fc4c71f05aeaf43a5943799d0b0e0192b8776 --- /dev/null +++ b/scripts/linux-updater-manifest.mjs @@ -0,0 +1,628 @@ +#!/usr/bin/env node + +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { parseArgs } from "node:util"; +import { isRecord } from "./lib/record-shared.mjs"; +import { + classifyReleaseTrain, + compareReleaseVersions, + parseReleaseVersion, +} from "./lib/release-version.mjs"; +import { + verifyReleaseToolingIdentity, + verifyReleaseWorkflowRun, +} from "./release-tooling-identity.mjs"; + +function gh(args) { + return execFileSync("gh", args, { + stdio: ["ignore", "pipe", "pipe"], + timeout: 60_000, + maxBuffer: 4 * 1024 * 1024, + killSignal: "SIGKILL", + }); +} + +function releaseVersion(tag) { + const version = + typeof tag === "string" && tag.startsWith("v") ? parseReleaseVersion(tag.slice(1)) : null; + return version && tag === `v${version.version}` ? version : null; +} + +function regularStable(tag) { + const version = releaseVersion(tag); + return version && classifyReleaseTrain(version) === "stable"; +} + +function readRelease(repository, tag, optional = false) { + let raw; + try { + raw = gh( + tag + ? [ + "release", + "view", + tag, + "--repo", + repository, + "--json", + "tagName,isDraft,isPrerelease,assets", + ] + : [ + "api", + `repos/${repository}/releases/latest`, + "--jq", + "{tagName: .tag_name, isDraft: .draft, isPrerelease: .prerelease, assets: .assets}", + ], + ); + } catch (error) { + if (optional && /release not found|HTTP 404/iu.test(String(error.stderr ?? ""))) { + return null; + } + throw error; + } + const release = JSON.parse(raw.toString("utf8")); + if ( + !isRecord(release) || + typeof release.tagName !== "string" || + typeof release.isDraft !== "boolean" || + typeof release.isPrerelease !== "boolean" || + !Array.isArray(release.assets) || + !release.assets.every((asset) => isRecord(asset) && typeof asset.name === "string") || + (tag && release.tagName !== tag) + ) { + throw new Error("GitHub returned invalid release metadata."); + } + return release; +} + +function publicStable(release) { + return release && regularStable(release.tagName) && !release.isDraft && !release.isPrerelease; +} + +function downloadAsset(repository, tag, name = "latest.json") { + return gh(["release", "download", tag, "--repo", repository, "--pattern", name, "--output", "-"]); +} + +function verifyManifestSource(repository, manifest, allowDraftTag) { + const source = readRelease(repository, manifest.sourceTag); + const assets = source.assets.filter((asset) => asset.name === manifest.assetName); + if ( + (!publicStable(source) && + !( + source.tagName === allowDraftTag && + regularStable(source.tagName) && + !source.isPrerelease + )) || + assets.length !== 1 || + assets[0].state !== "uploaded" || + !Number.isSafeInteger(assets[0].size) || + assets[0].size <= 0 + ) { + throw new Error( + `Linux manifest source ${manifest.sourceTag} must have its published AppImage.`, + ); + } +} + +function readManifest(repository, carrier, allowDraftTag, name = "latest.json") { + const assets = carrier.assets.filter((asset) => asset.name === name); + if (assets.length === 0) { + return null; + } + if (assets.length !== 1) { + throw new Error(`Release ${carrier.tagName} has ambiguous Linux manifests.`); + } + const bytes = downloadAsset(repository, carrier.tagName, name); + const value = JSON.parse(bytes.toString("utf8")); + const version = isRecord(value) ? value.version : undefined; + if (typeof version !== "string") { + throw new Error(`Release ${carrier.tagName} has no canonical Linux updater version.`); + } + const sourceTag = `v${version}`; + const platform = isRecord(value?.platforms) ? value.platforms["linux-x86_64"] : undefined; + const assetName = `OpenClaw-${version}-amd64.AppImage`; + const url = `https://github.com/${repository}/releases/download/${sourceTag}/${assetName}`; + const canonicalChannel = + carrier.tagName === "linux-stable" && !carrier.isDraft && carrier.isPrerelease; + if ( + !regularStable(sourceTag) || + (!canonicalChannel && + (!regularStable(carrier.tagName) || + compareReleaseVersions(version, carrier.tagName.slice(1)) > 0)) || + (name !== "latest.json" && + (name !== `OpenClaw-${version}-linux.json` || carrier.tagName !== sourceTag)) || + !isRecord(platform) || + Object.keys(value.platforms).length !== 1 || + typeof platform.signature !== "string" || + platform.signature.length === 0 || + !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/u.test(platform.signature) || + platform.url !== url + ) { + throw new Error(`Release ${carrier.tagName} has a noncanonical Linux updater manifest.`); + } + const manifest = { bytes, version, sourceTag, assetName, value }; + verifyManifestSource(repository, manifest, allowDraftTag); + return manifest; +} + +function readImmutableManifest(repository, source) { + const name = `OpenClaw-${source.tagName.slice(1)}-linux.json`; + const manifest = readManifest(repository, source, undefined, name); + if (!manifest) { + return null; + } + const current = JSON.parse( + gh(["api", `repos/${repository}/releases/tags/${source.tagName}`]).toString("utf8"), + ); + if ( + !isRecord(current) || + current.tag_name !== source.tagName || + current.draft !== false || + current.prerelease !== false || + !Number.isSafeInteger(current.id) || + current.id < 1 || + !Array.isArray(current.assets) || + !current.assets.every((entry) => isRecord(entry) && typeof entry.name === "string") + ) { + throw new Error("GitHub returned invalid immutable Linux release identity."); + } + const proof = manifest.value.linuxPublication; + const expected = [ + manifest.assetName, + `OpenClaw-${manifest.version}-amd64.deb`, + "SHA256SUMS.linux-app.txt", + ...(proof?.assets?.length === 6 + ? [ + `OpenClaw-${manifest.version}-darwin-aarch64.dmg`, + `OpenClaw-${manifest.version}-darwin-aarch64.app.tar.gz`, + `OpenClaw-${manifest.version}-windows-x86_64.exe`, + ] + : []), + ]; + const metadata = current.assets.filter((entry) => entry.name === name); + if ( + !publicStable(source) || + !isRecord(proof) || + proof.schemaVersion !== 1 || + !["sourceSha", "toolingSha", "channelSha"].every((key) => /^[a-f0-9]{40}$/u.test(proof[key])) || + !/^[a-f0-9]{64}$/u.test(proof.publicKeySha256) || + !Number.isSafeInteger(proof.releaseId) || + proof.releaseId < 1 || + proof.releaseId !== current.id || + !Array.isArray(proof.assets) || + proof.assets.length !== expected.length || + !expected.every( + (assetName) => proof.assets.filter((entry) => entry.name === assetName).length === 1, + ) || + metadata.length !== 1 || + metadata[0].state !== "uploaded" || + metadata[0].size !== manifest.bytes.length || + metadata[0].digest !== `sha256:${hash(manifest.bytes)}` + ) { + throw new Error("Immutable Linux publication identity or metadata digest is invalid."); + } + for (const entry of proof.assets) { + const matches = current.assets.filter((asset) => asset.name === entry.name); + if ( + !Number.isSafeInteger(entry.id) || + entry.id < 1 || + !Number.isSafeInteger(entry.size) || + entry.size < 1 || + !/^[a-f0-9]{64}$/u.test(entry.sha256) || + matches.length !== 1 || + matches[0].id !== entry.id || + matches[0].state !== "uploaded" || + matches[0].size !== entry.size || + matches[0].digest !== `sha256:${entry.sha256}` + ) { + throw new Error("Immutable Linux publication differs from its public asset inventory."); + } + } + const sourceSha = gh(["api", `repos/${repository}/commits/${source.tagName}`, "--jq", ".sha"]) + .toString("utf8") + .trim(); + if (sourceSha !== proof.sourceSha) { + throw new Error("Immutable Linux publication source changed."); + } + inspectLinuxSourceAssets(repository, source, manifest); + return manifest; +} + +function verifyAuthority(options) { + const runId = process.env.GITHUB_RUN_ID ?? ""; + const runAttempt = process.env.GITHUB_RUN_ATTEMPT ?? ""; + const prepared = Boolean(options["publication-request"]); + const workflowPath = prepared + ? ".github/workflows/openclaw-release-promote.yml" + : ".github/workflows/openclaw-release-publish.yml"; + const common = { repository: options.repository, workflowSha: process.env.GITHUB_WORKFLOW_SHA }; + + if (!prepared) { + if (!process.env.GITHUB_EVENT_PATH) { + throw new Error("Actions event payload is required for Linux manifest publication."); + } + const event = JSON.parse(readFileSync(process.env.GITHUB_EVENT_PATH, "utf8")); + if (event?.inputs?.tag !== options.tag || event?.inputs?.npm_dist_tag !== "latest") { + throw new Error("Actions release inputs differ from the Linux manifest operation."); + } + } + let parent = {}; + if (prepared) { + const request = JSON.parse(readFileSync(options["publication-request"], "utf8")); + if ( + request.repository !== options.repository || + request.inputs?.tag !== options.tag || + (options["source-sha"] && request.sourceSha !== options["source-sha"]) + ) { + throw new Error("Publication request differs from the Linux manifest operation."); + } + parent = { + releasePublishRunId: String(request.releaseRunId ?? ""), + releasePublishRunAttempt: String(request.releaseRunAttempt ?? ""), + releasePublishRef: request.tooling?.ref, + releasePublishFullRef: request.tooling?.fullRef, + releasePublishParentStatePolicy: "active-or-success", + }; + } + if (options["source-sha"]) { + const actual = gh(["api", `repos/${options.repository}/commits/${options.tag}`, "--jq", ".sha"]) + .toString("utf8") + .trim(); + if (actual !== options["source-sha"]) { + throw new Error(`Release tag ${options.tag} moved before Linux manifest publication.`); + } + } + const identity = verifyReleaseToolingIdentity({ + repository: options.repository, + workflowRef: process.env.GITHUB_REF_NAME, + workflowFullRef: process.env.GITHUB_REF, + workflowSha: process.env.GITHUB_WORKFLOW_SHA, + ...parent, + }); + verifyReleaseWorkflowRun({ + ...common, + runId, + runAttempt, + workflowPath, + workflowEvent: "workflow_dispatch", + workflowRef: process.env.GITHUB_REF_NAME, + workflowFullRef: process.env.GITHUB_REF, + runStatePolicy: "active", + }); + return identity; +} + +function hash(bytes) { + return createHash("sha256").update(bytes).digest("hex"); +} + +function uploadManifest(options, destination, current, candidate, latest, record) { + const candidatePath = join(options.output, "latest.json"); + writeFileSync(candidatePath, candidate.bytes, { flag: "wx" }); + if (current) { + writeFileSync(join(options.output, "previous.json"), current.bytes, { flag: "wx" }); + } + record({ + state: "prepared", + destinationTag: destination.tagName, + sourceTag: candidate.sourceTag, + version: candidate.version, + manifestSha256: hash(candidate.bytes), + previousManifestSha256: current ? hash(current.bytes) : null, + }); + + // Callers share one publication concurrency group; these rereads also catch + // manual release changes before this writer replaces the carried manifest. + const liveLatest = readRelease(options.repository); + if (liveLatest.tagName !== latest.tagName) { + throw new Error("GitHub latest changed before Linux manifest publication."); + } + const liveDestination = + destination.tagName === liveLatest.tagName + ? liveLatest + : readRelease(options.repository, destination.tagName); + if (!publicStable(liveLatest) || liveDestination.isPrerelease) { + throw new Error("Release eligibility changed before Linux manifest publication."); + } + const liveManifest = readManifest( + options.repository, + liveDestination, + options.command === "carry" ? destination.tagName : undefined, + ); + if ( + Boolean(current) !== Boolean(liveManifest) || + (current && !current.bytes.equals(liveManifest.bytes)) + ) { + throw new Error("Destination Linux manifest changed before publication."); + } + verifyManifestSource(options.repository, candidate); + record({ state: "uploading" }); + verifyAuthority(options); + + let uploadError; + try { + gh([ + "release", + "upload", + destination.tagName, + candidatePath, + "--repo", + options.repository, + "--clobber", + ]); + } catch (error) { + uploadError = error; + } + // A failed command may have uploaded successfully. Reconcile exact bytes + // once; never repeat an uncertain write or rebuild the signed bundle. + let remote; + try { + remote = downloadAsset(options.repository, destination.tagName); + } catch (error) { + throw new Error( + "Linux manifest upload could not be reconciled; inspect retained evidence before retrying.", + { + cause: error, + }, + ); + } + if (!remote.equals(candidate.bytes)) { + throw new Error( + "Linux manifest upload readback differs; inspect retained evidence before retrying.", + { + cause: uploadError, + }, + ); + } + record({ state: uploadError ? "reconciled" : "uploaded" }); +} + +function carry(options, record) { + const target = readRelease(options.repository, options.tag); + if (target.isPrerelease) { + throw new Error("Cannot carry the stable Linux manifest into a prerelease."); + } + const current = readManifest(options.repository, target, target.tagName); + const latest = readRelease(options.repository, undefined, true); + const previous = publicStable(latest) ? readManifest(options.repository, latest) : null; + if (!previous || (current && compareReleaseVersions(current.version, previous.version) >= 0)) { + record({ + state: current ? "unchanged" : "pending", + reason: current + ? "Target already has the newest usable Linux manifest." + : "No previous Linux manifest is available.", + }); + return; + } + if (compareReleaseVersions(previous.version, options.tag.slice(1)) > 0) { + throw new Error("Cannot carry a Linux version newer than the target release."); + } + uploadManifest(options, target, current, previous, latest, record); +} + +function inspectLinuxSourceAssets(repository, source, manifest) { + const version = source.tagName.slice(1); + const appimage = `OpenClaw-${version}-amd64.AppImage`; + const deb = `OpenClaw-${version}-amd64.deb`; + const checksumsName = "SHA256SUMS.linux-app.txt"; + const nativeAssets = source.assets.filter( + (asset) => + asset.name === checksumsName || + (asset.name.startsWith(`OpenClaw-${version}-`) && /\.(?:AppImage|deb)$/u.test(asset.name)), + ); + if (nativeAssets.length === 0 && (!manifest || manifest.sourceTag !== source.tagName)) { + return null; + } + if ( + !publicStable(source) || + !manifest || + manifest.sourceTag !== source.tagName || + nativeAssets.length !== 3 || + ![appimage, deb, checksumsName].every((name) => + nativeAssets.some((asset) => asset.name === name), + ) + ) { + throw new Error( + "Linux release assets are partial or inconsistent; inspect them before requesting another build.", + ); + } + const checksumBytes = downloadAsset(repository, source.tagName, checksumsName); + const checksums = new Map(); + for (const line of checksumBytes.toString("utf8").trimEnd().split("\n")) { + const match = /^([a-f0-9]{64}) [ *](?:\.\/)?([A-Za-z0-9._-]+)$/u.exec(line); + if (!match || checksums.has(match[2])) { + throw new Error("Linux release checksums are malformed or duplicated."); + } + checksums.set(match[2], match[1]); + } + for (const name of [appimage, deb]) { + const asset = nativeAssets.find((entry) => entry.name === name); + if (!checksums.has(name) || asset.digest !== `sha256:${checksums.get(name)}`) { + throw new Error(`Linux release checksum for ${name} differs from its GitHub asset digest.`); + } + } + return { checksumBytes, checksumsName }; +} + +export function inspectLinuxUpdaterManifest({ repository, carrierTag }) { + const carrier = readRelease(repository, carrierTag); + const immutable = readImmutableManifest(repository, carrier); + const manifest = readManifest(repository, carrier); + if (!manifest) { + return immutable + ? { + carrierTag, + immutableManifest: { + name: `OpenClaw-${immutable.version}-linux.json`, + sha256: hash(immutable.bytes), + }, + } + : null; + } + const source = readRelease(repository, manifest.sourceTag); + const publication = inspectLinuxSourceAssets(repository, source, manifest); + const currentCarrier = readRelease(repository, carrierTag); + const selectors = currentCarrier.assets.filter((asset) => asset.name === "latest.json"); + const manifestSha256 = hash(manifest.bytes); + if ( + !publicStable(currentCarrier) || + !publication || + selectors.length !== 1 || + selectors[0].state !== "uploaded" || + !Number.isSafeInteger(selectors[0].size) || + selectors[0].size <= 0 || + selectors[0].digest !== `sha256:${manifestSha256}` + ) { + throw new Error("Linux updater observation does not match the published carrier asset digest."); + } + return { + carrierTag, + manifestSha256, + sourceVersion: manifest.version, + ...(immutable + ? { + immutableManifest: { + name: `OpenClaw-${immutable.version}-linux.json`, + sha256: hash(immutable.bytes), + }, + } + : {}), + }; +} + +function status(options, record) { + const source = readRelease(options.repository, options.tag); + const immutable = readImmutableManifest(options.repository, source); + const manifest = immutable ?? readManifest(options.repository, source); + const publication = inspectLinuxSourceAssets(options.repository, source, manifest); + if (!publication) { + record({ + state: "pending", + assetsComplete: false, + reason: "No Linux bundles have been published for this tag.", + }); + return; + } + const version = options.tag.slice(1); + const channel = readRelease(options.repository, "linux-stable", true); + const canonical = channel ? readManifest(options.repository, channel) : null; + if (canonical) { + const canonicalSource = readRelease(options.repository, canonical.sourceTag); + const published = readImmutableManifest(options.repository, canonicalSource); + const channelSha = gh([ + "api", + `repos/${options.repository}/commits/linux-stable`, + "--jq", + ".sha", + ]) + .toString("utf8") + .trim(); + if ( + !published || + !published.bytes.equals(canonical.bytes) || + published.value.linuxPublication.channelSha !== channelSha + ) { + throw new Error("Canonical Linux metadata differs from its immutable publication."); + } + } + const needsChannelPublication = + !immutable || !canonical || compareReleaseVersions(canonical.version, version) < 0; + if ( + immutable && + canonical && + canonical.version === version && + !canonical.bytes.equals(immutable.bytes) + ) { + throw new Error("Same-version canonical Linux metadata conflicts with immutable publication."); + } + const latest = readRelease(options.repository, undefined, true); + let needsUpdaterPublication = false; + if (publicStable(latest) && compareReleaseVersions(version, latest.tagName.slice(1)) <= 0) { + const current = readManifest(options.repository, latest); + const selected = canonical ?? manifest; + needsUpdaterPublication = + !current || + compareReleaseVersions(current.version, selected.version) < 0 || + (current.version === selected.version && !current.bytes.equals(selected.bytes)); + } + writeFileSync(join(options.output, "latest.json"), manifest.bytes, { flag: "wx" }); + writeFileSync(join(options.output, publication.checksumsName), publication.checksumBytes, { + flag: "wx", + }); + record({ + state: "published", + assetsComplete: true, + version, + manifestSha256: hash(manifest.bytes), + needsUpdaterPublication, + needsChannelPublication, + }); +} + +function main() { + const { values, positionals } = parseArgs({ + allowPositionals: true, + options: Object.fromEntries( + ["tag", "repository", "output", "publication-request", "source-sha"].map((name) => [ + name, + { type: "string" }, + ]), + ), + }); + if ( + positionals.length !== 1 || + !["carry", "status"].includes(positionals[0]) || + !values.output || + !values.repository || + !/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(values.repository) || + !releaseVersion(values.tag) || + (values["source-sha"] && !/^[a-f0-9]{40}$/u.test(values["source-sha"])) + ) { + throw new Error( + "Usage: linux-updater-manifest.mjs <carry|status> --tag vYYYY.M.PATCH --repository owner/repo --output DIR [--publication-request FILE] [--source-sha SHA]", + ); + } + const options = { ...values, output: resolve(values.output), command: positionals[0] }; + mkdirSync(options.output, { recursive: true }); + const evidencePath = join(options.output, "evidence.json"); + const evidence = { + command: options.command, + repository: options.repository, + tag: options.tag, + state: "checking", + }; + writeFileSync(evidencePath, `${JSON.stringify(evidence, null, 2)}\n`, { flag: "wx" }); + const record = (update) => { + Object.assign(evidence, update); + writeFileSync(evidencePath, `${JSON.stringify(evidence, null, 2)}\n`); + }; + try { + if (!regularStable(options.tag)) { + record({ + state: "skipped", + reason: "Only regular stable releases use this Linux update channel.", + }); + } else if (options.command === "carry") { + carry(options, record); + } else if (options.command === "status") { + status(options, record); + } + } catch (error) { + record({ state: "failed", error: error.message }); + throw error; + } + console.log(JSON.stringify(evidence)); +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === resolve(process.argv[1])) { + try { + main(); + } catch (error) { + console.error(`linux-updater-manifest: ${error.message}`); + process.exitCode = 1; + } +} diff --git a/scripts/mac-elevation-host.sh b/scripts/mac-elevation-host.sh new file mode 100644 index 0000000000000000000000000000000000000000..ce374c6641c69709c6e71b9fb67a985efa090665 --- /dev/null +++ b/scripts/mac-elevation-host.sh @@ -0,0 +1,3497 @@ +#!/bin/bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +ELEVATION_LABEL="ai.openclaw.mac.elevation-host" +NORMAL_LABEL="ai.openclaw.mac" +EXPECTED_BUNDLE_ID="ai.openclaw.mac" +EXPECTED_TEAM_ID="FWJYW4S8P8" +EXPECTED_AUTHORITY="Developer ID Application: OpenClaw Foundation (FWJYW4S8P8)" +DEFAULT_APP="/Applications/OpenClaw.app" +RECOVERY_APP_PLAN_XATTR="com.openclaw.elevation.recovery-app-plan" +RECOVERY_MIGRATION_IDENTITY_XATTR="com.openclaw.elevation.recovery-migration-identity" + +COMMAND="${1:-}" +[[ -n "$COMMAND" ]] && shift || true +ARCHIVE="" +ARTIFACT_RECEIPT="" +EXPECTED_ARTIFACT_RECEIPT_SHA256="" +APP_PATH="$DEFAULT_APP" +STATE_DIR="${HOME}/.openclaw-elevation-host" +STATE_DIR_EXPLICIT=0 +CONFIG_PATH="" +CONFIG_PATH_EXPLICIT=0 +MIGRATE_LAUNCH_AGENT="" +MIGRATION_LABEL="" +MIGRATION_KIND="" +MIGRATION_NODE_ID="" +MIGRATION_NODE_ENV_PATH="" +MIGRATION_NODE_ENV_SHA="" +MIGRATION_NODE_ENV_IDENTITY="" +MIGRATION_NODE_WRAPPER_PATH="" +MIGRATION_NODE_WRAPPER_SHA="" +MIGRATION_NODE_WRAPPER_IDENTITY="" +MIGRATION_PLIST_SHA="" +MIGRATION_PLIST_IDENTITY="" +MIGRATION_CUSTODY_PATH="" +MIGRATION_CUSTODY_IDENTITY="" +MIGRATION_WAS_LOADED=0 +ADOPT_RUNNING_APP=0 +ADOPTION_PID="" +ADOPTION_ATTACH_ONLY=0 +EXPECTED_PEEKABOO_SOURCE_COMMIT="" +OUTPUT_DIR="$ROOT_DIR/dist/elevation-host" +WORK_ROOT="" +AUTHENTICATED_RENAME_HELPER="" +AUTHENTICATED_RENAME_HELPER_SHA="" +ARTIFACT_SNAPSHOT_ROOT="" +AUTHENTICATED_ARCHIVE_PATH="" +AUTHENTICATED_ARCHIVE_NAME="" +AUTHENTICATED_RECEIPT_PATH="" +AUTHENTICATED_INSTALLER_NAME="" +STAGED_APP_CONTAINER="" +STAGED_INSTALL_APP_PATH="" +NOTARY_RESULT_TEMP="" +CUTOVER_ACTIVE=0 +CUTOVER_COMMITTED=0 +CUTOVER_APP_MUTATED=0 +CUTOVER_MIGRATION_REMOVED=0 +CUTOVER_ADOPTION_STOPPED=0 +CUTOVER_ADOPTION_TERMINATION_SENT=0 +CUTOVER_RECOVERY_ATTEMPTED=0 +ROLLBACK_APP_PATH="" +ROLLBACK_APP_CDHASH_ARM64="" +ROLLBACK_APP_CDHASH_X86_64="" +ROLLBACK_ELEVATION_PLIST="" +ROLLBACK_ELEVATION_PLIST_SHA="" +ROLLBACK_ELEVATION_WAS_LOADED=0 +ROLLBACK_INSTALL_RECEIPT="" +ROLLBACK_INSTALL_RECEIPT_SHA="" +ROLLBACK_FAILED_SOURCE="" +RECOVERED_FAILED_APP_PATH="" +RECOVERY_FAILED_APP_PLANNED_PATH="" +ROLLBACK_MIGRATION_PLIST="" +ROLLBACK_MIGRATION_PLIST_SHA="" +ROLLBACK_MIGRATION_SOURCE="" +ROLLBACK_MIGRATION_LABEL="" +ROLLBACK_MIGRATION_WAS_LOADED=0 +ROLLBACK_ADOPTED_APP_WAS_RUNNING=0 +ROLLBACK_ADOPTED_APP_ATTACH_ONLY=0 +PREMUTATION_BACKUPS=() +VERIFIED_ARTIFACT_RECEIPT_SHA="" +VERIFIED_INSTALLER_SHA="" +INSTALL_RECEIPT_SCHEMA="" +INSTALL_RECEIPT_TRANSACTION_STATE="" +INSTALL_TRANSACTION_ID="" +FINAL_RECEIPT_PATH="" +PENDING_RECEIPT_PATH="" +RECOVERY_PENDING_INSTALL=0 +PENDING_RECEIPT_CREATED=0 +PENDING_RECEIPT_RETIRE_ID="" +EXPECTED_NODE_ID="" +EXPECTED_NODE_PROFILE="" +BEFORE_NODE_CONNECTED_AT=0 +UPGRADE_EXPECTED_NODE_ID="" +UPGRADE_EXPECTED_NODE_PROFILE="" +RECOVERY_CURRENT_APP_CDHASH_ARM64="" +RECOVERY_CURRENT_APP_CDHASH_X86_64="" +RECOVERY_CURRENT_APP_IDENTITY="" +RECOVERY_CURRENT_APP_STATE="" +RECOVERY_CURRENT_PLIST="" +RECOVERY_CURRENT_PLIST_SHA="" +RECOVERY_CURRENT_PLIST_WAS_LOADED=0 +RECOVERY_CURRENT_RECEIPT="" +RECOVERY_CURRENT_RECEIPT_SHA="" +RECOVERY_RESTORED_MIGRATION_IDENTITY="" +RECOVERY_RELAUNCHED_ADOPTED_PID="" +RECOVERY_RESUMED=0 +UNSAFE_ELEVATION_APP_QUARANTINE="" +UNSAFE_ELEVATION_APP_WAS_QUARANTINED=0 +ELEVATION_APP_OWNER_WAS_EVIDENCED=0 +OPENCLAW_CLI=() + +fail() { + printf 'ERROR: %s\n' "$*" >&2 + exit 1 +} + +usage() { + cat <<'HELP' +Usage: + scripts/mac-elevation-host.sh package --peekaboo-source-commit <sha> [--output-dir <dir>] + scripts/mac-elevation-host.sh verify --archive <zip> --receipt <json> --receipt-sha256 <sha256> + scripts/mac-elevation-host.sh install --archive <zip> --receipt <json> --receipt-sha256 <sha256> [--app <path>] [--state-dir <dir>] [--config-path <file>] [--migrate-launch-agent <plist>|--adopt-running-app] + scripts/mac-elevation-host.sh migration-plan [--migrate-launch-agent <plist>|--adopt-running-app] [--app <path>] [--state-dir <dir>] [--config-path <file>] + scripts/mac-elevation-host.sh status [--app <path>] [--state-dir <dir>] + scripts/mac-elevation-host.sh recover [--archive <zip> --receipt <json> --receipt-sha256 <sha256>] [--app <path>] [--state-dir <dir>] + scripts/mac-elevation-host.sh uninstall [--app <path>] [--state-dir <dir>] + scripts/mac-elevation-host.sh print-plist [--app <path>] [--state-dir <dir>] [--config-path <file>] + +The elevation host uses a separate launchd job, never rewrites ordinary OpenClaw +Launch at login, and never opens System Settings. Missing TCC is reported by status. +HELP +} + +case "$COMMAND" in + package|verify|install|migration-plan|status|recover|uninstall|print-plist) ;; + -h|--help|"") usage; exit 0 ;; + *) fail "unknown elevation-host command: $COMMAND" ;; +esac + +while [[ "$#" -gt 0 ]]; do + case "$1" in + --archive) [[ "$#" -ge 2 ]] || fail '--archive requires a path'; ARCHIVE="$2"; shift 2 ;; + --receipt) [[ "$#" -ge 2 ]] || fail '--receipt requires a path'; ARTIFACT_RECEIPT="$2"; shift 2 ;; + --receipt-sha256) [[ "$#" -ge 2 ]] || fail '--receipt-sha256 requires a digest'; EXPECTED_ARTIFACT_RECEIPT_SHA256="$2"; shift 2 ;; + --app) [[ "$#" -ge 2 ]] || fail '--app requires a path'; APP_PATH="$2"; shift 2 ;; + --state-dir) [[ "$#" -ge 2 ]] || fail '--state-dir requires a path'; STATE_DIR="$2"; STATE_DIR_EXPLICIT=1; shift 2 ;; + --config-path) [[ "$#" -ge 2 ]] || fail '--config-path requires a path'; CONFIG_PATH="$2"; CONFIG_PATH_EXPLICIT=1; shift 2 ;; + --migrate-launch-agent) [[ "$#" -ge 2 ]] || fail '--migrate-launch-agent requires a path'; MIGRATE_LAUNCH_AGENT="$2"; shift 2 ;; + --adopt-running-app) ADOPT_RUNNING_APP=1; shift ;; + --peekaboo-source-commit) [[ "$#" -ge 2 ]] || fail '--peekaboo-source-commit requires a SHA'; EXPECTED_PEEKABOO_SOURCE_COMMIT="$2"; shift 2 ;; + --output-dir) [[ "$#" -ge 2 ]] || fail '--output-dir requires a path'; OUTPUT_DIR="$2"; shift 2 ;; + -h|--help) usage; exit 0 ;; + *) fail "unknown elevation-host option: $1" ;; + esac +done + +case "$APP_PATH" in + /*.app) ;; + *) fail '--app must be an absolute .app path' ;; +esac +case "$STATE_DIR" in + /*) ;; + *) fail '--state-dir must be absolute' ;; +esac +if [[ -n "$CONFIG_PATH" ]]; then + case "$CONFIG_PATH" in /*) ;; *) fail '--config-path must be absolute' ;; esac +fi +if [[ -n "$MIGRATE_LAUNCH_AGENT" ]]; then + case "$MIGRATE_LAUNCH_AGENT" in /*.plist) ;; *) fail '--migrate-launch-agent must be an absolute .plist path' ;; esac +fi +if [[ -n "$EXPECTED_PEEKABOO_SOURCE_COMMIT" && ! "$EXPECTED_PEEKABOO_SOURCE_COMMIT" =~ ^[0-9a-f]{40}$ ]]; then + fail '--peekaboo-source-commit must be a full lowercase 40-character SHA' +fi +if [[ "$COMMAND" != "package" && -n "$EXPECTED_PEEKABOO_SOURCE_COMMIT" ]]; then + fail '--peekaboo-source-commit is valid only with package' +fi +if [[ "$COMMAND" != "verify" && "$COMMAND" != "install" && "$COMMAND" != "recover" && + -n "$ARTIFACT_RECEIPT" ]] +then + fail '--receipt is valid only with verify, install, or recover' +fi +if [[ -n "$EXPECTED_ARTIFACT_RECEIPT_SHA256" && + ! "$EXPECTED_ARTIFACT_RECEIPT_SHA256" =~ ^[0-9a-f]{64}$ ]] +then + fail '--receipt-sha256 must be a lowercase SHA-256 digest' +fi +if [[ "$COMMAND" != "verify" && "$COMMAND" != "install" && "$COMMAND" != "recover" && + -n "$EXPECTED_ARTIFACT_RECEIPT_SHA256" ]] +then + fail '--receipt-sha256 is valid only with verify, install, or recover' +fi +if [[ ("$COMMAND" == "verify" || "$COMMAND" == "install") && + -z "$EXPECTED_ARTIFACT_RECEIPT_SHA256" ]] +then + fail "$COMMAND requires --receipt-sha256 <sha256> from the authenticated release handoff" +fi +if [[ "$COMMAND" != "verify" && "$COMMAND" != "install" && "$COMMAND" != "recover" && -n "$ARCHIVE" ]]; then + fail '--archive is valid only with verify, install, or recover' +fi +if [[ "$COMMAND" == "recover" && + ( -n "$ARCHIVE" || -n "$ARTIFACT_RECEIPT" || -n "$EXPECTED_ARTIFACT_RECEIPT_SHA256" ) && + ( -z "$ARCHIVE" || -z "$ARTIFACT_RECEIPT" || -z "$EXPECTED_ARTIFACT_RECEIPT_SHA256" ) ]] +then + fail 'recover artifact helper requires --archive, --receipt, and --receipt-sha256 together' +fi +if [[ "$COMMAND" != "install" && "$COMMAND" != "migration-plan" && -n "$MIGRATE_LAUNCH_AGENT" ]]; then + fail '--migrate-launch-agent is valid only with install or migration-plan' +fi +if [[ "$COMMAND" != "install" && "$COMMAND" != "migration-plan" && "$ADOPT_RUNNING_APP" == "1" ]]; then + fail '--adopt-running-app is valid only with install or migration-plan' +fi +if [[ -n "$MIGRATE_LAUNCH_AGENT" && "$ADOPT_RUNNING_APP" == "1" ]]; then + fail 'choose either --migrate-launch-agent or --adopt-running-app' +fi + +PLIST_PATH="${HOME}/Library/LaunchAgents/${ELEVATION_LABEL}.plist" +NORMAL_PLIST_PATH="${HOME}/Library/LaunchAgents/${NORMAL_LABEL}.plist" +RECEIPT_PATH="${STATE_DIR}/elevation-host-install.json" +BRIDGE_SOCKET="${HOME}/Library/Application Support/OpenClaw/bridge.sock" + +cleanup_work_root() { + if [[ -n "$WORK_ROOT" && -d "$WORK_ROOT" ]]; then + rm -rf "$WORK_ROOT" + WORK_ROOT="" + fi + AUTHENTICATED_RENAME_HELPER="" + AUTHENTICATED_RENAME_HELPER_SHA="" +} + +cleanup_artifact_snapshot() { + [[ -n "$ARTIFACT_SNAPSHOT_ROOT" && -d "$ARTIFACT_SNAPSHOT_ROOT" ]] || return 0 + rm -rf "$ARTIFACT_SNAPSHOT_ROOT" + ARTIFACT_SNAPSHOT_ROOT="" +} + +cleanup_staged_install_app() { + [[ -n "$STAGED_APP_CONTAINER" && -d "$STAGED_APP_CONTAINER" ]] || return 0 + rm -rf "$STAGED_APP_CONTAINER" + STAGED_APP_CONTAINER="" + STAGED_INSTALL_APP_PATH="" +} + +cleanup() { + local exit_code=$? + set +e + if [[ "$CUTOVER_ACTIVE" == "1" && "$CUTOVER_COMMITTED" != "1" && + "$CUTOVER_RECOVERY_ATTEMPTED" != "1" ]] + then + if ! recover_install; then + printf 'ERROR: automatic elevation-host rollback was incomplete; run recover after inspecting preserved backups\n' >&2 + fi + fi + if [[ "$CUTOVER_COMMITTED" != "1" && "$CUTOVER_ACTIVE" != "1" ]]; then + if [[ "$COMMAND" == "install" && "$PENDING_RECEIPT_CREATED" == "1" ]]; then + remove_pending_receipt || + printf 'ERROR: could not remove the recovered pending install receipt\n' >&2 + fi + local backup + for backup in "${PREMUTATION_BACKUPS[@]:-}"; do + [[ -n "$backup" && -e "$backup" ]] && rm -f "$backup" + done + fi + cleanup_work_root + cleanup_artifact_snapshot + cleanup_staged_install_app + if [[ -n "$NOTARY_RESULT_TEMP" && -f "$NOTARY_RESULT_TEMP" ]]; then + rm -f "$NOTARY_RESULT_TEMP" + fi + return "$exit_code" +} +trap cleanup EXIT + +require_tool() { + command -v "$1" >/dev/null 2>&1 || fail "required tool not found: $1" +} + +case "$COMMAND" in + package) + required_tools=(codesign ditto file git jq lipo plutil shasum spctl xcrun) + ;; + verify) + required_tools=(codesign ditto file jq lipo plutil shasum spctl xcrun) + ;; + install) + required_tools=(codesign defaults df diskutil ditto file jq launchctl lipo lsof open pgrep plutil shasum spctl sqlite3 uuidgen xcrun) + ;; + migration-plan) + required_tools=(defaults df diskutil jq launchctl lsof pgrep plutil sqlite3) + ;; + status) + required_tools=(codesign file jq launchctl lipo plutil spctl xcrun) + ;; + recover) + required_tools=(codesign df diskutil ditto file jq launchctl lipo lsof open pgrep plutil shasum spctl xattr xcrun) + ;; + uninstall) + required_tools=(launchctl) + ;; + print-plist) + required_tools=(jq plutil) + ;; +esac +for tool in "${required_tools[@]}"; do + require_tool "$tool" +done + +plist_value() { + plist_file_value "$1/Contents/Info.plist" "$2" +} + +plist_file_value() { + local value + # Some macOS versions write failed-extraction diagnostics to stdout, not stderr. + if value="$(plutil -extract "$2" raw -o - "$1" 2>/dev/null)"; then + printf '%s\n' "$value" + fi +} + +codesign_metadata_value() { + local target="$1" key="$2" output + shift 2 + output="$(codesign -dv --verbose=4 "$@" "$target" 2>&1)" || { + local rc=$? + return "$rc" + } + # Executable/Identifier can contain newlines. Only the final CodeDirectory and + # its immediately preceding Format own signature fields; Authority is the leaf. + awk -F= -v key="$key" ' + /^(Identifier|Format)=/ { active = found = 0 } + /^CodeDirectory / { + active = 1; found = 0; format = previous; value = "" + if (key == "Format" && format ~ /^Format=/) { + value = substr(format, 8); found = 1 + } + } + active && key != "Format" && $1 == key && (!found || key != "Authority") { + value = substr($0, length(key) + 2); found = 1 + } + { previous = $0 } + END { if (format !~ /^Format=.*Mach-O / || !active || !found || value == "") exit 1; print value } + ' <<<"$output" +} + +codesign_value() { + codesign_metadata_value "$1" "$2" +} + +codesign_value_for_arch() { + codesign_metadata_value "$1" "$2" --arch "$3" +} + +entitlements_for() { + codesign -d --entitlements :- "$1" 2>/dev/null || true +} + +elevation_code_is_macho() { + local description="${2%%$'\n'*}" magic + [[ "$description" == *Mach-O* ]] && return 0 + # System file misses fat64; recognize native magics without a process per resource. + # CAFEBABE is also Java's magic. lipo remains the authority for native slices. + [[ "$description" != 'compiled Java class'* ]] || return 1 + LC_ALL=C IFS= read -r -n 4 magic <"$1" || return 1 + case "$magic" in + $'\xfe\xed\xfa\xce'|$'\xce\xfa\xed\xfe'|$'\xfe\xed\xfa\xcf'|$'\xcf\xfa\xed\xfe'|\ + $'\xca\xfe\xba\xbe'|$'\xbe\xba\xfe\xca'|$'\xca\xfe\xba\xbf'|$'\xbf\xba\xfe\xca') return 0 ;; + *) return 1 ;; + esac +} + +elevation_code_is_resource() { + local magic byte index header_size filetype='' + LC_ALL=C IFS= read -r -n 4 magic <&4 || return 1 + case "$magic" in + $'\xfe\xed\xfa\xce'|$'\xce\xfa\xed\xfe') header_size=28 ;; + $'\xfe\xed\xfa\xcf'|$'\xcf\xfa\xed\xfe') header_size=32 ;; + *) return 1 ;; + esac + # mach_header[_64].filetype is at byte 12. Objects (1), cores (4), stubs (9) + # and dSYMs (10) are resource-sealed, not native-signature candidates. Read NULs + # individually so Bash 3 preserves byte positions, including a complete header. + for ((index=4; index<header_size; index++)); do + LC_ALL=C IFS= read -r -d '' -n 1 byte <&4 || return 1 + if (( index >= 12 && index < 16 )); then + printf -v filetype '%s%02x' "$filetype" "'$byte" + fi + done + case "$magic:$filetype" in + $'\xce\xfa\xed\xfe':0[149a]000000|$'\xcf\xfa\xed\xfe':0[149a]000000|\ + $'\xfe\xed\xfa\xce':0000000[149a]|$'\xfe\xed\xfa\xcf':0000000[149a]) return 0 ;; + *) return 1 ;; + esac +} 4<"$1" + +verify_elevation_code() ( + local app workers worker arch signed_path resolved description inventory + local code_paths=() code_archs=() + app="$(cd "$1" && pwd -P)" || fail 'could not resolve elevation app' + workers="$app/Contents/Resources/node-worker" + # Shared app code stays universal; both private workers own a complete, build-matched + # native closure. Directory names alone never exempt code from slice validation. + # Filesystem aliases must not move worker paths outside the case-sensitive scope below. + for signed_path in "$app/Contents" "$app/Contents/Resources" "$workers" "$workers/arm64" "$workers/x86_64"; do + resolved="$(find "${signed_path%/*}" -mindepth 1 -maxdepth 1 -type d -name "${signed_path##*/}" -print)" || + fail 'could not scan elevation code' + [[ "$resolved" == "$signed_path" ]] || + fail "elevation worker directory missing or symlinked: $signed_path (canonical directory spelling required)" + done + inventory="$(mktemp -d "${TMPDIR:-/tmp}/openclaw-elevation-code.XXXXXX")" || + fail 'could not create elevation code inventory' + trap 'rm -rf "$inventory"' EXIT + find "$app" -print0 >"$inventory/paths" || fail 'could not scan elevation code' + if grep -q 'com.apple.security.automation.apple-events' <<<"$(entitlements_for "$app")"; then + fail "Apple Events entitlement remains on elevation app: $app" + fi + verify_elevation_code_batch() { + local index reported_path signed_path description arch archs mode prefix slice slice_arch kind slice_kind + local slice_archs=() + [[ "${#code_paths[@]}" -gt 0 ]] || return 0 + file -E -N -r -0 -0 -- "${code_paths[@]}" >"$inventory/classifications" || + fail "could not inspect elevation code: ${code_paths[0]}" + # Consume a completed classifier result, with one exact NUL-framed pair per + # queued path. Never trust a partial stream or a description's embedded filenames. + { + for ((index=0; index<${#code_paths[@]}; index++)); do + signed_path="${code_paths[index]}" + arch="${code_archs[index]}" + IFS= read -r -d '' reported_path <&3 && + IFS= read -r -d '' description <&3 && + [[ "$reported_path" == "$signed_path" && -n "$description" ]] || + fail "invalid elevation code classification: $signed_path" + [[ -f "$signed_path" && ! -L "$signed_path" ]] || + fail "elevation code changed type during classification: $signed_path" + # Fat-binary output repeats filenames after the classifier's first line. + description="${description%%$'\n'*}" + case "$description" in + ELF*|PE32*|*COFF*|MS-DOS\ executable*) + [[ -z "$arch" ]] || fail "elevation worker contains non-Mach-O native code: $signed_path" + ;; + esac + prefix=''; LC_ALL=C IFS= read -r -d '' -n 8 prefix <"$signed_path" || true + if [[ -n "$arch" && "$prefix" == $'!<thin>\n' ]]; then + fail "elevation worker contains unsupported thin archive: $signed_path" + fi + elevation_code_is_macho "$signed_path" "$description" || + [[ -n "$arch" && "$prefix" == $'!<arch>\n' ]] || continue + if grep -q 'com.apple.security.automation.apple-events' <<<"$(entitlements_for "$signed_path")"; then + fail "Apple Events entitlement remains on elevation code: $signed_path" + fi + if [[ -z "$arch" ]]; then + mode="$(stat -f '%Lp' -- "$signed_path")" || fail "could not inspect elevation code mode: $signed_path" + fi + # lipo -archs concatenates fat MH_CORE names. -info keeps separators; + # strip the exact input-path prefix so filename text cannot supply slices. + archs="$(lipo -info "$signed_path")" || fail "could not inspect elevation code slices: $signed_path" + case "$archs" in + "Architectures in the fat file: $signed_path are: "*) archs="${archs#"Architectures in the fat file: $signed_path are: "}" ;; + "Non-fat file: $signed_path is architecture: "*) archs="${archs#"Non-fat file: $signed_path is architecture: "}" ;; + *) fail "could not inspect elevation code slices: $signed_path" ;; + esac + [[ -n "$archs" && "$archs" != *$'\n'* ]] || fail "invalid elevation code slices: $signed_path" + if [[ -n "$arch" ]]; then + [[ " $archs " == *" $arch "* ]] || + fail "elevation worker Mach-O lacks $arch: $signed_path ($archs)" + elif (( (8#$mode & 0111) == 0111 )); then + [[ " $archs " == *' x86_64 '* && " $archs " == *' arm64 '* ]] || + fail "elevation Mach-O is not universal: $signed_path ($archs)" + fi + # Archives and resource Mach types are sealed, not signable images. Inspect fat + # slices in owned scratch space: neither filenames nor file(1) identify + # fat64 archives reliably, and mixed archive/native containers are invalid. + kind=native + [[ "$prefix" != $'!<arch>\n' ]] || kind=archive + if elevation_code_is_resource "$signed_path"; then kind=resource; fi + read -r -a slice_archs <<<"$archs" + case "${prefix:0:4}" in + $'\xca\xfe\xba\xbe'|$'\xbe\xba\xfe\xca'|$'\xca\xfe\xba\xbf'|$'\xbf\xba\xfe\xca') + kind='' + slice="$inventory/slice" + for slice_arch in "${slice_archs[@]}"; do + lipo "$signed_path" -thin "$slice_arch" -output "$slice" || + fail "could not inspect elevation code slice: $signed_path ($slice_arch)" + prefix=''; LC_ALL=C IFS= read -r -d '' -n 8 prefix <"$slice" || true + case "$prefix" in + $'!<arch>\n') slice_kind=archive ;; + $'\xfe\xed\xfa\xce'*|$'\xce\xfa\xed\xfe'*|$'\xfe\xed\xfa\xcf'*|$'\xcf\xfa\xed\xfe'*) slice_kind=native ;; + *) fail "invalid elevation code slice: $signed_path ($slice_arch)" ;; + esac + if elevation_code_is_resource "$slice"; then slice_kind=resource; fi + [[ "$(lipo -archs "$slice")" == "$slice_arch" ]] || + fail "invalid elevation code slice: $signed_path ($slice_arch)" + if [[ -n "$kind" && "$kind" != "$slice_kind" ]]; then + [[ "$kind" != resource && "$slice_kind" != resource ]] || + fail "mixed resource/native elevation code: $signed_path" + fail "mixed archive/native elevation code: $signed_path" + fi + kind="$slice_kind" + done + ;; + esac + if [[ "$kind" == native ]]; then + for slice_arch in "${slice_archs[@]}"; do + codesign_value_for_arch "$signed_path" Format "$slice_arch" >/dev/null || + fail "elevation code lacks native signature format: $signed_path ($slice_arch)" + done + fi + done + if IFS= read -r -d '' reported_path <&3 || [[ -n "$reported_path" ]]; then + fail 'unexpected trailing elevation code classification' + fi + } 3<"$inventory/classifications" + code_paths=() + code_archs=() + } + while IFS= read -r -d '' signed_path; do + arch="" + case "$signed_path" in + "$workers") ;; + "$workers"/*) + arch="${signed_path#"$workers/"}" + arch="${arch%%/*}" + [[ "$arch" == arm64 || "$arch" == x86_64 ]] || + fail "unexpected elevation worker architecture entry: $signed_path" + if [[ -L "$signed_path" ]]; then + [[ -e "$signed_path" ]] || fail "broken or cyclic elevation worker symlink: $signed_path" + resolved="$(stat -f '%R/' -- "$signed_path")" || fail "could not resolve elevation worker symlink: $signed_path" + resolved="${resolved%/}" + case "$resolved" in + "$workers/$arch"|"$workers/$arch"/*) ;; + *) fail "elevation worker symlink escapes its architecture tree: $signed_path" ;; + esac + fi + ;; + esac + if [[ -f "$signed_path" && ! -L "$signed_path" ]]; then + code_paths+=("$signed_path") + code_archs+=("$arch") + if [[ "${#code_paths[@]}" -ge 64 ]]; then verify_elevation_code_batch; fi + elif [[ -d "$signed_path" && ! -L "$signed_path" ]]; then + case "$signed_path" in + *.app|*.framework|*.xpc) + if codesign -dv "$signed_path" >/dev/null 2>&1 && + grep -q 'com.apple.security.automation.apple-events' <<<"$(entitlements_for "$signed_path")" + then + fail "Apple Events entitlement remains on elevation bundle: $signed_path" + fi + ;; + esac + fi + done <"$inventory/paths" + verify_elevation_code_batch + # BSD find can silently skip cycles. Canonical traversal ancestors must still + # reject a cyclic worker rather than accepting it as a closed payload. + find -L "$workers" -type d -print0 | while IFS= read -r -d '' signed_path; do + [[ -L "$signed_path" ]] || continue + resolved="$(stat -f '%R/' -- "$signed_path")" || fail "could not resolve elevation worker directory: $signed_path" + [[ "$signed_path/" != "$resolved"* ]] || fail "cyclic elevation worker directory: $signed_path" + done || fail 'cyclic or unreadable elevation worker tree' + + local node entry metadata version commit built_at build_id + version="$(plist_value "$app" CFBundleShortVersionString)" + commit="$(plist_value "$app" OpenClawGitCommit)" + built_at="$(plist_value "$app" OpenClawBuildTimestamp)" + build_id="$(plist_value "$app" OpenClawWorkerBuildID)" + [[ -n "$version" && -n "$commit" && -n "$built_at" && -n "$build_id" ]] || + fail 'elevation app is missing worker build identity' + for arch in arm64 x86_64; do + worker="$workers/$arch" + node="$worker/bin/node" + entry="$worker/lib/node_modules/openclaw/dist/entry.js" + metadata="$worker/lib/node_modules/openclaw/dist/build-info.json" + [[ -f "$node" && -x "$node" && -f "$entry" && -r "$entry" && -f "$metadata" ]] || + fail "elevation worker payload is incomplete: $worker" + resolved="$(stat -f '%R/' -- "$node")" || fail "could not resolve elevation worker Node: $node" + resolved="${resolved%/}" + description="$(file -b -E "$resolved")" || fail "could not inspect elevation worker Node: $node" + elevation_code_is_macho "$resolved" "$description" && + codesign_value_for_arch "$resolved" Format "$arch" >/dev/null || + fail "elevation worker Node must be Mach-O: $node" + jq -e -s --arg version "$version" --arg commit "$commit" --arg builtAt "$built_at" --arg buildId "$build_id" ' + length == 1 and (.[0] | + .version == $version and .commit == $commit and .builtAt == $builtAt and .buildId == $buildId) + ' "$metadata" >/dev/null 2>&1 || fail "elevation worker build metadata does not match app: $worker" + done + + local helper="$app/Contents/MacOS/openclaw-mlx-tts" + if [[ -f "$helper" ]] && grep -q '<key>' <<<"$(entitlements_for "$helper")"; then + fail "MLX helper must be signed without app entitlements: $helper" + fi +) + +elevation_app_is_cua_free() { + local app="$1" + local cua_driver="$app/Contents/Resources/cua-driver" + [[ ! -e "$cua_driver" && ! -L "$cua_driver" ]] +} + +elevation_plist_binds_app() { + local plist="$1" args executable label program + [[ -n "$plist" && -f "$plist" && ! -L "$plist" ]] || return 1 + label="$(plist_file_value "$plist" Label)" + [[ "$label" == "$ELEVATION_LABEL" ]] || return 1 + executable="$APP_PATH/Contents/MacOS/OpenClaw" + if program="$(plutil -extract Program raw -o - "$plist" 2>/dev/null)"; then + [[ "$program" == "$executable" ]] || return 1 + elif plutil -extract Program xml1 -o /dev/null "$plist" 2>/dev/null; then + return 1 + fi + args="$(plutil -extract ProgramArguments json -o - "$plist" 2>/dev/null)" || return 1 + [[ "$(jq -c . <<<"$args")" == \ + "$(jq -cn --arg executable "$executable" '[$executable,"--elevation-host"]')" ]] +} + +elevation_receipt_binds_app() { + local receipt="$1" + [[ -n "$receipt" && -f "$receipt" && ! -L "$receipt" ]] || return 1 + jq -e \ + --arg appPath "$APP_PATH" \ + --arg plistPath "$PLIST_PATH" ' + type == "object" and + .appPath == $appPath and + .plistPath == $plistPath and + ( + (.schemaVersion == 3 and .kind == "openclaw-elevation-install") or + ( + (has("schemaVersion") | not) and (has("kind") | not) and + keys == ["appPath","archiveSha256","backupPath","peekabooCommit","plistPath","previousPlist","sourceCommit"] + ) + ) + ' "$receipt" >/dev/null 2>&1 +} + +loaded_elevation_job_binds_app() { + local snapshot program + snapshot="$(job_snapshot "$job_domain")" + [[ -n "$snapshot" ]] || return 1 + program="$(awk -F' = ' '/^[[:space:]]*program = / {print $2; exit}' <<<"$snapshot")" + [[ "$program" == "$APP_PATH/Contents/MacOS/OpenClaw" ]] || return 1 + grep -Eq '^[[:space:]]*--elevation-host[[:space:]]*$' <<<"$snapshot" +} + +elevation_ownership_is_evidenced() { + local candidate + loaded_elevation_job_binds_app && return 0 + for candidate in "$PLIST_PATH" "$ROLLBACK_ELEVATION_PLIST" "$RECOVERY_CURRENT_PLIST"; do + elevation_plist_binds_app "$candidate" && return 0 + done + for candidate in \ + "$RECEIPT_PATH" \ + "$FINAL_RECEIPT_PATH" \ + "$PENDING_RECEIPT_PATH" \ + "$ROLLBACK_INSTALL_RECEIPT" \ + "$RECOVERY_CURRENT_RECEIPT" + do + elevation_receipt_binds_app "$candidate" && return 0 + done + return 1 +} + +quarantine_elevation_plist() { + local description="$1" quarantine_path source_identity source_sha="" source_kind + [[ -e "$PLIST_PATH" || -L "$PLIST_PATH" ]] || return 0 + source_identity="$(path_identity "$PLIST_PATH")" || return 1 + if [[ -f "$PLIST_PATH" && ! -L "$PLIST_PATH" ]]; then + source_kind="file" + source_sha="$(shasum -a 256 "$PLIST_PATH" | awk '{print $1}')" || return 1 + [[ "$source_sha" =~ ^[0-9a-f]{64}$ ]] || return 1 + elif [[ -L "$PLIST_PATH" ]]; then + source_kind="symlink" + else + return 1 + fi + quarantine_path="$(mktemp -u "$STATE_DIR/elevation-host.quarantined-launch-agent.XXXXXX")" || return 1 + [[ ! -e "$quarantine_path" && ! -L "$quarantine_path" ]] || return 1 + rename_app_exclusively "$PLIST_PATH" "$quarantine_path" || return 1 + [[ ! -e "$PLIST_PATH" && ! -L "$PLIST_PATH" ]] || return 1 + [[ "$(path_identity "$quarantine_path")" == "$source_identity" ]] || return 1 + if [[ "$source_kind" == "file" ]]; then + backup_file_matches "$quarantine_path" "$source_sha" || return 1 + else + [[ -L "$quarantine_path" ]] || return 1 + fi + fsync_parent "$PLIST_PATH" || return 1 + printf 'Quarantined %s elevation LaunchAgent outside launchd discovery at %s\n' \ + "$description" "$quarantine_path" >&2 +} + +ensure_elevation_job_absent() { + local state + state="$(job_loaded_state "$job_domain")" + if [[ "$state" != "absent" ]]; then + launchctl bootout "$job_domain" >/dev/null 2>&1 || true + state="$(job_loaded_state "$job_domain")" + fi + [[ "$state" == "absent" ]] +} + +neutralize_unsafe_elevation_launch_agent() { + local description="$1" evidence_path="$2" evidence_sha="$3" neutralization_failed=0 + if ! quarantine_elevation_plist "$description"; then + if [[ -e "$PLIST_PATH" || -L "$PLIST_PATH" ]]; then + rm -f -- "$PLIST_PATH" || neutralization_failed=1 + fi + fsync_parent "$PLIST_PATH" || neutralization_failed=1 + printf 'Removed unquarantinable %s elevation LaunchAgent; exact evidence remains at %s\n' \ + "$description" "$evidence_path" >&2 + fi + [[ ! -e "$PLIST_PATH" && ! -L "$PLIST_PATH" ]] || neutralization_failed=1 + ensure_elevation_job_absent || neutralization_failed=1 + if [[ -n "$evidence_path" ]]; then + backup_file_matches "$evidence_path" "$evidence_sha" || neutralization_failed=1 + fi + [[ "$neutralization_failed" == "0" ]] +} + +quarantine_entry_unsafe_elevation_app() { + local app_identity app_kind quarantine_container quarantine_app quarantine_failed=0 + ELEVATION_APP_OWNER_WAS_EVIDENCED=0 + elevation_ownership_is_evidenced || return 0 + # Keep this fact across neutralization: launchd and its plist may be gone before + # rollback decides whether the displaced CUA-bearing app can return to APP_PATH. + ELEVATION_APP_OWNER_WAS_EVIDENCED=1 + [[ -e "$APP_PATH" || -L "$APP_PATH" ]] || return 0 + elevation_app_is_cua_free "$APP_PATH" && return 0 + + neutralize_unsafe_elevation_launch_agent 'entry for unsafe elevation app' '' '' || + quarantine_failed=1 + if [[ -L "$APP_PATH" ]]; then + app_kind="symlink" + app_identity="$(path_identity "$APP_PATH")" || quarantine_failed=1 + elif [[ -d "$APP_PATH" ]]; then + app_kind="bundle" + app_identity="$(durable_path_identity "$APP_PATH")" || quarantine_failed=1 + else + quarantine_failed=1 + fi + if [[ "$quarantine_failed" == "0" ]]; then + if quarantine_container="$(mktemp -d "$STATE_DIR/elevation-host.quarantined-app.XXXXXX")"; then + quarantine_app="$quarantine_container/OpenClaw.app" + rename_app_exclusively "$APP_PATH" "$quarantine_app" || quarantine_failed=1 + if path_matches_identity "$quarantine_app" "$app_identity" && + { [[ "$app_kind" == "symlink" && -L "$quarantine_app" ]] || + { [[ "$app_kind" == "bundle" && -d "$quarantine_app" && ! -L "$quarantine_app" ]] && + ! elevation_app_is_cua_free "$quarantine_app"; }; } && + [[ ! -e "$APP_PATH" && ! -L "$APP_PATH" ]] + then + UNSAFE_ELEVATION_APP_QUARANTINE="$quarantine_app" + UNSAFE_ELEVATION_APP_WAS_QUARANTINED=1 + fsync_parent "$APP_PATH" || quarantine_failed=1 + else + quarantine_failed=1 + fi + else + quarantine_failed=1 + fi + fi + [[ ! -e "$APP_PATH" && ! -L "$APP_PATH" ]] || quarantine_failed=1 + [[ ! -e "$PLIST_PATH" && ! -L "$PLIST_PATH" ]] || quarantine_failed=1 + if [[ "$UNSAFE_ELEVATION_APP_WAS_QUARANTINED" == "1" ]]; then + printf 'Quarantined CUA-bearing elevation app outside its launchd path at %s\n' \ + "$UNSAFE_ELEVATION_APP_QUARANTINE" >&2 + fi + [[ "$quarantine_failed" == "0" ]] +} + +# Canonical elevation identity check: a strict superset of verify_elevation_signature in +# codesign-mac-app.sh, and the only one that runs post-notarization and on the target Mac at install +# time. Dropping it lets the portable installer accept an archive nobody re-verified after signing. +verify_elevation_app() { + local app="$1" + [[ -d "$app" && ! -L "$app" ]] || fail "elevation app not found or symlinked: $app" + local cua_driver="$app/Contents/Resources/cua-driver" + elevation_app_is_cua_free "$app" || + fail "elevation app must not contain bundled CUA driver: $cua_driver" + [[ "$(plist_value "$app" CFBundleIdentifier)" == "$EXPECTED_BUNDLE_ID" ]] || + fail "elevation app bundle id must be $EXPECTED_BUNDLE_ID" + local source_commit peekaboo_commit + source_commit="$(plist_value "$app" OpenClawGitCommit)" + peekaboo_commit="$(plist_value "$app" PeekabooSourceCommit)" + [[ "$source_commit" =~ ^[0-9a-f]{40}$ ]] || fail 'elevation app has invalid OpenClawGitCommit' + [[ "$peekaboo_commit" =~ ^[0-9a-f]{40}$ ]] || fail 'elevation app has invalid PeekabooSourceCommit' + verify_signed_app_identity "$app" || + fail "elevation app must be signed for every architecture by $EXPECTED_AUTHORITY" + # Conditional callers suppress errexit, so policy failures must return explicitly. + codesign --verify --strict --test-requirement='=notarized' "$app" || return $? + xcrun stapler validate "$app" >/dev/null || return $? + spctl --assess --type execute "$app" || return $? + verify_elevation_code "$app" +} + +verify_signed_app_identity() { + local app="$1" arch + codesign --verify --deep --strict --all-architectures "$app" >/dev/null 2>&1 || return 1 + for arch in arm64 x86_64; do + [[ "$(codesign_value_for_arch "$app" TeamIdentifier "$arch")" == "$EXPECTED_TEAM_ID" ]] || return 1 + [[ "$(codesign_value_for_arch "$app" Authority "$arch")" == "$EXPECTED_AUTHORITY" ]] || return 1 + done +} + +verify_rollback_app() { + local app="$1" expected_arm64_cdhash="$2" expected_x86_64_cdhash="$3" + [[ -d "$app" && ! -L "$app" ]] || return 1 + [[ "$(plist_value "$app" CFBundleIdentifier)" == "$EXPECTED_BUNDLE_ID" ]] || return 1 + [[ "$(plist_value "$app" OpenClawGitCommit)" =~ ^[0-9a-f]{40}$ ]] || return 1 + verify_signed_app_identity "$app" || return 1 + [[ -n "$expected_arm64_cdhash" && -n "$expected_x86_64_cdhash" ]] || return 1 + [[ "$(codesign_value_for_arch "$app" CDHash arm64)" == "$expected_arm64_cdhash" ]] || return 1 + [[ "$(codesign_value_for_arch "$app" CDHash x86_64)" == "$expected_x86_64_cdhash" ]] +} + +verify_recorded_rollback_app() { + verify_rollback_app "$1" "$ROLLBACK_APP_CDHASH_ARM64" "$ROLLBACK_APP_CDHASH_X86_64" +} + +verify_recorded_current_app() { + verify_rollback_app "$1" "$RECOVERY_CURRENT_APP_CDHASH_ARM64" "$RECOVERY_CURRENT_APP_CDHASH_X86_64" +} + +backup_file_matches() { + local path="$1" expected_sha="$2" + [[ -f "$path" && ! -L "$path" && "$expected_sha" =~ ^[0-9a-f]{64}$ && + "$(shasum -a 256 "$path" | awk '{print $1}')" == "$expected_sha" ]] +} + +path_identity() { + local target="$1" + [[ -e "$target" || -L "$target" ]] || return 1 + stat -f '%d:%i' -- "$target" 2>/dev/null +} + +durable_path_identity() { + local target="$1" device file_identity volume_uuid + [[ -e "$target" || -L "$target" ]] || return 1 + file_identity="$(stat -f '%i:%v' -- "$target" 2>/dev/null)" || return 1 + device="$(df -P "$target" 2>/dev/null | awk 'NR == 2 {print $1}')" || return 1 + [[ "$device" == /dev/* ]] || return 1 + volume_uuid="$(diskutil info -plist "$device" 2>/dev/null | + plutil -extract VolumeUUID raw -o - - 2>/dev/null | + tr '[:lower:]' '[:upper:]')" || return 1 + [[ "$volume_uuid" =~ ^[0-9A-Fa-f-]{36}$ && "$file_identity" =~ ^[0-9]+:[0-9]+$ ]] || return 1 + printf '%s:%s' "$volume_uuid" "$file_identity" +} + +path_matches_identity() { + local target="$1" expected_identity="$2" + if [[ "$expected_identity" =~ ^[0-9]+:[0-9]+$ ]]; then + [[ "$(path_identity "$target")" == "$expected_identity" ]] + elif [[ "$expected_identity" =~ ^[0-9A-F-]{36}:[0-9]+:[0-9]+$ ]]; then + [[ "$(durable_path_identity "$target")" == "$expected_identity" ]] + else + return 1 + fi +} + +read_optional_receipt_xattr() { + local output_variable="$1" attribute="$2" value attributes + if value="$(xattr -p "$attribute" "$RECEIPT_PATH" 2>/dev/null)"; then + printf -v "$output_variable" '%s' "$value" + return 0 + fi + attributes="$(xattr "$RECEIPT_PATH" 2>/dev/null)" || return 1 + grep -Fqx "$attribute" <<<"$attributes" && return 1 + printf -v "$output_variable" '' +} + +write_receipt_xattr() { + local attribute="$1" value="$2" + xattr -w "$attribute" "$value" "$RECEIPT_PATH" || return 1 + [[ "$(xattr -p "$attribute" "$RECEIPT_PATH" 2>/dev/null)" == "$value" ]] || return 1 + fsync_file_and_parent "$RECEIPT_PATH" +} + +record_recovery_app_plan() { + local value="${RECOVERY_CURRENT_APP_STATE}|${RECOVERY_CURRENT_APP_IDENTITY}" + [[ "$value" =~ ^(absent[|]|valid[|][0-9A-F-]{36}:[0-9]+:[0-9]+|damaged[|][0-9A-F-]{36}:[0-9]+:[0-9]+)$ ]] || return 1 + write_receipt_xattr "$RECOVERY_APP_PLAN_XATTR" "$value" +} + +record_recovery_migration_identity() { + local identity="$1" + [[ "$identity" =~ ^[0-9A-F-]{36}:[0-9]+:[0-9]+$ ]] || return 1 + write_receipt_xattr "$RECOVERY_MIGRATION_IDENTITY_XATTR" "$identity" +} + +preserve_file_by_exclusive_custody() { + local source="$1" expected_sha="$2" expected_identity="$3" custody + custody="$(mktemp -u "${source}.reversal-custody.${ROLLBACK_FAILED_SOURCE}.XXXXXX")" || return 1 + [[ ! -e "$custody" && ! -L "$custody" ]] || return 1 + rename_app_exclusively "$source" "$custody" || return 1 + if [[ -f "$custody" && ! -L "$custody" ]] && + path_matches_identity "$custody" "$expected_identity" && + backup_file_matches "$custody" "$expected_sha" + then + if [[ ! -e "$source" && ! -L "$source" ]]; then + printf 'Preserved reversed migration plist at %s\n' "$custody" >&2 + return 0 + fi + printf 'Preserved reversed migration plist at %s; replacement remains at %s\n' \ + "$custody" "$source" >&2 + return 1 + fi + # Preserve an unexpected replacement: restore it exclusively when possible, otherwise + # leave it in the private custody path for operator inspection. Never unlink it. + if [[ ! -e "$source" && ! -L "$source" ]]; then + rename_app_exclusively "$custody" "$source" || true + fi + if [[ -e "$custody" || -L "$custody" ]]; then + printf 'Preserved unexpected reversal custody at %s\n' "$custody" >&2 + elif [[ -e "$source" || -L "$source" ]]; then + printf 'Restored unexpected reversal entry at %s\n' "$source" >&2 + fi + return 1 +} + +restore_file_atomically() { + local source="$1" destination="$2" expected_sha="$3" mode="$4" staged="" + backup_file_matches "$source" "$expected_sha" || return 1 + staged="$(mktemp "${destination}.restore.XXXXXX")" || return 1 + if ! cp -p "$source" "$staged" || + ! chmod "$mode" "$staged" || + ! backup_file_matches "$staged" "$expected_sha" + then + rm -f "$staged" + return 1 + fi + mv "$staged" "$destination" || true + rm -f "$staged" + backup_file_matches "$destination" "$expected_sha" && + [[ "$(stat -f '%Lp' "$destination")" == "$mode" ]] +} + +restore_install_receipt_after_rollback() { + if [[ -n "$ROLLBACK_INSTALL_RECEIPT" ]]; then + restore_file_atomically \ + "$ROLLBACK_INSTALL_RECEIPT" \ + "$RECEIPT_PATH" \ + "$ROLLBACK_INSTALL_RECEIPT_SHA" \ + 600 + elif [[ -e "$RECEIPT_PATH" || -L "$RECEIPT_PATH" ]]; then + [[ "$ELEVATION_APP_OWNER_WAS_EVIDENCED" == "1" ]] || return 1 + [[ -f "$RECEIPT_PATH" && ! -L "$RECEIPT_PATH" ]] || return 1 + rm "$RECEIPT_PATH" + fi +} + +receipt_string() { + local receipt="$1" filter="$2" label="$3" + local value + value="$(jq -er "$filter | select(type == \"string\")" "$receipt" 2>/dev/null || true)" + [[ -n "$value" ]] || fail "artifact receipt has invalid $label" + printf '%s' "$value" +} + +state_backup_path_is_canonical() { + local candidate="$1" modern_basename_regex="$2" legacy_basename="$3" + [[ -d "$STATE_DIR" && ! -L "$STATE_DIR" && -f "$candidate" && ! -L "$candidate" ]] || return 1 + local canonical_state canonical_parent candidate_basename + canonical_state="$(cd "$STATE_DIR" && pwd -P)" || return 1 + canonical_parent="$(cd "$(dirname "$candidate")" && pwd -P)" || return 1 + [[ "$canonical_parent" == "$canonical_state" ]] || return 1 + candidate_basename="$(basename "$candidate")" + [[ -n "$legacy_basename" && "$candidate_basename" == "$legacy_basename" ]] || + [[ "$candidate_basename" =~ $modern_basename_regex ]] +} + +restore_file_without_overwrite() { + local source="$1" destination="$2" expected_sha="$3" output_variable="$4" + local restore_tmp restored_identity + [[ ! -e "$destination" && ! -L "$destination" ]] || return 1 + restore_tmp="$(mktemp "${destination}.restore.XXXXXX")" || return 1 + if ! cp -p "$source" "$restore_tmp" || + [[ "$(shasum -a 256 "$restore_tmp" | awk '{print $1}')" != "$expected_sha" ]] + then + rm -f "$restore_tmp" + return 1 + fi + if [[ "$CUTOVER_RECOVERY_ATTEMPTED" == "1" ]]; then + restored_identity="$(durable_path_identity "$restore_tmp")" || { + rm -f "$restore_tmp" + return 1 + } + if ! record_recovery_migration_identity "$restored_identity"; then + rm -f "$restore_tmp" + return 1 + fi + else + restored_identity="$(path_identity "$restore_tmp")" || { + rm -f "$restore_tmp" + return 1 + } + fi + if ! /bin/link "$restore_tmp" "$destination"; then + rm -f "$restore_tmp" + return 1 + fi + if ! path_matches_identity "$destination" "$restored_identity"; then + rm -f "$restore_tmp" + return 1 + fi + rm -f "$restore_tmp" + printf -v "$output_variable" '%s' "$restored_identity" +} + +finish_custody_signal_deferral() { + local deferred_signal="$1" + trap - HUP INT TERM + if [[ -n "$deferred_signal" ]]; then + kill -s "$deferred_signal" "$$" + fi +} + +take_migration_plist_custody() { + local source="$1" expected_sha="$2" custody_signal="" move_status=0 custody_sha="" + local source_identity observed_custody_identity="" + # Keep cooperative termination pending until rollback can identify the moved path. + # A signed same-directory exclusive rename takes the exact current plist without + # overwriting a raced destination or unlinking a later replacement. + trap 'custody_signal=INT' INT + trap 'custody_signal=TERM' TERM + trap 'custody_signal=HUP' HUP + source_identity="$(durable_path_identity "$source")" || + fail 'could not inspect the migration LaunchAgent before custody' + [[ "$source_identity" == "$MIGRATION_PLIST_IDENTITY" ]] || + fail 'migration LaunchAgent identity changed before custody' + if [[ -z "$MIGRATION_CUSTODY_PATH" ]]; then + MIGRATION_CUSTODY_PATH="$(mktemp -u "${source}.custody.XXXXXX")" || + fail 'could not reserve migration LaunchAgent custody' + fi + [[ ! -e "$MIGRATION_CUSTODY_PATH" && ! -L "$MIGRATION_CUSTODY_PATH" ]] || + fail 'migration LaunchAgent custody path is already occupied' + rename_app_exclusively "$source" "$MIGRATION_CUSTODY_PATH" || move_status=$? + if [[ -f "$MIGRATION_CUSTODY_PATH" && ! -L "$MIGRATION_CUSTODY_PATH" ]]; then + custody_sha="$(shasum -a 256 "$MIGRATION_CUSTODY_PATH" | awk '{print $1}')" + observed_custody_identity="$(durable_path_identity "$MIGRATION_CUSTODY_PATH")" || true + fi + if [[ ! -e "$source" && ! -L "$source" ]]; then + # Even a reported move failure may have removed the source. Arm rollback from the + # already-verified backup before propagating the failure. This also covers a raced + # source replacement that was moved into custody instead of the planned identity. + CUTOVER_ACTIVE=1 + CUTOVER_MIGRATION_REMOVED=1 + fi + + if [[ "$move_status" != "0" || "$custody_sha" != "$expected_sha" || + "$observed_custody_identity" != "$source_identity" || -e "$source" || -L "$source" ]] + then + finish_custody_signal_deferral "$custody_signal" + fail 'could not take exact custody of the migration LaunchAgent; rerun migration-plan' + fi + finish_custody_signal_deferral "$custody_signal" +} + +prepare_authenticated_artifact_inputs() { + local receipt="$1" archive="$2" installer="$3" + [[ -f "$receipt" && ! -L "$receipt" ]] || fail "artifact receipt not found or symlinked: $receipt" + [[ -f "$archive" && ! -L "$archive" ]] || fail "archive not found or symlinked: $archive" + [[ -f "$installer" && ! -L "$installer" ]] || fail "elevation installer not found or symlinked: $installer" + + cleanup_artifact_snapshot + ARTIFACT_SNAPSHOT_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/openclaw-elevation-inputs.XXXXXX")" + AUTHENTICATED_RECEIPT_PATH="$ARTIFACT_SNAPSHOT_ROOT/receipt.json" + cp -p "$receipt" "$AUTHENTICATED_RECEIPT_PATH" + local receipt_sha receipt_archive_checksum receipt_installer_checksum + local receipt_installer_sha installer_sha receipt_archive_sha archive_sha + receipt_sha="$(shasum -a 256 "$AUTHENTICATED_RECEIPT_PATH" | awk '{print $1}')" + [[ -n "$EXPECTED_ARTIFACT_RECEIPT_SHA256" && + "$receipt_sha" == "$EXPECTED_ARTIFACT_RECEIPT_SHA256" ]] || + fail 'artifact receipt does not match the authenticated release handoff digest' + jq -e ' + type == "object" and + keys == ["architectures","archive","archiveChecksum","archiveSha256","authority","build","cdhashes","entitlementsSha256","installer","installerChecksum","installerSha256","kind","notarizationId","peekabooCommit","schemaVersion","sourceCommit","teamIdentifier","version"] and + .schemaVersion == 1 and .kind == "openclaw-elevation-artifact" and + (.architectures | type == "object" and keys == ["helper","main"]) and + (.cdhashes | type == "object" and keys == ["arm64","x86_64"]) and + (.entitlementsSha256 | type == "object" and keys == ["helper","main"]) and + (.notarizationId | type == "string" and test("^[0-9a-fA-F-]{36}$")) + ' "$AUTHENTICATED_RECEIPT_PATH" >/dev/null 2>&1 || fail 'artifact receipt schema is invalid' + + AUTHENTICATED_ARCHIVE_NAME="$(receipt_string "$AUTHENTICATED_RECEIPT_PATH" '.archive' archive)" + AUTHENTICATED_INSTALLER_NAME="$(receipt_string "$AUTHENTICATED_RECEIPT_PATH" '.installer' installer)" + [[ "$(basename "$archive")" == "$AUTHENTICATED_ARCHIVE_NAME" ]] || + fail 'artifact receipt archive name mismatch' + [[ "$(basename "$installer")" == "$AUTHENTICATED_INSTALLER_NAME" ]] || + fail 'artifact receipt installer name mismatch' + receipt_archive_checksum="$(receipt_string "$AUTHENTICATED_RECEIPT_PATH" '.archiveChecksum' archiveChecksum)" + receipt_installer_checksum="$(receipt_string "$AUTHENTICATED_RECEIPT_PATH" '.installerChecksum' installerChecksum)" + [[ "$receipt_archive_checksum" == "${AUTHENTICATED_ARCHIVE_NAME}.sha256" ]] || + fail 'artifact receipt archive checksum name mismatch' + [[ "$receipt_installer_checksum" == "${AUTHENTICATED_INSTALLER_NAME}.sha256" ]] || + fail 'artifact receipt installer checksum name mismatch' + receipt_installer_sha="$(receipt_string "$AUTHENTICATED_RECEIPT_PATH" '.installerSha256' installerSha256)" + installer_sha="$(shasum -a 256 "$installer" | awk '{print $1}')" + [[ "$receipt_installer_sha" == "$installer_sha" ]] || fail 'artifact receipt installer digest mismatch' + + AUTHENTICATED_ARCHIVE_PATH="$ARTIFACT_SNAPSHOT_ROOT/archive.zip" + cp -p "$archive" "$AUTHENTICATED_ARCHIVE_PATH" + receipt_archive_sha="$(receipt_string "$AUTHENTICATED_RECEIPT_PATH" '.archiveSha256' archiveSha256)" + archive_sha="$(shasum -a 256 "$AUTHENTICATED_ARCHIVE_PATH" | awk '{print $1}')" + [[ "$receipt_archive_sha" == "$archive_sha" ]] || fail 'artifact receipt archive digest mismatch' + VERIFIED_ARTIFACT_RECEIPT_SHA="$receipt_sha" + VERIFIED_INSTALLER_SHA="$installer_sha" +} + +verify_artifact_receipt() { + local receipt="$1" archive="$2" app="$3" installer="$4" + [[ -f "$receipt" && ! -L "$receipt" ]] || fail "artifact receipt not found or symlinked: $receipt" + [[ -f "$installer" && ! -L "$installer" ]] || fail "elevation installer not found or symlinked: $installer" + verify_elevation_app "$app" || return $? + local receipt_sha + receipt_sha="$(shasum -a 256 "$receipt" | awk '{print $1}')" + [[ -n "$EXPECTED_ARTIFACT_RECEIPT_SHA256" && + "$receipt_sha" == "$EXPECTED_ARTIFACT_RECEIPT_SHA256" ]] || + fail 'artifact receipt does not match the authenticated release handoff digest' + jq -e ' + type == "object" and + keys == ["architectures","archive","archiveChecksum","archiveSha256","authority","build","cdhashes","entitlementsSha256","installer","installerChecksum","installerSha256","kind","notarizationId","peekabooCommit","schemaVersion","sourceCommit","teamIdentifier","version"] and + .schemaVersion == 1 and .kind == "openclaw-elevation-artifact" and + (.architectures | type == "object" and keys == ["helper","main"]) and + (.cdhashes | type == "object" and keys == ["arm64","x86_64"]) and + (.entitlementsSha256 | type == "object" and keys == ["helper","main"]) and + (.notarizationId | type == "string" and test("^[0-9a-fA-F-]{36}$")) + ' "$receipt" >/dev/null 2>&1 || fail 'artifact receipt schema is invalid' + + local archive_name installer_name archive_sha installer_sha source_commit peekaboo_commit + local arm64_cdhash x86_64_cdhash + archive_name="$AUTHENTICATED_ARCHIVE_NAME" + installer_name="$AUTHENTICATED_INSTALLER_NAME" + archive_sha="$(shasum -a 256 "$archive" | awk '{print $1}')" + installer_sha="$(shasum -a 256 "$installer" | awk '{print $1}')" + source_commit="$(plist_value "$app" OpenClawGitCommit)" + peekaboo_commit="$(plist_value "$app" PeekabooSourceCommit)" + + [[ "$(receipt_string "$receipt" '.archive' archive)" == "$archive_name" ]] || fail 'artifact receipt archive name mismatch' + [[ "$(receipt_string "$receipt" '.archiveChecksum' archiveChecksum)" == "${archive_name}.sha256" ]] || + fail 'artifact receipt archive checksum name mismatch' + [[ "$(receipt_string "$receipt" '.archiveSha256' archiveSha256)" == "$archive_sha" ]] || fail 'artifact receipt archive digest mismatch' + [[ "$(receipt_string "$receipt" '.installer' installer)" == "$installer_name" ]] || + fail 'artifact receipt installer name mismatch' + [[ "$(receipt_string "$receipt" '.installerChecksum' installerChecksum)" == "${installer_name}.sha256" ]] || + fail 'artifact receipt installer checksum name mismatch' + [[ "$(receipt_string "$receipt" '.installerSha256' installerSha256)" == "$installer_sha" ]] || + fail 'artifact receipt installer digest mismatch' + [[ "$(receipt_string "$receipt" '.sourceCommit' sourceCommit)" == "$source_commit" ]] || fail 'artifact receipt OpenClaw source mismatch' + [[ "$(receipt_string "$receipt" '.peekabooCommit' peekabooCommit)" == "$peekaboo_commit" ]] || fail 'artifact receipt Peekaboo source mismatch' + [[ "$(receipt_string "$receipt" '.version' version)" == "$(plist_value "$app" CFBundleShortVersionString)" ]] || fail 'artifact receipt version mismatch' + [[ "$(receipt_string "$receipt" '.build' build)" == "$(plist_value "$app" CFBundleVersion)" ]] || fail 'artifact receipt build mismatch' + [[ "$(receipt_string "$receipt" '.authority' authority)" == "$(codesign_value "$app" Authority)" ]] || fail 'artifact receipt signing authority mismatch' + [[ "$(receipt_string "$receipt" '.teamIdentifier' teamIdentifier)" == "$(codesign_value "$app" TeamIdentifier)" ]] || fail 'artifact receipt TeamIdentifier mismatch' + arm64_cdhash="$(codesign_value_for_arch "$app" CDHash arm64)" + x86_64_cdhash="$(codesign_value_for_arch "$app" CDHash x86_64)" + [[ "$(receipt_string "$receipt" '.cdhashes.arm64' cdhashes.arm64)" == "$arm64_cdhash" ]] || + fail 'artifact receipt arm64 CDHash mismatch' + [[ "$(receipt_string "$receipt" '.cdhashes.x86_64' cdhashes.x86_64)" == "$x86_64_cdhash" ]] || + fail 'artifact receipt x86_64 CDHash mismatch' + [[ "$(receipt_string "$receipt" '.architectures.main' architectures.main)" == "$(lipo -archs "$app/Contents/MacOS/OpenClaw")" ]] || fail 'artifact receipt main architecture mismatch' + [[ "$(receipt_string "$receipt" '.architectures.helper' architectures.helper)" == "$(lipo -archs "$app/Contents/MacOS/openclaw-mlx-tts")" ]] || fail 'artifact receipt helper architecture mismatch' + [[ "$(receipt_string "$receipt" '.entitlementsSha256.main' entitlementsSha256.main)" == "$(entitlements_for "$app/Contents/MacOS/OpenClaw" | shasum -a 256 | awk '{print $1}')" ]] || fail 'artifact receipt main entitlement mismatch' + [[ "$(receipt_string "$receipt" '.entitlementsSha256.helper' entitlementsSha256.helper)" == "$(entitlements_for "$app/Contents/MacOS/openclaw-mlx-tts" | shasum -a 256 | awk '{print $1}')" ]] || fail 'artifact receipt helper entitlement mismatch' + + VERIFIED_ARTIFACT_RECEIPT_SHA="$receipt_sha" + VERIFIED_INSTALLER_SHA="$installer_sha" +} +verify_artifact_set() { + [[ -n "$ARCHIVE" ]] || fail 'verify requires --archive <zip>' + [[ -n "$ARTIFACT_RECEIPT" ]] || fail 'verify requires --receipt <json>' + local staged_app + prepare_authenticated_artifact_inputs "$ARTIFACT_RECEIPT" "$ARCHIVE" "${BASH_SOURCE[0]}" + extract_verified_artifact "${BASH_SOURCE[0]}" staged_app + printf 'Elevation artifact verified: source=%s peekaboo=%s\n' \ + "$(plist_value "$staged_app" OpenClawGitCommit)" "$(plist_value "$staged_app" PeekabooSourceCommit)" +} + +extract_verified_artifact() { + local installer="$1" output_variable="$2" candidate_helper candidate_helper_sha entries + [[ -f "$AUTHENTICATED_ARCHIVE_PATH" ]] || fail "archive not found: $AUTHENTICATED_ARCHIVE_PATH" + cleanup_work_root + WORK_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/openclaw-elevation.XXXXXX")" + ditto -x -k "$AUTHENTICATED_ARCHIVE_PATH" "$WORK_ROOT" || return $? + entries="$(find "$WORK_ROOT" -mindepth 1 -maxdepth 1 -print | sort)" + [[ "$entries" == "$WORK_ROOT/OpenClaw.app" && -d "$WORK_ROOT/OpenClaw.app" && + ! -L "$WORK_ROOT/OpenClaw.app" ]] || fail 'elevation archive root must contain exactly physical OpenClaw.app' + # Capture untrusted helper bytes before the final audit; publish authority only + # after the complete app/receipt check. Later helper use rechecks this digest. + candidate_helper="$WORK_ROOT/OpenClaw.app/Contents/MacOS/OpenClaw" + [[ -f "$candidate_helper" && ! -L "$candidate_helper" && -x "$candidate_helper" ]] || + fail 'authenticated elevation rename helper is unavailable' + candidate_helper_sha="$(shasum -a 256 "$candidate_helper" | awk '{print $1}')" || return $? + verify_artifact_receipt \ + "$AUTHENTICATED_RECEIPT_PATH" "$AUTHENTICATED_ARCHIVE_PATH" "$WORK_ROOT/OpenClaw.app" "$installer" || return $? + AUTHENTICATED_RENAME_HELPER="$candidate_helper" + AUTHENTICATED_RENAME_HELPER_SHA="$candidate_helper_sha" + printf -v "$output_variable" '%s' "$WORK_ROOT/OpenClaw.app" +} + +stage_verified_app_for_install() { + local source_app="$1" source_commit="$2" peekaboo_commit="$3" + cleanup_staged_install_app + STAGED_APP_CONTAINER="$(mktemp -d "${APP_PATH}.incoming-${source_commit}.XXXXXX")" + STAGED_INSTALL_APP_PATH="$STAGED_APP_CONTAINER/OpenClaw.app" + ditto "$source_app" "$STAGED_INSTALL_APP_PATH" + [[ "$(plist_value "$STAGED_INSTALL_APP_PATH" OpenClawGitCommit)" == "$source_commit" ]] || + fail 'same-filesystem staged app source mismatch' + [[ "$(plist_value "$STAGED_INSTALL_APP_PATH" PeekabooSourceCommit)" == "$peekaboo_commit" ]] || + fail 'same-filesystem staged Peekaboo source mismatch' + verify_artifact_receipt \ + "$AUTHENTICATED_RECEIPT_PATH" \ + "$AUTHENTICATED_ARCHIVE_PATH" \ + "$STAGED_INSTALL_APP_PATH" \ + "${BASH_SOURCE[0]}" +} + +prepare_current_app_rename_helper() { + local source_app="$APP_PATH" + [[ -z "$RECOVERED_FAILED_APP_PATH" ]] || source_app="$RECOVERED_FAILED_APP_PATH" + verify_recorded_current_app "$source_app" || + fail 'current recovery app cannot authenticate the exclusive rename helper' + cleanup_work_root + WORK_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/openclaw-elevation-current.XXXXXX")" + ditto "$source_app" "$WORK_ROOT/OpenClaw.app" + verify_recorded_current_app "$WORK_ROOT/OpenClaw.app" || + fail 'copied recovery app cannot authenticate the exclusive rename helper' + AUTHENTICATED_RENAME_HELPER="$WORK_ROOT/OpenClaw.app/Contents/MacOS/OpenClaw" + [[ -f "$AUTHENTICATED_RENAME_HELPER" && ! -L "$AUTHENTICATED_RENAME_HELPER" && + -x "$AUTHENTICATED_RENAME_HELPER" ]] || fail 'authenticated elevation rename helper is unavailable' + AUTHENTICATED_RENAME_HELPER_SHA="$(shasum -a 256 "$AUTHENTICATED_RENAME_HELPER" | awk '{print $1}')" +} + +run_authenticated_elevation_helper() { + local helper_sha + [[ -n "$AUTHENTICATED_RENAME_HELPER" && -f "$AUTHENTICATED_RENAME_HELPER" && + ! -L "$AUTHENTICATED_RENAME_HELPER" && -x "$AUTHENTICATED_RENAME_HELPER" && + "$AUTHENTICATED_RENAME_HELPER_SHA" =~ ^[0-9a-f]{64}$ ]] || return 1 + helper_sha="$(shasum -a 256 "$AUTHENTICATED_RENAME_HELPER" | awk '{print $1}')" || return 1 + [[ "$helper_sha" == "$AUTHENTICATED_RENAME_HELPER_SHA" ]] || return 1 + "$AUTHENTICATED_RENAME_HELPER" "$@" +} + +rename_app_exclusively() { + run_authenticated_elevation_helper --elevation-rename-exclusive "$1" "$2" || return 1 + fsync_parent "$2" +} + +preserve_current_app_for_recovery() { + local custody_label="$1" failed_container failed_path + [[ -n "$RECOVERY_CURRENT_APP_IDENTITY" ]] || return 1 + if [[ -n "$RECOVERY_FAILED_APP_PLANNED_PATH" ]]; then + failed_path="$RECOVERY_FAILED_APP_PLANNED_PATH" + failed_container="$(dirname "$failed_path")" + if [[ ! -e "$failed_container" && ! -L "$failed_container" ]]; then + mkdir "$failed_container" || return 1 + elif [[ ! -d "$failed_container" || -L "$failed_container" || + -n "$(find "$failed_container" -mindepth 1 -maxdepth 1 -print -quit)" ]] + then + return 1 + fi + else + failed_container="$(mktemp -d "${APP_PATH}.failed-elevation-host-${ROLLBACK_FAILED_SOURCE}.XXXXXX")" || + return 1 + failed_path="$failed_container/OpenClaw.app" + fi + if ! rename_app_exclusively "$APP_PATH" "$failed_path"; then + rmdir "$failed_container" 2>/dev/null || true + return 1 + fi + if [[ ! -d "$failed_path" || -L "$failed_path" ]] || + ! path_matches_identity "$failed_path" "$RECOVERY_CURRENT_APP_IDENTITY" + then + if [[ (-e "$failed_path" || -L "$failed_path") && ! -e "$APP_PATH" && ! -L "$APP_PATH" ]]; then + rename_app_exclusively "$failed_path" "$APP_PATH" || true + fi + if [[ -e "$failed_path" || -L "$failed_path" ]]; then + printf 'Preserved unexpected app custody at %s\n' "$failed_path" >&2 + elif [[ -e "$APP_PATH" || -L "$APP_PATH" ]]; then + printf 'Restored replacement app entry at %s\n' "$APP_PATH" >&2 + fi + rmdir "$failed_container" 2>/dev/null || true + return 1 + fi + RECOVERED_FAILED_APP_PATH="$failed_path" + if [[ -e "$APP_PATH" || -L "$APP_PATH" ]]; then + printf 'Preserved %s at %s; replacement remains at %s\n' \ + "$custody_label" "$RECOVERED_FAILED_APP_PATH" "$APP_PATH" >&2 + return 1 + fi +} + +render_plist() { + local destination="$1" + local executable="$APP_PATH/Contents/MacOS/OpenClaw" + local log_path="$STATE_DIR/logs/mac-app.log" + local environment_json + environment_json="$(jq -cn \ + --arg path '/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin' \ + --arg state "$STATE_DIR" \ + --arg config "$CONFIG_PATH" \ + '{PATH:$path,OPENCLAW_STATE_DIR:$state} + (if $config == "" then {} else {OPENCLAW_CONFIG_PATH:$config} end)')" + + plutil -create xml1 "$destination" + plutil -insert Label -string "$ELEVATION_LABEL" "$destination" + plutil -insert ProgramArguments -json "$(jq -cn --arg executable "$executable" '[$executable,"--elevation-host"]')" "$destination" + plutil -insert WorkingDirectory -string "$HOME" "$destination" + plutil -insert RunAtLoad -bool true "$destination" + plutil -insert KeepAlive -bool true "$destination" + plutil -insert EnvironmentVariables -json "$environment_json" "$destination" + plutil -insert StandardOutPath -string "$log_path" "$destination" + plutil -insert StandardErrorPath -string "$log_path" "$destination" +} + +launch_domain="gui/$(id -u)" +job_domain="$launch_domain/$ELEVATION_LABEL" +normal_domain="$launch_domain/$NORMAL_LABEL" + +job_snapshot() { + launchctl print "$1" 2>/dev/null || true +} + +job_loaded_state() { + local output + if output="$(launchctl print "$1" 2>&1)"; then + printf 'loaded' + elif grep -q 'Could not find service' <<<"$output"; then + printf 'absent' + else + printf 'unknown' + fi +} + +job_pid_for_domain() { + awk -F' = ' '/^[[:space:]]*pid = / {print $2; exit}' <<<"$(job_snapshot "$1")" +} + +job_pid() { + job_pid_for_domain "$job_domain" +} + +refresh_runtime_paths() { + PLIST_PATH="${HOME}/Library/LaunchAgents/${ELEVATION_LABEL}.plist" + NORMAL_PLIST_PATH="${HOME}/Library/LaunchAgents/${NORMAL_LABEL}.plist" + FINAL_RECEIPT_PATH="${STATE_DIR}/elevation-host-install.json" + PENDING_RECEIPT_PATH="${STATE_DIR}/elevation-host-install.pending.json" + RECEIPT_PATH="$FINAL_RECEIPT_PATH" +} + +resolve_reusable_openclaw_cli() { + local cli + cli="$(command -v openclaw 2>/dev/null || true)" + case "$cli" in /*) ;; *) fail 'openclaw CLI is required for gateway node attestation' ;; esac + [[ -f "$cli" && -x "$cli" ]] || fail 'openclaw CLI is required for gateway node attestation' + OPENCLAW_CLI=("$cli") +} + +resolve_migration_inputs() { + [[ -n "$MIGRATE_LAUNCH_AGENT" ]] || return 0 + [[ -f "$MIGRATE_LAUNCH_AGENT" && ! -L "$MIGRATE_LAUNCH_AGENT" ]] || + fail "migration LaunchAgent is missing or symlinked: $MIGRATE_LAUNCH_AGENT" + [[ "$(dirname "$MIGRATE_LAUNCH_AGENT")" == "${HOME}/Library/LaunchAgents" ]] || + fail '--migrate-launch-agent must name a plist directly under the current user LaunchAgents directory' + [[ "$MIGRATE_LAUNCH_AGENT" != "$PLIST_PATH" && "$MIGRATE_LAUNCH_AGENT" != "$NORMAL_PLIST_PATH" ]] || + fail 'the elevation and ordinary login LaunchAgents cannot be migration sources' + + MIGRATION_LABEL="$(plist_file_value "$MIGRATE_LAUNCH_AGENT" Label)" + [[ "$MIGRATION_LABEL" =~ ^[A-Za-z0-9._-]+$ ]] || fail 'migration LaunchAgent has an invalid Label' + [[ "$(basename "$MIGRATE_LAUNCH_AGENT")" == "${MIGRATION_LABEL}.plist" ]] || + fail 'migration LaunchAgent filename must match its Label' + MIGRATION_PLIST_SHA="$(shasum -a 256 "$MIGRATE_LAUNCH_AGENT" | awk '{print $1}')" + MIGRATION_PLIST_IDENTITY="$(durable_path_identity "$MIGRATE_LAUNCH_AGENT")" || + fail 'migration LaunchAgent identity could not be inspected' + + local args app_binary inline_state inline_config environment node_env node_paths + args="$(plutil -extract ProgramArguments json -o - "$MIGRATE_LAUNCH_AGENT" 2>/dev/null || true)" + jq -e 'type == "array" and length > 0 and all(.[]; type == "string")' <<<"$args" >/dev/null 2>&1 || + fail 'migration LaunchAgent must have a string ProgramArguments array' + app_binary="$APP_PATH/Contents/MacOS/OpenClaw" + if jq -e --arg appBinary "$app_binary" ' + .[0] == $appBinary and + (.[1:] | index("--background-only") != null) and + (.[1:] | all(. == "--background-only" or . == "--attach-only" or . == "--no-launchd")) + ' <<<"$args" >/dev/null 2>&1 + then + MIGRATION_KIND="app-launch-agent" + environment="$(plutil -extract EnvironmentVariables json -o - "$MIGRATE_LAUNCH_AGENT" 2>/dev/null || true)" + jq -e ' + type == "object" and + all(keys[]; . == "PATH" or . == "OPENCLAW_STATE_DIR" or . == "OPENCLAW_CONFIG_PATH") + ' <<<"$environment" >/dev/null 2>&1 || + fail 'app migration LaunchAgent has unsupported environment keys; move routing into its state/config first' + inline_state="$(plist_file_value "$MIGRATE_LAUNCH_AGENT" EnvironmentVariables.OPENCLAW_STATE_DIR)" + case "$inline_state" in /*) ;; *) fail 'app migration LaunchAgent must expose an absolute OPENCLAW_STATE_DIR' ;; esac + inline_config="$(plist_file_value "$MIGRATE_LAUNCH_AGENT" EnvironmentVariables.OPENCLAW_CONFIG_PATH)" + resolve_reusable_openclaw_cli + else + MIGRATION_KIND="canonical-node" + [[ "$MIGRATION_LABEL" == "ai.openclaw.node" ]] || + fail 'non-app migration sources must be the canonical ai.openclaw.node LaunchAgent' + jq -e ' + def validTail: + length == 0 or + ((.[0] == "--tls" or .[0] == "--no-tls" or .[0] == "--share-installed-apps" or .[0] == "--no-share-installed-apps") and (.[1:] | validTail)) or + ((.[0] == "--tls-fingerprint" or .[0] == "--context-path" or .[0] == "--node-id" or .[0] == "--display-name") and length >= 2 and (.[2:] | validTail)); + length >= 11 and .[0] == "/bin/sh" and + (.[3] | startswith("/")) and (.[4] | startswith("/")) and + .[5] == "node" and .[6] == "run" and .[7] == "--host" and (.[8] | length > 0) and + .[9] == "--port" and (.[10] | test("^[0-9]+$")) and (.[11:] | validTail) + ' <<<"$args" >/dev/null 2>&1 || fail 'canonical node LaunchAgent arguments are not recognized' + local node_wrapper node_id_count + node_id_count="$(jq -r '[range(11; length) as $i | select(.[$i] == "--node-id")] | length' <<<"$args")" + [[ "$node_id_count" == '0' || "$node_id_count" == '1' ]] || + fail 'canonical node LaunchAgent has duplicate --node-id overrides' + MIGRATION_NODE_ID="$(jq -r '[range(11; length) as $i | select(.[$i] == "--node-id") | .[$i + 1]] | first // empty' <<<"$args")" + [[ "$node_id_count" == '0' || -n "$MIGRATION_NODE_ID" ]] || + fail 'canonical node LaunchAgent has an empty --node-id override' + node_env="$(jq -r '.[2]' <<<"$args")" + node_wrapper="$(jq -r '.[1]' <<<"$args")" + case "$node_env" in /*) ;; *) fail 'canonical node environment path must be absolute' ;; esac + case "$node_wrapper" in /*) ;; *) fail 'canonical node wrapper path must be absolute' ;; esac + [[ "$node_wrapper" == "${node_env%.env}-env-wrapper.sh" ]] || + fail 'canonical node LaunchAgent wrapper and environment paths do not match' + [[ -f "$node_wrapper" && ! -L "$node_wrapper" && -x "$node_wrapper" ]] || + fail 'canonical node environment wrapper is missing, symlinked, or not executable' + [[ "$(stat -f '%Lp' "$node_wrapper")" == '700' && "$(stat -f '%u' "$node_wrapper")" == "$(id -u)" ]] || + fail 'canonical node environment wrapper must be current-user owned with mode 0700' + canonical_node_wrapper_is_canonical "$node_wrapper" || + fail 'canonical node environment wrapper has custom behavior' + node_paths="$(read_generated_node_paths "$node_env")" + inline_state="$(jq -r '.stateDir' <<<"$node_paths")" + inline_config="$(jq -r '.configPath' <<<"$node_paths")" + case "$inline_state" in /*) ;; *) fail 'canonical node OPENCLAW_STATE_DIR must be absolute' ;; esac + [[ "$node_env" == "${inline_state}/service-env/${MIGRATION_LABEL}.env" ]] || + fail 'canonical node environment file is outside its state-owned service-env directory' + MIGRATION_NODE_ENV_PATH="$node_env" + MIGRATION_NODE_ENV_SHA="$(shasum -a 256 "$node_env" | awk '{print $1}')" + MIGRATION_NODE_ENV_IDENTITY="$(durable_path_identity "$node_env")" || + fail 'canonical node environment identity could not be inspected' + MIGRATION_NODE_WRAPPER_PATH="$node_wrapper" + MIGRATION_NODE_WRAPPER_SHA="$(shasum -a 256 "$node_wrapper" | awk '{print $1}')" + MIGRATION_NODE_WRAPPER_IDENTITY="$(durable_path_identity "$node_wrapper")" || + fail 'canonical node wrapper identity could not be inspected' + resolve_reusable_openclaw_cli + fi + if [[ "$STATE_DIR_EXPLICIT" == "1" && "$STATE_DIR" != "$inline_state" ]]; then + fail '--state-dir does not match the migration LaunchAgent OPENCLAW_STATE_DIR' + fi + STATE_DIR="$inline_state" + local effective_config="${inline_config:-$STATE_DIR/openclaw.json}" + case "$effective_config" in /*) ;; *) fail 'migration LaunchAgent OPENCLAW_CONFIG_PATH must be absolute' ;; esac + if [[ "$CONFIG_PATH_EXPLICIT" == "1" && "$CONFIG_PATH" != "$effective_config" ]]; then + fail '--config-path does not match the migration LaunchAgent OPENCLAW_CONFIG_PATH' + fi + CONFIG_PATH="$effective_config" + refresh_runtime_paths + local migration_state + migration_state="$(job_loaded_state "$launch_domain/$MIGRATION_LABEL")" + case "$migration_state" in + loaded) MIGRATION_WAS_LOADED=1 ;; + absent) MIGRATION_WAS_LOADED=0 ;; + *) fail 'launchd ownership state could not be inspected' ;; + esac +} + +canonical_node_wrapper_is_canonical() { + local wrapper_path="$1" + local expected_wrapper=$'#!/bin/sh\nset -eu\nenv_file="$1"\nshift\nif [ -f "$env_file" ]; then\n . "$env_file"\nfi\nexec "$@"' + [[ "$(<"$wrapper_path")" == "$expected_wrapper" ]] +} + +read_generated_node_paths() { + local env_file="$1" + [[ -x /usr/bin/python3 ]] || fail 'system python3 is required to inspect the canonical node environment' + [[ -f "$env_file" && ! -L "$env_file" ]] || fail 'canonical node environment file is missing or symlinked' + [[ "$(stat -f '%Lp' "$env_file")" == '600' && "$(stat -f '%u' "$env_file")" == "$(id -u)" ]] || + fail 'canonical node environment file must be current-user owned with mode 0600' + /usr/bin/python3 - "$env_file" <<'PY' +import json +import re +import sys +from pathlib import Path + +raw = Path(sys.argv[1]).read_bytes() +if b"\r" in raw: + raise SystemExit("ERROR: canonical node environment must use LF line endings") +try: + lines = raw.decode("utf-8").split("\n") +except UnicodeDecodeError: + raise SystemExit("ERROR: canonical node environment must be UTF-8") from None +header = "# Generated by OpenClaw. Do not edit while the gateway service is installed." +if not lines or lines[0] != header: + raise SystemExit("ERROR: canonical node environment lacks its generated header") + +selected = {} +seen = set() +for line in lines[1:]: + if not line: + continue + match = re.fullmatch(r"export ([A-Za-z_][A-Za-z0-9_]*)=(.*)", line) + if not match: + raise SystemExit("ERROR: canonical node environment contains a noncanonical statement") + key, encoded = match.groups() + if key in seen: + raise SystemExit(f"ERROR: duplicate key in canonical node environment: {key}") + seen.add(key) + if len(encoded) < 2 or not encoded.startswith("'") or not encoded.endswith("'"): + raise SystemExit("ERROR: canonical node environment contains a noncanonical value") + value = encoded[1:-1].replace("'\\''", "'") + canonical = "'" + value.replace("'", "'\\''") + "'" + if encoded != canonical: + raise SystemExit("ERROR: canonical node environment contains a noncanonical value") + if key in {"OPENCLAW_STATE_DIR", "OPENCLAW_CONFIG_PATH"}: + selected[key] = value +if not selected.get("OPENCLAW_STATE_DIR") or not selected.get("OPENCLAW_CONFIG_PATH"): + raise SystemExit("ERROR: canonical node environment lacks state/config paths") +print(json.dumps({"stateDir": selected["OPENCLAW_STATE_DIR"], "configPath": selected["OPENCLAW_CONFIG_PATH"]})) +PY +} + +verify_canonical_node_sidecars() { + [[ "$MIGRATION_KIND" == "canonical-node" ]] || return 0 + [[ "$MIGRATION_NODE_WRAPPER_PATH" == "${MIGRATION_NODE_ENV_PATH%.env}-env-wrapper.sh" ]] || return 1 + [[ -f "$MIGRATION_NODE_ENV_PATH" && ! -L "$MIGRATION_NODE_ENV_PATH" && + "$(stat -f '%Lp' "$MIGRATION_NODE_ENV_PATH")" == '600' && + "$(stat -f '%u' "$MIGRATION_NODE_ENV_PATH")" == "$(id -u)" ]] || return 1 + [[ -f "$MIGRATION_NODE_WRAPPER_PATH" && ! -L "$MIGRATION_NODE_WRAPPER_PATH" && + -x "$MIGRATION_NODE_WRAPPER_PATH" && + "$(stat -f '%Lp' "$MIGRATION_NODE_WRAPPER_PATH")" == '700' && + "$(stat -f '%u' "$MIGRATION_NODE_WRAPPER_PATH")" == "$(id -u)" ]] || return 1 + [[ "$MIGRATION_NODE_ENV_SHA" =~ ^[0-9a-f]{64}$ && + "$(shasum -a 256 "$MIGRATION_NODE_ENV_PATH" | awk '{print $1}')" == "$MIGRATION_NODE_ENV_SHA" ]] || + return 1 + [[ "$MIGRATION_NODE_WRAPPER_SHA" =~ ^[0-9a-f]{64}$ && + "$(shasum -a 256 "$MIGRATION_NODE_WRAPPER_PATH" | awk '{print $1}')" == "$MIGRATION_NODE_WRAPPER_SHA" ]] || + return 1 + if [[ -n "$MIGRATION_NODE_ENV_IDENTITY" ]]; then + path_matches_identity "$MIGRATION_NODE_ENV_PATH" "$MIGRATION_NODE_ENV_IDENTITY" || return 1 + fi + if [[ -n "$MIGRATION_NODE_WRAPPER_IDENTITY" ]]; then + path_matches_identity "$MIGRATION_NODE_WRAPPER_PATH" "$MIGRATION_NODE_WRAPPER_IDENTITY" || return 1 + fi + canonical_node_wrapper_is_canonical "$MIGRATION_NODE_WRAPPER_PATH" || return 1 + local node_paths + node_paths="$(read_generated_node_paths "$MIGRATION_NODE_ENV_PATH")" || return 1 + [[ "$(jq -r '.stateDir' <<<"$node_paths")" == "$STATE_DIR" && + "$(jq -r '.configPath' <<<"$node_paths")" == "$CONFIG_PATH" ]] +} + +migration_receipt_matches_backup_plist() { + local plist_path="$1" args app_binary plist_label node_env node_wrapper + plist_label="$(plist_file_value "$plist_path" Label)" || return 1 + [[ "$plist_label" == "$ROLLBACK_MIGRATION_LABEL" ]] || return 1 + args="$(plutil -extract ProgramArguments json -o - "$plist_path" 2>/dev/null)" || return 1 + jq -e 'type == "array" and length > 0 and all(.[]; type == "string")' <<<"$args" >/dev/null 2>&1 || + return 1 + app_binary="$APP_PATH/Contents/MacOS/OpenClaw" + if jq -e --arg appBinary "$app_binary" ' + .[0] == $appBinary and + (.[1:] | index("--background-only") != null) and + (.[1:] | all(. == "--background-only" or . == "--attach-only" or . == "--no-launchd")) + ' <<<"$args" >/dev/null 2>&1 + then + [[ "$MIGRATION_KIND" == "app-launch-agent" && + -z "$MIGRATION_NODE_ENV_PATH" && -z "$MIGRATION_NODE_ENV_SHA" && + -z "$MIGRATION_NODE_WRAPPER_PATH" && -z "$MIGRATION_NODE_WRAPPER_SHA" ]] + return + fi + [[ "$MIGRATION_KIND" == "canonical-node" && "$plist_label" == "ai.openclaw.node" ]] || return 1 + jq -e ' + def validTail: + length == 0 or + ((.[0] == "--tls" or .[0] == "--no-tls" or .[0] == "--share-installed-apps" or .[0] == "--no-share-installed-apps") and (.[1:] | validTail)) or + ((.[0] == "--tls-fingerprint" or .[0] == "--context-path" or .[0] == "--node-id" or .[0] == "--display-name") and length >= 2 and (.[2:] | validTail)); + length >= 11 and .[0] == "/bin/sh" and + (.[3] | startswith("/")) and (.[4] | startswith("/")) and + .[5] == "node" and .[6] == "run" and .[7] == "--host" and (.[8] | length > 0) and + .[9] == "--port" and (.[10] | test("^[0-9]+$")) and (.[11:] | validTail) + ' <<<"$args" >/dev/null 2>&1 || return 1 + node_wrapper="$(jq -r '.[1]' <<<"$args")" + node_env="$(jq -r '.[2]' <<<"$args")" + [[ "$node_env" == "$MIGRATION_NODE_ENV_PATH" && + "$node_wrapper" == "$MIGRATION_NODE_WRAPPER_PATH" ]] || return 1 + verify_canonical_node_sidecars +} + +background_app_records() { + local app_binary="$APP_PATH/Contents/MacOS/OpenClaw" pid command_line executable attach_only + while IFS= read -r pid; do + [[ "$pid" =~ ^[0-9]+$ ]] || continue + executable="$(lsof -a -p "$pid" -d txt -Fn 2>/dev/null | sed -n 's/^n//p' | head -n 1)" + [[ "$executable" == "$app_binary" ]] || continue + command_line="$(ps -p "$pid" -o command= 2>/dev/null || true)" + case "$command_line" in + "$app_binary --attach-only --background-only"|"$app_binary --background-only --attach-only") + attach_only=1 ;; + "$app_binary --background-only") attach_only=0 ;; + *) continue ;; + esac + printf '%s %s\n' "$pid" "$attach_only" + done < <(pgrep -x OpenClaw 2>/dev/null || true) +} + +app_binary_pids() { + local app_binary="$APP_PATH/Contents/MacOS/OpenClaw" pid executable lsof_output listed_status + local pids="" pgrep_status=0 + pids="$(pgrep -x OpenClaw 2>/dev/null)" || pgrep_status=$? + case "$pgrep_status" in + 0) ;; + 1) return 0 ;; + *) return 1 ;; + esac + while IFS= read -r pid; do + [[ "$pid" =~ ^[0-9]+$ ]] || continue + if ! lsof_output="$(lsof -a -p "$pid" -d txt -Fn 2>/dev/null)"; then + listed_status=0 + openclaw_pid_is_listed "$pid" || listed_status=$? + case "$listed_status" in + 0|2) return 1 ;; + 1) continue ;; + esac + fi + executable="$(sed -n 's/^n//p' <<<"$lsof_output" | head -n 1)" + if [[ -z "$executable" ]]; then + listed_status=0 + openclaw_pid_is_listed "$pid" || listed_status=$? + case "$listed_status" in + 0|2) return 1 ;; + 1) continue ;; + esac + fi + [[ "$executable" == "$app_binary" ]] && printf '%s\n' "$pid" + done <<<"$pids" +} + +openclaw_pid_is_listed() { + local _expected_pid="$1" pgrep_status=0 + pgrep -x OpenClaw >/dev/null 2>&1 || pgrep_status=$? + case "$pgrep_status" in + 0) return 0 ;; + 1) return 1 ;; + *) return 2 ;; + esac +} + +wait_for_app_binary_exit() { + local remaining="" + for _ in $(seq 1 80); do + if remaining="$(app_binary_pids)" && [[ -z "$remaining" ]]; then + return 0 + fi + sleep 0.25 + done + return 1 +} + +resolve_adoption_inputs() { + [[ "$ADOPT_RUNNING_APP" == "1" ]] || return 0 + MIGRATION_KIND="running-app" + [[ -n "$CONFIG_PATH" ]] || CONFIG_PATH="$STATE_DIR/openclaw.json" + refresh_runtime_paths + resolve_reusable_openclaw_cli + local records=() record + while IFS= read -r record; do + [[ -n "$record" ]] && records+=("$record") + done < <(background_app_records) + [[ "${#records[@]}" == "1" ]] || + fail 'adoption requires exactly one unsupervised background-only OpenClaw process' + ADOPTION_PID="${records[0]%% *}" + ADOPTION_ATTACH_ONLY="${records[0]##* }" + local elevation_pid + elevation_pid="$(job_pid)" + [[ -z "$elevation_pid" || "$ADOPTION_PID" != "$elevation_pid" ]] || + fail 'adoption refuses the launchd-owned elevation process' +} + +resolve_managed_upgrade_inputs() { + [[ "$COMMAND" == "install" && -z "$MIGRATE_LAUNCH_AGENT" && "$ADOPT_RUNNING_APP" != "1" && + -e "$RECEIPT_PATH" ]] || return 0 + verify_install_receipt + require_committed_install_receipt + local recorded_config recorded_state + if [[ "$INSTALL_RECEIPT_SCHEMA" == "legacy" ]]; then + [[ -f "$PLIST_PATH" && ! -L "$PLIST_PATH" ]] || + fail 'legacy elevation upgrade requires its installed LaunchAgent plist' + recorded_state="$(plist_file_value "$PLIST_PATH" EnvironmentVariables.OPENCLAW_STATE_DIR)" + [[ "$recorded_state" == "$STATE_DIR" ]] || + fail 'legacy elevation LaunchAgent state directory does not match --state-dir' + recorded_config="$(plist_file_value "$PLIST_PATH" EnvironmentVariables.OPENCLAW_CONFIG_PATH)" + recorded_config="${recorded_config:-$STATE_DIR/openclaw.json}" + else + recorded_config="$(jq -r '.configPath' "$RECEIPT_PATH")" + fi + if [[ "$CONFIG_PATH_EXPLICIT" == "1" && "$CONFIG_PATH" != "$recorded_config" ]]; then + fail '--config-path does not match the existing elevation install receipt' + fi + CONFIG_PATH="$recorded_config" + if [[ "$INSTALL_RECEIPT_SCHEMA" != "legacy" ]]; then + UPGRADE_EXPECTED_NODE_ID="$(jq -r '.nodeId' "$RECEIPT_PATH")" + UPGRADE_EXPECTED_NODE_PROFILE="$(jq -r '.nodeProfile' "$RECEIPT_PATH")" + fi + refresh_runtime_paths +} + +adopted_app_is_current() { + local records=() record + while IFS= read -r record; do + [[ -n "$record" ]] && records+=("$record") + done < <(background_app_records) + [[ "${#records[@]}" == "1" && + "${records[0]%% *}" == "$ADOPTION_PID" && + "${records[0]##* }" == "$ADOPTION_ATTACH_ONLY" ]] +} + +wait_for_adopted_app_resume() { + local records=() record + for _ in $(seq 1 80); do + records=() + while IFS= read -r record; do + [[ -n "$record" ]] && records+=("$record") + done < <(background_app_records) + if [[ "${#records[@]}" == "1" && "${records[0]##* }" == "$ROLLBACK_ADOPTED_APP_ATTACH_ONLY" ]]; then + ADOPTION_PID="${records[0]%% *}" + return 0 + fi + sleep 0.25 + done + return 1 +} + +relaunch_adopted_app() { + local open_args=( + -n + -g + --env "OPENCLAW_STATE_DIR=$STATE_DIR" + --env "OPENCLAW_CONFIG_PATH=$CONFIG_PATH" + "$APP_PATH" + --args + ) + if [[ "$ROLLBACK_ADOPTED_APP_ATTACH_ONLY" == "1" ]]; then + open_args+=(--attach-only) + fi + open_args+=(--background-only) + open "${open_args[@]}" >/dev/null 2>&1 || return 1 + wait_for_adopted_app_resume +} + +restore_adopted_app_after_cutover() { + if [[ "$CUTOVER_ADOPTION_TERMINATION_SENT" != "1" && -n "$ADOPTION_PID" ]] && + adopted_app_is_current + then + return 0 + fi + if [[ "$ADOPTION_PID" =~ ^[0-9]+$ ]]; then + for _ in $(seq 1 80); do + kill -0 "$ADOPTION_PID" 2>/dev/null || break + sleep 0.25 + done + kill -0 "$ADOPTION_PID" 2>/dev/null && return 1 + fi + relaunch_adopted_app || return 1 + RECOVERY_RELAUNCHED_ADOPTED_PID="$ADOPTION_PID" +} +run_openclaw_cli() { + [[ "${#OPENCLAW_CLI[@]}" -gt 0 && -n "${OPENCLAW_CLI[0]}" ]] || + fail 'openclaw CLI is required for gateway node attestation' + local env_args=( + -u OPENCLAW_GATEWAY_URL + -u OPENCLAW_GATEWAY_PORT + -u OPENCLAW_GATEWAY_TOKEN + -u OPENCLAW_GATEWAY_PASSWORD + "OPENCLAW_STATE_DIR=$STATE_DIR" + "OPENCLAW_CONFIG_PATH=$CONFIG_PATH" + ) + env "${env_args[@]}" "${OPENCLAW_CLI[@]}" "$@" +} + +prepare_gateway_attestation() { + [[ -d "$STATE_DIR" && ! -L "$STATE_DIR" ]] || fail "state directory is missing or symlinked: $STATE_DIR" + [[ -f "$CONFIG_PATH" && ! -L "$CONFIG_PATH" ]] || fail "config is missing or symlinked: $CONFIG_PATH" + local mode remote_url token password profile database node_id nodes_json + mode="$(run_openclaw_cli config get gateway.mode --json 2>/dev/null || true)" + jq -e '. == "remote"' <<<"$mode" >/dev/null 2>&1 || + fail 'elevation host requires app-readable gateway.mode=remote in the selected config' + remote_url="$(run_openclaw_cli config get gateway.remote.url --json 2>/dev/null || true)" + jq -e 'type == "string" and length > 0' <<<"$remote_url" >/dev/null 2>&1 || + fail 'elevation host requires a nonempty app-readable gateway.remote.url' + token="$(run_openclaw_cli config get gateway.remote.token --json 2>/dev/null || true)" + password="$(run_openclaw_cli config get gateway.remote.password --json 2>/dev/null || true)" + if ! jq -e 'type == "string" and length > 0' <<<"$token" >/dev/null 2>&1 && + ! jq -e 'type == "string" and length > 0' <<<"$password" >/dev/null 2>&1 + then + fail 'elevation host requires app-readable string auth in gateway.remote.token or gateway.remote.password' + fi + + profile="$(defaults read "$EXPECTED_BUNDLE_ID" openclaw.macNodeIdentityProfile 2>/dev/null || true)" + [[ "$profile" == 'primary' || "$profile" == 'node' ]] || + fail 'macOS app node identity profile is unavailable; launch and pair the app before migration' + if [[ "$MIGRATION_KIND" == 'canonical-node' && "$profile" != 'primary' ]]; then + fail 'canonical node migration requires the macOS app to select the primary identity profile' + fi + database="$STATE_DIR/state/openclaw.sqlite" + [[ -f "$database" && ! -L "$database" ]] || fail 'state database is missing or symlinked' + node_id="$(sqlite3 -readonly -batch -noheader "$database" \ + "SELECT device_id FROM device_identities WHERE identity_key = '$profile';" 2>/dev/null || true)" + [[ "${#node_id}" -ge 8 && "${#node_id}" -le 256 && "$node_id" =~ ^[A-Za-z0-9._:-]+$ ]] || + fail 'selected macOS node identity is missing or invalid' + if [[ "$MIGRATION_KIND" == 'canonical-node' && -n "$MIGRATION_NODE_ID" && + "$MIGRATION_NODE_ID" != "$node_id" ]] + then + fail 'canonical node LaunchAgent --node-id does not match the selected paired macOS identity' + fi + if [[ -n "$UPGRADE_EXPECTED_NODE_ID" && + ("$UPGRADE_EXPECTED_NODE_ID" != "$node_id" || "$UPGRADE_EXPECTED_NODE_PROFILE" != "$profile") ]] + then + fail 'managed upgrade identity does not match the existing elevation install receipt' + fi + nodes_json="$(run_openclaw_cli nodes status --json --timeout 5000 2>/dev/null || true)" + jq -e --arg nodeId "$node_id" ' + [.nodes[]? | select(.nodeId == $nodeId)] as $matches | + ($matches | length) == 1 and + ($matches[0].paired == true or $matches[0].approvalState == "approved") + ' <<<"$nodes_json" >/dev/null 2>&1 || + fail 'selected macOS node identity is not paired on the configured gateway' + EXPECTED_NODE_PROFILE="$profile" + EXPECTED_NODE_ID="$node_id" + BEFORE_NODE_CONNECTED_AT="$(jq -r --arg nodeId "$node_id" \ + '[.nodes[]? | select(.nodeId == $nodeId and .connected == true) | .connectedAtMs // 0] | max // 0' \ + <<<"$nodes_json")" + [[ "$BEFORE_NODE_CONNECTED_AT" =~ ^[0-9]+$ ]] || BEFORE_NODE_CONNECTED_AT=0 +} + +verify_gateway_node_readiness() { + local expected_version="$1" nodes_json + for _ in $(seq 1 30); do + nodes_json="$(run_openclaw_cli nodes status --connected --json --timeout 3000 2>/dev/null || true)" + if jq -e \ + --arg nodeId "$EXPECTED_NODE_ID" \ + --arg version "$expected_version" \ + --argjson previousConnectedAt "$BEFORE_NODE_CONNECTED_AT" ' + [.nodes[]? | select(.nodeId == $nodeId)] as $matches | + ($matches | length) == 1 and + $matches[0].connected == true and + $matches[0].clientId == "openclaw-macos" and + $matches[0].clientMode == "node" and + (($matches[0].uiVersion == $version) or ($matches[0].version == $version)) and + (($matches[0].connectedAtMs | numbers) > $previousConnectedAt) and + (($matches[0].caps | arrays | index("computer")) != null) and + (($matches[0].commands | arrays | index("screen.snapshot")) != null) and + (($matches[0].commands | arrays | index("computer.act")) != null) and + ($matches[0].computerUse | type) == "object" + ' <<<"$nodes_json" >/dev/null 2>&1 + then + return 0 + fi + sleep 0.5 + done + return 1 +} + +print_migration_plan() { + [[ -n "$MIGRATE_LAUNCH_AGENT" || "$ADOPT_RUNNING_APP" == "1" ]] || + fail 'migration-plan requires --migrate-launch-agent <plist> or --adopt-running-app' + jq -n \ + --arg kind "$MIGRATION_KIND" \ + --arg label "$MIGRATION_LABEL" \ + --arg sourcePlist "$MIGRATE_LAUNCH_AGENT" \ + --arg stateDir "$STATE_DIR" \ + --arg configPath "$CONFIG_PATH" \ + --arg expectedNodeId "$EXPECTED_NODE_ID" \ + --argjson loaded "$MIGRATION_WAS_LOADED" \ + '{kind:$kind,label:(if $label == "" then null else $label end),sourcePlist:(if $sourcePlist == "" then null else $sourcePlist end),stateDir:$stateDir,configPath:$configPath,expectedNodeId:$expectedNodeId,loaded:($loaded == 1),action:"replace-with-elevation-host"}' +} + +ensure_no_normal_owner() { + [[ ! -f "$NORMAL_PLIST_PATH" ]] || fail "ordinary Launch at login is installed at $NORMAL_PLIST_PATH" + [[ -z "$(job_snapshot "$normal_domain")" ]] || fail "ordinary Launch at login job is loaded: $NORMAL_LABEL" + + local candidate_plist candidate_label candidate_program + while IFS= read -r -d '' candidate_plist; do + [[ "$candidate_plist" == "$PLIST_PATH" || "$candidate_plist" == "$NORMAL_PLIST_PATH" ]] && continue + [[ -n "$MIGRATE_LAUNCH_AGENT" && "$candidate_plist" == "$MIGRATE_LAUNCH_AGENT" ]] && continue + candidate_label="$(plutil -extract Label raw -o - "$candidate_plist" 2>/dev/null || true)" + candidate_program="$(plutil -extract ProgramArguments.0 raw -o - "$candidate_plist" 2>/dev/null || true)" + if [[ "$candidate_program" == "$APP_PATH/Contents/MacOS/OpenClaw" ]]; then + fail "conflicting OpenClaw launch agent is installed: ${candidate_label:-$candidate_plist}" + fi + done < <(find "$(dirname "$PLIST_PATH")" -maxdepth 1 -type f -name '*.plist' -print0 2>/dev/null) + + local pid command_line elevation_pid migration_pid + elevation_pid="$(job_pid)" + migration_pid="" + if [[ -n "$MIGRATION_LABEL" ]]; then + migration_pid="$(job_pid_for_domain "$launch_domain/$MIGRATION_LABEL")" + fi + while IFS= read -r pid; do + [[ -n "$pid" ]] || continue + command_line="$(ps -p "$pid" -o command= 2>/dev/null || true)" + [[ "$command_line" == "$APP_PATH/Contents/MacOS/OpenClaw"* ]] || continue + [[ -n "$elevation_pid" && "$pid" == "$elevation_pid" ]] || + [[ -n "$migration_pid" && "$pid" == "$migration_pid" ]] || + [[ -n "$ADOPTION_PID" && "$pid" == "$ADOPTION_PID" ]] || + fail "unsupervised or conflicting OpenClaw process is running: $pid" + done < <(pgrep -x OpenClaw 2>/dev/null || true) +} + +peekaboo_bin() { + if [[ -x "$HOME/bin/peekaboo" ]]; then printf '%s\n' "$HOME/bin/peekaboo"; return; fi + command -v peekaboo 2>/dev/null || true +} + +verify_bridge_readiness() { + local expected_pid="$1" + local pb bridge_json + pb="$(peekaboo_bin)" + [[ -n "$pb" ]] || fail 'peekaboo CLI is required to verify elevation-host readiness' + bridge_json="$($pb bridge status --bridge-socket "$BRIDGE_SOCKET" --json 2>/dev/null || true)" + [[ "$(jq -r '.success // false' <<<"$bridge_json")" == 'true' ]] || return 1 + [[ "$(jq -r '.data.selected.handshake.hostIdentity.processIdentifier // 0' <<<"$bridge_json")" == "$expected_pid" ]] || return 1 +} + +tcc_summary() { + local pb permissions_json missing + pb="$(peekaboo_bin)" + [[ -n "$pb" ]] || { printf 'peekaboo CLI unavailable\n'; return 4; } + if ! permissions_json="$($pb permissions status --all-sources --bridge-socket "$BRIDGE_SOCKET" --json 2>/dev/null)"; then + printf 'TCC: unknown (permission probe failed)\n' + return 4 + fi + if ! jq -e ' + (.success == true) and + (.data.sources | type == "array") and + ([.data.sources[]? | select(.isSelected == true)] | length == 1) and + ([.data.sources[]? | select(.isSelected == true) | .permissions] | length == 1) and + ([.data.sources[]? | select(.isSelected == true) | .permissions | type] == ["array"]) and + ([.data.sources[]? | select(.isSelected == true) | .permissions[]?] | length > 0) and + all( + .data.sources[]? | select(.isSelected == true) | .permissions[]?; + (.name | type) == "string" and (.isGranted | type) == "boolean" + ) + ' <<<"$permissions_json" >/dev/null 2>&1; then + printf 'TCC: unknown (permission probe returned invalid status)\n' + return 4 + fi + missing="$(jq -r '[.data.sources[]? | select(.isSelected == true) | .permissions[]? | select(.isGranted != true) | .name] | unique | join(", ")' <<<"$permissions_json")" + if [[ -n "$missing" ]]; then + printf 'missing TCC: %s\n' "$missing" + return 4 + fi + printf 'TCC: ready\n' +} + +fsync_file_and_parent() { + run_authenticated_elevation_helper --elevation-sync-file "$1" +} + +fsync_parent() { + run_authenticated_elevation_helper --elevation-sync-directory "$(dirname "$1")" +} + +fsync_tree() { + run_authenticated_elevation_helper --elevation-sync-tree "$1" +} + +write_install_receipt() { + local target="$1" transaction_state="$2" source_commit="$3" peekaboo_commit="$4" archive_sha="$5" + local arm64_cdhash="$6" x86_64_cdhash="$7" + [[ "$transaction_state" == "installing" || "$transaction_state" == "installed" ]] || + fail 'internal install receipt transaction state is invalid' + [[ "$INSTALL_TRANSACTION_ID" =~ ^[0-9A-F-]{36}$ ]] || + fail 'internal install transaction identity is invalid' + mkdir -p "$STATE_DIR" + local tmp="${target}.tmp.$$" + jq -n \ + --argjson schemaVersion 3 \ + --arg kind 'openclaw-elevation-install' \ + --arg transactionState "$transaction_state" \ + --arg transactionId "$INSTALL_TRANSACTION_ID" \ + --arg sourceCommit "$source_commit" \ + --arg peekabooCommit "$peekaboo_commit" \ + --arg appPath "$APP_PATH" \ + --arg stateDir "$STATE_DIR" \ + --arg configPath "$CONFIG_PATH" \ + --arg backupPath "$ROLLBACK_APP_PATH" \ + --arg backupArm64CDHash "$ROLLBACK_APP_CDHASH_ARM64" \ + --arg backupX8664CDHash "$ROLLBACK_APP_CDHASH_X86_64" \ + --arg plistPath "$PLIST_PATH" \ + --arg previousPlist "$ROLLBACK_ELEVATION_PLIST" \ + --arg previousPlistSha256 "$ROLLBACK_ELEVATION_PLIST_SHA" \ + --argjson previousPlistWasLoaded "$ROLLBACK_ELEVATION_WAS_LOADED" \ + --arg previousReceipt "$ROLLBACK_INSTALL_RECEIPT" \ + --arg previousReceiptSha256 "$ROLLBACK_INSTALL_RECEIPT_SHA" \ + --arg archiveSha256 "$archive_sha" \ + --arg artifactReceiptSha256 "$VERIFIED_ARTIFACT_RECEIPT_SHA" \ + --arg installerSha256 "$VERIFIED_INSTALLER_SHA" \ + --arg arm64CDHash "$arm64_cdhash" \ + --arg x8664CDHash "$x86_64_cdhash" \ + --arg nodeId "$EXPECTED_NODE_ID" \ + --arg nodeProfile "$EXPECTED_NODE_PROFILE" \ + --arg migrationSource "$ROLLBACK_MIGRATION_SOURCE" \ + --arg migrationBackup "$ROLLBACK_MIGRATION_PLIST" \ + --arg migrationBackupSha256 "$ROLLBACK_MIGRATION_PLIST_SHA" \ + --arg migrationCustodyPath "$MIGRATION_CUSTODY_PATH" \ + --arg migrationSourceIdentity "$MIGRATION_CUSTODY_IDENTITY" \ + --arg migrationKind "$MIGRATION_KIND" \ + --arg migrationLabel "$ROLLBACK_MIGRATION_LABEL" \ + --arg migrationNodeEnvPath "$MIGRATION_NODE_ENV_PATH" \ + --arg migrationNodeEnvSha256 "$MIGRATION_NODE_ENV_SHA" \ + --arg migrationNodeEnvIdentity "$MIGRATION_NODE_ENV_IDENTITY" \ + --arg migrationNodeWrapperPath "$MIGRATION_NODE_WRAPPER_PATH" \ + --arg migrationNodeWrapperSha256 "$MIGRATION_NODE_WRAPPER_SHA" \ + --arg migrationNodeWrapperIdentity "$MIGRATION_NODE_WRAPPER_IDENTITY" \ + --argjson migrationWasLoaded "$ROLLBACK_MIGRATION_WAS_LOADED" \ + --argjson adoptedAppWasRunning "$ROLLBACK_ADOPTED_APP_WAS_RUNNING" \ + --argjson adoptedAppAttachOnly "$ROLLBACK_ADOPTED_APP_ATTACH_ONLY" \ + '{schemaVersion:$schemaVersion,kind:$kind,transactionState:$transactionState,transactionId:$transactionId,sourceCommit:$sourceCommit,peekabooCommit:$peekabooCommit,archiveSha256:$archiveSha256,artifactReceiptSha256:$artifactReceiptSha256,installerSha256:$installerSha256,cdhashes:{arm64:$arm64CDHash,x86_64:$x8664CDHash},nodeId:$nodeId,nodeProfile:$nodeProfile,appPath:$appPath,stateDir:$stateDir,configPath:$configPath,backupPath:$backupPath,backupCDHashes:{arm64:$backupArm64CDHash,x86_64:$backupX8664CDHash},plistPath:$plistPath,previousPlist:$previousPlist,previousPlistSha256:$previousPlistSha256,previousPlistWasLoaded:($previousPlistWasLoaded == 1),previousReceipt:$previousReceipt,previousReceiptSha256:$previousReceiptSha256,migration:(if $migrationSource == "" then null else {kind:$migrationKind,sourcePlist:$migrationSource,sourceIdentity:$migrationSourceIdentity,custodyPath:$migrationCustodyPath,backupPlist:$migrationBackup,backupSha256:$migrationBackupSha256,label:$migrationLabel,wasLoaded:($migrationWasLoaded == 1),nodeEnvPath:$migrationNodeEnvPath,nodeEnvSha256:$migrationNodeEnvSha256,nodeEnvIdentity:$migrationNodeEnvIdentity,nodeWrapperPath:$migrationNodeWrapperPath,nodeWrapperSha256:$migrationNodeWrapperSha256,nodeWrapperIdentity:$migrationNodeWrapperIdentity} end),adoptedApp:{wasRunning:($adoptedAppWasRunning == 1),attachOnly:($adoptedAppAttachOnly == 1)}}' >"$tmp" + chmod 600 "$tmp" + if ! fsync_file_and_parent "$tmp"; then + rm -f "$tmp" + fail 'authenticated elevation helper could not sync the install receipt' + fi + if [[ "$transaction_state" == "installing" ]]; then + if ! rename_app_exclusively "$tmp" "$target"; then + rm -f "$tmp" + fail 'could not exclusively publish the prepared install receipt' + fi + elif ! mv "$tmp" "$target"; then + rm -f "$tmp" + fail 'could not atomically publish the install receipt' + fi + fsync_file_and_parent "$target" || + fail 'authenticated elevation helper could not commit the install receipt' +} + +write_receipt() { + write_install_receipt "$FINAL_RECEIPT_PATH" installed "$@" +} + +remove_pending_receipt() { + local expected_transaction_id="${1:-$INSTALL_TRANSACTION_ID}" + [[ -e "$PENDING_RECEIPT_PATH" || -L "$PENDING_RECEIPT_PATH" ]] || return 0 + [[ -f "$PENDING_RECEIPT_PATH" && ! -L "$PENDING_RECEIPT_PATH" ]] || return 1 + [[ "$expected_transaction_id" =~ ^[0-9A-F-]{36}$ && + "$(jq -r '.transactionId // empty' "$PENDING_RECEIPT_PATH" 2>/dev/null)" == "$expected_transaction_id" ]] || + return 1 + rm "$PENDING_RECEIPT_PATH" || return 1 + fsync_parent "$PENDING_RECEIPT_PATH" +} + +verify_install_receipt() { + [[ -f "$RECEIPT_PATH" && ! -L "$RECEIPT_PATH" ]] || fail "elevation install receipt not found or symlinked: $RECEIPT_PATH" + local verify_current_app="${1:-1}" + [[ "$verify_current_app" == "0" || "$verify_current_app" == "1" ]] || + fail 'internal install receipt verification mode is invalid' + local receipt_arm64_cdhash receipt_x86_64_cdhash + INSTALL_RECEIPT_SCHEMA="" + INSTALL_RECEIPT_TRANSACTION_STATE="" + INSTALL_TRANSACTION_ID="" + # Origin main shipped exactly this unversioned seven-key receipt. Numeric schemas 1 and 2 + # existed only on the unmerged hardening branch and intentionally have no compatibility path. + if jq -e ' + type == "object" and + keys == ["appPath","archiveSha256","backupPath","peekabooCommit","plistPath","previousPlist","sourceCommit"] and + (.sourceCommit | type == "string" and test("^[0-9a-f]{40}$")) and + (.peekabooCommit | type == "string" and test("^[0-9a-f]{40}$")) and + (.archiveSha256 | type == "string" and test("^[0-9a-f]{64}$")) and + (.appPath | type == "string" and startswith("/")) and + (.backupPath | type == "string") and + (.plistPath | type == "string" and startswith("/")) and + (.previousPlist | type == "string") + ' "$RECEIPT_PATH" >/dev/null 2>&1 + then + INSTALL_RECEIPT_SCHEMA="legacy" + INSTALL_RECEIPT_TRANSACTION_STATE="installed" + # Schema 3 first ships with the exact migration-sidecar binding below. Earlier + # five-key migration objects existed only on this unmerged branch and cannot + # safely authorize recovery, so they are intentionally not compatibility input. + elif jq -e ' + type == "object" and + keys == ["adoptedApp","appPath","archiveSha256","artifactReceiptSha256","backupCDHashes","backupPath","cdhashes","configPath","installerSha256","kind","migration","nodeId","nodeProfile","peekabooCommit","plistPath","previousPlist","previousPlistSha256","previousPlistWasLoaded","previousReceipt","previousReceiptSha256","schemaVersion","sourceCommit","stateDir","transactionId","transactionState"] and + .schemaVersion == 3 and + .kind == "openclaw-elevation-install" and + (.transactionState == "installing" or .transactionState == "installed") and + (.transactionId | type == "string" and test("^[0-9A-F-]{36}$")) and + (.sourceCommit | type == "string" and test("^[0-9a-f]{40}$")) and + (.peekabooCommit | type == "string" and test("^[0-9a-f]{40}$")) and + (.archiveSha256 | type == "string" and test("^[0-9a-f]{64}$")) and + (.artifactReceiptSha256 | type == "string" and test("^[0-9a-f]{64}$")) and + (.installerSha256 | type == "string" and test("^[0-9a-f]{64}$")) and + (.cdhashes | type == "object" and keys == ["arm64","x86_64"] and all(.[]; type == "string" and length > 0)) and + (.nodeId | type == "string" and length > 0) and + (.nodeProfile == "primary" or .nodeProfile == "node") and + (.backupCDHashes | type == "object" and keys == ["arm64","x86_64"] and all(.[]; type == "string")) and + ( + (.backupPath == "" and all(.backupCDHashes[]; . == "")) or + ((.backupPath | type == "string" and startswith("/")) and all(.backupCDHashes[]; length > 0)) + ) and + (.previousPlistSha256 | type == "string") and + (.previousReceipt | type == "string") and + (.previousReceiptSha256 | type == "string") and + (.stateDir | type == "string" and startswith("/")) and + (.configPath | type == "string" and startswith("/")) and + (.previousPlistWasLoaded | type == "boolean") and + (.migration == null or ( + .migration | type == "object" and + keys == ["backupPlist","backupSha256","custodyPath","kind","label","nodeEnvIdentity","nodeEnvPath","nodeEnvSha256","nodeWrapperIdentity","nodeWrapperPath","nodeWrapperSha256","sourceIdentity","sourcePlist","wasLoaded"] and + (.kind == "app-launch-agent" or .kind == "canonical-node") and + (.backupSha256 | type == "string") and + (.custodyPath | type == "string" and startswith("/")) and + (.sourceIdentity | type == "string" and test("^[0-9A-F-]{36}:[0-9]+:[0-9]+$")) and + (.wasLoaded | type == "boolean") and + ( + (.kind == "app-launch-agent" and .nodeEnvPath == "" and .nodeEnvSha256 == "" and .nodeEnvIdentity == "" and .nodeWrapperPath == "" and .nodeWrapperSha256 == "" and .nodeWrapperIdentity == "") or + (.kind == "canonical-node" and + (.nodeEnvPath | type == "string" and startswith("/")) and + (.nodeEnvSha256 | type == "string" and test("^[0-9a-f]{64}$")) and + (.nodeEnvIdentity | type == "string" and test("^[0-9A-F-]{36}:[0-9]+:[0-9]+$")) and + (.nodeWrapperPath | type == "string" and startswith("/")) and + (.nodeWrapperSha256 | type == "string" and test("^[0-9a-f]{64}$")) and + (.nodeWrapperIdentity | type == "string" and test("^[0-9A-F-]{36}:[0-9]+:[0-9]+$"))) + ) + )) and + (.adoptedApp | type == "object" and (.wasRunning | type == "boolean") and (.attachOnly | type == "boolean")) + ' "$RECEIPT_PATH" >/dev/null 2>&1 + then + INSTALL_RECEIPT_SCHEMA="3" + INSTALL_RECEIPT_TRANSACTION_STATE="$(jq -r '.transactionState' "$RECEIPT_PATH")" + INSTALL_TRANSACTION_ID="$(jq -r '.transactionId' "$RECEIPT_PATH")" + else + fail 'elevation install receipt schema is invalid' + fi + if [[ "$verify_current_app" == "1" ]]; then + [[ "$(jq -r '.sourceCommit' "$RECEIPT_PATH")" == "$(plist_value "$APP_PATH" OpenClawGitCommit)" ]] || + fail 'installed app source does not match the elevation install receipt' + [[ "$(jq -r '.peekabooCommit' "$RECEIPT_PATH")" == "$(plist_value "$APP_PATH" PeekabooSourceCommit)" ]] || + fail 'installed Peekaboo source does not match the elevation install receipt' + fi + [[ "$(jq -r '.appPath' "$RECEIPT_PATH")" == "$APP_PATH" ]] || fail 'elevation install receipt app path mismatch' + [[ "$(jq -r '.plistPath' "$RECEIPT_PATH")" == "$PLIST_PATH" ]] || fail 'elevation install receipt plist path mismatch' + if [[ "$INSTALL_RECEIPT_SCHEMA" != 'legacy' ]]; then + if [[ "$verify_current_app" == "1" ]]; then + receipt_arm64_cdhash="$(jq -r '.cdhashes.arm64' "$RECEIPT_PATH")" + receipt_x86_64_cdhash="$(jq -r '.cdhashes.x86_64' "$RECEIPT_PATH")" + [[ "$receipt_arm64_cdhash" == "$(codesign_value_for_arch "$APP_PATH" CDHash arm64)" ]] || + fail 'installed app arm64 CDHash does not match the elevation install receipt' + [[ "$receipt_x86_64_cdhash" == "$(codesign_value_for_arch "$APP_PATH" CDHash x86_64)" ]] || + fail 'installed app x86_64 CDHash does not match the elevation install receipt' + fi + [[ "$(jq -r '.stateDir' "$RECEIPT_PATH")" == "$STATE_DIR" ]] || + fail 'elevation install receipt state directory mismatch' + fi +} + +require_committed_install_receipt() { + [[ "$INSTALL_RECEIPT_TRANSACTION_STATE" == "installed" ]] || + fail 'elevation install receipt is an incomplete install transaction; run recover' +} + +package_host() { + [[ "$(uname -s)" == 'Darwin' ]] || fail 'elevation packaging requires macOS' + [[ -n "$EXPECTED_PEEKABOO_SOURCE_COMMIT" ]] || + fail 'package requires --peekaboo-source-commit <full-sha>' + local source_commit prefix zip_path receipt_path checksum_path installer_path installer_checksum_path notary_result + source_commit="$(git -C "$ROOT_DIR" rev-parse HEAD)" + [[ "$source_commit" =~ ^[0-9a-f]{40}$ ]] || fail 'could not resolve exact source commit' + prefix="OpenClaw-${source_commit}-Peekaboo-${EXPECTED_PEEKABOO_SOURCE_COMMIT}-stable" + zip_path="$OUTPUT_DIR/${prefix}.zip" + receipt_path="$OUTPUT_DIR/${prefix}.json" + checksum_path="$zip_path.sha256" + installer_path="$OUTPUT_DIR/${prefix}-installer.sh" + installer_checksum_path="$installer_path.sha256" + for output in "$zip_path" "$receipt_path" "$checksum_path" "$installer_path" "$installer_checksum_path"; do + [[ ! -e "$output" ]] || fail "immutable elevation output already exists: $output" + done + mkdir -p "$OUTPUT_DIR" + NOTARY_RESULT_TEMP="$(mktemp "${TMPDIR:-/tmp}/openclaw-elevation-notary.XXXXXX")" + notary_result="$NOTARY_RESULT_TEMP" + + SIGN_IDENTITY="$EXPECTED_AUTHORITY" \ + OPENCLAW_EXPECTED_PEEKABOO_SOURCE_COMMIT="$EXPECTED_PEEKABOO_SOURCE_COMMIT" \ + OPENCLAW_MAC_SIGNING_VARIANT=elevation-host \ + NOTARY_RESULT_FILE="$notary_result" \ + SKIP_DMG=1 \ + SKIP_DSYM=1 \ + "$ROOT_DIR/scripts/package-mac-dist.sh" + + local app="$ROOT_DIR/dist/OpenClaw.app" + verify_elevation_app "$app" + [[ "$(plist_value "$app" OpenClawGitCommit)" == "$source_commit" ]] || + fail 'packaged elevation source does not match HEAD' + [[ "$(plist_value "$app" PeekabooSourceCommit)" == "$EXPECTED_PEEKABOO_SOURCE_COMMIT" ]] || + fail 'packaged elevation Peekaboo source does not match the requested release head' + local version source_zip + version="$(plist_value "$app" CFBundleShortVersionString)" + source_zip="$ROOT_DIR/dist/OpenClaw-${version}.zip" + [[ -f "$source_zip" ]] || fail "distribution zip missing: $source_zip" + local tmp_zip="${zip_path}.tmp.$$" + cp "$source_zip" "$tmp_zip" + chmod 444 "$tmp_zip" + + local archive_sha installer_sha committed_installer_sha notary_id + local main_archs helper_archs main_entitlements helper_entitlements arm64_cdhash x86_64_cdhash + archive_sha="$(shasum -a 256 "$tmp_zip" | awk '{print $1}')" + local tmp_installer="${installer_path}.tmp.$$" + cp "$ROOT_DIR/scripts/mac-elevation-host.sh" "$tmp_installer" + chmod 555 "$tmp_installer" + installer_sha="$(shasum -a 256 "$tmp_installer" | awk '{print $1}')" + committed_installer_sha="$(git -C "$ROOT_DIR" show "${source_commit}:scripts/mac-elevation-host.sh" | shasum -a 256 | awk '{print $1}')" + [[ "$installer_sha" == "$committed_installer_sha" ]] || + fail 'portable installer does not match the selected source commit' + notary_id="$(jq -r '.id // empty' "$notary_result")" + [[ -n "$notary_id" ]] || fail 'accepted notarization id was not recorded' + main_archs="$(lipo -archs "$app/Contents/MacOS/OpenClaw")" + helper_archs="$(lipo -archs "$app/Contents/MacOS/openclaw-mlx-tts")" + main_entitlements="$(entitlements_for "$app/Contents/MacOS/OpenClaw" | shasum -a 256 | awk '{print $1}')" + helper_entitlements="$(entitlements_for "$app/Contents/MacOS/openclaw-mlx-tts" | shasum -a 256 | awk '{print $1}')" + arm64_cdhash="$(codesign_value_for_arch "$app" CDHash arm64)" + x86_64_cdhash="$(codesign_value_for_arch "$app" CDHash x86_64)" + [[ -n "$arm64_cdhash" && -n "$x86_64_cdhash" ]] || fail 'could not resolve per-architecture app CDHashes' + mv "$tmp_zip" "$zip_path" + mv "$tmp_installer" "$installer_path" + jq -n \ + --argjson schemaVersion 1 \ + --arg kind 'openclaw-elevation-artifact' \ + --arg archive "$(basename "$zip_path")" \ + --arg archiveSha256 "$archive_sha" \ + --arg archiveChecksum "$(basename "$checksum_path")" \ + --arg installer "$(basename "$installer_path")" \ + --arg installerSha256 "$installer_sha" \ + --arg installerChecksum "$(basename "$installer_checksum_path")" \ + --arg sourceCommit "$source_commit" \ + --arg peekabooCommit "$(plist_value "$app" PeekabooSourceCommit)" \ + --arg version "$version" \ + --arg build "$(plist_value "$app" CFBundleVersion)" \ + --arg authority "$EXPECTED_AUTHORITY" \ + --arg teamIdentifier "$EXPECTED_TEAM_ID" \ + --arg arm64CDHash "$arm64_cdhash" \ + --arg x8664CDHash "$x86_64_cdhash" \ + --arg mainArchitectures "$main_archs" \ + --arg helperArchitectures "$helper_archs" \ + --arg mainEntitlementsSha256 "$main_entitlements" \ + --arg helperEntitlementsSha256 "$helper_entitlements" \ + --arg notarizationId "$notary_id" \ + '{schemaVersion:$schemaVersion,kind:$kind,archive:$archive,archiveSha256:$archiveSha256,archiveChecksum:$archiveChecksum,installer:$installer,installerSha256:$installerSha256,installerChecksum:$installerChecksum,sourceCommit:$sourceCommit,peekabooCommit:$peekabooCommit,version:$version,build:$build,authority:$authority,teamIdentifier:$teamIdentifier,cdhashes:{arm64:$arm64CDHash,x86_64:$x8664CDHash},architectures:{main:$mainArchitectures,helper:$helperArchitectures},entitlementsSha256:{main:$mainEntitlementsSha256,helper:$helperEntitlementsSha256},notarizationId:$notarizationId}' >"${receipt_path}.tmp.$$" + chmod 444 "${receipt_path}.tmp.$$" + mv "${receipt_path}.tmp.$$" "$receipt_path" + EXPECTED_ARTIFACT_RECEIPT_SHA256="$(shasum -a 256 "$receipt_path" | awk '{print $1}')" + printf '%s %s\n' "$archive_sha" "$(basename "$zip_path")" >"${checksum_path}.tmp.$$" + chmod 444 "${checksum_path}.tmp.$$" + mv "${checksum_path}.tmp.$$" "$checksum_path" + printf '%s %s\n' "$installer_sha" "$(basename "$installer_path")" >"${installer_checksum_path}.tmp.$$" + chmod 444 "${installer_checksum_path}.tmp.$$" + mv "${installer_checksum_path}.tmp.$$" "$installer_checksum_path" + prepare_authenticated_artifact_inputs "$receipt_path" "$zip_path" "$installer_path" + local extracted + extract_verified_artifact "$installer_path" extracted + [[ "$(plist_value "$extracted" OpenClawGitCommit)" == "$source_commit" ]] || + fail 'extracted elevation source mismatch' + printf 'Elevation archive: %s\nInstaller: %s\nReceipt: %s\nArchive SHA-256: %s\nInstaller SHA-256: %s\nReceipt SHA-256: %s\n' \ + "$zip_path" "$installer_path" "$receipt_path" "$archive_sha" "$installer_sha" \ + "$EXPECTED_ARTIFACT_RECEIPT_SHA256" +} + +install_host() { + [[ -n "$ARCHIVE" ]] || fail 'install requires --archive <zip>' + [[ -n "$ARTIFACT_RECEIPT" ]] || fail 'install requires --receipt <json>' + ensure_no_normal_owner + local staged_app source_commit peekaboo_commit old_pid migration_pid plist_tmp staged_install_identity + local current_migration_state elevation_state adoption_signal="" commit_signal="" + local planned_archive_sha planned_arm64_cdhash planned_x86_64_cdhash + prepare_authenticated_artifact_inputs "$ARTIFACT_RECEIPT" "$ARCHIVE" "${BASH_SOURCE[0]}" + extract_verified_artifact "${BASH_SOURCE[0]}" staged_app + source_commit="$(plist_value "$staged_app" OpenClawGitCommit)" + peekaboo_commit="$(plist_value "$staged_app" PeekabooSourceCommit)" + planned_archive_sha="$(shasum -a 256 "$AUTHENTICATED_ARCHIVE_PATH" | awk '{print $1}')" + planned_arm64_cdhash="$(receipt_string "$AUTHENTICATED_RECEIPT_PATH" '.cdhashes.arm64' cdhashes.arm64)" + planned_x86_64_cdhash="$(receipt_string "$AUTHENTICATED_RECEIPT_PATH" '.cdhashes.x86_64' cdhashes.x86_64)" + stage_verified_app_for_install "$staged_app" "$source_commit" "$peekaboo_commit" + mkdir -p "$STATE_DIR/logs" "$(dirname "$PLIST_PATH")" + [[ ! -e "$PENDING_RECEIPT_PATH" && ! -L "$PENDING_RECEIPT_PATH" ]] || + fail 'an incomplete elevation install transaction exists; run recover before installing' + if [[ -n "$MIGRATE_LAUNCH_AGENT" ]]; then + [[ -d "$STATE_DIR" && ! -L "$STATE_DIR" ]] || + fail "migration state directory is missing or symlinked: $STATE_DIR" + fi + + ROLLBACK_APP_PATH="" + ROLLBACK_APP_CDHASH_ARM64="" + ROLLBACK_APP_CDHASH_X86_64="" + if [[ -e "$APP_PATH" || -L "$APP_PATH" ]]; then + [[ -d "$APP_PATH" && ! -L "$APP_PATH" ]] || + fail 'installed OpenClaw app is missing, symlinked, or not a bundle directory' + local installed_commit + installed_commit="$(plist_value "$APP_PATH" OpenClawGitCommit)" + [[ "$installed_commit" =~ ^[0-9a-f]{40}$ ]] || fail 'installed OpenClaw app has no exact source receipt' + ROLLBACK_APP_PATH="$(mktemp -u "${APP_PATH}.rollback-elevation-host-${installed_commit}.XXXXXX")" + # The unpredictable name is consumed only by renamex_np(RENAME_EXCL); any entry that + # appears before custody atomically refuses without nesting or overwrite. + [[ ! -e "$ROLLBACK_APP_PATH" && ! -L "$ROLLBACK_APP_PATH" ]] || + fail "could not reserve a unique elevation backup path: $ROLLBACK_APP_PATH" + ROLLBACK_APP_CDHASH_ARM64="$(codesign_value_for_arch "$APP_PATH" CDHash arm64)" + ROLLBACK_APP_CDHASH_X86_64="$(codesign_value_for_arch "$APP_PATH" CDHash x86_64)" + [[ -n "$ROLLBACK_APP_CDHASH_ARM64" && -n "$ROLLBACK_APP_CDHASH_X86_64" ]] || + fail 'installed OpenClaw app has no signed per-architecture CDHashes' + verify_recorded_rollback_app "$APP_PATH" || + fail 'installed OpenClaw app does not pass strict signature and identity validation' + fi + ROLLBACK_INSTALL_RECEIPT="" + ROLLBACK_INSTALL_RECEIPT_SHA="" + if [[ -e "$RECEIPT_PATH" ]]; then + [[ -f "$RECEIPT_PATH" && ! -L "$RECEIPT_PATH" ]] || + fail 'existing elevation install receipt is not a regular file' + verify_install_receipt + require_committed_install_receipt + ROLLBACK_INSTALL_RECEIPT="$(mktemp "$STATE_DIR/elevation-host.previous-receipt.${source_commit}.XXXXXX")" + cp -p "$RECEIPT_PATH" "$ROLLBACK_INSTALL_RECEIPT" + ROLLBACK_INSTALL_RECEIPT_SHA="$(shasum -a 256 "$ROLLBACK_INSTALL_RECEIPT" | awk '{print $1}')" + PREMUTATION_BACKUPS+=("$ROLLBACK_INSTALL_RECEIPT") + fi + ROLLBACK_ELEVATION_PLIST="" + ROLLBACK_ELEVATION_PLIST_SHA="" + ROLLBACK_ELEVATION_WAS_LOADED=0 + elevation_state="$(job_loaded_state "$job_domain")" + [[ "$elevation_state" != 'unknown' ]] || fail 'previous elevation launchd state could not be inspected' + if [[ "$elevation_state" == 'loaded' && ! -f "$PLIST_PATH" ]]; then + fail 'loaded elevation job has no recoverable LaunchAgent plist' + fi + if [[ -f "$PLIST_PATH" ]]; then + ROLLBACK_ELEVATION_PLIST="$(mktemp "$STATE_DIR/elevation-host.previous-plist.${source_commit}.XXXXXX")" + cp -p "$PLIST_PATH" "$ROLLBACK_ELEVATION_PLIST" + ROLLBACK_ELEVATION_PLIST_SHA="$(shasum -a 256 "$ROLLBACK_ELEVATION_PLIST" | awk '{print $1}')" + PREMUTATION_BACKUPS+=("$ROLLBACK_ELEVATION_PLIST") + [[ "$elevation_state" == 'loaded' ]] && ROLLBACK_ELEVATION_WAS_LOADED=1 + fi + ROLLBACK_MIGRATION_SOURCE="$MIGRATE_LAUNCH_AGENT" + ROLLBACK_MIGRATION_LABEL="$MIGRATION_LABEL" + ROLLBACK_MIGRATION_WAS_LOADED="$MIGRATION_WAS_LOADED" + ROLLBACK_MIGRATION_PLIST="" + ROLLBACK_MIGRATION_PLIST_SHA="" + if [[ -n "$MIGRATE_LAUNCH_AGENT" ]]; then + ROLLBACK_MIGRATION_PLIST="$(mktemp "$STATE_DIR/elevation-host.previous-launch-agent.${source_commit}.XXXXXX")" + cp -p "$MIGRATE_LAUNCH_AGENT" "$ROLLBACK_MIGRATION_PLIST" + ROLLBACK_MIGRATION_PLIST_SHA="$MIGRATION_PLIST_SHA" + [[ "$(shasum -a 256 "$ROLLBACK_MIGRATION_PLIST" | awk '{print $1}')" == "$MIGRATION_PLIST_SHA" ]] || + fail 'migration LaunchAgent changed while creating the rollback copy; rerun migration-plan' + PREMUTATION_BACKUPS+=("$ROLLBACK_MIGRATION_PLIST") + fi + ROLLBACK_ADOPTED_APP_WAS_RUNNING="$ADOPT_RUNNING_APP" + ROLLBACK_ADOPTED_APP_ATTACH_ONLY="$ADOPTION_ATTACH_ONLY" + + plist_tmp="${PLIST_PATH}.tmp.$$" + render_plist "$plist_tmp" + chmod 644 "$plist_tmp" + + old_pid="$(job_pid)" + migration_pid="" + if [[ -n "$MIGRATION_LABEL" ]]; then + migration_pid="$(job_pid_for_domain "$launch_domain/$MIGRATION_LABEL")" + [[ "$(shasum -a 256 "$MIGRATE_LAUNCH_AGENT" | awk '{print $1}')" == "$MIGRATION_PLIST_SHA" ]] || + fail 'migration LaunchAgent changed after planning; rerun migration-plan' + current_migration_state="$(job_loaded_state "$launch_domain/$MIGRATION_LABEL")" + case "$current_migration_state" in + loaded) [[ "$MIGRATION_WAS_LOADED" == '1' ]] || fail 'migration LaunchAgent loaded after planning; rerun migration-plan' ;; + absent) [[ "$MIGRATION_WAS_LOADED" == '0' ]] || fail 'migration LaunchAgent stopped after planning; rerun migration-plan' ;; + *) fail 'migration launchd state became unreadable after planning' ;; + esac + verify_canonical_node_sidecars || + fail 'canonical node sidecars changed after migration planning; rerun migration-plan' + fi + ROLLBACK_FAILED_SOURCE="$source_commit" + INSTALL_TRANSACTION_ID="$(uuidgen)" + [[ "$INSTALL_TRANSACTION_ID" =~ ^[0-9A-F-]{36}$ ]] || + fail 'could not create an install transaction identity' + if [[ -n "$MIGRATE_LAUNCH_AGENT" ]]; then + MIGRATION_CUSTODY_PATH="$(mktemp -u "${MIGRATE_LAUNCH_AGENT}.custody.XXXXXX")" || + fail 'could not reserve migration LaunchAgent custody' + [[ ! -e "$MIGRATION_CUSTODY_PATH" && ! -L "$MIGRATION_CUSTODY_PATH" ]] || + fail 'migration LaunchAgent custody path is already occupied' + MIGRATION_CUSTODY_IDENTITY="$MIGRATION_PLIST_IDENTITY" + fi + local rollback_payload + for rollback_payload in "${PREMUTATION_BACKUPS[@]:-}"; do + [[ -n "$rollback_payload" ]] || continue + fsync_file_and_parent "$rollback_payload" || + fail 'authenticated elevation helper could not sync a rollback payload' + done + fsync_tree "$STAGED_INSTALL_APP_PATH" || + fail 'authenticated elevation helper could not sync the staged install app' + fsync_parent "$STAGED_INSTALL_APP_PATH" || + fail 'authenticated elevation helper could not sync the staged app namespace' + if [[ -e "$APP_PATH" || -L "$APP_PATH" ]]; then + fsync_parent "$APP_PATH" || + fail 'authenticated elevation helper could not sync the installed app namespace' + fi + if [[ -n "$MIGRATE_LAUNCH_AGENT" ]]; then + fsync_parent "$MIGRATE_LAUNCH_AGENT" || + fail 'authenticated elevation helper could not sync the migration namespace' + fi + write_install_receipt \ + "$PENDING_RECEIPT_PATH" \ + installing \ + "$source_commit" \ + "$peekaboo_commit" \ + "$planned_archive_sha" \ + "$planned_arm64_cdhash" \ + "$planned_x86_64_cdhash" + PENDING_RECEIPT_CREATED=1 + CUTOVER_ACTIVE=1 + if [[ -n "$MIGRATE_LAUNCH_AGENT" ]]; then + take_migration_plist_custody "$MIGRATE_LAUNCH_AGENT" "$MIGRATION_PLIST_SHA" + fi + if [[ "$ROLLBACK_ELEVATION_WAS_LOADED" == "1" ]]; then + launchctl bootout "$job_domain" >/dev/null 2>&1 || fail 'could not stop previous elevation host' + fi + if [[ "$MIGRATION_WAS_LOADED" == "1" ]]; then + launchctl bootout "$launch_domain/$MIGRATION_LABEL" >/dev/null 2>&1 || + fail "could not stop migration LaunchAgent: $MIGRATION_LABEL" + fi + if [[ -n "$ADOPTION_PID" ]]; then + adopted_app_is_current || fail 'adopted OpenClaw process changed after migration planning' + trap 'adoption_signal=INT' INT + trap 'adoption_signal=TERM' TERM + trap 'adoption_signal=HUP' HUP + CUTOVER_ADOPTION_STOPPED=1 + kill "$ADOPTION_PID" 2>/dev/null || fail "could not stop adopted OpenClaw process: $ADOPTION_PID" + CUTOVER_ADOPTION_TERMINATION_SENT=1 + finish_custody_signal_deferral "$adoption_signal" + fi + if [[ "$old_pid" =~ ^[0-9]+$ ]]; then + for _ in $(seq 1 80); do + kill -0 "$old_pid" 2>/dev/null || break + sleep 0.25 + done + kill -0 "$old_pid" 2>/dev/null && fail "previous elevation host did not exit: $old_pid" + fi + if [[ "$migration_pid" =~ ^[0-9]+$ ]]; then + for _ in $(seq 1 80); do + kill -0 "$migration_pid" 2>/dev/null || break + sleep 0.25 + done + kill -0 "$migration_pid" 2>/dev/null && + fail "migration LaunchAgent did not exit: $MIGRATION_LABEL ($migration_pid)" + fi + if [[ "$ADOPTION_PID" =~ ^[0-9]+$ ]]; then + for _ in $(seq 1 80); do + kill -0 "$ADOPTION_PID" 2>/dev/null || break + sleep 0.25 + done + kill -0 "$ADOPTION_PID" 2>/dev/null && fail "adopted OpenClaw process did not exit: $ADOPTION_PID" + fi + if [[ -n "$MIGRATION_LABEL" ]]; then + current_migration_state="$(job_loaded_state "$launch_domain/$MIGRATION_LABEL")" + [[ "$current_migration_state" == "absent" ]] || + fail 'migration LaunchAgent remained loaded after bootout' + fi + wait_for_app_binary_exit || fail 'an OpenClaw app process survived owner shutdown' + if [[ -n "$MIGRATION_LABEL" ]]; then + current_migration_state="$(job_loaded_state "$launch_domain/$MIGRATION_LABEL")" + [[ "$current_migration_state" == "absent" ]] || + fail 'migration LaunchAgent reloaded during owner shutdown' + verify_canonical_node_sidecars || + fail 'canonical node sidecars changed during owner shutdown' + fi + CUTOVER_APP_MUTATED=1 + if [[ -n "$ROLLBACK_APP_PATH" ]]; then + verify_recorded_rollback_app "$APP_PATH" || + fail 'installed OpenClaw app changed before rollback custody' + rename_app_exclusively "$APP_PATH" "$ROLLBACK_APP_PATH" || true + verify_recorded_rollback_app "$ROLLBACK_APP_PATH" || + fail 'could not take verified custody of the installed OpenClaw app' + [[ ! -e "$APP_PATH" && ! -L "$APP_PATH" ]] || + fail 'installed OpenClaw app path was recreated during rollback custody' + fi + staged_install_identity="$(path_identity "$STAGED_INSTALL_APP_PATH")" || + fail 'same-filesystem staged app identity could not be inspected before install' + rename_app_exclusively "$STAGED_INSTALL_APP_PATH" "$APP_PATH" || true + path_matches_identity "$APP_PATH" "$staged_install_identity" || + fail 'same-filesystem staged app identity changed during install' + RECOVERY_CURRENT_APP_IDENTITY="$staged_install_identity" + verify_artifact_receipt \ + "$AUTHENTICATED_RECEIPT_PATH" \ + "$AUTHENTICATED_ARCHIVE_PATH" \ + "$APP_PATH" \ + "${BASH_SOURCE[0]}" + [[ ! -e "$STAGED_INSTALL_APP_PATH" && ! -L "$STAGED_INSTALL_APP_PATH" ]] || + fail 'same-filesystem staged app was not atomically installed' + rmdir "$STAGED_APP_CONTAINER" + STAGED_APP_CONTAINER="" + STAGED_INSTALL_APP_PATH="" + mv "$plist_tmp" "$PLIST_PATH" + fsync_file_and_parent "$PLIST_PATH" || + fail 'authenticated elevation helper could not sync the elevation LaunchAgent' + + if ! launchctl bootstrap "$launch_domain" "$PLIST_PATH" || + ! launchctl kickstart -k "$job_domain" + then + fail 'could not bootstrap elevation host' + fi + + local ready_pid="" + for _ in $(seq 1 80); do + ready_pid="$(job_pid)" + if [[ "$ready_pid" =~ ^[0-9]+$ ]] && verify_bridge_readiness "$ready_pid"; then break; fi + ready_pid="" + sleep 0.25 + done + if [[ -z "$ready_pid" ]]; then + fail 'elevation host did not become Bridge-ready' + fi + verify_gateway_node_readiness "$(plist_value "$APP_PATH" CFBundleShortVersionString)" || + fail 'elevation host did not reconnect the expected macOS computer-use node' + + if [[ "$CUTOVER_MIGRATION_REMOVED" == "1" && + (-e "$ROLLBACK_MIGRATION_SOURCE" || -L "$ROLLBACK_MIGRATION_SOURCE") ]] + then + fail 'migration LaunchAgent path was recreated before cutover commit' + fi + if [[ -n "$ROLLBACK_MIGRATION_LABEL" && + ("$CUTOVER_MIGRATION_REMOVED" == "1" || "$CUTOVER_RECOVERY_ATTEMPTED" == "1") ]] + then + [[ "$(job_loaded_state "$launch_domain/$ROLLBACK_MIGRATION_LABEL")" == "absent" ]] || + fail 'migration LaunchAgent reloaded before cutover commit' + fi + + trap 'commit_signal=INT' INT + trap 'commit_signal=TERM' TERM + trap 'commit_signal=HUP' HUP + verify_artifact_receipt \ + "$AUTHENTICATED_RECEIPT_PATH" \ + "$AUTHENTICATED_ARCHIVE_PATH" \ + "$APP_PATH" \ + "${BASH_SOURCE[0]}" + local final_arm64_cdhash final_x86_64_cdhash + if ! final_arm64_cdhash="$(codesign_value_for_arch "$APP_PATH" CDHash arm64)" || + ! final_x86_64_cdhash="$(codesign_value_for_arch "$APP_PATH" CDHash x86_64)" + then + fail 'could not resolve final installed app per-architecture CDHashes' + fi + [[ -n "$final_arm64_cdhash" && -n "$final_x86_64_cdhash" ]] || + fail 'could not resolve final installed app per-architecture CDHashes' + [[ "$final_arm64_cdhash" == "$(receipt_string "$AUTHENTICATED_RECEIPT_PATH" '.cdhashes.arm64' cdhashes.arm64)" && + "$final_x86_64_cdhash" == "$(receipt_string "$AUTHENTICATED_RECEIPT_PATH" '.cdhashes.x86_64' cdhashes.x86_64)" ]] || + fail 'final installed app CDHashes do not match the authenticated artifact receipt' + write_receipt \ + "$source_commit" \ + "$peekaboo_commit" \ + "$planned_archive_sha" \ + "$final_arm64_cdhash" \ + "$final_x86_64_cdhash" + remove_pending_receipt || fail 'could not retire the prepared install transaction' + CUTOVER_COMMITTED=1 + CUTOVER_ACTIVE=0 + finish_custody_signal_deferral "$commit_signal" + printf 'Elevation host installed: pid=%s source=%s\n' "$ready_pid" "$source_commit" + if [[ -n "$MIGRATION_CUSTODY_PATH" ]]; then + printf 'Preserved migrated LaunchAgent custody at %s\n' "$MIGRATION_CUSTODY_PATH" + fi + # Commit only after the launchd-owned Bridge and exact Gateway node are ready. Missing TCC is + # degraded capability, not a failed cutover; `status` remains the final readiness gate. + tcc_summary || true +} + +recover_install() { + local recovery_failed=0 elevation_state restored_state prior_owner_state + local unsafe_previous_elevation=0 rollback_app_candidate="" + quarantine_entry_unsafe_elevation_app || return 1 + [[ "$UNSAFE_ELEVATION_APP_WAS_QUARANTINED" == "0" ]] || recovery_failed=1 + if [[ "$CUTOVER_APP_MUTATED" == "1" && -n "$ROLLBACK_APP_PATH" ]]; then + verify_recorded_rollback_app "$APP_PATH" || + verify_recorded_rollback_app "$ROLLBACK_APP_PATH" || return 1 + fi + if [[ "$ELEVATION_APP_OWNER_WAS_EVIDENCED" == "1" && -n "$ROLLBACK_APP_PATH" ]] + then + if [[ -d "$ROLLBACK_APP_PATH" && ! -L "$ROLLBACK_APP_PATH" ]] && + ! elevation_app_is_cua_free "$ROLLBACK_APP_PATH" + then + rollback_app_candidate="$ROLLBACK_APP_PATH" + elif [[ -d "$APP_PATH" && ! -L "$APP_PATH" ]] && ! elevation_app_is_cua_free "$APP_PATH"; then + rollback_app_candidate="$APP_PATH" + fi + [[ -z "$rollback_app_candidate" ]] || unsafe_previous_elevation=1 + fi + if [[ "$unsafe_previous_elevation" == "0" ]]; then + [[ -z "$ROLLBACK_ELEVATION_PLIST" ]] || + backup_file_matches "$ROLLBACK_ELEVATION_PLIST" "$ROLLBACK_ELEVATION_PLIST_SHA" || return 1 + fi + [[ -z "$ROLLBACK_MIGRATION_PLIST" ]] || + backup_file_matches "$ROLLBACK_MIGRATION_PLIST" "$ROLLBACK_MIGRATION_PLIST_SHA" || return 1 + [[ -z "$ROLLBACK_INSTALL_RECEIPT" ]] || + backup_file_matches "$ROLLBACK_INSTALL_RECEIPT" "$ROLLBACK_INSTALL_RECEIPT_SHA" || return 1 + elevation_state="$(job_loaded_state "$job_domain")" + [[ "$elevation_state" != 'unknown' ]] || return 1 + if [[ "$elevation_state" == 'loaded' ]]; then + launchctl bootout "$job_domain" >/dev/null 2>&1 || return 1 + fi + if [[ -n "$ROLLBACK_MIGRATION_LABEL" ]]; then + prior_owner_state="$(job_loaded_state "$launch_domain/$ROLLBACK_MIGRATION_LABEL")" + [[ "$prior_owner_state" != 'unknown' ]] || return 1 + if [[ "$prior_owner_state" == 'loaded' ]]; then + launchctl bootout "$launch_domain/$ROLLBACK_MIGRATION_LABEL" >/dev/null 2>&1 || return 1 + fi + fi + if [[ "$ROLLBACK_ADOPTED_APP_WAS_RUNNING" == "1" ]]; then + local adopted_records=() adopted_record + while IFS= read -r adopted_record; do + [[ -n "$adopted_record" ]] && adopted_records+=("$adopted_record") + done < <(background_app_records) + if [[ "${#adopted_records[@]}" == "1" && + "${adopted_records[0]##* }" == "$ROLLBACK_ADOPTED_APP_ATTACH_ONLY" ]] + then + ADOPTION_PID="${adopted_records[0]%% *}" + kill "$ADOPTION_PID" 2>/dev/null || return 1 + elif [[ "${#adopted_records[@]}" != "0" ]]; then + return 1 + fi + fi + # Do not mutate app custody until the exact adopted PID and every other process + # still executing this app binary have actually exited. + wait_for_app_binary_exit || return 1 + [[ "$(job_loaded_state "$job_domain")" == "absent" ]] || return 1 + if [[ "$CUTOVER_APP_MUTATED" == "1" && -d "$APP_PATH" && + -n "$ROLLBACK_APP_CDHASH_ARM64" && -n "$ROLLBACK_APP_CDHASH_X86_64" && + ! -d "$ROLLBACK_APP_PATH" ]] && + verify_recorded_rollback_app "$APP_PATH" + then + : # The pre-armed move failed before displacing the already-restored prior app. + elif [[ "$CUTOVER_APP_MUTATED" == "1" && "$CUTOVER_RECOVERY_ATTEMPTED" == "1" && + "$RECOVERY_CURRENT_APP_STATE" == "damaged" && -d "$APP_PATH" && ! -L "$APP_PATH" ]] + then + preserve_current_app_for_recovery 'damaged current app' || return 1 + elif [[ "$CUTOVER_APP_MUTATED" == "1" && -d "$APP_PATH" && + "$(plist_value "$APP_PATH" OpenClawGitCommit)" == "$ROLLBACK_FAILED_SOURCE" ]] + then + preserve_current_app_for_recovery 'failed elevation app' || return 1 + elif [[ "$CUTOVER_APP_MUTATED" == "1" && -d "$APP_PATH" ]]; then + [[ -n "$ROLLBACK_APP_PATH" ]] && + verify_recorded_rollback_app "$APP_PATH" || return 1 + fi + if [[ "$unsafe_previous_elevation" == "1" ]]; then + if [[ -d "$ROLLBACK_APP_PATH" && ! -L "$ROLLBACK_APP_PATH" ]]; then + verify_recorded_rollback_app "$ROLLBACK_APP_PATH" || recovery_failed=1 + [[ ! -e "$APP_PATH" && ! -L "$APP_PATH" ]] || recovery_failed=1 + elif verify_recorded_rollback_app "$APP_PATH"; then + [[ ! -e "$ROLLBACK_APP_PATH" && ! -L "$ROLLBACK_APP_PATH" ]] || recovery_failed=1 + if [[ "$recovery_failed" == "0" ]]; then + rename_app_exclusively "$APP_PATH" "$ROLLBACK_APP_PATH" || true + verify_recorded_rollback_app "$ROLLBACK_APP_PATH" || recovery_failed=1 + [[ ! -e "$APP_PATH" && ! -L "$APP_PATH" ]] || recovery_failed=1 + fi + else + recovery_failed=1 + fi + elif [[ -n "$ROLLBACK_APP_PATH" && -d "$ROLLBACK_APP_PATH" ]]; then + [[ ! -e "$APP_PATH" && ! -L "$APP_PATH" ]] || return 1 + rename_app_exclusively "$ROLLBACK_APP_PATH" "$APP_PATH" || true + verify_recorded_rollback_app "$APP_PATH" || recovery_failed=1 + [[ ! -e "$ROLLBACK_APP_PATH" && ! -L "$ROLLBACK_APP_PATH" ]] || recovery_failed=1 + elif [[ -n "$ROLLBACK_APP_PATH" ]]; then + verify_recorded_rollback_app "$APP_PATH" || recovery_failed=1 + fi + if [[ "$unsafe_previous_elevation" == "1" ]]; then + if [[ -n "$ROLLBACK_ELEVATION_PLIST" ]]; then + neutralize_unsafe_elevation_launch_agent \ + 'replacement for unsafe previous' \ + "$ROLLBACK_ELEVATION_PLIST" \ + "$ROLLBACK_ELEVATION_PLIST_SHA" || recovery_failed=1 + printf 'Preserved previous elevation app with bundled CUA driver at %s and LaunchAgent evidence at %s; refusing to restore it as elevation host\n' \ + "$ROLLBACK_APP_PATH" "$ROLLBACK_ELEVATION_PLIST" >&2 + else + neutralize_unsafe_elevation_launch_agent \ + 'replacement for unsafe previous' \ + '' \ + '' || recovery_failed=1 + printf 'Preserved previous elevation app with bundled CUA driver at %s; no prior elevation LaunchAgent was recorded, and it will not be restored as elevation host\n' \ + "$ROLLBACK_APP_PATH" >&2 + fi + recovery_failed=1 + elif [[ -n "$ROLLBACK_ELEVATION_PLIST" && -f "$ROLLBACK_ELEVATION_PLIST" ]]; then + restore_file_atomically \ + "$ROLLBACK_ELEVATION_PLIST" \ + "$PLIST_PATH" \ + "$ROLLBACK_ELEVATION_PLIST_SHA" \ + 644 || recovery_failed=1 + restored_state="$(job_loaded_state "$job_domain")" + [[ "$restored_state" != 'unknown' ]] || recovery_failed=1 + if [[ "$recovery_failed" == "0" && + "$ROLLBACK_ELEVATION_WAS_LOADED" == "1" && "$restored_state" == 'absent' ]] + then + launchctl bootstrap "$launch_domain" "$PLIST_PATH" >/dev/null 2>&1 || recovery_failed=1 + elif [[ "$ROLLBACK_ELEVATION_WAS_LOADED" == "0" && "$restored_state" != 'absent' ]]; then + recovery_failed=1 + fi + elif [[ "$ELEVATION_APP_OWNER_WAS_EVIDENCED" == "1" ]]; then + [[ ! -f "$PLIST_PATH" ]] || rm -f "$PLIST_PATH" || recovery_failed=1 + elif [[ -e "$PLIST_PATH" || -L "$PLIST_PATH" ]]; then + recovery_failed=1 + fi + if [[ -n "$ROLLBACK_MIGRATION_SOURCE" && -f "$ROLLBACK_MIGRATION_PLIST" ]]; then + if [[ "$CUTOVER_MIGRATION_REMOVED" != "1" && + -z "$RECOVERY_RESTORED_MIGRATION_IDENTITY" ]] && + path_matches_identity "$ROLLBACK_MIGRATION_SOURCE" "$MIGRATION_PLIST_IDENTITY" && + backup_file_matches "$ROLLBACK_MIGRATION_SOURCE" "$ROLLBACK_MIGRATION_PLIST_SHA" + then + RECOVERY_RESTORED_MIGRATION_IDENTITY="$MIGRATION_PLIST_IDENTITY" + fi + if [[ "$CUTOVER_MIGRATION_REMOVED" == "1" ]]; then + if [[ -e "$ROLLBACK_MIGRATION_SOURCE" || -L "$ROLLBACK_MIGRATION_SOURCE" ]]; then + if [[ ! -f "$ROLLBACK_MIGRATION_SOURCE" || -L "$ROLLBACK_MIGRATION_SOURCE" ]] || + ! path_matches_identity \ + "$ROLLBACK_MIGRATION_SOURCE" \ + "$RECOVERY_RESTORED_MIGRATION_IDENTITY" || + ! backup_file_matches "$ROLLBACK_MIGRATION_SOURCE" "$ROLLBACK_MIGRATION_PLIST_SHA" + then + recovery_failed=1 + fi + elif [[ -n "$MIGRATION_CUSTODY_PATH" && -f "$MIGRATION_CUSTODY_PATH" && + ! -L "$MIGRATION_CUSTODY_PATH" && + "$MIGRATION_CUSTODY_IDENTITY" =~ ^[0-9A-F-]{36}:[0-9]+:[0-9]+$ ]] && + path_matches_identity "$MIGRATION_CUSTODY_PATH" "$MIGRATION_CUSTODY_IDENTITY" && + backup_file_matches "$MIGRATION_CUSTODY_PATH" "$ROLLBACK_MIGRATION_PLIST_SHA" + then + if [[ "$CUTOVER_RECOVERY_ATTEMPTED" == "1" ]]; then + record_recovery_migration_identity "$MIGRATION_CUSTODY_IDENTITY" || recovery_failed=1 + fi + if [[ "$recovery_failed" == "0" ]]; then + rename_app_exclusively "$MIGRATION_CUSTODY_PATH" "$ROLLBACK_MIGRATION_SOURCE" || + recovery_failed=1 + fi + if [[ "$recovery_failed" == "0" ]] && + path_matches_identity "$ROLLBACK_MIGRATION_SOURCE" "$MIGRATION_CUSTODY_IDENTITY" && + backup_file_matches "$ROLLBACK_MIGRATION_SOURCE" "$ROLLBACK_MIGRATION_PLIST_SHA" + then + RECOVERY_RESTORED_MIGRATION_IDENTITY="$MIGRATION_CUSTODY_IDENTITY" + MIGRATION_CUSTODY_PATH="" + MIGRATION_CUSTODY_IDENTITY="" + else + recovery_failed=1 + fi + else + restore_file_without_overwrite \ + "$ROLLBACK_MIGRATION_PLIST" \ + "$ROLLBACK_MIGRATION_SOURCE" \ + "$ROLLBACK_MIGRATION_PLIST_SHA" \ + RECOVERY_RESTORED_MIGRATION_IDENTITY || recovery_failed=1 + fi + elif [[ ! -f "$ROLLBACK_MIGRATION_SOURCE" || + "$(shasum -a 256 "$ROLLBACK_MIGRATION_SOURCE" | awk '{print $1}')" != "$ROLLBACK_MIGRATION_PLIST_SHA" ]] + then + recovery_failed=1 + fi + if [[ "$recovery_failed" == "0" ]] && + ! path_matches_identity \ + "$ROLLBACK_MIGRATION_SOURCE" \ + "$RECOVERY_RESTORED_MIGRATION_IDENTITY" + then + recovery_failed=1 + fi + if [[ "$recovery_failed" == "0" ]] && ! verify_canonical_node_sidecars; then + recovery_failed=1 + fi + restored_state="$(job_loaded_state "$launch_domain/$ROLLBACK_MIGRATION_LABEL")" + [[ "$restored_state" != 'unknown' ]] || recovery_failed=1 + if [[ "$recovery_failed" == "0" && + "$ROLLBACK_MIGRATION_WAS_LOADED" == "1" && "$restored_state" == 'absent' ]] + then + launchctl bootstrap "$launch_domain" "$ROLLBACK_MIGRATION_SOURCE" >/dev/null 2>&1 || + recovery_failed=1 + elif [[ "$ROLLBACK_MIGRATION_WAS_LOADED" == "0" && "$restored_state" != 'absent' ]]; then + recovery_failed=1 + fi + if [[ "$recovery_failed" == "0" ]] && + ! path_matches_identity \ + "$ROLLBACK_MIGRATION_SOURCE" \ + "$RECOVERY_RESTORED_MIGRATION_IDENTITY" + then + recovery_failed=1 + fi + fi + if [[ -n "$MIGRATION_CUSTODY_PATH" ]]; then + if [[ ! -e "$MIGRATION_CUSTODY_PATH" && ! -L "$MIGRATION_CUSTODY_PATH" && + -f "$ROLLBACK_MIGRATION_SOURCE" && ! -L "$ROLLBACK_MIGRATION_SOURCE" ]] && + path_matches_identity "$ROLLBACK_MIGRATION_SOURCE" "$RECOVERY_RESTORED_MIGRATION_IDENTITY" && + backup_file_matches "$ROLLBACK_MIGRATION_SOURCE" "$ROLLBACK_MIGRATION_PLIST_SHA" + then + MIGRATION_CUSTODY_PATH="" + elif [[ -f "$MIGRATION_CUSTODY_PATH" && ! -L "$MIGRATION_CUSTODY_PATH" && + "$MIGRATION_CUSTODY_IDENTITY" =~ ^[0-9A-F-]{36}:[0-9]+:[0-9]+$ ]] && + path_matches_identity "$MIGRATION_CUSTODY_PATH" "$MIGRATION_CUSTODY_IDENTITY" && + backup_file_matches "$MIGRATION_CUSTODY_PATH" "$ROLLBACK_MIGRATION_PLIST_SHA" + then + printf 'Preserved migration custody at %s\n' "$MIGRATION_CUSTODY_PATH" >&2 + recovery_failed=1 + else + recovery_failed=1 + fi + fi + if [[ "$CUTOVER_RECOVERY_ATTEMPTED" != "1" ]]; then + restore_install_receipt_after_rollback || recovery_failed=1 + fi + if [[ "$CUTOVER_ADOPTION_STOPPED" == "1" && "$recovery_failed" == "0" ]]; then + restore_adopted_app_after_cutover || recovery_failed=1 + fi + if [[ "$recovery_failed" == "0" ]]; then + CUTOVER_ACTIVE=0 + CUTOVER_APP_MUTATED=0 + CUTOVER_MIGRATION_REMOVED=0 + CUTOVER_ADOPTION_STOPPED=0 + CUTOVER_ADOPTION_TERMINATION_SENT=0 + return 0 + fi + return 1 +} + +restore_current_generation_after_recovery_failure() { + local restore_failed=0 app_restore_failed=0 state + local unsafe_current_elevation=0 current_app_evidence_path="" + quarantine_entry_unsafe_elevation_app || return 1 + if [[ "$UNSAFE_ELEVATION_APP_WAS_QUARANTINED" == "1" ]]; then + unsafe_current_elevation=1 + current_app_evidence_path="$UNSAFE_ELEVATION_APP_QUARANTINE" + app_restore_failed=1 + fi + + if [[ "$RECOVERY_RELAUNCHED_ADOPTED_PID" =~ ^[0-9]+$ ]]; then + ADOPTION_PID="$RECOVERY_RELAUNCHED_ADOPTED_PID" + if adopted_app_is_current; then + kill "$ADOPTION_PID" 2>/dev/null || restore_failed=1 + for _ in $(seq 1 80); do + kill -0 "$ADOPTION_PID" 2>/dev/null || break + sleep 0.25 + done + kill -0 "$ADOPTION_PID" 2>/dev/null && restore_failed=1 + else + restore_failed=1 + fi + fi + + if [[ -n "$ROLLBACK_MIGRATION_LABEL" ]]; then + state="$(job_loaded_state "$launch_domain/$ROLLBACK_MIGRATION_LABEL")" + if [[ "$state" == "loaded" ]]; then + launchctl bootout "$launch_domain/$ROLLBACK_MIGRATION_LABEL" >/dev/null 2>&1 || restore_failed=1 + elif [[ "$state" == "unknown" ]]; then + restore_failed=1 + fi + fi + if [[ -n "$ROLLBACK_MIGRATION_SOURCE" && + (-e "$ROLLBACK_MIGRATION_SOURCE" || -L "$ROLLBACK_MIGRATION_SOURCE") ]] + then + if [[ -n "$RECOVERY_RESTORED_MIGRATION_IDENTITY" ]]; then + preserve_file_by_exclusive_custody \ + "$ROLLBACK_MIGRATION_SOURCE" \ + "$ROLLBACK_MIGRATION_PLIST_SHA" \ + "$RECOVERY_RESTORED_MIGRATION_IDENTITY" || restore_failed=1 + else + printf 'Preserved unrelated migration source at %s\n' "$ROLLBACK_MIGRATION_SOURCE" >&2 + restore_failed=1 + fi + fi + + state="$(job_loaded_state "$job_domain")" + if [[ "$state" == "loaded" ]]; then + launchctl bootout "$job_domain" >/dev/null 2>&1 || restore_failed=1 + elif [[ "$state" == "unknown" ]]; then + restore_failed=1 + fi + wait_for_app_binary_exit || return 1 + [[ "$(job_loaded_state "$job_domain")" == "absent" ]] || return 1 + if [[ -n "$ROLLBACK_MIGRATION_LABEL" ]]; then + [[ "$(job_loaded_state "$launch_domain/$ROLLBACK_MIGRATION_LABEL")" == "absent" ]] || return 1 + fi + + if [[ "$unsafe_current_elevation" == "0" && + ( "$RECOVERY_CURRENT_APP_STATE" == "valid" || + "$RECOVERY_CURRENT_APP_STATE" == "damaged" ) ]] + then + if [[ -n "$RECOVERED_FAILED_APP_PATH" && -d "$RECOVERED_FAILED_APP_PATH" && + ! -L "$RECOVERED_FAILED_APP_PATH" ]] && + ! elevation_app_is_cua_free "$RECOVERED_FAILED_APP_PATH" + then + unsafe_current_elevation=1 + current_app_evidence_path="$RECOVERED_FAILED_APP_PATH" + elif [[ -d "$APP_PATH" && ! -L "$APP_PATH" ]] && ! elevation_app_is_cua_free "$APP_PATH"; then + unsafe_current_elevation=1 + if preserve_current_app_for_recovery 'unsafe current elevation app'; then + current_app_evidence_path="$RECOVERED_FAILED_APP_PATH" + elif [[ -n "$RECOVERED_FAILED_APP_PATH" ]] && + verify_recorded_current_app "$RECOVERED_FAILED_APP_PATH" + then + current_app_evidence_path="$RECOVERED_FAILED_APP_PATH" + app_restore_failed=1 + else + current_app_evidence_path="$APP_PATH" + app_restore_failed=1 + fi + fi + fi + + if [[ "$unsafe_current_elevation" == "1" && "$app_restore_failed" != "0" ]]; then + : # Preserve the unsafe classification and skip every normal restoration path. + elif [[ "$RECOVERY_CURRENT_APP_STATE" == "absent" ]]; then + if [[ ! -e "$APP_PATH" && ! -L "$APP_PATH" ]]; then + [[ -z "$ROLLBACK_APP_PATH" ]] || + verify_recorded_rollback_app "$ROLLBACK_APP_PATH" || app_restore_failed=1 + elif [[ ! -e "$ROLLBACK_APP_PATH" && ! -L "$ROLLBACK_APP_PATH" ]] && + verify_recorded_rollback_app "$APP_PATH" + then + rename_app_exclusively "$APP_PATH" "$ROLLBACK_APP_PATH" || true + verify_recorded_rollback_app "$ROLLBACK_APP_PATH" || app_restore_failed=1 + [[ ! -e "$APP_PATH" && ! -L "$APP_PATH" ]] || app_restore_failed=1 + else + app_restore_failed=1 + fi + elif [[ -n "$RECOVERED_FAILED_APP_PATH" ]]; then + if [[ -e "$ROLLBACK_APP_PATH" || -L "$ROLLBACK_APP_PATH" ]]; then + app_restore_failed=1 + elif [[ -d "$APP_PATH" && ! -L "$APP_PATH" ]]; then + rename_app_exclusively "$APP_PATH" "$ROLLBACK_APP_PATH" || true + verify_recorded_rollback_app "$ROLLBACK_APP_PATH" || app_restore_failed=1 + [[ ! -e "$APP_PATH" && ! -L "$APP_PATH" ]] || app_restore_failed=1 + fi + if [[ "$app_restore_failed" == "0" && -d "$RECOVERED_FAILED_APP_PATH" && + ! -L "$RECOVERED_FAILED_APP_PATH" ]] && + path_matches_identity "$RECOVERED_FAILED_APP_PATH" "$RECOVERY_CURRENT_APP_IDENTITY" + then + if [[ "$unsafe_current_elevation" == "1" ]]; then + verify_recorded_current_app "$RECOVERED_FAILED_APP_PATH" || app_restore_failed=1 + [[ ! -e "$APP_PATH" && ! -L "$APP_PATH" ]] || app_restore_failed=1 + else + rename_app_exclusively "$RECOVERED_FAILED_APP_PATH" "$APP_PATH" || true + path_matches_identity "$APP_PATH" "$RECOVERY_CURRENT_APP_IDENTITY" || app_restore_failed=1 + if [[ "$app_restore_failed" == "0" && "$RECOVERY_CURRENT_APP_STATE" == "valid" ]]; then + verify_recorded_current_app "$APP_PATH" || app_restore_failed=1 + fi + [[ ! -e "$RECOVERED_FAILED_APP_PATH" && ! -L "$RECOVERED_FAILED_APP_PATH" ]] || + app_restore_failed=1 + rmdir "$(dirname "$RECOVERED_FAILED_APP_PATH")" 2>/dev/null || true + fi + else + app_restore_failed=1 + fi + fi + case "$RECOVERY_CURRENT_APP_STATE" in + absent) [[ ! -e "$APP_PATH" && ! -L "$APP_PATH" ]] || app_restore_failed=1 ;; + damaged) + if [[ "$unsafe_current_elevation" == "1" ]]; then + [[ -n "$current_app_evidence_path" && ! -e "$APP_PATH" && ! -L "$APP_PATH" ]] && + path_matches_identity "$current_app_evidence_path" "$RECOVERY_CURRENT_APP_IDENTITY" && + verify_recorded_current_app "$current_app_evidence_path" || app_restore_failed=1 + else + [[ -d "$APP_PATH" && ! -L "$APP_PATH" ]] && + path_matches_identity "$APP_PATH" "$RECOVERY_CURRENT_APP_IDENTITY" || app_restore_failed=1 + fi + ;; + valid) + if [[ "$unsafe_current_elevation" == "1" ]]; then + [[ -n "$current_app_evidence_path" && ! -e "$APP_PATH" && ! -L "$APP_PATH" ]] && + path_matches_identity "$current_app_evidence_path" "$RECOVERY_CURRENT_APP_IDENTITY" && + verify_recorded_current_app "$current_app_evidence_path" || app_restore_failed=1 + else + path_matches_identity "$APP_PATH" "$RECOVERY_CURRENT_APP_IDENTITY" && + verify_recorded_current_app "$APP_PATH" || app_restore_failed=1 + fi + ;; + *) app_restore_failed=1 ;; + esac + if [[ "$app_restore_failed" != "0" && "$unsafe_current_elevation" == "0" ]]; then + return 1 + fi + + if [[ "$unsafe_current_elevation" == "1" ]]; then + if [[ -n "$RECOVERY_CURRENT_PLIST" ]]; then + neutralize_unsafe_elevation_launch_agent \ + 'replacement for unsafe current' \ + "$RECOVERY_CURRENT_PLIST" \ + "$RECOVERY_CURRENT_PLIST_SHA" || restore_failed=1 + printf 'Preserved current elevation app with bundled CUA driver at %s and LaunchAgent evidence at %s; refusing to restore it as elevation host\n' \ + "$current_app_evidence_path" "$RECOVERY_CURRENT_PLIST" >&2 + else + neutralize_unsafe_elevation_launch_agent \ + 'replacement for unsafe current' \ + '' \ + '' || restore_failed=1 + printf 'Preserved current elevation app with bundled CUA driver at %s; no current elevation LaunchAgent was recorded, and it will not be restored as elevation host\n' \ + "$current_app_evidence_path" >&2 + fi + restore_failed=1 + elif [[ -n "$RECOVERY_CURRENT_PLIST" ]]; then + restore_file_atomically \ + "$RECOVERY_CURRENT_PLIST" \ + "$PLIST_PATH" \ + "$RECOVERY_CURRENT_PLIST_SHA" \ + 644 || restore_failed=1 + elif [[ -e "$PLIST_PATH" || -L "$PLIST_PATH" ]]; then + rm -f "$PLIST_PATH" || restore_failed=1 + fi + + restore_file_atomically \ + "$RECOVERY_CURRENT_RECEIPT" \ + "$RECEIPT_PATH" \ + "$RECOVERY_CURRENT_RECEIPT_SHA" \ + 600 || restore_failed=1 + + if [[ "$restore_failed" == "0" && "$RECOVERY_CURRENT_PLIST_WAS_LOADED" == "1" ]]; then + launchctl bootstrap "$launch_domain" "$PLIST_PATH" >/dev/null 2>&1 || restore_failed=1 + fi + if [[ -n "$ROLLBACK_MIGRATION_SOURCE" && + (-e "$ROLLBACK_MIGRATION_SOURCE" || -L "$ROLLBACK_MIGRATION_SOURCE") ]] + then + printf 'Preserved unexpected migration source at %s\n' "$ROLLBACK_MIGRATION_SOURCE" >&2 + restore_failed=1 + fi + if [[ "$restore_failed" == "0" ]]; then + CUTOVER_ACTIVE=0 + CUTOVER_APP_MUTATED=0 + CUTOVER_MIGRATION_REMOVED=0 + CUTOVER_ADOPTION_STOPPED=0 + CUTOVER_ADOPTION_TERMINATION_SENT=0 + return 0 + fi + return 1 +} + +status_host() { + [[ ! -e "$PENDING_RECEIPT_PATH" && ! -L "$PENDING_RECEIPT_PATH" ]] || + fail 'an incomplete elevation install transaction exists; run recover' + ensure_no_normal_owner + verify_elevation_app "$APP_PATH" + verify_install_receipt + require_committed_install_receipt + [[ -f "$PLIST_PATH" ]] || fail "elevation launch agent is not installed: $PLIST_PATH" + local args loaded_pid plist_config + args="$(plutil -extract ProgramArguments json -o - "$PLIST_PATH")" + [[ "$(jq -c . <<<"$args")" == "$(jq -cn --arg executable "$APP_PATH/Contents/MacOS/OpenClaw" '[$executable,"--elevation-host"]')" ]] || + fail 'elevation launch agent arguments are not canonical' + [[ "$(plutil -extract RunAtLoad raw -o - "$PLIST_PATH")" == 'true' ]] || fail 'RunAtLoad is not enabled' + [[ "$(plutil -extract KeepAlive raw -o - "$PLIST_PATH")" == 'true' ]] || fail 'KeepAlive is not enabled' + [[ "$(plist_file_value "$PLIST_PATH" EnvironmentVariables.OPENCLAW_STATE_DIR)" == "$STATE_DIR" ]] || + fail 'elevation launch agent state directory is not canonical' + plist_config="$(plist_file_value "$PLIST_PATH" EnvironmentVariables.OPENCLAW_CONFIG_PATH)" + if [[ "$INSTALL_RECEIPT_SCHEMA" == 'legacy' ]]; then + CONFIG_PATH="${plist_config:-$STATE_DIR/openclaw.json}" + else + CONFIG_PATH="$(jq -r '.configPath' "$RECEIPT_PATH")" + [[ "$plist_config" == "$CONFIG_PATH" ]] || fail 'elevation launch agent config path is not canonical' + fi + loaded_pid="$(job_pid)" + [[ "$loaded_pid" =~ ^[0-9]+$ ]] || fail 'elevation launch agent is not running' + verify_bridge_readiness "$loaded_pid" || fail 'elevation Bridge is not ready for the launchd-owned process' + resolve_reusable_openclaw_cli + if [[ "$INSTALL_RECEIPT_SCHEMA" == 'legacy' ]]; then + prepare_gateway_attestation + else + EXPECTED_NODE_ID="$(jq -r '.nodeId' "$RECEIPT_PATH")" + EXPECTED_NODE_PROFILE="$(jq -r '.nodeProfile' "$RECEIPT_PATH")" + fi + BEFORE_NODE_CONNECTED_AT=-1 + verify_gateway_node_readiness "$(plist_value "$APP_PATH" CFBundleShortVersionString)" || + fail 'elevation macOS computer-use node is not ready on the configured gateway' + printf 'Elevation host ready: pid=%s source=%s\n' "$loaded_pid" "$(plist_value "$APP_PATH" OpenClawGitCommit)" + tcc_summary || return $? +} + +select_recovery_receipt() { + RECEIPT_PATH="$FINAL_RECEIPT_PATH" + RECOVERY_PENDING_INSTALL=0 + [[ -e "$PENDING_RECEIPT_PATH" || -L "$PENDING_RECEIPT_PATH" ]] || return 0 + [[ -f "$PENDING_RECEIPT_PATH" && ! -L "$PENDING_RECEIPT_PATH" ]] || + fail 'pending elevation install receipt is not a regular file' + local current_source final_transaction_id pending_source pending_transaction_id + current_source="$(plist_value "$APP_PATH" OpenClawGitCommit)" + pending_source="$(jq -r '.sourceCommit // empty' "$PENDING_RECEIPT_PATH" 2>/dev/null || true)" + pending_transaction_id="$(jq -r '.transactionId // empty' "$PENDING_RECEIPT_PATH" 2>/dev/null || true)" + final_transaction_id="$(jq -r '.transactionId // empty' "$FINAL_RECEIPT_PATH" 2>/dev/null || true)" + if [[ "$pending_source" =~ ^[0-9a-f]{40}$ && "$pending_source" == "$current_source" && + "$pending_transaction_id" =~ ^[0-9A-F-]{36}$ && + "$pending_transaction_id" == "$final_transaction_id" && + -f "$FINAL_RECEIPT_PATH" && ! -L "$FINAL_RECEIPT_PATH" ]] && + (verify_install_receipt 1 && require_committed_install_receipt) >/dev/null 2>&1 + then + PENDING_RECEIPT_RETIRE_ID="$pending_transaction_id" + RECEIPT_PATH="$FINAL_RECEIPT_PATH" + return 0 + fi + RECEIPT_PATH="$PENDING_RECEIPT_PATH" + RECOVERY_PENDING_INSTALL=1 +} + +recover_host() { + local current_app_valid=0 current_app_matches_receipt=0 current_receipt_sha plan_value plan_state plan_identity + local migration_identity recovery_helper_app + local pending_migration_identity_needs_record=0 + if [[ ! -e "$APP_PATH" && ! -L "$APP_PATH" ]]; then + RECOVERY_CURRENT_APP_STATE="absent" + elif [[ -L "$APP_PATH" || ! -d "$APP_PATH" ]]; then + fail 'current OpenClaw app has an unsupported entry type; inspect it before recovery' + else + RECOVERY_CURRENT_APP_IDENTITY="$(durable_path_identity "$APP_PATH")" || + fail 'current OpenClaw app identity could not be inspected before recovery' + if (verify_elevation_app "$APP_PATH") >/dev/null 2>&1; then + RECOVERY_CURRENT_APP_STATE="valid" + current_app_valid=1 + else + RECOVERY_CURRENT_APP_STATE="damaged" + fi + path_matches_identity "$APP_PATH" "$RECOVERY_CURRENT_APP_IDENTITY" || + fail 'current OpenClaw app changed during recovery planning' + fi + select_recovery_receipt + verify_install_receipt 0 + if [[ "$RECOVERY_PENDING_INSTALL" == "1" ]]; then + [[ "$INSTALL_RECEIPT_TRANSACTION_STATE" == "installing" ]] || + fail 'pending elevation receipt is not an install transaction' + else + require_committed_install_receipt + fi + if [[ "$current_app_valid" == "1" ]] && + (verify_install_receipt 1) >/dev/null 2>&1 + then + current_app_matches_receipt=1 + fi + ROLLBACK_APP_PATH="$(jq -r '.backupPath // empty' "$RECEIPT_PATH")" + ROLLBACK_APP_CDHASH_ARM64="$(jq -r '.backupCDHashes.arm64 // empty' "$RECEIPT_PATH")" + ROLLBACK_APP_CDHASH_X86_64="$(jq -r '.backupCDHashes.x86_64 // empty' "$RECEIPT_PATH")" + ROLLBACK_ELEVATION_PLIST="$(jq -r '.previousPlist // empty' "$RECEIPT_PATH")" + ROLLBACK_ELEVATION_PLIST_SHA="$(jq -r '.previousPlistSha256 // empty' "$RECEIPT_PATH")" + ROLLBACK_ELEVATION_WAS_LOADED="$(jq -r 'if .previousPlistWasLoaded then 1 else 0 end' "$RECEIPT_PATH")" + ROLLBACK_INSTALL_RECEIPT="$(jq -r '.previousReceipt // empty' "$RECEIPT_PATH")" + ROLLBACK_INSTALL_RECEIPT_SHA="$(jq -r '.previousReceiptSha256 // empty' "$RECEIPT_PATH")" + ROLLBACK_FAILED_SOURCE="$(jq -r '.sourceCommit' "$RECEIPT_PATH")" + ROLLBACK_MIGRATION_SOURCE="$(jq -r '.migration.sourcePlist // empty' "$RECEIPT_PATH")" + ROLLBACK_MIGRATION_PLIST="$(jq -r '.migration.backupPlist // empty' "$RECEIPT_PATH")" + ROLLBACK_MIGRATION_PLIST_SHA="$(jq -r '.migration.backupSha256 // empty' "$RECEIPT_PATH")" + MIGRATION_KIND="$(jq -r '.migration.kind // empty' "$RECEIPT_PATH")" + ROLLBACK_MIGRATION_LABEL="$(jq -r '.migration.label // empty' "$RECEIPT_PATH")" + ROLLBACK_MIGRATION_WAS_LOADED="$(jq -r 'if .migration.wasLoaded then 1 else 0 end' "$RECEIPT_PATH")" + MIGRATION_CUSTODY_PATH="$(jq -r '.migration.custodyPath // empty' "$RECEIPT_PATH")" + MIGRATION_CUSTODY_IDENTITY="$(jq -r '.migration.sourceIdentity // empty' "$RECEIPT_PATH")" + MIGRATION_NODE_ENV_PATH="$(jq -r '.migration.nodeEnvPath // empty' "$RECEIPT_PATH")" + MIGRATION_NODE_ENV_SHA="$(jq -r '.migration.nodeEnvSha256 // empty' "$RECEIPT_PATH")" + MIGRATION_NODE_ENV_IDENTITY="$(jq -r '.migration.nodeEnvIdentity // empty' "$RECEIPT_PATH")" + MIGRATION_NODE_WRAPPER_PATH="$(jq -r '.migration.nodeWrapperPath // empty' "$RECEIPT_PATH")" + MIGRATION_NODE_WRAPPER_SHA="$(jq -r '.migration.nodeWrapperSha256 // empty' "$RECEIPT_PATH")" + MIGRATION_NODE_WRAPPER_IDENTITY="$(jq -r '.migration.nodeWrapperIdentity // empty' "$RECEIPT_PATH")" + ROLLBACK_ADOPTED_APP_WAS_RUNNING="$(jq -r 'if .adoptedApp.wasRunning then 1 else 0 end' "$RECEIPT_PATH")" + ROLLBACK_ADOPTED_APP_ATTACH_ONLY="$(jq -r 'if .adoptedApp.attachOnly then 1 else 0 end' "$RECEIPT_PATH")" + current_receipt_sha="$(shasum -a 256 "$RECEIPT_PATH" | awk '{print $1}')" + RECOVERY_FAILED_APP_PLANNED_PATH="${APP_PATH}.failed-elevation-host-${ROLLBACK_FAILED_SOURCE}.${current_receipt_sha:0:12}/OpenClaw.app" + read_optional_receipt_xattr plan_value "$RECOVERY_APP_PLAN_XATTR" || + fail 'could not inspect the recovery app transaction binding' + if [[ -n "$plan_value" ]]; then + [[ "$plan_value" =~ ^(absent[|]|valid[|][0-9A-F-]{36}:[0-9]+:[0-9]+|damaged[|][0-9A-F-]{36}:[0-9]+:[0-9]+)$ ]] || + fail 'recovery app transaction binding is invalid' + plan_state="${plan_value%%|*}" + plan_identity="${plan_value#*|}" + if [[ -e "$RECOVERY_FAILED_APP_PLANNED_PATH" || -L "$RECOVERY_FAILED_APP_PLANNED_PATH" ]]; then + if [[ "$plan_state" == "absent" || ! -d "$RECOVERY_FAILED_APP_PLANNED_PATH" || + -L "$RECOVERY_FAILED_APP_PLANNED_PATH" ]] || + ! path_matches_identity "$RECOVERY_FAILED_APP_PLANNED_PATH" "$plan_identity" + then + fail 'recovery app custody no longer matches its durable transaction binding' + fi + RECOVERY_CURRENT_APP_STATE="$plan_state" + RECOVERY_CURRENT_APP_IDENTITY="$plan_identity" + RECOVERED_FAILED_APP_PATH="$RECOVERY_FAILED_APP_PLANNED_PATH" + RECOVERY_RESUMED=1 + else + if [[ "$plan_state" == "absent" ]]; then + if [[ "$RECOVERY_CURRENT_APP_STATE" == "absent" ]]; then + : + elif [[ "$INSTALL_RECEIPT_SCHEMA" != "legacy" && -n "$ROLLBACK_APP_PATH" && + ! -e "$ROLLBACK_APP_PATH" && ! -L "$ROLLBACK_APP_PATH" ]] && + verify_recorded_rollback_app "$APP_PATH" + then + RECOVERY_RESUMED=1 + else + fail 'current app state no longer matches the durable recovery transaction' + fi + elif [[ "$RECOVERY_CURRENT_APP_STATE" != "$plan_state" ]]; then + fail 'current app state no longer matches the durable recovery transaction' + else + path_matches_identity "$APP_PATH" "$plan_identity" || + fail 'current app no longer matches the durable recovery transaction' + fi + RECOVERY_CURRENT_APP_STATE="$plan_state" + RECOVERY_CURRENT_APP_IDENTITY="$plan_identity" + fi + elif [[ "$current_app_valid" == "1" && "$RECOVERY_PENDING_INSTALL" != "1" ]]; then + [[ "$current_app_matches_receipt" == "1" ]] || + fail 'installed app source does not match the elevation install receipt' + fi + read_optional_receipt_xattr migration_identity "$RECOVERY_MIGRATION_IDENTITY_XATTR" || + fail 'could not inspect the recovery migration transaction binding' + if [[ -z "$migration_identity" && "$RECOVERY_PENDING_INSTALL" == "1" && + -n "$ROLLBACK_MIGRATION_SOURCE" ]] && + backup_file_matches "$ROLLBACK_MIGRATION_SOURCE" "$ROLLBACK_MIGRATION_PLIST_SHA" + then + migration_identity="$(durable_path_identity "$ROLLBACK_MIGRATION_SOURCE")" || + fail 'could not bind the unchanged migration source to the pending transaction' + [[ "$migration_identity" == "$MIGRATION_CUSTODY_IDENTITY" ]] || + fail 'pending migration source identity no longer matches the prepared transaction' + pending_migration_identity_needs_record=1 + fi + if [[ -n "$migration_identity" ]]; then + [[ "$migration_identity" =~ ^[0-9A-F-]{36}:[0-9]+:[0-9]+$ ]] || + fail 'recovery migration transaction binding is invalid' + RECOVERY_RESTORED_MIGRATION_IDENTITY="$migration_identity" + fi + if [[ "$INSTALL_RECEIPT_SCHEMA" != "legacy" ]]; then + RECOVERY_CURRENT_APP_CDHASH_ARM64="$(jq -r '.cdhashes.arm64' "$RECEIPT_PATH")" + RECOVERY_CURRENT_APP_CDHASH_X86_64="$(jq -r '.cdhashes.x86_64' "$RECEIPT_PATH")" + fi + if [[ "$INSTALL_RECEIPT_SCHEMA" == 'legacy' || "$RECOVERY_CURRENT_APP_STATE" != "valid" || + ("$RECOVERY_PENDING_INSTALL" == "1" && "$current_app_matches_receipt" != "1") ]] + then + [[ -n "$ARCHIVE" && -n "$ARTIFACT_RECEIPT" && -n "$EXPECTED_ARTIFACT_RECEIPT_SHA256" ]] || + fail 'recovery requires the authenticated elevation archive, receipt, and receipt digest when the current app cannot supply a trusted rename helper' + prepare_authenticated_artifact_inputs "$ARTIFACT_RECEIPT" "$ARCHIVE" "${BASH_SOURCE[0]}" + extract_verified_artifact "${BASH_SOURCE[0]}" recovery_helper_app + if [[ "$INSTALL_RECEIPT_SCHEMA" == 'legacy' ]]; then + CONFIG_PATH="$STATE_DIR/openclaw.json" + else + CONFIG_PATH="$(jq -r '.configPath' "$RECEIPT_PATH")" + fi + if [[ "$RECOVERY_PENDING_INSTALL" == "1" && "$current_app_valid" == "1" && + "$current_app_matches_receipt" != "1" ]] + then + RECOVERY_CURRENT_APP_CDHASH_ARM64="$(codesign_value_for_arch "$APP_PATH" CDHash arm64)" + RECOVERY_CURRENT_APP_CDHASH_X86_64="$(codesign_value_for_arch "$APP_PATH" CDHash x86_64)" + fi + else + [[ -z "$ARCHIVE" && -z "$ARTIFACT_RECEIPT" && -z "$EXPECTED_ARTIFACT_RECEIPT_SHA256" ]] || + fail 'artifact helper inputs are valid only for legacy recovery' + CONFIG_PATH="$(jq -r '.configPath' "$RECEIPT_PATH")" + prepare_current_app_rename_helper + fi + if [[ "$pending_migration_identity_needs_record" == "1" ]]; then + record_recovery_migration_identity "$migration_identity" || + fail 'could not persist the pending migration source identity' + fi + if [[ -n "$PENDING_RECEIPT_RETIRE_ID" ]]; then + remove_pending_receipt "$PENDING_RECEIPT_RETIRE_ID" || + fail 'could not remove a completed pending install receipt' + PENDING_RECEIPT_RETIRE_ID="" + fi + if [[ -n "$ROLLBACK_APP_PATH" ]]; then + local backup_generation="${ROLLBACK_APP_PATH#"$APP_PATH.rollback-elevation-host-"}" + if [[ "$INSTALL_RECEIPT_SCHEMA" == 'legacy' ]]; then + [[ "$backup_generation" =~ ^[0-9a-f]{40}$ ]] || + fail 'legacy receipt app backup path is outside the canonical rollback namespace' + else + # Schema 3 first ships with generation-unique custody; earlier numeric schemas were + # confined to this unmerged branch and are intentionally not compatibility formats. + [[ "$backup_generation" =~ ^[0-9a-f]{40}[.][A-Za-z0-9]{6}$ ]] || + fail 'receipt app backup path is outside the canonical rollback namespace' + fi + if [[ "$INSTALL_RECEIPT_SCHEMA" == 'legacy' ]]; then + [[ -d "$ROLLBACK_APP_PATH" && ! -L "$ROLLBACK_APP_PATH" ]] || + fail 'legacy recovery cannot resume after its unauthenticated backup path moved' + ROLLBACK_APP_CDHASH_ARM64="$(codesign_value_for_arch "$ROLLBACK_APP_PATH" CDHash arm64)" + ROLLBACK_APP_CDHASH_X86_64="$(codesign_value_for_arch "$ROLLBACK_APP_PATH" CDHash x86_64)" + fi + [[ -n "$ROLLBACK_APP_CDHASH_ARM64" && -n "$ROLLBACK_APP_CDHASH_X86_64" ]] || + fail 'receipt has no recoverable per-architecture app CDHashes' + if [[ -e "$ROLLBACK_APP_PATH" || -L "$ROLLBACK_APP_PATH" ]]; then + if [[ ! -d "$ROLLBACK_APP_PATH" || -L "$ROLLBACK_APP_PATH" ]] || + ! verify_recorded_rollback_app "$ROLLBACK_APP_PATH" + then + fail 'receipt app backup does not pass strict signature and identity validation' + fi + elif [[ "$RECOVERY_PENDING_INSTALL" == "1" ]] && + verify_recorded_rollback_app "$APP_PATH" + then + # Before custody, the prior app is authenticated by its recorded CDHashes, not + # the new elevation payload contract. It may legitimately have no bundled worker. + : + elif [[ "$RECOVERY_RESUMED" == "1" ]]; then + verify_recorded_rollback_app "$APP_PATH" || + fail 'resumed recovery has no authenticated prior app at the canonical path' + else + fail 'receipt app backup is missing, symlinked, or not a bundle directory' + fi + else + [[ -z "$ROLLBACK_APP_CDHASH_ARM64" && -z "$ROLLBACK_APP_CDHASH_X86_64" ]] || + fail 'receipt has per-architecture CDHashes without an app backup' + fi + if [[ -n "$ROLLBACK_ELEVATION_PLIST" ]]; then + state_backup_path_is_canonical \ + "$ROLLBACK_ELEVATION_PLIST" \ + '^elevation-host[.]previous-plist[.][0-9a-f]{40}[.][A-Za-z0-9]{6}$' \ + 'elevation-host.previous.plist' || + fail 'receipt elevation plist backup path is not canonical' + if [[ "$INSTALL_RECEIPT_SCHEMA" == 'legacy' ]]; then + ROLLBACK_ELEVATION_PLIST_SHA="$(shasum -a 256 "$ROLLBACK_ELEVATION_PLIST" | awk '{print $1}')" + ROLLBACK_ELEVATION_WAS_LOADED=1 + fi + [[ "$ROLLBACK_ELEVATION_PLIST_SHA" =~ ^[0-9a-f]{64}$ ]] || + fail 'receipt elevation plist backup digest is invalid' + backup_file_matches "$ROLLBACK_ELEVATION_PLIST" "$ROLLBACK_ELEVATION_PLIST_SHA" || + fail 'receipt elevation plist backup failed digest validation' + fi + if [[ -n "$ROLLBACK_INSTALL_RECEIPT" ]]; then + state_backup_path_is_canonical \ + "$ROLLBACK_INSTALL_RECEIPT" \ + '^elevation-host[.]previous-receipt[.][0-9a-f]{40}[.][A-Za-z0-9]{6}$' \ + '' || + fail 'receipt previous install receipt path is not canonical' + [[ "$ROLLBACK_INSTALL_RECEIPT_SHA" =~ ^[0-9a-f]{64}$ ]] || + fail 'receipt previous install receipt digest is invalid' + backup_file_matches "$ROLLBACK_INSTALL_RECEIPT" "$ROLLBACK_INSTALL_RECEIPT_SHA" || + fail 'receipt previous install receipt failed digest validation' + fi + if [[ -n "$ROLLBACK_MIGRATION_SOURCE" ]]; then + [[ "$(dirname "$ROLLBACK_MIGRATION_SOURCE")" == "$HOME/Library/LaunchAgents" ]] || + fail 'receipt migration source is outside the current user LaunchAgents directory' + [[ "$ROLLBACK_MIGRATION_LABEL" =~ ^[A-Za-z0-9._-]+$ ]] || fail 'receipt migration label is invalid' + [[ "$ROLLBACK_MIGRATION_LABEL" != "$ELEVATION_LABEL" && "$ROLLBACK_MIGRATION_LABEL" != "$NORMAL_LABEL" ]] || + fail 'receipt migration label targets a protected LaunchAgent' + [[ "$(basename "$ROLLBACK_MIGRATION_SOURCE")" == "${ROLLBACK_MIGRATION_LABEL}.plist" ]] || + fail 'receipt migration source filename does not match its label' + [[ "$(dirname "$MIGRATION_CUSTODY_PATH")" == "$(dirname "$ROLLBACK_MIGRATION_SOURCE")" && + "$MIGRATION_CUSTODY_PATH" == "$ROLLBACK_MIGRATION_SOURCE".custody.* ]] || + fail 'receipt migration custody path is outside the source LaunchAgent directory' + state_backup_path_is_canonical \ + "$ROLLBACK_MIGRATION_PLIST" \ + '^elevation-host[.]previous-launch-agent[.][0-9a-f]{40}[.][A-Za-z0-9]{6}$' \ + 'elevation-host.previous-launch-agent.plist' || + fail 'receipt migration plist backup path is not canonical' + [[ "$ROLLBACK_MIGRATION_PLIST_SHA" =~ ^[0-9a-f]{64}$ ]] || + fail 'receipt migration plist backup digest is invalid' + backup_file_matches "$ROLLBACK_MIGRATION_PLIST" "$ROLLBACK_MIGRATION_PLIST_SHA" || + fail 'receipt migration plist backup failed digest validation' + migration_receipt_matches_backup_plist "$ROLLBACK_MIGRATION_PLIST" || + fail 'receipt migration metadata no longer matches its authenticated LaunchAgent generation' + if [[ -e "$ROLLBACK_MIGRATION_SOURCE" || -L "$ROLLBACK_MIGRATION_SOURCE" ]]; then + if [[ ! -f "$ROLLBACK_MIGRATION_SOURCE" || -L "$ROLLBACK_MIGRATION_SOURCE" ]] || + ! path_matches_identity \ + "$ROLLBACK_MIGRATION_SOURCE" \ + "$RECOVERY_RESTORED_MIGRATION_IDENTITY" || + ! backup_file_matches "$ROLLBACK_MIGRATION_SOURCE" "$ROLLBACK_MIGRATION_PLIST_SHA" + then + fail 'could not restore the previous OpenClaw installation completely: migration source no longer matches its durable recovery transaction' + fi + RECOVERY_RESUMED=1 + fi + local recovery_migration_state + recovery_migration_state="$(job_loaded_state "$launch_domain/$ROLLBACK_MIGRATION_LABEL")" + [[ "$recovery_migration_state" == "absent" ]] || + [[ "$RECOVERY_PENDING_INSTALL" == "1" && "$ROLLBACK_MIGRATION_WAS_LOADED" == "1" && + "$recovery_migration_state" == "loaded" ]] || + [[ "$RECOVERY_RESUMED" == "1" && "$ROLLBACK_MIGRATION_WAS_LOADED" == "1" && + "$recovery_migration_state" == "loaded" ]] || + fail 'migration LaunchAgent is already loaded before recovery' + fi + local recovered_receipt receipt_restore_tmp="" current_job_state recovery_signal="" + recovered_receipt="$(mktemp "$STATE_DIR/elevation-host.recovered-receipt.${ROLLBACK_FAILED_SOURCE}.XXXXXX")" + cp -p "$RECEIPT_PATH" "$recovered_receipt" + [[ "$(shasum -a 256 "$recovered_receipt" | awk '{print $1}')" == "$current_receipt_sha" ]] || + fail 'copied recovered install receipt failed digest verification' + PREMUTATION_BACKUPS+=("$recovered_receipt") + RECOVERY_CURRENT_RECEIPT="$recovered_receipt" + RECOVERY_CURRENT_RECEIPT_SHA="$current_receipt_sha" + if [[ "$INSTALL_RECEIPT_SCHEMA" == "legacy" && "$RECOVERY_CURRENT_APP_STATE" == "valid" ]]; then + local recovery_current_app_candidate="$APP_PATH" + [[ -z "$RECOVERED_FAILED_APP_PATH" ]] || + recovery_current_app_candidate="$RECOVERED_FAILED_APP_PATH" + RECOVERY_CURRENT_APP_CDHASH_ARM64="$(codesign_value_for_arch "$recovery_current_app_candidate" CDHash arm64)" + RECOVERY_CURRENT_APP_CDHASH_X86_64="$(codesign_value_for_arch "$recovery_current_app_candidate" CDHash x86_64)" + elif [[ "$INSTALL_RECEIPT_SCHEMA" == "legacy" ]]; then + RECOVERY_CURRENT_APP_CDHASH_ARM64="" + RECOVERY_CURRENT_APP_CDHASH_X86_64="" + fi + if [[ -n "$RECOVERED_FAILED_APP_PATH" && "$RECOVERY_CURRENT_APP_STATE" == "valid" ]]; then + verify_recorded_current_app "$RECOVERED_FAILED_APP_PATH" || + fail 'resumed recovery app custody no longer matches the install receipt' + fi + current_job_state="$(job_loaded_state "$job_domain")" + case "$current_job_state" in + loaded) RECOVERY_CURRENT_PLIST_WAS_LOADED=1 ;; + absent) RECOVERY_CURRENT_PLIST_WAS_LOADED=0 ;; + *) fail 'current elevation launchd state could not be inspected before recovery' ;; + esac + if [[ -e "$PLIST_PATH" || -L "$PLIST_PATH" ]]; then + [[ -f "$PLIST_PATH" && ! -L "$PLIST_PATH" ]] || + fail 'current elevation LaunchAgent plist is not a regular file' + RECOVERY_CURRENT_PLIST="$(mktemp "$STATE_DIR/elevation-host.recovery-current-plist.${ROLLBACK_FAILED_SOURCE}.XXXXXX")" + cp -p "$PLIST_PATH" "$RECOVERY_CURRENT_PLIST" + RECOVERY_CURRENT_PLIST_SHA="$(shasum -a 256 "$RECOVERY_CURRENT_PLIST" | awk '{print $1}')" + PREMUTATION_BACKUPS+=("$RECOVERY_CURRENT_PLIST") + elif [[ "$RECOVERY_CURRENT_PLIST_WAS_LOADED" == "1" ]]; then + fail 'loaded elevation job has no plist to restore after a failed recovery' + fi + if [[ -n "$ROLLBACK_INSTALL_RECEIPT" ]]; then + receipt_restore_tmp="$(mktemp "$STATE_DIR/elevation-host.restore-receipt.${ROLLBACK_FAILED_SOURCE}.XXXXXX")" + cp -p "$ROLLBACK_INSTALL_RECEIPT" "$receipt_restore_tmp" + [[ "$(shasum -a 256 "$receipt_restore_tmp" | awk '{print $1}')" == "$ROLLBACK_INSTALL_RECEIPT_SHA" ]] || + fail 'copied previous install receipt failed digest verification' + PREMUTATION_BACKUPS+=("$receipt_restore_tmp") + fi + if [[ -z "$plan_value" ]]; then + record_recovery_app_plan || fail 'could not durably bind the recovery app transaction' + fi + CUTOVER_APP_MUTATED=1 + [[ -z "$ROLLBACK_MIGRATION_SOURCE" ]] || CUTOVER_MIGRATION_REMOVED=1 + CUTOVER_ADOPTION_STOPPED="$ROLLBACK_ADOPTED_APP_WAS_RUNNING" + CUTOVER_ADOPTION_TERMINATION_SENT="$ROLLBACK_ADOPTED_APP_WAS_RUNNING" + CUTOVER_ACTIVE=1 + CUTOVER_RECOVERY_ATTEMPTED=1 + trap 'recovery_signal=INT' INT + trap 'recovery_signal=TERM' TERM + trap 'recovery_signal=HUP' HUP + if ! recover_install; then + if [[ "$RECOVERY_RESUMED" == "1" ]]; then + finish_custody_signal_deferral "$recovery_signal" + fail 'resumed recovery remains incomplete; retry using the preserved transaction state' + fi + if restore_current_generation_after_recovery_failure; then + finish_custody_signal_deferral "$recovery_signal" + fail 'could not restore the previous OpenClaw installation completely' + fi + finish_custody_signal_deferral "$recovery_signal" + fail 'recovery failed and the current OpenClaw installation could not be restored completely' + fi + + if [[ -n "$receipt_restore_tmp" ]]; then + if ! mv "$receipt_restore_tmp" "$FINAL_RECEIPT_PATH"; then + if restore_current_generation_after_recovery_failure; then + finish_custody_signal_deferral "$recovery_signal" + fail 'could not restore the previous elevation install receipt' + fi + finish_custody_signal_deferral "$recovery_signal" + fail 'receipt restoration failed and the current OpenClaw installation could not be restored completely' + fi + if ! fsync_file_and_parent "$FINAL_RECEIPT_PATH"; then + CUTOVER_COMMITTED=1 + CUTOVER_ACTIVE=0 + finish_custody_signal_deferral "$recovery_signal" + fail 'previous receipt was restored but its durability sync failed; inspect status before retrying' + fi + elif [[ -e "$FINAL_RECEIPT_PATH" || -L "$FINAL_RECEIPT_PATH" ]]; then + if ! rm "$FINAL_RECEIPT_PATH"; then + if restore_current_generation_after_recovery_failure; then + finish_custody_signal_deferral "$recovery_signal" + fail 'could not remove the replaced elevation install receipt' + fi + finish_custody_signal_deferral "$recovery_signal" + fail 'receipt removal failed and the current OpenClaw installation could not be restored completely' + fi + if ! fsync_parent "$FINAL_RECEIPT_PATH"; then + CUTOVER_COMMITTED=1 + CUTOVER_ACTIVE=0 + finish_custody_signal_deferral "$recovery_signal" + fail 'replaced receipt was removed but its durability sync failed; inspect status before retrying' + fi + fi + if [[ "$RECOVERY_PENDING_INSTALL" == "1" ]]; then + remove_pending_receipt || { + finish_custody_signal_deferral "$recovery_signal" + fail 'could not remove the recovered install transaction receipt' + } + fi + CUTOVER_COMMITTED=1 + CUTOVER_ACTIVE=0 + finish_custody_signal_deferral "$recovery_signal" + printf 'Recovered previous OpenClaw app from %s; replaced app preserved at %s; receipt preserved at %s\n' \ + "$ROLLBACK_APP_PATH" "$RECOVERED_FAILED_APP_PATH" "$recovered_receipt" +} + +uninstall_host() { + launchctl bootout "$job_domain" >/dev/null 2>&1 || true + if [[ -f "$PLIST_PATH" ]]; then rm -f "$PLIST_PATH"; fi + printf 'Elevation launch agent removed; app, state, TCC, Keychain, and recovery receipt preserved.\n' + if [[ -f "$RECEIPT_PATH" && "$(jq -r '.migration.sourcePlist // empty' "$RECEIPT_PATH" 2>/dev/null)" != "" ]]; then + printf 'A migrated LaunchAgent remains preserved in the receipt; run recover to restore it with the prior app.\n' + fi +} + +refresh_runtime_paths +resolve_migration_inputs +resolve_adoption_inputs +resolve_managed_upgrade_inputs +if [[ "$COMMAND" == "install" || "$COMMAND" == "migration-plan" ]]; then + [[ -n "$CONFIG_PATH" ]] || CONFIG_PATH="$STATE_DIR/openclaw.json" + refresh_runtime_paths + if [[ "${#OPENCLAW_CLI[@]}" == "0" ]]; then + resolve_reusable_openclaw_cli + fi + prepare_gateway_attestation +fi + +case "$COMMAND" in + package) package_host ;; + verify) verify_artifact_set ;; + install) install_host ;; + migration-plan) print_migration_plan ;; + status) status_host ;; + recover) recover_host ;; + uninstall) uninstall_host ;; + print-plist) + WORK_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/openclaw-elevation-plist.XXXXXX")" + render_plist "$WORK_ROOT/agent.plist" + cat "$WORK_ROOT/agent.plist" + ;; +esac diff --git a/scripts/macos-health-render.sb b/scripts/macos-health-render.sb new file mode 100644 index 0000000000000000000000000000000000000000..847631af11478cca4cbb1b4766f35a09bdc937da --- /dev/null +++ b/scripts/macos-health-render.sb @@ -0,0 +1,18 @@ +(version 1) +(allow default) +(deny network*) +(deny file-write*) +(deny file-read* (subpath (param "ORIGINAL_HOME"))) +(allow file-read* (subpath (param "REPO"))) +(allow file-read* file-write* (subpath (param "ISOLATED_ROOT"))) +(allow file-write* (literal "/dev/null")) +; A redirected HOME does not isolate daemon-backed preferences or Keychain IPC. +(deny mach-lookup + (global-name "com.apple.cfprefsd.daemon") + (global-name "com.apple.cfprefsd.agent") + (local-name "com.apple.cfprefsd.agent") + (global-name "com.apple.SecurityServer") + (global-name "com.apple.securityd") + (global-name "com.apple.securityd.xpc")) +(deny ipc-posix-shm-read* (ipc-posix-name-prefix "apple.cfprefs.")) +(deny file-read* file-write* (subpath "/Library/Keychains")) diff --git a/scripts/materialize-mac-node-worker.py b/scripts/materialize-mac-node-worker.py new file mode 100644 index 0000000000000000000000000000000000000000..3ae262686711a32d2a0f1cb6bb75fb810bfefc1c --- /dev/null +++ b/scripts/materialize-mac-node-worker.py @@ -0,0 +1,262 @@ +#!/usr/bin/python3 +"""Construct a fresh target-native worker without pruning the installed package.""" +from contextlib import ExitStack +import importlib.util +import json +import os +import re +import shutil +import stat +import subprocess +import sys + +spec = importlib.util.spec_from_file_location( + "mac_native_inventory", os.path.join(os.path.dirname(__file__), "lib/mac-native-inventory.py") +) +native = importlib.util.module_from_spec(spec) +spec.loader.exec_module(native) + +# PE/COFF machine identifiers (winnt.h), including bigobj/import-object headers; +# XCOFF is big endian. Weak prefix hits must not override file's positive text verdict. +COFF_MAGICS = {value.to_bytes(2, "little") for value in ( + 0x14c, 0x166, 0x169, 0x184, 0x1a2, 0x1a3, 0x1a6, 0x1a8, + 0x1c0, 0x1c2, 0x1c4, 0x1d3, 0x1f0, 0x1f1, 0x200, 0x266, + 0x366, 0x466, 0x5032, 0x5064, 0x5128, 0x6232, 0x6264, + 0x8664, 0x9041, 0xa641, 0xa64e, 0xaa64, 0xebc, +)} | {b"\x01\xdf", b"\x01\xf7"} + + +def candidate(header): + return (header[:4] in native.MACHO_MAGICS or header[:4] in (b"\x7fELF", b"\0\0\xff\xff") + or header[:2] == b"MZ" or header[:2] in COFF_MAGICS + or header[:8] in (b"!<arch>\n", b"!<thin>\n")) + + +def classify(batch, architecture): + # Inherited /dev/fd names bind tools to the same objects later copied. + # Darwin dup/dev-fd readers share cursors: rewind before EVERY operation. + fds = tuple(stream.fileno() for _, stream, _ in batch) + for _, stream, _ in batch: + stream.seek(0) + result = subprocess.run( + ["/usr/bin/file", "-L", "-E", "-b", "-0", "-0", "--", + *(f"/dev/fd/{fd}" for fd in fds)], + pass_fds=fds, stdout=subprocess.PIPE, check=True, + ) + descriptions = result.stdout.split(b"\0") + if len(descriptions) != len(batch) + 1 or descriptions[-1] != b"": + raise ValueError("Incomplete worker file classification") + for (entry, stream, header), description in zip(batch, descriptions): + description = description.split(b"\n", 1)[0] + if not description or re.search(rb"ERROR|cannot (?:read|open)", description, re.I): + raise ValueError(f"Invalid worker file classification: {entry.parts!r}") + reason = None + if header[:4] == b"\xca\xfe\xba\xbe" and description.startswith(b"compiled Java class"): + pass + elif header[:4] in native.MACHO_MAGICS or header[:8] in (b"!<arch>\n", b"!<thin>\n"): + if not (description.startswith(b"Mach-O") or description == b"data" or b"ar archive" in description): + raise ValueError(f"Unclassified worker native header: {entry.parts!r}") + stream.seek(0) + fd_path = f"/dev/fd/{stream.fileno()}" + info = subprocess.run( + ["/usr/bin/lipo", "-info", fd_path], + pass_fds=(stream.fileno(),), stdout=subprocess.PIPE, check=True, + ).stdout + # -archs can concatenate fat resource slices; -info separates them. + # Bind its complete, single-line result to the requested descriptor. + fd_pattern = re.escape(fd_path.encode()) + match = re.fullmatch( + rb"(?:Architectures in the fat file: " + fd_pattern + + rb" are: ([a-zA-Z0-9_]+(?: [a-zA-Z0-9_]+)*) ?|Non-fat file: " + + fd_pattern + rb" is architecture: ([a-zA-Z0-9_]+))\n", info, + ) + # lipo can report an unknown CPU with exit 0; it is not an omission verdict. + if not match or b"unknown" in info: + raise ValueError(f"Uncertain worker native slices: {entry.parts!r}") + slices = match[1] or match[2] + if architecture.encode() not in slices.split(): + reason = f"lacks {architecture} ({slices.decode()})" + elif re.search(rb"^(?:ELF|PE32|MS-DOS executable)|\b(?:COFF|XCOFF)\b", description): + minimum = (64 if header[4:5] == b"\x02" else 52) if header[:4] == b"\x7fELF" else (64 if header[:2] == b"MZ" else 20) + if len(header) < minimum or re.search(rb"invalid|corrupt|truncated|unknown|missing", description, re.I): + raise ValueError(f"Malformed worker native image: {entry.parts!r}") + reason = f"not Darwin ({description.decode(errors='replace')})" + # file trims trailing NULs and guesses unknown 8-bit encodings as text. + # Neither proves that a binary-prefix candidate is an ordinary resource. + elif b"\0" in header or not re.search( + rb"(?:^|, )(?:ASCII|Unicode|ISO-8859|(?:International )?EBCDIC) text\b", description + ): + raise ValueError(f"Unclassified worker native header: {entry.parts!r}") + yield entry, stream, reason + + +def validate_links(entries, tree): + """Resolve filesystem-equivalent retained targets, then audit every target edge.""" + children = {parts: set() for parts in entries} + for parts in entries: + if parts: + children[parts[:-1]].add(parts) + targets = {} + resolving = set() + + def lookup(parent, name): + exact = (*parent, name) + if exact in entries: + return {exact} + observed = native.content_identity(tree.lstat_child(parent, name)) + # Existing same-parent hardlinks have identical metadata, but copying + # splits their output inodes. Retain every equivalent leaf, not one spelling. + matches = {child for child in children[parent] + if native.content_identity(entries[child].info) == observed} + if not matches: + raise ValueError(f"Worker symlink has no retained target: {exact!r}") + return matches + + def resolve_link(parts): + if parts in resolving: + raise ValueError(f"Cyclic worker symlink: {parts!r}") + if parts in targets: + return targets[parts] + resolving.add(parts) + target = entries[parts].target + if os.path.isabs(target): + raise ValueError(f"Worker symlink escapes input: {parts!r}") + resolved = {parts[:-1]} + for component in target.split("/"): + following = set() + for parent in resolved: + if not isinstance(entries[parent], native.NativeInventoryDirectory): + raise ValueError(f"Worker symlink traverses a non-directory: {parts!r}") + if component == "..": + if not parent: + raise ValueError(f"Worker symlink escapes input: {parts!r}") + following.add(parent[:-1]) + elif component in ("", "."): + following.add(parent) + else: + for child in lookup(parent, component): + following.update(resolve_link(child) if isinstance( + entries[child], native.NativeInventorySymlink) else {child}) + resolved = following + resolving.remove(parts) + targets[parts] = resolved + return resolved + + for parts, entry in entries.items(): + if isinstance(entry, native.NativeInventorySymlink): + resolve_link(parts) + visiting, visited = set(), set() + + def visit(parts): + if parts in visiting: + raise ValueError(f"Cyclic worker tree: {parts!r}") + if parts in visited: + return + visiting.add(parts) + for child in targets.get(parts, children[parts]): + visit(child) + visiting.remove(parts) + visited.add(parts) + + visit(()) + return targets + + +def materialize(source, destination, parent, architecture): + source = os.path.abspath(source) + source = os.path.join(os.path.realpath(os.path.dirname(source)), os.path.basename(source)) + parent_path = os.path.abspath(parent) + parent = os.path.realpath(parent_path) + destination = os.path.join(os.path.realpath(os.path.dirname(os.path.abspath(destination))), + os.path.basename(destination)) + if (not stat.S_ISDIR(os.lstat(parent_path).st_mode) + or os.path.dirname(destination) != parent + or os.path.commonpath((source, destination)) in (source, destination)): + raise ValueError("Worker output must be disjoint from input and directly inside its staging parent") + + entries, directories, batch = {}, [], [] + retained_files = omitted = 0 + + def copy(entry, stream): + nonlocal retained_files + stream.seek(0) + with open(os.path.join(destination, *entry.parts), "xb") as output: + shutil.copyfileobj(stream, output) + if output.tell() != entry.info.st_size: + raise ValueError(f"Short worker copy: {entry.parts!r}") + # Darwin writes clear setuid; drain buffered bytes before restoring mode. + output.flush() + os.fchmod(output.fileno(), stat.S_IMODE(entry.info.st_mode)) + retained_files += 1 + + def flush(): + nonlocal omitted + for entry, stream, reason in classify(batch, architecture) if batch else (): + if reason is None: + copy(entry, stream) + else: + del entries[entry.parts] + omitted += 1 + if omitted <= 40: + print(f"Omitting native {json.dumps('/'.join(entry.parts))}: {reason[:240]}", file=sys.stderr) + handles.close() + batch.clear() + + # The build owns the private staging parent exclusively until return. Source + # substitution is untrusted; concurrent writers to output/parent are not supported. + # mkdir claims only a fresh output; cleanup never removes an existing occupant. + os.mkdir(destination, 0o700) + directories.append(destination) + try: + with ExitStack() as handles, native.open_native_inventory_tree(source) as tree: + for entry in tree.entries(): + entries[entry.parts] = entry + output = os.path.join(destination, *entry.parts) + if isinstance(entry, native.NativeInventoryDirectory): + if entry.parts: + os.mkdir(output, 0o700) + directories.append(output) + elif isinstance(entry, native.NativeInventoryFile): + header = os.pread(entry.stream.fileno(), 64, 0) + if candidate(header): + stream = handles.enter_context(os.fdopen(os.dup(entry.stream.fileno()), "rb", buffering=0)) + batch.append((entry, stream, header)) + if len(batch) == native.CLASSIFIER_BATCH_SIZE: + flush() + else: + copy(entry, entry.stream) + elif isinstance(entry, native.NativeInventorySpecial): + raise ValueError(f"Unsupported worker filesystem entry: {entry.parts!r}") + flush() + targets = validate_links(entries, tree) + for entry in entries.values(): + if isinstance(entry, native.NativeInventorySymlink): + output = os.path.join(destination, *entry.parts) + os.symlink(entry.target, output) + os.lchmod(output, stat.S_IMODE(entry.info.st_mode)) + # Source and output volumes may have different name equivalence. Check + # literal links only after all exist, while output directories are accessible. + for parts, equivalents in targets.items(): + actual = native.identity(os.stat(os.path.join(destination, *parts))) + if not any(actual == native.identity(os.lstat(os.path.join(destination, *target))) + for target in equivalents): + raise ValueError(f"Worker symlink has no equivalent output target: {parts!r}") + for entry in reversed(list(entries.values())): + if isinstance(entry, native.NativeInventoryDirectory): + os.chmod(os.path.join(destination, *entry.parts), stat.S_IMODE(entry.info.st_mode)) + tree.validate() + except BaseException: + for directory in directories: + os.chmod(directory, 0o700) + shutil.rmtree(destination) + raise + print(f"Materialized {architecture} worker: retained {retained_files} files; omitted {omitted} native images (first 40 paths logged)", file=sys.stderr) + + +if __name__ == "__main__": + try: + if sys.platform != "darwin" or len(sys.argv) != 5 or sys.argv[4] not in ("arm64", "x86_64"): + raise ValueError("Usage: materialize-mac-node-worker.py <source> <fresh-output> <staging-parent> <arm64|x86_64> (macOS only)") + materialize(*sys.argv[1:]) + except Exception as error: + sys.exit(f"[materialize-mac-node-worker] FAILED: {error}") diff --git a/scripts/materialize-vercel-cli.sh b/scripts/materialize-vercel-cli.sh new file mode 100644 index 0000000000000000000000000000000000000000..0e9026567d315c3492bf4dd4730e836e1bf84c24 --- /dev/null +++ b/scripts/materialize-vercel-cli.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source_root="${1:?trusted Vercel CLI source root is required}" +destination="${2:?Vercel CLI destination is required}" +github_output="${3:-}" + +package_json="${source_root}/package.json" +package_lock="${source_root}/package-lock.json" +expected_lock_sha256="2aac03328880441f2b423496ed52dbb638fa866c4cc6d8ba76909ec7c4fc702b" +expected_vercel_integrity="sha512-C+L/JKmlGDypKGcTU/atckydeK/AKa/7fKwUbvcwveguV1QPlY8beiIGgbwkdbb80bbIpPFHRQYrhi5XPAmCBA==" +test -f "${package_json}" +test -f "${package_lock}" +if [[ -e "${destination}" || -L "${destination}" ]]; then + echo "Vercel CLI destination must not already exist: ${destination}" >&2 + exit 1 +fi + +install -d -m 0700 "${destination}" +install -m 0600 "${package_json}" "${destination}/package.json" +install -m 0600 "${package_lock}" "${destination}/package-lock.json" + +lock_sha256="$( + VERCEL_CLI_LOCK="${package_lock}" \ + node -e "const { createHash } = require('node:crypto'); const { readFileSync } = require('node:fs'); process.stdout.write(createHash('sha256').update(readFileSync(process.env.VERCEL_CLI_LOCK)).digest('hex'));" +)" +[[ "${lock_sha256}" == "${expected_lock_sha256}" ]] || { + echo "Pinned Vercel CLI lock SHA-256 mismatch." >&2 + exit 1 +} +vercel_integrity="$( + VERCEL_CLI_LOCK="${package_lock}" \ + node -p "require(require('node:path').resolve(process.env.VERCEL_CLI_LOCK)).packages['node_modules/vercel'].integrity" +)" +[[ "${vercel_integrity}" == "${expected_vercel_integrity}" ]] || { + echo "Pinned Vercel CLI integrity mismatch." >&2 + exit 1 +} + +# Install with lifecycle scripts disabled before any credential is exposed to +# the CLI process. The committed lock fixes the complete dependency closure. +( + cd "${destination}" + npm ci \ + --ignore-scripts \ + --no-audit \ + --no-fund \ + --omit=dev +) + +vercel_version="$( + VERCEL_CLI_ROOT="${destination}" \ + node -p "require(require('node:path').join(process.env.VERCEL_CLI_ROOT, 'node_modules/vercel/package.json')).version" +)" +[[ "${vercel_version}" == "59.11.7" ]] || { + echo "Pinned Vercel CLI version mismatch: ${vercel_version}" >&2 + exit 1 +} +test -x "${destination}/node_modules/.bin/vercel" +vercel_cli="${destination}/node_modules/.bin/vercel" +# Use Sandbox directly: Vercel's sandbox wrapper overwrites failed remote exits. +test -x "${destination}/node_modules/.bin/sandbox" +sandbox_cli="${destination}/node_modules/.bin/sandbox" + +echo "Materialized vercel@${vercel_version} from lock ${lock_sha256}." +if [[ -n "${github_output}" ]]; then + { + echo "cli=${vercel_cli}" + echo "sandbox_cli=${sandbox_cli}" + echo "integrity=${vercel_integrity}" + echo "lock_sha256=${lock_sha256}" + echo "version=${vercel_version}" + } >> "${github_output}" +fi diff --git a/scripts/mobile-reauth.sh b/scripts/mobile-reauth.sh new file mode 100644 index 0000000000000000000000000000000000000000..ad7d3d05ac0b5b4097ea5abfa9b90c70cf9cfa15 --- /dev/null +++ b/scripts/mobile-reauth.sh @@ -0,0 +1,84 @@ +#!/bin/bash +# Mobile-friendly Claude Code re-authentication +# Designed for use via SSH from Termux +# +# This script handles the authentication flow in a way that works +# from a mobile device by: +# 1. Checking if auth is needed +# 2. Running claude setup-token for long-lived auth +# 3. Outputting URLs that can be easily opened on phone + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Colors +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +CYAN='\033[0;36m' +NC='\033[0m' + +echo "=== Claude Code Mobile Re-Auth ===" +echo "" + +# Check current auth status +echo "Checking auth status..." +AUTH_STATUS=$("$SCRIPT_DIR/claude-auth-status.sh" simple 2>/dev/null || echo "ERROR") + +case "$AUTH_STATUS" in + OK) + echo -e "${GREEN}Auth is valid!${NC}" + "$SCRIPT_DIR/claude-auth-status.sh" full + exit 0 + ;; + CLAUDE_EXPIRING|OPENCLAW_EXPIRING) + echo -e "${YELLOW}Auth is expiring soon.${NC}" + echo "" + ;; + *) + echo -e "${RED}Auth needs refresh.${NC}" + echo "" + ;; +esac + +echo "Starting long-lived token setup..." +echo "" +echo -e "${CYAN}Instructions:${NC}" +echo "1. Open this URL on your phone:" +echo "" +echo -e " ${CYAN}https://console.anthropic.com/settings/api-keys${NC}" +echo "" +echo "2. Sign in if needed" +echo "3. Create a new API key or use existing 'Claude Code' key" +echo "4. Copy the key (starts with sk-ant-...)" +echo "5. When prompted below, paste the key" +echo "" +echo "Press Enter when ready to continue..." +read -r + +# Run setup-token interactively +echo "" +echo "Running 'claude setup-token'..." +echo "(Follow the prompts and paste your API key when asked)" +echo "" + +if claude setup-token; then + echo "" + echo -e "${GREEN}Authentication successful!${NC}" + echo "" + "$SCRIPT_DIR/claude-auth-status.sh" full + + # Restart openclaw service if running + if systemctl --user is-active openclaw >/dev/null 2>&1; then + echo "" + echo "Restarting openclaw service..." + systemctl --user restart openclaw + echo -e "${GREEN}Service restarted.${NC}" + fi +else + echo "" + echo -e "${RED}Authentication failed.${NC}" + echo "Please try again or check the Claude Code documentation." + exit 1 +fi diff --git a/scripts/mobile-release-intent.mjs b/scripts/mobile-release-intent.mjs new file mode 100644 index 0000000000000000000000000000000000000000..b43eb41fce3f2693c7c4b2de9948bef39a5e5d0a --- /dev/null +++ b/scripts/mobile-release-intent.mjs @@ -0,0 +1,327 @@ +#!/usr/bin/env node +import crypto from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; + +const MOBILE_RELEASE_INTENT_KIND = "openclaw-mobile-release-ref-intent"; +export const MOBILE_RELEASE_INTENT_MAX_BYTES = 4 * 1024; + +const VERSION_RE = /^20[0-9]{2}\.(?:[1-9]|1[0-2])\.[1-9][0-9]*$/u; +const SHA_RE = /^[0-9a-f]{40}$/u; +const DIGEST_RE = /^sha256:[0-9a-f]{64}$/u; +const POSITIVE_INTEGER_RE = /^[1-9][0-9]*$/u; +const TARGET_REF_RE = /^release\/(20[0-9]{2}\.(?:[1-9]|1[0-2])\.[1-9][0-9]*)-mobile$/u; + +function fail(message) { + throw new Error(message); +} + +function exactKeys(value, keys, label = "Mobile release intent") { + if ( + value === null || + typeof value !== "object" || + Array.isArray(value) || + JSON.stringify(Object.keys(value).toSorted()) !== JSON.stringify(keys.toSorted()) + ) { + fail(`${label} has an unexpected shape.`); + } +} + +function hasAsciiControlCharacter(value) { + for (let index = 0; index < value.length; index += 1) { + const codeUnit = value.charCodeAt(index); + if (codeUnit <= 0x1f || codeUnit === 0x7f) { + return true; + } + } + return false; +} + +function boundedString(value, name, maxLength = 200) { + const normalized = String(value ?? ""); + if ( + normalized.length < 1 || + normalized.length > maxLength || + normalized.trim() !== normalized || + hasAsciiControlCharacter(normalized) + ) { + fail(`${name} must be a bounded printable string.`); + } + return normalized; +} + +function positiveInteger(value, name) { + const normalized = String(value ?? ""); + if (!POSITIVE_INTEGER_RE.test(normalized)) { + fail(`${name} must be a positive integer.`); + } + return normalized; +} + +function releaseVersion(value, name = "Mobile gateway version") { + const normalized = String(value ?? ""); + if (!VERSION_RE.test(normalized)) { + fail(`${name} must match YYYY.M.PATCH.`); + } + return normalized; +} + +function targetRef(value, gatewayVersion) { + const normalized = String(value ?? ""); + const match = TARGET_REF_RE.exec(normalized); + if (!match || match[1] !== gatewayVersion) { + fail("Mobile release target ref must exactly match release/YYYY.M.PATCH-mobile."); + } + return normalized; +} + +function targetSha(value) { + const normalized = String(value ?? ""); + if (!SHA_RE.test(normalized)) { + fail("Mobile release target SHA must be a full lowercase commit SHA."); + } + return normalized; +} + +function digest(value, name) { + const normalized = String(value ?? ""); + if (!DIGEST_RE.test(normalized)) { + fail(`${name} must be a canonical sha256 digest.`); + } + return normalized; +} + +function iosStoreIdentity(value, gatewayVersion) { + const appStoreVersion = releaseVersion(value.appStoreVersion, "iOS App Store version"); + const escapedGateway = gatewayVersion.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"); + if (!new RegExp(`^${escapedGateway}[0-9]$`, "u").test(appStoreVersion)) { + fail("iOS App Store version must be the gateway version plus one revision digit."); + } + return { + appStoreVersion, + buildNumber: positiveInteger(value.buildNumber, "iOS build number"), + internalGroupId: boundedString(value.internalGroupId, "TestFlight internal group ID"), + internalGroupName: boundedString(value.internalGroupName, "TestFlight internal group name"), + }; +} + +function androidVersionCodePrefix(version) { + const [year, month, patch] = version.split("."); + return `${year}${month.padStart(2, "0")}${patch.padStart(2, "0")}`; +} + +function androidStoreIdentity(value, gatewayVersion) { + const versionName = releaseVersion(value.versionName, "Android versionName"); + if (versionName !== gatewayVersion) { + fail("Android versionName must exactly match the mobile gateway version."); + } + if ( + value.phoneTrack !== "internal" || + value.wearTrack !== "wear:internal" || + value.releaseStatus !== "completed" || + value.playEditState !== "committed" + ) { + fail( + "Android beta intent must bind internal and wear:internal tracks, completed status, and a committed Play edit.", + ); + } + const phoneVersionCode = positiveInteger(value.phoneVersionCode, "Android phone versionCode"); + const wearVersionCode = positiveInteger(value.wearVersionCode, "Android Wear versionCode"); + const prefix = androidVersionCodePrefix(gatewayVersion); + const phoneSuffix = Number.parseInt(phoneVersionCode.slice(prefix.length), 10); + if ( + !phoneVersionCode.startsWith(prefix) || + phoneVersionCode.length !== prefix.length + 2 || + !Number.isInteger(phoneSuffix) || + phoneSuffix < 1 || + phoneSuffix > 49 + ) { + fail(`Android phone versionCode must match ${prefix}01 through ${prefix}49.`); + } + if (BigInt(wearVersionCode) !== BigInt(phoneVersionCode) + 50n) { + fail("Android Wear versionCode must be the phone versionCode plus 50."); + } + return { + phoneTrack: value.phoneTrack, + phoneVersionCode, + playEditState: value.playEditState, + releaseStatus: value.releaseStatus, + versionName, + wearTrack: value.wearTrack, + wearVersionCode, + }; +} + +function planPayload(value) { + const gatewayVersion = releaseVersion(value.gatewayVersion); + const platform = value.platform; + if (platform !== "ios" && platform !== "android") { + fail("Mobile release intent platform must be ios or android."); + } + return { + authorityReceiptDigest: digest( + value.authorityReceiptDigest, + "Mobile release authority receipt digest", + ), + gatewayVersion, + platform, + storeIdentity: + platform === "ios" + ? iosStoreIdentity(value, gatewayVersion) + : androidStoreIdentity(value, gatewayVersion), + targetRef: targetRef(value.targetRef, gatewayVersion), + targetSha: targetSha(value.targetSha), + }; +} + +export function mobileReleasePlanDigest(value) { + const bytes = JSON.stringify(planPayload(value)); + return `sha256:${crypto.createHash("sha256").update(bytes).digest("hex")}`; +} + +export function validateMobileReleaseIntent(value) { + const platform = value?.platform; + const commonKeys = [ + "authorityReceiptDigest", + "gatewayVersion", + "kind", + "planDigest", + "platform", + "schemaVersion", + "targetRef", + "targetSha", + ]; + const platformKeys = + platform === "ios" + ? ["appStoreVersion", "buildNumber", "internalGroupId", "internalGroupName"] + : platform === "android" + ? [ + "phoneTrack", + "phoneVersionCode", + "playEditState", + "releaseStatus", + "versionName", + "wearTrack", + "wearVersionCode", + ] + : []; + exactKeys(value, [...commonKeys, ...platformKeys]); + const payload = planPayload(value); + const intent = { + authorityReceiptDigest: payload.authorityReceiptDigest, + gatewayVersion: payload.gatewayVersion, + kind: MOBILE_RELEASE_INTENT_KIND, + planDigest: digest(value.planDigest, "Mobile release plan digest"), + platform, + schemaVersion: 2, + targetRef: payload.targetRef, + targetSha: payload.targetSha, + ...payload.storeIdentity, + }; + if (value.kind !== intent.kind || value.schemaVersion !== intent.schemaVersion) { + fail("Mobile release intent kind or schema version is invalid."); + } + if (intent.planDigest !== mobileReleasePlanDigest(intent)) { + fail("Mobile release plan digest does not match the signed release tuple."); + } + return intent; +} + +export function mobileReleaseRefForIntent(value) { + const intent = validateMobileReleaseIntent(value); + if (intent.platform === "ios") { + return `refs/openclaw/mobile-releases/ios/${intent.appStoreVersion}-${intent.buildNumber}`; + } + return `refs/openclaw/mobile-releases/android/${intent.versionName}-${intent.phoneVersionCode}`; +} + +function canonicalMobileReleaseIntentBytes(value) { + return `${JSON.stringify(validateMobileReleaseIntent(value))}\n`; +} + +export function readMobileReleaseIntent(file) { + const stat = fs.lstatSync(file); + if ( + !stat.isFile() || + stat.isSymbolicLink() || + stat.size < 2 || + stat.size > MOBILE_RELEASE_INTENT_MAX_BYTES + ) { + fail("Mobile release intent must be a regular file no larger than 4 KiB."); + } + const source = fs.readFileSync(file, "utf8"); + const intent = validateMobileReleaseIntent(JSON.parse(source)); + if (source !== canonicalMobileReleaseIntentBytes(intent)) { + fail("Mobile release intent is not in canonical closed-schema form."); + } + return intent; +} + +export function writeMobileReleaseIntent(file, value) { + const intent = validateMobileReleaseIntent(value); + const source = canonicalMobileReleaseIntentBytes(intent); + if (Buffer.byteLength(source) > MOBILE_RELEASE_INTENT_MAX_BYTES) { + fail("Mobile release intent exceeds 4 KiB."); + } + fs.mkdirSync(path.dirname(file), { recursive: true }); + const temporary = `${file}.${process.pid}.tmp`; + fs.writeFileSync(temporary, source, { flag: "wx", mode: 0o600 }); + fs.renameSync(temporary, file); + return intent; +} + +function option(argv, name) { + const index = argv.indexOf(name); + if (index < 0 || !argv[index + 1] || argv[index + 1].startsWith("-")) { + fail(`Missing ${name}.`); + } + return argv[index + 1]; +} + +function main(argv) { + if (argv[0] !== "write") { + fail("Expected the write command."); + } + const platform = option(argv, "--platform"); + const common = { + authorityReceiptDigest: option(argv, "--authority-receipt-digest"), + gatewayVersion: option(argv, "--gateway-version"), + kind: MOBILE_RELEASE_INTENT_KIND, + platform, + schemaVersion: 2, + targetRef: option(argv, "--target-ref"), + targetSha: option(argv, "--target-sha"), + }; + const storeIdentity = + platform === "ios" + ? { + appStoreVersion: option(argv, "--app-store-version"), + buildNumber: option(argv, "--build-number"), + internalGroupId: option(argv, "--internal-group-id"), + internalGroupName: option(argv, "--internal-group-name"), + } + : { + phoneTrack: option(argv, "--phone-track"), + phoneVersionCode: option(argv, "--phone-version-code"), + playEditState: option(argv, "--play-edit-state"), + releaseStatus: option(argv, "--release-status"), + versionName: option(argv, "--version-name"), + wearTrack: option(argv, "--wear-track"), + wearVersionCode: option(argv, "--wear-version-code"), + }; + const unsigned = { ...common, ...storeIdentity }; + writeMobileReleaseIntent(option(argv, "--output"), { + ...unsigned, + planDigest: mobileReleasePlanDigest(unsigned), + }); +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { + try { + main(process.argv.slice(2)); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + } +} diff --git a/scripts/mobile-release-ref.ts b/scripts/mobile-release-ref.ts new file mode 100644 index 0000000000000000000000000000000000000000..d91c35086b0b5d0fd85808c68168c3aa52ada804 --- /dev/null +++ b/scripts/mobile-release-ref.ts @@ -0,0 +1,376 @@ +// Tracks uploaded mobile store builds with non-tag Git refs. +import { execFileSync } from "node:child_process"; +import { existsSync } from "node:fs"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; + +type MobileReleasePlatform = "ios" | "android"; +type MobileReleaseCommand = "preflight" | "record" | "resolve"; + +type GitDeps = { + execFileSync?: typeof execFileSync; +}; + +type MobileReleaseOptions = { + build: string | null; + command: MobileReleaseCommand; + platform: MobileReleasePlatform; + remote: string; + rootDir: string; + sha: string; + version: string; + versionCode: string | null; +}; + +type RemoteRefState = { + ref: string; + sha: string; +} | null; + +const REF_PREFIX = "refs/openclaw/mobile-releases"; +const VERSION_RE = /^20\d{2}\.(?:[1-9]\d?)\.(?:[1-9]\d*)$/u; +const POSITIVE_INTEGER_RE = /^[1-9]\d*$/u; + +function git(args: string[], rootDir: string, deps: GitDeps = {}): string { + const exec = deps.execFileSync ?? execFileSync; + return exec("git", args, { + cwd: rootDir, + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + maxBuffer: 16 * 1024 * 1024, + }); +} + +function errorOutput(value: unknown): string { + if (Buffer.isBuffer(value)) { + return value.toString("utf8"); + } + if (typeof value === "string") { + return value; + } + if (value == null) { + return ""; + } + return JSON.stringify(value) ?? Object.prototype.toString.call(value); +} + +function gitAllowFailure( + args: string[], + rootDir: string, + deps: GitDeps = {}, +): { ok: boolean; stdout: string; stderr: string } { + try { + return { ok: true, stdout: git(args, rootDir, deps), stderr: "" }; + } catch (error) { + const e = error as { stdout?: unknown; stderr?: unknown }; + const stdout = errorOutput(e.stdout); + const stderr = errorOutput(e.stderr); + return { ok: false, stdout, stderr }; + } +} + +function readOptionValue(argv: string[], index: number, flag: string): string { + const value = argv[index + 1]; + if (!value || value.startsWith("-")) { + throw new Error(`Missing value for ${flag}.`); + } + return value; +} + +function parsePlatform(raw: string | null): MobileReleasePlatform { + if (raw === "ios" || raw === "android") { + return raw; + } + throw new Error("Missing or invalid --platform. Expected ios or android."); +} + +function parseCommand(raw: string | undefined): MobileReleaseCommand { + if (raw === "-h" || raw === "--help") { + throw new Error(usage()); + } + if (raw === "preflight" || raw === "record" || raw === "resolve") { + return raw; + } + throw new Error(`Unknown command '${raw ?? ""}'. Expected preflight, record, or resolve.`); +} + +export function parseArgs(argv: string[]): MobileReleaseOptions { + const command = parseCommand(argv[0]); + let build: string | null = null; + let platform: string | null = null; + let remote = "origin"; + let rootDir = path.resolve("."); + let sha = "HEAD"; + let version = ""; + let versionCode: string | null = null; + + for (let index = 1; index < argv.length; index += 1) { + const arg = argv[index]; + switch (arg) { + case "--": + break; + case "--platform": + platform = readOptionValue(argv, index, arg); + index += 1; + break; + case "--version": + version = readOptionValue(argv, index, arg); + index += 1; + break; + case "--build": + build = readOptionValue(argv, index, arg); + index += 1; + break; + case "--version-code": + versionCode = readOptionValue(argv, index, arg); + index += 1; + break; + case "--sha": + sha = readOptionValue(argv, index, arg); + index += 1; + break; + case "--remote": + remote = readOptionValue(argv, index, arg); + index += 1; + break; + case "--root": + rootDir = path.resolve(readOptionValue(argv, index, arg)); + index += 1; + break; + case "-h": + case "--help": + throw new Error(usage()); + default: + throw new Error(`Unknown argument: ${arg}`); + } + } + + return { + build, + command, + platform: parsePlatform(platform), + remote, + rootDir, + sha, + version, + versionCode, + }; +} + +function validateVersion(version: string): string { + const trimmed = version.trim(); + if (!VERSION_RE.test(trimmed)) { + throw new Error(`Invalid mobile release version '${version}'. Expected YYYY.M.D.`); + } + return trimmed; +} + +function validatePositiveInteger(label: string, value: string | null): string { + const trimmed = value?.trim() ?? ""; + if (!POSITIVE_INTEGER_RE.test(trimmed)) { + throw new Error(`Invalid ${label} '${value ?? ""}'. Expected a positive integer.`); + } + return trimmed; +} + +function androidVersionCodePrefix(version: string): string { + const [year, rawMonth, rawPatch] = version.split("."); + return `${year}${rawMonth?.padStart(2, "0")}${rawPatch?.padStart(2, "0")}`; +} + +function validateAndroidVersionCode(version: string, versionCode: string | null): string { + const code = validatePositiveInteger("Android versionCode", versionCode); + const prefix = androidVersionCodePrefix(version); + const suffix = Number.parseInt(code.slice(prefix.length), 10); + if ( + !code.startsWith(prefix) || + code.length !== prefix.length + 2 || + !Number.isInteger(suffix) || + suffix < 1 || + suffix > 99 + ) { + throw new Error( + `Invalid Android versionCode '${code}'. Expected ${prefix}01 through ${prefix}99 for version ${version}.`, + ); + } + return code; +} + +export function mobileReleaseRefFor(options: { + build?: string | null; + platform: MobileReleasePlatform; + version: string; + versionCode?: string | null; +}): string { + const version = validateVersion(options.version); + if (options.platform === "ios") { + const build = validatePositiveInteger("iOS build", options.build ?? null); + return `${REF_PREFIX}/ios/${version}-${build}`; + } + + const versionCode = validateAndroidVersionCode(version, options.versionCode ?? null); + return `${REF_PREFIX}/android/${version}-${versionCode}`; +} + +function assertRootDir(rootDir: string): void { + if (!existsSync(path.join(rootDir, ".git"))) { + throw new Error(`Not a Git checkout root: ${rootDir}`); + } +} + +function resolveCommitSha(sha: string, rootDir: string, deps: GitDeps = {}): string { + return git(["rev-parse", "--verify", `${sha}^{commit}`], rootDir, deps).trim(); +} + +function readRemoteRef( + remote: string, + ref: string, + rootDir: string, + deps: GitDeps = {}, +): RemoteRefState { + const result = gitAllowFailure(["ls-remote", "--refs", remote, ref], rootDir, deps); + if (!result.ok) { + const detail = (result.stderr || result.stdout).trim(); + throw new Error(`Failed to inspect remote release ref ${ref}: ${detail}`); + } + + const line = result.stdout.trim(); + if (!line) { + return null; + } + + const [sha, remoteRef] = line.split(/\s+/u); + if (!sha || remoteRef !== ref) { + throw new Error(`Unexpected remote ref lookup output for ${ref}: ${line}`); + } + return { ref: remoteRef, sha }; +} + +function shortSha(sha: string): string { + return sha.slice(0, 12); +} + +function recoveryCommand(options: { ref: string; remote: string; sha: string }): string { + return `git push --force-with-lease=${options.ref}: ${options.remote} ${options.sha}:${options.ref}`; +} + +export function preflightMobileReleaseRef( + options: MobileReleaseOptions, + deps: GitDeps = {}, +): { ref: string; sha: string; status: "available" | "already-recorded" } { + assertRootDir(options.rootDir); + const ref = mobileReleaseRefFor(options); + const sha = resolveCommitSha(options.sha, options.rootDir, deps); + const existing = readRemoteRef(options.remote, ref, options.rootDir, deps); + + if (!existing) { + return { ref, sha, status: "available" }; + } + if (existing.sha === sha) { + return { ref, sha, status: "already-recorded" }; + } + + throw new Error( + `Mobile release ref ${ref} already points at ${existing.sha}; refusing to record ${sha}.`, + ); +} + +export function recordMobileReleaseRef( + options: MobileReleaseOptions, + deps: GitDeps = {}, +): { ref: string; sha: string; status: "created" | "already-recorded" } { + const preflight = preflightMobileReleaseRef(options, deps); + if (preflight.status === "already-recorded") { + return { ...preflight, status: "already-recorded" }; + } + + const pushArgs = [ + "push", + `--force-with-lease=${preflight.ref}:`, + options.remote, + `${preflight.sha}:${preflight.ref}`, + ]; + const result = gitAllowFailure(pushArgs, options.rootDir, deps); + if (!result.ok) { + const detail = (result.stderr || result.stdout).trim(); + throw new Error( + `Failed to create mobile release ref ${preflight.ref}. Recovery command:\n${recoveryCommand({ + ref: preflight.ref, + remote: options.remote, + sha: preflight.sha, + })}\n${detail}`, + ); + } + + const recorded = readRemoteRef(options.remote, preflight.ref, options.rootDir, deps); + if (recorded?.sha !== preflight.sha) { + throw new Error( + `Mobile release ref ${preflight.ref} was not recorded at ${preflight.sha}; remote has ${recorded?.sha ?? "no ref"}.`, + ); + } + + return { ref: preflight.ref, sha: preflight.sha, status: "created" }; +} + +export function resolveMobileReleaseRef( + options: MobileReleaseOptions, + deps: GitDeps = {}, +): { ref: string; sha: string } { + assertRootDir(options.rootDir); + const ref = mobileReleaseRefFor(options); + const existing = readRemoteRef(options.remote, ref, options.rootDir, deps); + if (!existing) { + throw new Error(`Mobile release ref ${ref} does not exist on ${options.remote}.`); + } + return { ref, sha: existing.sha }; +} + +function usage(): string { + return [ + "Usage:", + " node --import tsx scripts/mobile-release-ref.ts preflight --platform ios --version YYYY.M.D --build N [--sha HEAD] [--remote origin]", + " node --import tsx scripts/mobile-release-ref.ts record --platform android --version YYYY.M.D --version-code YYYYMMDDNN [--sha HEAD] [--remote origin]", + " node --import tsx scripts/mobile-release-ref.ts resolve --platform ios --version YYYY.M.D --build N [--remote origin]", + ].join("\n"); +} + +async function main(argv: string[]): Promise<number> { + try { + const options = parseArgs(argv); + if (options.command === "preflight") { + const result = preflightMobileReleaseRef(options); + const suffix = + result.status === "already-recorded" + ? `already records ${shortSha(result.sha)}` + : `available for ${shortSha(result.sha)}`; + process.stdout.write(`Mobile release ref ${result.ref} is ${suffix}.\n`); + return 0; + } + + if (options.command === "record") { + const result = recordMobileReleaseRef(options); + const verb = result.status === "already-recorded" ? "already records" : "recorded"; + process.stdout.write(`Mobile release ref ${result.ref} ${verb} ${result.sha}.\n`); + return 0; + } + + const result = resolveMobileReleaseRef(options); + process.stdout.write(`${result.sha}\t${result.ref}\n`); + return 0; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + if (message.startsWith("Usage:")) { + process.stdout.write(`${message}\n`); + return 0; + } + process.stderr.write(`${message}\n`); + return 1; + } +} + +if (import.meta.url === pathToFileURL(path.resolve(process.argv[1] ?? "")).href) { + const exitCode = await main(process.argv.slice(2)); + if (exitCode !== 0) { + process.exit(exitCode); + } +} diff --git a/scripts/native-i18n-locales.ts b/scripts/native-i18n-locales.ts new file mode 100644 index 0000000000000000000000000000000000000000..f4725bd7691b68ec2bb274fbb0e7db4b6ae3c21c --- /dev/null +++ b/scripts/native-i18n-locales.ts @@ -0,0 +1,29 @@ +/** + * Shared native-app locale list. Lives outside native-app-i18n.ts so the + * derived generators (android-app-i18n, apple-app-i18n) can import it without + * creating a cycle with native-app-i18n's top-level CLI await, which chains + * those generators after rewriting the inventory. + */ +export const NATIVE_I18N_LOCALES = [ + "zh-CN", + "zh-TW", + "pt-BR", + "de", + "es", + "ja-JP", + "ko", + "fr", + "hi", + "ar", + "it", + "tr", + "uk", + "id", + "pl", + "th", + "vi", + "nl", + "fa", + "ru", + "sv", +] as const; diff --git a/scripts/notarize-mac-artifact.sh b/scripts/notarize-mac-artifact.sh new file mode 100644 index 0000000000000000000000000000000000000000..c92f3d828de344a8b5ac590e8ed1706a7c179f42 --- /dev/null +++ b/scripts/notarize-mac-artifact.sh @@ -0,0 +1,223 @@ +#!/bin/bash +set -euo pipefail + +# Notarize a macOS artifact (zip/dmg/pkg) and optionally staple the app bundle. +# +# Usage: +# STAPLE_APP_PATH=dist/OpenClaw.app scripts/notarize-mac-artifact.sh <artifact> +# +# Auth (pick one): +# NOTARYTOOL_PROFILE keychain profile created via `xcrun notarytool store-credentials` +# NOTARYTOOL_KEY path to App Store Connect API key (.p8) +# NOTARYTOOL_KEY_ID API key ID +# NOTARYTOOL_ISSUER API issuer ID +# NOTARY_RESULT_FILE optional mode-0600 JSON result path + +ARTIFACT="" +SUBMISSION_FILE="" +STAPLE_APP_PATH="${STAPLE_APP_PATH:-}" +NOTARY_RESULT_FILE="${NOTARY_RESULT_FILE:-}" + +usage() { + cat <<'HELP' +Usage: scripts/notarize-mac-artifact.sh <artifact> [--submission-file <checkpoint.json>] + +Options: + --submission-file <path> Save and resume the Apple submission for these exact artifact bytes. + +Env: + STAPLE_APP_PATH=dist/OpenClaw.app + NOTARYTOOL_PROFILE=<keychain-profile> + NOTARYTOOL_KEY=<api-key.p8> + NOTARYTOOL_KEY_ID=<api-key-id> + NOTARYTOOL_ISSUER=<issuer-id> + NOTARY_RESULT_FILE=<accepted-result.json> +HELP +} + +while [[ "$#" -gt 0 ]]; do + case "$1" in + --help|-h) usage; exit 0 ;; + --submission-file) + [[ "$#" -ge 2 && -n "$2" && "$2" != -* && -z "$SUBMISSION_FILE" ]] || { + echo "Error: --submission-file requires one checkpoint path." >&2; exit 1; + } + SUBMISSION_FILE="$2"; shift 2 ;; + --) shift; break ;; + -*) echo "Error: unknown notarization option: $1" >&2; exit 1 ;; + *) + [[ -z "$ARTIFACT" ]] || { echo "Error: unexpected notarization argument: $1" >&2; exit 1; } + ARTIFACT="$1"; shift ;; + esac +done +if [[ "$#" -gt 0 ]]; then + [[ "$#" -eq 1 && -z "$ARTIFACT" ]] || { echo "Error: unexpected notarization argument: $1" >&2; exit 1; } + ARTIFACT="$1" +fi + +if [[ -z "$ARTIFACT" ]]; then + usage >&2 + exit 1 +fi +if [[ ! -e "$ARTIFACT" ]]; then + echo "Error: artifact not found: $ARTIFACT" >&2 + exit 1 +fi +if [[ -n "$STAPLE_APP_PATH" && ! -d "$STAPLE_APP_PATH" ]]; then + echo "Error: STAPLE_APP_PATH not found: $STAPLE_APP_PATH" >&2 + exit 1 +fi + +if ! command -v xcrun >/dev/null 2>&1; then + echo "Error: xcrun not found; install Xcode command line tools." >&2 + exit 1 +fi +if ! command -v jq >/dev/null 2>&1; then + echo "Error: jq not found; install jq to validate notarization results." >&2 + exit 1 +fi +if [[ -n "$NOTARY_RESULT_FILE" && ! -d "$(dirname "$NOTARY_RESULT_FILE")" ]]; then + echo "Error: NOTARY_RESULT_FILE parent directory not found: $(dirname "$NOTARY_RESULT_FILE")" >&2 + exit 1 +fi + +notary_json_tmp="" +staple_dir="" +cleanup() { + [[ -z "$notary_json_tmp" ]] || rm -f "$notary_json_tmp" + [[ -z "$staple_dir" ]] || rm -rf "$staple_dir" + return 0 +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM HUP + +write_json() { + local destination="$1" value="$2" + notary_json_tmp="$(mktemp "${destination}.tmp.XXXXXX")" + chmod 600 "$notary_json_tmp" + printf '%s\n' "$value" >"$notary_json_tmp" + mv "$notary_json_tmp" "$destination" + notary_json_tmp="" +} + +checkpoint="" +artifact_sha="" +if [[ -n "$SUBMISSION_FILE" ]]; then + [[ -d "$(dirname "$SUBMISSION_FILE")" && ! -L "$SUBMISSION_FILE" ]] || { + echo "Error: invalid notarization checkpoint path: $SUBMISSION_FILE" >&2; exit 1; + } + canonical_submission="$(cd "$(dirname "$SUBMISSION_FILE")" && pwd -P)/$(basename "$SUBMISSION_FILE")" + canonical_artifact="$(cd "$(dirname "$ARTIFACT")" && pwd -P)/$(basename "$ARTIFACT")" + [[ "$canonical_submission" != "$canonical_artifact" ]] || { + echo "Error: notarization checkpoint must be separate from the artifact." >&2; exit 1; + } + if [[ -n "$NOTARY_RESULT_FILE" ]]; then + canonical_result="$(cd "$(dirname "$NOTARY_RESULT_FILE")" && pwd -P)/$(basename "$NOTARY_RESULT_FILE")" + [[ "$canonical_submission" != "$canonical_result" ]] || { + echo "Error: notarization checkpoint must be separate from NOTARY_RESULT_FILE." >&2; exit 1; + } + fi + artifact_sha="$(shasum -a 256 "$ARTIFACT" | awk '{print $1}')" + if [[ -e "$SUBMISSION_FILE" ]]; then + checkpoint="$(cat "$SUBMISSION_FILE")" + if ! jq -e --arg name "$(basename "$ARTIFACT")" --arg sha "$artifact_sha" ' + def digest: type == "string" and test("^[0-9a-f]{64}$"); + type == "object" and .version == 1 and .artifactName == $name and + (.artifactSha256 | digest) and + (.submissionId | type == "string" and test("^[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}$")) and + ((has("result") | not) or (.result.id == .submissionId and + (.result.status == "Accepted" or .result.status == "Invalid" or .result.status == "Rejected"))) and + ((has("stapledSha256") | not) or (.result.status == "Accepted" and (.stapledSha256 | digest))) and + (.artifactSha256 == $sha or .stapledSha256 == $sha) + ' <<<"$checkpoint" >/dev/null 2>&1; then + echo "Error: notarization checkpoint is corrupt or belongs to different artifact bytes: $SUBMISSION_FILE" >&2 + exit 1 + fi + fi +fi + +auth_args=() +if [[ -n "${NOTARYTOOL_PROFILE:-}" ]]; then + auth_args+=(--keychain-profile "$NOTARYTOOL_PROFILE") +elif [[ -n "${NOTARYTOOL_KEY:-}" && -n "${NOTARYTOOL_KEY_ID:-}" && -n "${NOTARYTOOL_ISSUER:-}" ]]; then + auth_args+=(--key "$NOTARYTOOL_KEY" --key-id "$NOTARYTOOL_KEY_ID" --issuer "$NOTARYTOOL_ISSUER") +else + echo "Error: Notary auth missing. Set NOTARYTOOL_PROFILE or NOTARYTOOL_KEY/NOTARYTOOL_KEY_ID/NOTARYTOOL_ISSUER." >&2 + exit 1 +fi + +echo "🧾 Notarizing: $ARTIFACT" +if [[ -z "$SUBMISSION_FILE" ]]; then + notary_result="$(xcrun notarytool submit "$ARTIFACT" "${auth_args[@]}" \ + --wait --no-s3-acceleration --output-format json)" +else + if [[ -z "$checkpoint" ]]; then + submission_exit=0 + submission_result="$(xcrun notarytool submit "$ARTIFACT" "${auth_args[@]}" \ + --no-wait --no-s3-acceleration --output-format json)" || submission_exit=$? + notary_id="$(jq -er '.id | select(type == "string" and test("^[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}$"))' <<<"$submission_result")" + checkpoint="$(jq -n --arg name "$(basename "$ARTIFACT")" --arg sha "$artifact_sha" --arg id "$notary_id" \ + '{version: 1, artifactName: $name, artifactSha256: $sha, submissionId: $id}')" + # Persist before waiting: a transport failure must never create a second submission. + write_json "$SUBMISSION_FILE" "$checkpoint" + [[ "$submission_exit" -eq 0 ]] || exit "$submission_exit" + fi + notary_id="$(jq -r .submissionId <<<"$checkpoint")" + notary_result="$(jq -c '.result // empty' <<<"$checkpoint")" + if [[ -z "$notary_result" ]]; then + wait_exit=0 + notary_result="$(xcrun notarytool wait "$notary_id" "${auth_args[@]}" --output-format json)" || wait_exit=$? + jq -e --arg id "$notary_id" '.id == $id and (.status == "Accepted" or .status == "Invalid" or .status == "Rejected")' \ + <<<"$notary_result" >/dev/null || { echo "Error: unexpected notarization wait result." >&2; exit 1; } + checkpoint="$(jq --argjson result "$notary_result" '.result = $result' <<<"$checkpoint")" + write_json "$SUBMISSION_FILE" "$checkpoint" + [[ "$wait_exit" -eq 0 ]] || exit "$wait_exit" + fi +fi +printf '%s\n' "$notary_result" +notary_status="$(jq -r '.status // empty' <<<"$notary_result")" +notary_id="$(jq -r '.id // empty' <<<"$notary_result")" +if [[ "$notary_status" != "Accepted" || -z "$notary_id" ]]; then + echo "Error: notarization did not return an accepted result with an id." >&2 + exit 1 +fi +if [[ -n "$NOTARY_RESULT_FILE" ]]; then + write_json "$NOTARY_RESULT_FILE" "$notary_result" +fi + +case "$ARTIFACT" in + *.dmg|*.pkg) + echo "📌 Stapling artifact: $ARTIFACT" + if [[ -n "$SUBMISSION_FILE" ]]; then + if [[ "$artifact_sha" != "$(jq -r '.stapledSha256 // empty' <<<"$checkpoint")" ]]; then + # Staple a copy, then record its hash before replacement so interruption + # accepts either known version, never arbitrary post-submission changes. + staple_dir="$(mktemp -d "$(dirname "$ARTIFACT")/.notary-staple.XXXXXX")" + staple_artifact="$staple_dir/$(basename "$ARTIFACT")" + cp -p "$ARTIFACT" "$staple_artifact" + xcrun stapler staple "$staple_artifact" + xcrun stapler validate "$staple_artifact" + stapled_sha="$(shasum -a 256 "$staple_artifact" | awk '{print $1}')" + checkpoint="$(jq --arg sha "$stapled_sha" '.stapledSha256 = $sha' <<<"$checkpoint")" + write_json "$SUBMISSION_FILE" "$checkpoint" + mv "$staple_artifact" "$ARTIFACT" + rmdir "$staple_dir" + staple_dir="" + fi + else + xcrun stapler staple "$ARTIFACT" + fi + xcrun stapler validate "$ARTIFACT" + ;; + *) + ;; +esac + +if [[ -n "$STAPLE_APP_PATH" ]]; then + echo "📌 Stapling app: $STAPLE_APP_PATH" + xcrun stapler staple "$STAPLE_APP_PATH" + xcrun stapler validate "$STAPLE_APP_PATH" +fi + +echo "✅ Notarization complete" diff --git a/scripts/npm-placeholder-publication.mjs b/scripts/npm-placeholder-publication.mjs new file mode 100644 index 0000000000000000000000000000000000000000..4a0dbf251f4c1bee47c7b00bc8cbc40d0dbece05 --- /dev/null +++ b/scripts/npm-placeholder-publication.mjs @@ -0,0 +1,825 @@ +#!/usr/bin/env node + +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { + lstatSync, + mkdtempSync, + mkdirSync, + readFileSync, + readdirSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { basename, join, resolve } from "node:path"; +import { pathToFileURL } from "node:url"; +import { gzipSync } from "node:zlib"; +import { readPublicationArtifactArchive, sha256Digest } from "./lib/actions-artifact-archive.mjs"; +import { fetchNpmRegistryPackumentWithRetry } from "./lib/npm-publish-plan.mjs"; + +const MANIFEST_FILENAME = "npm-placeholder-manifest.json"; +const MANIFEST_SCHEMA = "openclaw.npm-placeholder-publication/v1"; +const PACKAGE_VERSION = "0.0.0"; +const PUBLISH_TAG = "placeholder"; +const WORKFLOW_PATH = ".github/workflows/npm-placeholder-bootstrap.yml"; +const PACKAGE_NAME_RE = /^@openclaw\/[a-z0-9][a-z0-9._-]*$/u; +const SHA_RE = /^[0-9a-f]{40}$/u; +const SHA256_RE = /^[0-9a-f]{64}$/u; +const MAX_ARTIFACT_BYTES = 32 * 1024 * 1024; +const MAX_FILE_BYTES = 2 * 1024 * 1024; + +/** @typedef {typeof fetch} PlaceholderFetch */ +/** + * @typedef {object} CreatePlaceholderPublicationParams + * @property {PlaceholderFetch} [fetchImpl] + * @property {string} outputDir + * @property {string} packages + * @property {string} repoRoot + * @property {string} targetSha + * @property {string} workflowSha + */ +/** + * @typedef {object} PublishPlaceholdersParams + * @property {string} artifactDir + * @property {PlaceholderFetch} [fetchImpl] + * @property {string} npmToken + * @property {(args: string[], options: { cwd: string; env: NodeJS.ProcessEnv }) => void} [npmRunner] + * @property {number} [registryAttempts] + * @property {(delayMs: number) => Promise<void>} [sleep] + * @property {string} targetSha + * @property {string} [tempRoot] + * @property {string} workflowSha + */ + +function compareCodeUnits(left, right) { + return left < right ? -1 : left > right ? 1 : 0; +} + +function assertTrimmedString(value, label) { + if (typeof value !== "string" || value.length === 0 || value.trim() !== value) { + throw new Error(`${label} must be a non-empty trimmed string.`); + } + return value; +} + +function assertCommitSha(value, label) { + const sha = assertTrimmedString(value, label); + if (!SHA_RE.test(sha)) { + throw new Error(`${label} must be a full lowercase commit SHA.`); + } + return sha; +} + +function sha256(bytes) { + return createHash("sha256").update(bytes).digest("hex"); +} + +function npmIntegrity(bytes) { + return `sha512-${createHash("sha512").update(bytes).digest("base64")}`; +} + +function npmShasum(bytes) { + return createHash("sha1").update(bytes).digest("hex"); +} + +function canonicalJson(value) { + return `${JSON.stringify(value, null, 2)}\n`; +} + +function placeholderPackageJson(packageName) { + return { + name: packageName, + version: PACKAGE_VERSION, + description: "Reserved package name for an official OpenClaw plugin.", + license: "MIT", + repository: { + type: "git", + url: "git+https://github.com/openclaw/openclaw.git", + }, + publishConfig: { + access: "public", + tag: PUBLISH_TAG, + }, + }; +} + +function placeholderReadme(packageName) { + return `# ${packageName}\n\nReserved placeholder for the official OpenClaw plugin package. Use a published release version instead.\n`; +} + +function writeTarString(header, offset, length, value) { + const bytes = Buffer.from(value, "utf8"); + if (bytes.length > length) { + throw new Error(`Tar header value exceeds ${length} bytes.`); + } + bytes.copy(header, offset); +} + +function writeTarOctal(header, offset, length, value) { + const encoded = value.toString(8).padStart(length - 1, "0"); + writeTarString(header, offset, length, `${encoded}\0`); +} + +function tarEntry(path, content) { + const header = Buffer.alloc(512); + writeTarString(header, 0, 100, path); + writeTarOctal(header, 100, 8, 0o644); + writeTarOctal(header, 108, 8, 0); + writeTarOctal(header, 116, 8, 0); + writeTarOctal(header, 124, 12, content.length); + writeTarOctal(header, 136, 12, 0); + header.fill(0x20, 148, 156); + header[156] = "0".charCodeAt(0); + writeTarString(header, 257, 6, "ustar\0"); + writeTarString(header, 263, 2, "00"); + const checksum = header.reduce((sum, byte) => sum + byte, 0); + writeTarString(header, 148, 8, `${checksum.toString(8).padStart(6, "0")}\0 `); + const padding = Buffer.alloc((512 - (content.length % 512)) % 512); + return Buffer.concat([header, content, padding]); +} + +export function createPlaceholderTarball(packageName) { + if (!PACKAGE_NAME_RE.test(packageName)) { + throw new Error(`Invalid OpenClaw package name: ${packageName}`); + } + const packageJson = Buffer.from(canonicalJson(placeholderPackageJson(packageName)), "utf8"); + const readme = Buffer.from(placeholderReadme(packageName), "utf8"); + const tar = Buffer.concat([ + tarEntry("package/package.json", packageJson), + tarEntry("package/README.md", readme), + Buffer.alloc(1024), + ]); + return gzipSync(tar, { level: 9, mtime: 0 }); +} + +export function parseSelectedPackages(input) { + const raw = assertTrimmedString(input, "package selection"); + const packages = raw.split(",").map((value) => value.trim()); + if (packages.some((value) => value.length === 0)) { + throw new Error("Package selection must not contain empty entries."); + } + for (const packageName of packages) { + if (!PACKAGE_NAME_RE.test(packageName)) { + throw new Error(`Invalid OpenClaw package name: ${packageName}`); + } + } + if (new Set(packages).size !== packages.length) { + throw new Error("Package selection must not contain duplicates."); + } + return packages; +} + +function readJsonRegularFile(path, label) { + const info = lstatSync(path); + if (!info.isFile() || info.isSymbolicLink() || info.size === 0 || info.size > MAX_FILE_BYTES) { + throw new Error(`${label} must be a bounded regular file.`); + } + let value; + try { + value = JSON.parse(readFileSync(path, "utf8")); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + throw new Error(`${label} is invalid JSON: ${detail}`, { cause: error }); + } + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error(`${label} must contain an object.`); + } + return value; +} + +export function resolveSelectedPackageSources(repoRoot, packageNames) { + const extensionsDir = resolve(repoRoot, "extensions"); + const officialPackageNames = new Set(); + for (const catalogName of ["plugin", "provider", "channel"]) { + const catalogPath = resolve( + repoRoot, + "scripts", + "lib", + `official-external-${catalogName}-catalog.json`, + ); + const catalog = readJsonRegularFile(catalogPath, `Official external ${catalogName} catalog`); + if (!Array.isArray(catalog.entries)) { + throw new Error(`Official external ${catalogName} catalog entries must be an array.`); + } + for (const entry of catalog.entries) { + if ( + entry?.source === "official" && + typeof entry.name === "string" && + entry.openclaw?.install?.npmSpec === entry.name + ) { + officialPackageNames.add(entry.name); + } + } + } + const candidates = []; + for (const entry of readdirSync(extensionsDir, { withFileTypes: true })) { + if (!entry.isDirectory() || entry.isSymbolicLink()) { + continue; + } + const packageDir = `extensions/${entry.name}`; + const packageJsonPath = resolve(repoRoot, packageDir, "package.json"); + let info; + try { + info = lstatSync(packageJsonPath); + } catch (error) { + if (error && typeof error === "object" && error.code === "ENOENT") { + continue; + } + throw error; + } + if (!info.isFile() || info.isSymbolicLink()) { + throw new Error(`${packageDir}/package.json must be a regular file.`); + } + const packageJsonBytes = readFileSync(packageJsonPath); + const packageJson = readJsonRegularFile(packageJsonPath, `${packageDir}/package.json`); + candidates.push({ packageDir, packageJson, packageJsonBytes }); + } + + return packageNames.map((packageName) => { + const matches = candidates.filter((candidate) => candidate.packageJson.name === packageName); + if (matches.length !== 1) { + throw new Error(`${packageName} must map uniquely to extensions/*/package.json.`); + } + const [match] = matches; + if ( + match.packageJson.private === true || + ["private", "restricted"].includes(match.packageJson.publishConfig?.access) || + match.packageJson.openclaw?.install?.npmSpec !== packageName || + match.packageJson.openclaw?.build?.bundledDist !== false || + match.packageJson.openclaw?.release?.publishToNpm !== true + ) { + throw new Error(`${packageName} is not a public release-enabled npm plugin.`); + } + if (!officialPackageNames.has(packageName)) { + throw new Error(`${packageName} is not present in an official external package catalog.`); + } + return { + packageDir: match.packageDir, + packageName, + sourcePackageJsonSha256: sha256(match.packageJsonBytes), + }; + }); +} + +function normalizeDistTags(value, packageName) { + if (value === undefined) { + return {}; + } + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error(`${packageName}: npm dist-tags must be an object.`); + } + const entries = []; + for (const [tag, version] of Object.entries(value)) { + if ( + typeof tag !== "string" || + tag.length === 0 || + typeof version !== "string" || + version.length === 0 + ) { + throw new Error(`${packageName}: npm dist-tags contain an invalid entry.`); + } + entries.push([tag, version]); + } + return Object.fromEntries(entries.toSorted(([left], [right]) => compareCodeUnits(left, right))); +} + +export function classifyRegistryState(params) { + const { expectedIntegrity, expectedShasum, packageName, registry } = params; + if (registry.status === 404) { + return { action: "publish", newPackage: true, nonPlaceholderTags: {} }; + } + if (registry.status !== 200 || !registry.packument || typeof registry.packument !== "object") { + throw new Error(`${packageName}: npm registry returned HTTP ${registry.status}.`); + } + const distTags = normalizeDistTags(registry.packument["dist-tags"], packageName); + if (distTags[PUBLISH_TAG] !== undefined && distTags[PUBLISH_TAG] !== PACKAGE_VERSION) { + throw new Error( + `${packageName}: placeholder dist-tag points to ${distTags[PUBLISH_TAG]}, expected ${PACKAGE_VERSION}.`, + ); + } + const nonPlaceholderTags = Object.fromEntries( + Object.entries(distTags).filter(([tag]) => tag !== PUBLISH_TAG), + ); + const publishedDist = registry.packument.versions?.[PACKAGE_VERSION]?.dist; + if (publishedDist !== undefined) { + if ( + publishedDist?.integrity !== expectedIntegrity || + publishedDist?.shasum !== expectedShasum + ) { + throw new Error(`${packageName}@${PACKAGE_VERSION}: npm registry tarball bytes differ.`); + } + return { + action: distTags[PUBLISH_TAG] === PACKAGE_VERSION ? "skip" : "tag", + newPackage: false, + nonPlaceholderTags, + }; + } + return { action: "publish", newPackage: false, nonPlaceholderTags }; +} + +async function readRegistry(packageName, fetchImpl = fetch) { + const result = await fetchNpmRegistryPackumentWithRetry({ + packageName, + packageUrl: `https://registry.npmjs.org/${encodeURIComponent(packageName)}`, + fetchImpl, + }); + return { packument: result.packument, status: result.status }; +} + +async function readStableRegistry(packageName, fetchImpl = fetch) { + const observations = []; + for (let attempt = 0; attempt < 3; attempt += 1) { + observations.push(await readRegistry(packageName, fetchImpl)); + } + const serialized = observations.map((observation) => JSON.stringify(observation)); + if (new Set(serialized).size !== 1) { + throw new Error(`${packageName}: npm registry state changed during placeholder planning.`); + } + return observations[0]; +} + +function assertFreshDirectory(path) { + try { + lstatSync(path); + throw new Error(`Output directory already exists: ${path}`); + } catch (error) { + if (!(error && typeof error === "object" && error.code === "ENOENT")) { + throw error; + } + } + mkdirSync(path, { recursive: true, mode: 0o700 }); +} + +/** @param {CreatePlaceholderPublicationParams} params */ +export async function createPlaceholderPublication(params) { + const repoRoot = resolve(params.repoRoot); + const outputDir = resolve(params.outputDir); + const targetSha = assertCommitSha(params.targetSha, "target SHA"); + const workflowSha = assertCommitSha(params.workflowSha, "workflow SHA"); + const packageNames = parseSelectedPackages(params.packages); + const sources = resolveSelectedPackageSources(repoRoot, packageNames); + assertFreshDirectory(outputDir); + + const packages = []; + for (const source of sources) { + const tarball = createPlaceholderTarball(source.packageName); + const tarballName = `${source.packageName.slice(1).replaceAll("/", "-")}-${PACKAGE_VERSION}.tgz`; + const identity = { + integrity: npmIntegrity(tarball), + sha256: sha256(tarball), + shasum: npmShasum(tarball), + sizeBytes: tarball.length, + }; + const registry = await readStableRegistry(source.packageName, params.fetchImpl); + const state = classifyRegistryState({ + expectedIntegrity: identity.integrity, + expectedShasum: identity.shasum, + packageName: source.packageName, + registry, + }); + writeFileSync(join(outputDir, tarballName), tarball, { flag: "wx", mode: 0o600 }); + packages.push({ + ...source, + action: state.action, + newPackage: state.newPackage, + preExistingDistTags: state.nonPlaceholderTags, + tarball: { name: tarballName, ...identity }, + }); + } + + const manifest = { + schema: MANIFEST_SCHEMA, + targetSha, + workflowPath: WORKFLOW_PATH, + workflowSha, + version: PACKAGE_VERSION, + publishTag: PUBLISH_TAG, + packages, + }; + writeFileSync(join(outputDir, MANIFEST_FILENAME), canonicalJson(manifest), { + flag: "wx", + mode: 0o600, + }); + return manifest; +} + +function validateManifest(value, params) { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error("Placeholder manifest must be an object."); + } + const topLevelKeys = Object.keys(value).toSorted(); + if ( + JSON.stringify(topLevelKeys) !== + JSON.stringify([ + "packages", + "publishTag", + "schema", + "targetSha", + "version", + "workflowPath", + "workflowSha", + ]) || + value.schema !== MANIFEST_SCHEMA || + value.targetSha !== params.targetSha || + value.workflowSha !== params.workflowSha || + value.workflowPath !== WORKFLOW_PATH || + value.version !== PACKAGE_VERSION || + value.publishTag !== PUBLISH_TAG || + !Array.isArray(value.packages) || + value.packages.length === 0 + ) { + throw new Error("Placeholder manifest identity does not match the approved publication."); + } + const names = value.packages.map((entry) => entry?.packageName); + if (new Set(names).size !== names.length || names.some((name) => !PACKAGE_NAME_RE.test(name))) { + throw new Error("Placeholder manifest package inventory is invalid."); + } + for (const entry of value.packages) { + const entryKeys = Object.keys(entry).toSorted(); + if ( + JSON.stringify(entryKeys) !== + JSON.stringify([ + "action", + "newPackage", + "packageDir", + "packageName", + "preExistingDistTags", + "sourcePackageJsonSha256", + "tarball", + ]) + ) { + throw new Error(`${entry.packageName}: placeholder manifest entry shape is invalid.`); + } + const expectedTarballName = `${entry.packageName.slice(1).replaceAll("/", "-")}-${PACKAGE_VERSION}.tgz`; + const tarballKeys = + entry.tarball && typeof entry.tarball === "object" + ? Object.keys(entry.tarball).toSorted() + : []; + if ( + !["publish", "skip", "tag"].includes(entry.action) || + typeof entry.newPackage !== "boolean" || + !/^extensions\/[a-z0-9][a-z0-9._-]*$/u.test(entry.packageDir) || + !SHA256_RE.test(entry.sourcePackageJsonSha256) || + JSON.stringify(tarballKeys) !== + JSON.stringify(["integrity", "name", "sha256", "shasum", "sizeBytes"]) || + entry.tarball.name !== expectedTarballName || + !/^sha512-[A-Za-z0-9+/]+={0,2}$/u.test(entry.tarball.integrity) || + !SHA256_RE.test(entry.tarball.sha256) || + !/^[0-9a-f]{40}$/u.test(entry.tarball.shasum) || + !Number.isSafeInteger(entry.tarball.sizeBytes) || + entry.tarball.sizeBytes <= 0 || + entry.tarball.sizeBytes > MAX_FILE_BYTES + ) { + throw new Error(`${entry.packageName}: placeholder manifest package binding is invalid.`); + } + const normalizedTags = normalizeDistTags(entry.preExistingDistTags, entry.packageName); + if ( + Object.hasOwn(entry.preExistingDistTags, PUBLISH_TAG) || + JSON.stringify(normalizedTags) !== JSON.stringify(entry.preExistingDistTags) || + (entry.newPackage && + (entry.action !== "publish" || Object.keys(entry.preExistingDistTags).length !== 0)) || + ((entry.action === "tag" || entry.action === "skip") && entry.newPackage) + ) { + throw new Error( + `${entry.packageName}: placeholder manifest dist-tag snapshot is not canonical.`, + ); + } + } + return value; +} + +export async function verifyPlaceholderArtifact(params) { + const targetSha = assertCommitSha(params.targetSha, "target SHA"); + const workflowSha = assertCommitSha(params.workflowSha, "workflow SHA"); + const artifactName = assertTrimmedString(params.artifactName, "artifact name"); + const runId = Number(params.runId); + const producerRunAttempt = Number(params.producerRunAttempt); + if (artifactName !== `npm-placeholder-publication-${runId}-${producerRunAttempt}`) { + throw new Error("Placeholder artifact name does not match its workflow run and attempt."); + } + const expected = { + artifactDigest: assertTrimmedString(params.artifactDigest, "artifact digest"), + artifactId: Number(params.artifactId), + artifactName, + artifactSizeBytes: Number(params.artifactSizeBytes), + consumerRunAttempt: Number(params.consumerRunAttempt), + producerJobName: "Plan npm placeholder publication", + repository: assertTrimmedString(params.repository, "repository"), + runAttempt: producerRunAttempt, + runId, + runStatePolicy: "same-run-producer-success", + workflowEvent: "workflow_dispatch", + workflowHeadBranch: "main", + workflowPath: WORKFLOW_PATH, + workflowSha, + }; + const archive = await readPublicationArtifactArchive({ + archivePolicy: { + minEntries: 2, + maxEntries: 101, + maxArchiveBytes: MAX_ARTIFACT_BYTES, + maxExpandedBytes: MAX_ARTIFACT_BYTES, + allowPath: (name) => + basename(name) === name && + (name === MANIFEST_FILENAME || /^[A-Za-z0-9._-]+\.tgz$/u.test(name)), + maxEntryBytes: (name) => (name === MANIFEST_FILENAME ? MAX_FILE_BYTES : MAX_FILE_BYTES), + }, + expected, + maxArchiveBytes: MAX_ARTIFACT_BYTES, + token: assertTrimmedString(params.token, "GitHub token"), + }); + const manifestBytes = archive.files.get(MANIFEST_FILENAME); + if (!manifestBytes) { + throw new Error("Placeholder artifact is missing its manifest."); + } + const manifest = validateManifest(JSON.parse(manifestBytes.toString("utf8")), { + targetSha, + workflowSha, + }); + if (archive.files.size !== manifest.packages.length + 1) { + throw new Error("Placeholder artifact file count does not match its manifest."); + } + + const sources = resolveSelectedPackageSources( + params.targetRoot, + manifest.packages.map((entry) => entry.packageName), + ); + const sourceByName = new Map(sources.map((source) => [source.packageName, source])); + const outputDir = resolve(params.outputDir); + assertFreshDirectory(outputDir); + for (const entry of manifest.packages) { + const source = sourceByName.get(entry.packageName); + if ( + source?.packageDir !== entry.packageDir || + source?.sourcePackageJsonSha256 !== entry.sourcePackageJsonSha256 || + !SHA256_RE.test(entry.tarball?.sha256) + ) { + throw new Error(`${entry.packageName}: source package binding changed.`); + } + const tarball = archive.files.get(entry.tarball.name); + const expectedTarball = createPlaceholderTarball(entry.packageName); + if ( + !tarball || + !tarball.equals(expectedTarball) || + entry.tarball.sizeBytes !== tarball.length || + entry.tarball.sha256 !== sha256(tarball) || + entry.tarball.integrity !== npmIntegrity(tarball) || + entry.tarball.shasum !== npmShasum(tarball) + ) { + throw new Error(`${entry.packageName}: placeholder tarball is not canonical.`); + } + writeFileSync(join(outputDir, entry.tarball.name), tarball, { flag: "wx", mode: 0o600 }); + } + writeFileSync(join(outputDir, MANIFEST_FILENAME), canonicalJson(manifest), { + flag: "wx", + mode: 0o600, + }); + return { artifactSha256: sha256Digest(archive.archiveBytes), manifest }; +} + +function sameTags(left, right) { + return JSON.stringify(left) === JSON.stringify(right); +} + +export function assertFinalRegistryState(entry, registry) { + const state = classifyRegistryState({ + expectedIntegrity: entry.tarball.integrity, + expectedShasum: entry.tarball.shasum, + packageName: entry.packageName, + registry, + }); + if (state.action !== "skip") { + throw new Error(`${entry.packageName}: placeholder publication did not reach its final state.`); + } + if (!sameTags(state.nonPlaceholderTags, entry.preExistingDistTags)) { + throw new Error(`${entry.packageName}: a non-placeholder dist-tag changed during publication.`); + } +} + +function defaultNpmRunner(args, options) { + const result = spawnSync("npm", args, { + cwd: options.cwd, + encoding: "utf8", + env: options.env, + killSignal: "SIGTERM", + stdio: ["ignore", "pipe", "pipe"], + timeout: 300_000, + }); + if (result.status !== 0) { + throw new Error( + `npm ${args[0]} failed (${result.status ?? "signal"}): ${(result.stderr || result.stdout).trim()}`, + ); + } +} + +async function readFinalRegistry(entry, params) { + const attempts = params.registryAttempts ?? 8; + const sleep = + params.sleep ?? + ((delayMs) => + new Promise((done) => { + setTimeout(done, delayMs); + })); + let lastError; + for (let attempt = 1; attempt <= attempts; attempt += 1) { + try { + const registry = await readRegistry(entry.packageName, params.fetchImpl); + assertFinalRegistryState(entry, registry); + return; + } catch (error) { + lastError = error; + if (attempt < attempts) { + await sleep(attempt * 1000); + } + } + } + throw lastError; +} + +/** @param {PublishPlaceholdersParams} params */ +export async function publishPlaceholders(params) { + const artifactDir = resolve(params.artifactDir); + const manifest = validateManifest( + readJsonRegularFile(join(artifactDir, MANIFEST_FILENAME), "Placeholder manifest"), + { + targetSha: params.targetSha, + workflowSha: params.workflowSha, + }, + ); + const npmToken = assertTrimmedString(params.npmToken, "NPM token"); + const publishHome = mkdtempSync( + join(resolve(params.tempRoot ?? tmpdir()), "openclaw-npm-placeholder-"), + ); + try { + const npmrc = join(publishHome, "npmrc"); + writeFileSync( + npmrc, + `registry=https://registry.npmjs.org/\n//registry.npmjs.org/:_authToken=${npmToken}\n`, + { + mode: 0o600, + }, + ); + const childEnv = { ...process.env }; + delete childEnv.NPM_TOKEN; + delete childEnv.NODE_AUTH_TOKEN; + delete childEnv.NODE_OPTIONS; + Object.assign(childEnv, { + HOME: publishHome, + NPM_CONFIG_GLOBALCONFIG: "/dev/null", + NPM_CONFIG_IGNORE_SCRIPTS: "true", + NPM_CONFIG_REGISTRY: "https://registry.npmjs.org/", + NPM_CONFIG_USERCONFIG: npmrc, + }); + const npmRunner = params.npmRunner ?? defaultNpmRunner; + const results = []; + + for (const entry of manifest.packages) { + const current = await readRegistry(entry.packageName, params.fetchImpl); + const state = classifyRegistryState({ + expectedIntegrity: entry.tarball.integrity, + expectedShasum: entry.tarball.shasum, + packageName: entry.packageName, + registry: current, + }); + if (!sameTags(state.nonPlaceholderTags, entry.preExistingDistTags)) { + throw new Error(`${entry.packageName}: npm dist-tags changed after the immutable plan.`); + } + let mutationError; + try { + if (state.action === "publish") { + npmRunner( + [ + "publish", + join(artifactDir, entry.tarball.name), + "--access", + "public", + "--ignore-scripts", + "--provenance", + "--tag", + PUBLISH_TAG, + ], + { cwd: artifactDir, env: childEnv }, + ); + } else if (state.action === "tag") { + npmRunner(["dist-tag", "add", `${entry.packageName}@${PACKAGE_VERSION}`, PUBLISH_TAG], { + cwd: artifactDir, + env: childEnv, + }); + } + } catch (error) { + mutationError = error; + } + try { + await readFinalRegistry(entry, params); + } catch (readbackError) { + if (mutationError) { + throw new Error( + `${entry.packageName}: npm mutation failed and exact registry readback did not converge: ${mutationError.message}`, + { cause: readbackError }, + ); + } + throw readbackError; + } + results.push({ + action: state.action, + newPackage: entry.newPackage, + packageName: entry.packageName, + }); + } + return { results }; + } finally { + rmSync(publishHome, { force: true, recursive: true }); + } +} + +function parseCliArgs(argv) { + const [command, ...rest] = argv; + if (!["create", "verify-artifact", "publish"].includes(command)) { + throw new Error("Usage: npm-placeholder-publication.mjs <create|verify-artifact|publish> ..."); + } + const values = {}; + for (let index = 0; index < rest.length; index += 2) { + const key = rest[index]; + const value = rest[index + 1]; + if (!key?.startsWith("--") || value === undefined || value.startsWith("--")) { + throw new Error(`Invalid ${command} argument near ${key ?? "<missing>"}.`); + } + const name = key.slice(2).replace(/-([a-z])/gu, (_, letter) => letter.toUpperCase()); + if (values[name] !== undefined) { + throw new Error(`Duplicate ${command} option: ${key}`); + } + values[name] = value; + } + return { command, values }; +} + +function appendGithubOutput(path, values) { + if (!path) { + return; + } + writeFileSync( + path, + `${Object.entries(values) + .map(([key, value]) => `${key}=${value}`) + .join("\n")}\n`, + { flag: "a" }, + ); +} + +export async function main(argv = process.argv.slice(2)) { + const { command, values } = parseCliArgs(argv); + if (command === "create") { + const manifest = await createPlaceholderPublication({ + outputDir: values.outputDir, + packages: values.packages, + repoRoot: values.repoRoot, + targetSha: values.targetSha, + workflowSha: values.workflowSha, + }); + appendGithubOutput(values.githubOutput, { + existing_without_zero_count: manifest.packages.filter( + (entry) => !entry.newPackage && entry.action === "publish", + ).length, + new_package_count: manifest.packages.filter((entry) => entry.newPackage).length, + package_count: manifest.packages.length, + }); + return; + } + if (command === "verify-artifact") { + const result = await verifyPlaceholderArtifact({ + artifactDigest: values.artifactDigest, + artifactId: values.artifactId, + artifactName: values.artifactName, + artifactSizeBytes: values.artifactSizeBytes, + consumerRunAttempt: values.consumerRunAttempt, + outputDir: values.outputDir, + producerRunAttempt: values.producerRunAttempt, + repository: values.repository, + runId: values.runId, + targetRoot: values.targetRoot, + targetSha: values.targetSha, + token: process.env.GH_TOKEN, + workflowSha: values.workflowSha, + }); + appendGithubOutput(values.githubOutput, { artifact_sha256: result.artifactSha256 }); + return; + } + const result = await publishPlaceholders({ + artifactDir: values.artifactDir, + npmToken: process.env.NPM_TOKEN, + targetSha: values.targetSha, + workflowSha: values.workflowSha, + }); + const resultPath = resolve(values.resultPath); + writeFileSync(resultPath, canonicalJson(result), { flag: "wx", mode: 0o600 }); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch( + /** @param {unknown} error */ (error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + }, + ); +} diff --git a/scripts/npm-preflight-tooling-identity.mjs b/scripts/npm-preflight-tooling-identity.mjs new file mode 100644 index 0000000000000000000000000000000000000000..2179f27405f11485413dd83c4825e36d72419866 --- /dev/null +++ b/scripts/npm-preflight-tooling-identity.mjs @@ -0,0 +1,460 @@ +#!/usr/bin/env node + +import { createHash } from "node:crypto"; +import { mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { isDeepStrictEqual, parseArgs } from "node:util"; +import { + downloadExactActionsArtifactArchive, + inspectActionsArtifactZipWithPolicy, +} from "./lib/actions-artifact-archive.mjs"; +import { isRecord } from "./lib/record-shared.mjs"; +import { verifyNpmBundleProducer } from "./npm-prepared-bundle.mjs"; +import { + runReleaseToolingGh, + validateReleaseToolingIdentity, +} from "./release-tooling-identity.mjs"; + +const FULL_RELEASE_WORKFLOW = ".github/workflows/full-release-validation.yml"; +const ARTIFACT_WORKFLOW = ".github/workflows/full-release-artifacts.yml"; +const QUALIFIED_WORKFLOWS = new Set([FULL_RELEASE_WORKFLOW, ARTIFACT_WORKFLOW]); + +/** + * @typedef {object} NpmPreflightProducerOptions + * @property {unknown} manifest + * @property {string} repository + * @property {string} workflowFullRef + * @property {string} workflowSha + * @property {string | number} runId + * @property {string | number} runAttempt + * @property {string} [workflowPath] + * @property {unknown} [fullReleaseManifest] + * @property {string | number} [fullReleaseRunId] + * @property {string | number} [fullReleaseRunAttempt] + */ + +/** + * @typedef {object} FullReleaseNpmPreflightOptions + * @property {unknown} manifest + * @property {string} [repository] + * @property {string | number} runId + * @property {string | number} runAttempt + * @property {string} sourceSha + * @property {string} toolingSha + */ + +function parseJson(raw, label) { + try { + return JSON.parse(raw); + } catch (error) { + throw new Error(`${label} returned invalid JSON.`, { cause: error }); + } +} + +/** @param {NpmPreflightProducerOptions} options */ +export function validateNpmPreflightProducer({ + manifest, + repository, + workflowFullRef, + workflowSha, + runId, + runAttempt, + workflowPath = ".github/workflows/openclaw-npm-release.yml", + fullReleaseManifest, + fullReleaseRunId, + fullReleaseRunAttempt, +}) { + if (QUALIFIED_WORKFLOWS.has(workflowPath) && manifest?.version !== 3) { + throw new Error("FRV npm preflight requires qualified version 3 producer evidence."); + } + // Published v1 preflights did not record the original ref qualifier. Keep + // their existing recovery contract without inferring historical provenance. + if (manifest?.version === 1 && !Object.hasOwn(manifest, "producer")) { + return { originalWorkflowRef: null, provenance: "legacy-unrecorded" }; + } + if (![2, 3].includes(manifest?.version) || !isRecord(manifest.producer)) { + throw new Error("npm preflight producer metadata is missing or unsupported."); + } + if ( + !/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(repository ?? "") || + !/^refs\/(?:heads|tags)\/.+$/u.test(workflowFullRef ?? "") || + !/^[a-f0-9]{40}$/u.test(workflowSha ?? "") || + !/^[1-9][0-9]*$/u.test(String(runId ?? "")) || + !/^[1-9][0-9]*$/u.test(String(runAttempt ?? "")) || + ![".github/workflows/openclaw-npm-release.yml", ...QUALIFIED_WORKFLOWS].includes( + workflowPath, + ) || + (manifest.version === 2 && workflowPath !== ".github/workflows/openclaw-npm-release.yml") + ) { + throw new Error("npm preflight expected producer identity is invalid."); + } + const expected = { + repository, + workflowRef: `${repository}/${workflowPath}@${workflowFullRef}`, + workflowSha, + runId: String(runId), + runAttempt: String(runAttempt), + ...(manifest.version === 3 + ? { + producerWorkflowPath: ".github/workflows/openclaw-npm-preflight.yml", + jobId: manifest.producer.jobId, + jobName: manifest.producer.jobName, + } + : {}), + }; + if ( + manifest.version === 3 && + (!/^[1-9][0-9]*$/u.test(manifest.producer.jobId ?? "") || + typeof manifest.producer.jobName !== "string" || + !/(?:^| \/ )Qualify prepared npm package$/u.test(manifest.producer.jobName) || + manifest.preparedBundle?.schema !== "openclaw.prepared-npm-bundle/v1" || + manifest.preparedBundle.source?.sha !== manifest.releaseSha || + manifest.preparedBundle.package?.sha256 !== manifest.tarballSha256 || + manifest.preparedBundle.producer?.repository !== repository || + manifest.preparedBundle.producer?.workflowSha !== workflowSha) + ) { + throw new Error("npm preflight qualification does not bind the prepared package producer."); + } + if ( + Object.keys(manifest.producer).length !== Object.keys(expected).length || + Object.entries(expected).some(([key, value]) => manifest.producer[key] !== value) + ) { + throw new Error("npm preflight immutable producer identity mismatch."); + } + if (workflowPath === ARTIFACT_WORKFLOW) { + const qualified = validateFullReleaseNpmPreflight({ + manifest: fullReleaseManifest, + repository, + runId: fullReleaseRunId, + runAttempt: fullReleaseRunAttempt, + sourceSha: manifest.releaseSha, + toolingSha: workflowSha, + }); + if (!isDeepStrictEqual(qualified.producer, manifest.producer)) { + throw new Error("npm artifact producer is not the selected full release qualification."); + } + } + return { originalWorkflowRef: expected.workflowRef, provenance: "immutable-manifest" }; +} + +/** @param {NpmPreflightProducerOptions & {manifestSha256?: string, runGh?: typeof runReleaseToolingGh}} options */ +export function verifyNpmPreflightProducer({ runGh = runReleaseToolingGh, ...options }) { + const identity = validateNpmPreflightProducer(options); + if (options.manifest.version !== 3) { + return identity; + } + const producer = options.manifest.producer; + if (QUALIFIED_WORKFLOWS.has(options.workflowPath)) { + const { descriptor: qualified } = resolveFullReleaseNpmPreflight({ + manifest: options.fullReleaseManifest, + repository: options.repository, + runId: options.fullReleaseRunId ?? options.runId, + runAttempt: options.fullReleaseRunAttempt ?? options.runAttempt, + sourceSha: options.manifest.releaseSha, + toolingSha: options.workflowSha, + runGh, + }); + if ( + !isDeepStrictEqual(qualified.producer, producer) || + qualified.manifestSha256 !== options.manifestSha256 + ) { + throw new Error("npm preflight differs from the exact full release qualification."); + } + return identity; + } + verifyNpmBundleProducer({ + producer, + repository: options.repository, + toolingSha: options.workflowSha, + qualified: true, + requireCompletedParent: true, + runGh, + }); + return identity; +} + +/** @param {FullReleaseNpmPreflightOptions} options */ +export function validateFullReleaseNpmPreflight({ + manifest, + repository, + runId, + runAttempt, + sourceSha, + toolingSha, +}) { + const qualified = manifest?.publicationArtifacts?.npmPreflight; + const producer = qualified?.producer; + const producerRepository = repository ?? producer?.repository; + const independentProducer = + producer?.workflowRef === + `${producerRepository}/${ARTIFACT_WORKFLOW}@${manifest?.workflowFullRef}`; + if ( + !/^[1-9][0-9]*$/u.test(String(runId ?? "")) || + !/^[1-9][0-9]*$/u.test(String(runAttempt ?? "")) || + manifest?.workflowName !== "Full Release Validation" || + String(manifest.runId) !== String(runId) || + String(manifest.runAttempt) !== String(runAttempt) || + manifest.targetSha !== sourceSha || + manifest.workflowSha !== toolingSha || + qualified?.schema !== "openclaw.qualified-npm-preflight/v1" || + qualified.source?.sha !== sourceSha || + !/^[a-f0-9]{64}$/u.test(qualified.manifestSha256 ?? "") || + !/^[1-9][0-9]*$/u.test(qualified.artifact?.id ?? "") || + !/^[a-f0-9]{64}$/u.test(qualified.artifact?.digest ?? "") || + !/^openclaw-npm-preflight-[A-Za-z0-9_.-]+$/u.test(qualified.artifact?.name ?? "") || + !/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(producerRepository ?? "") || + producer?.repository !== producerRepository || + !/^[1-9][0-9]*$/u.test(producer?.runId ?? "") || + !/^[1-9][0-9]*$/u.test(producer?.runAttempt ?? "") || + qualified.artifact.runId !== producer?.runId || + qualified.artifact.runAttempt !== producer?.runAttempt || + producer?.workflowSha !== toolingSha || + (independentProducer + ? !/^refs\/(?:heads|tags)\/.+$/u.test(manifest.workflowFullRef ?? "") + : producer?.runId !== String(runId) || + producer?.runAttempt !== String(runAttempt) || + !producer?.workflowRef?.startsWith(`${producerRepository}/${FULL_RELEASE_WORKFLOW}@refs/`)) + ) { + throw new Error( + "Full Release Validation does not bind a qualified npm preflight for this exact release and attempt; supply its historical separate preflight run when recovering an older release.", + ); + } + return qualified; +} + +// The FRV run authorizes publication; its descriptor identifies the independent +// producer whose immutable archive survives a rerun of that authorization. +/** @param {FullReleaseNpmPreflightOptions & {repository: string, runGh?: typeof runReleaseToolingGh}} options */ +export function resolveFullReleaseNpmPreflight({ runGh = runReleaseToolingGh, ...options }) { + const descriptor = validateFullReleaseNpmPreflight(options); + const producer = descriptor.producer; + const { run } = verifyNpmBundleProducer({ + producer, + repository: options.repository, + toolingSha: options.toolingSha, + qualified: true, + requireCompletedParent: true, + runGh, + }); + const artifact = parseJson( + runGh(["api", `repos/${options.repository}/actions/artifacts/${descriptor.artifact.id}`]), + "qualified npm preflight artifact", + ); + if ( + String(artifact.id) !== descriptor.artifact.id || + artifact.name !== descriptor.artifact.name || + artifact.digest !== `sha256:${descriptor.artifact.digest}` || + artifact.expired !== false || + String(artifact.workflow_run?.id) !== producer.runId || + artifact.workflow_run?.head_sha !== options.toolingSha + ) { + throw new Error("Qualified npm preflight artifact identity changed."); + } + return { descriptor, producer, run, artifact }; +} + +/** @param {Parameters<typeof resolveFullReleaseNpmPreflight>[0] & {outputDir: string, token: string, fetchImpl?: typeof fetch, archivePath?: string, deadlineMs?: number}} options */ +export async function downloadFullReleaseNpmPreflight({ + outputDir, + token, + fetchImpl, + archivePath, + deadlineMs, + ...options +}) { + const resolved = resolveFullReleaseNpmPreflight(options); + const { descriptor, producer, artifact } = resolved; + const { archiveBytes } = await downloadExactActionsArtifactArchive({ + token, + fetchImpl, + archivePath, + deadlineMs, + expected: { + repository: options.repository, + artifactId: Number(descriptor.artifact.id), + artifactName: descriptor.artifact.name, + artifactDigest: `sha256:${descriptor.artifact.digest}`, + artifactSizeBytes: artifact.size_in_bytes, + artifactExpiresAt: artifact.expires_at, + runId: Number(producer.runId), + workflowSha: options.toolingSha, + }, + }); + const files = inspectActionsArtifactZipWithPolicy(archiveBytes, { + minEntries: 1, + maxEntries: 1024, + allowPath: (name) => + /^[A-Za-z0-9_.-]+\.(?:tgz|json|txt)$|^core-packages-SHA256SUMS$|^dependency-evidence\/[A-Za-z0-9_.-]+\.(?:json|md)$/u.test( + name, + ), + maxEntryBytes: (name) => (name.endsWith(".tgz") ? 192 : 17) * 1024 * 1024, + }); + const manifestBytes = files.get("preflight-manifest.json"); + if ( + !manifestBytes || + createHash("sha256").update(manifestBytes).digest("hex") !== descriptor.manifestSha256 + ) { + throw new Error("Downloaded npm preflight manifest differs from its qualified descriptor."); + } + mkdirSync(outputDir, { recursive: true }); + if (readdirSync(outputDir).length !== 0) { + throw new Error("Qualified npm preflight destination must be empty."); + } + for (const [name, bytes] of files) { + const path = join(outputDir, name); + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, bytes, { flag: "wx" }); + } + return resolved; +} + +// Actions exposes a short head branch for tags too; a matching branch makes +// producer provenance ambiguous even if both refs currently point at one SHA. +export function validateReleasePreflightTagIdentity({ branches, ...identity }) { + if ( + !Array.isArray(branches) || + branches.some( + (branch) => + !isRecord(branch) || + typeof branch.ref !== "string" || + branch.ref === `refs/heads/${identity.workflowRef}`, + ) + ) { + throw new Error("npm preflight has ambiguous protected tag provenance."); + } + const validated = validateReleaseToolingIdentity(identity); + if (validated.route !== "protected-tag") { + throw new Error("npm preflight producer must use a protected tag."); + } + return validated; +} + +export function verifyReleasePreflightToolingIdentity({ + repository, + publisherSha, + runGh = runReleaseToolingGh, + workflowFullRef, + workflowRef, + workflowSha, +}) { + if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(repository ?? "")) { + throw new Error("npm preflight repository must be owner/name."); + } + if (!/^[a-f0-9]{40}$/u.test(publisherSha ?? "")) { + throw new Error("publisher workflow SHA must be a lowercase 40-character commit SHA."); + } + const normalizedRepository = repository; + const targetSha = publisherSha; + const identity = { workflowFullRef, workflowRef, workflowSha }; + const tagRef = parseJson( + runGh(["api", `repos/${normalizedRepository}/git/ref/tags/${workflowRef}`, "--method", "GET"]), + "npm preflight producer tag", + ); + const branches = parseJson( + runGh([ + "api", + `repos/${normalizedRepository}/git/matching-refs/heads/${workflowRef}`, + "--method", + "GET", + ]), + "npm preflight producer branches", + ); + const validated = validateReleasePreflightTagIdentity({ ...identity, tagRef, branches }); + // Producer evidence and current publication authority are distinct. Require + // the producer on both trusted main and the current publisher's ancestry. + for (const target of ["main", targetSha]) { + const comparison = parseJson( + runGh([ + "api", + `repos/${normalizedRepository}/compare/${validated.sha}...${target}`, + "--method", + "GET", + "--jq", + "{status}", + ]), + "npm preflight producer ancestry", + ); + if (comparison?.status !== "ahead" && comparison?.status !== "identical") { + throw new Error(`npm preflight producer is not reachable from ${target}.`); + } + } + return validated; +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) { + try { + const { values } = parseArgs({ + options: { + repository: { type: "string" }, + "workflow-ref": { type: "string" }, + "workflow-full-ref": { type: "string" }, + "workflow-sha": { type: "string" }, + "publisher-sha": { type: "string" }, + manifest: { type: "string" }, + "run-id": { type: "string" }, + "run-attempt": { type: "string" }, + "workflow-path": { type: "string" }, + "full-release-manifest": { type: "string" }, + "full-release-run-id": { type: "string" }, + "full-release-run-attempt": { type: "string" }, + "resolve-full-release-manifest": { type: "string" }, + "source-sha": { type: "string" }, + "output-dir": { type: "string" }, + }, + }); + const options = { + repository: values.repository, + workflowRef: values["workflow-ref"], + workflowFullRef: values["workflow-full-ref"], + workflowSha: values["workflow-sha"], + publisherSha: values["publisher-sha"], + workflowPath: values["workflow-path"], + }; + let identity; + if (values["resolve-full-release-manifest"]) { + const resolution = { + manifest: parseJson( + readFileSync(values["resolve-full-release-manifest"], "utf8"), + "full release manifest", + ), + repository: values.repository, + runId: values["run-id"], + runAttempt: values["run-attempt"], + sourceSha: values["source-sha"], + toolingSha: values["workflow-sha"], + }; + identity = values["output-dir"] + ? await downloadFullReleaseNpmPreflight({ + ...resolution, + outputDir: values["output-dir"], + token: process.env.GH_TOKEN, + }) + : resolveFullReleaseNpmPreflight(resolution); + } else { + identity = values.manifest + ? verifyNpmPreflightProducer({ + ...options, + manifest: parseJson(readFileSync(values.manifest, "utf8"), "npm preflight manifest"), + manifestSha256: createHash("sha256") + .update(readFileSync(values.manifest)) + .digest("hex"), + fullReleaseManifest: values["full-release-manifest"] + ? parseJson( + readFileSync(values["full-release-manifest"], "utf8"), + "full release manifest", + ) + : undefined, + runId: values["run-id"], + runAttempt: values["run-attempt"], + fullReleaseRunId: values["full-release-run-id"], + fullReleaseRunAttempt: values["full-release-run-attempt"], + }) + : verifyReleasePreflightToolingIdentity(options); + } + process.stdout.write(`${JSON.stringify(identity)}\n`); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); + } +} diff --git a/scripts/npm-trusted-publisher-preflight.mjs b/scripts/npm-trusted-publisher-preflight.mjs new file mode 100644 index 0000000000000000000000000000000000000000..d78696232c6e7c2415f478d303da2c6f3ec769e7 --- /dev/null +++ b/scripts/npm-trusted-publisher-preflight.mjs @@ -0,0 +1,118 @@ +#!/usr/bin/env node + +import { readBoundedResponseText } from "./lib/bounded-response.mjs"; +import { isDirectRunUrl } from "./lib/direct-run.mjs"; + +const AUDIENCE = "npm:registry.npmjs.org"; +const NPM_REGISTRY = "https://registry.npmjs.org"; +const REQUEST_TIMEOUT_MS = 20_000; +const RESPONSE_BODY_MAX_BYTES = 16 * 1024; +const PACKAGE_NAME_PATTERN = /^(?:@[a-z0-9][a-z0-9._~-]*\/)?[a-z0-9][a-z0-9._~-]*$/u; + +async function requestToken(url, init, label, tokenField) { + const signal = AbortSignal.timeout(REQUEST_TIMEOUT_MS); + let response; + try { + response = await fetch(url, { + ...init, + signal, + }); + } catch (error) { + const detail = + error instanceof Error && error.name === "TimeoutError" ? " timed out" : " failed"; + throw new Error(`${label}${detail}.`, { cause: error }); + } + if (!response.ok) { + await response.body?.cancel().catch(() => undefined); + throw new Error(`${label} failed (HTTP ${response.status}).`); + } + + let text; + try { + text = await readBoundedResponseText(response, label, RESPONSE_BODY_MAX_BYTES, { signal }); + } catch (error) { + const detail = + error instanceof Error && error.name === "TimeoutError" + ? " timed out" + : " returned an invalid response body"; + throw new Error(`${label}${detail}.`, { cause: error }); + } + + let body; + try { + body = JSON.parse(text); + } catch (error) { + throw new Error(`${label} returned invalid JSON.`, { cause: error }); + } + if (!body || typeof body !== "object" || Array.isArray(body)) { + throw new Error(`${label} returned an invalid response shape.`); + } + const token = body[tokenField]; + if (typeof token !== "string" || token.length === 0) { + throw new Error(`${label} response is missing ${tokenField}.`); + } + return token; +} + +export async function preflightNpmTrustedPublisher(packageName) { + if ( + typeof packageName !== "string" || + packageName.length > 214 || + !PACKAGE_NAME_PATTERN.test(packageName) + ) { + throw new Error("A canonical npm package name is required."); + } + + const requestUrl = process.env.ACTIONS_ID_TOKEN_REQUEST_URL; + const githubRequestToken = process.env.ACTIONS_ID_TOKEN_REQUEST_TOKEN; + if (!requestUrl || !githubRequestToken) { + throw new Error("GitHub OIDC request credentials are unavailable; grant id-token: write."); + } + + let githubOidcUrl; + try { + githubOidcUrl = new URL(requestUrl); + } catch { + throw new Error("GitHub OIDC request URL is invalid."); + } + githubOidcUrl.searchParams.append("audience", AUDIENCE); + const githubOidcToken = await requestToken( + githubOidcUrl.href, + { + headers: { + Accept: "application/json", + Authorization: `Bearer ${githubRequestToken}`, + }, + }, + "GitHub OIDC token request", + "value", + ); + + // npm-package-arg's escapedName preserves the scope and escapes only its slash. + const escapedPackageName = packageName.replaceAll("/", "%2f"); + await requestToken( + `${NPM_REGISTRY}/-/npm/v1/oidc/token/exchange/package/${escapedPackageName}`, + { + method: "POST", + headers: { + Accept: "application/json", + Authorization: `Bearer ${githubOidcToken}`, + }, + }, + `npm trusted-publisher exchange for ${packageName}`, + "token", + ); + + console.log(`npm trusted-publisher OIDC exchange verified for ${packageName}.`); +} + +if (isDirectRunUrl(process.argv[1], import.meta.url)) { + try { + await preflightNpmTrustedPublisher(process.argv[2]); + } catch (error) { + console.error( + error instanceof Error ? error.message : "npm trusted-publisher preflight failed.", + ); + process.exitCode = 1; + } +} diff --git a/scripts/openclaw-npm-extended-stable-release.mjs b/scripts/openclaw-npm-extended-stable-release.mjs new file mode 100644 index 0000000000000000000000000000000000000000..fde83fa4e53378da58c1dbfd013142f87a742d0d --- /dev/null +++ b/scripts/openclaw-npm-extended-stable-release.mjs @@ -0,0 +1,621 @@ +#!/usr/bin/env node + +import { execFileSync, spawnSync } from "node:child_process"; +import { appendFileSync, readFileSync } from "node:fs"; +import { pathToFileURL } from "node:url"; +import { classifyReleaseTrain, parseReleaseVersion } from "./lib/release-version.mjs"; + +const SUPPORTED_DIST_TAGS = new Set(["alpha", "beta", "latest", "extended-stable"]); + +export function parseExtendedStableGuardBypass(value = "") { + if (value === "" || value === "false") { + return false; + } + if (value === "true") { + return true; + } + throw new Error(`BYPASS_EXTENDED_STABLE_GUARD must be "true" or "false"; got "${value}".`); +} + +function requireExtendedStableBypassTag(npmDistTag, bypassExtendedStableGuard) { + if (bypassExtendedStableGuard && npmDistTag !== "extended-stable") { + throw new Error( + "BYPASS_EXTENDED_STABLE_GUARD may only be used with the extended-stable npm dist-tag.", + ); + } +} + +export function validateNpmPublishBoundary( + packageVersion, + npmDistTag, + { bypassExtendedStableGuard = false } = {}, +) { + if (!SUPPORTED_DIST_TAGS.has(npmDistTag)) { + throw new Error(`Unsupported npm dist-tag "${npmDistTag}".`); + } + requireExtendedStableBypassTag(npmDistTag, bypassExtendedStableGuard); + const parsed = parseReleaseVersion(packageVersion); + if (parsed === null) { + throw new Error(`Unsupported release version "${packageVersion}".`); + } + const releaseTrain = classifyReleaseTrain(parsed); + + if (releaseTrain === "alpha") { + if (npmDistTag !== "alpha") { + throw new Error("Alpha prereleases must publish to the alpha npm dist-tag."); + } + return parsed; + } + if (releaseTrain === "beta") { + if (npmDistTag !== "beta") { + throw new Error("Beta prereleases must publish to the beta npm dist-tag."); + } + return parsed; + } + + if (npmDistTag === "extended-stable") { + if (parsed.correctionNumber !== undefined) { + throw new Error("Extended-stable npm publication does not allow correction suffixes."); + } + if (!bypassExtendedStableGuard && releaseTrain !== "extended-stable") { + throw new Error("Extended-stable npm publication requires release patch 33 or above."); + } + return parsed; + } + if ( + releaseTrain === "extended-stable" || + releaseTrain === "unsupported-extended-stable-correction" + ) { + throw new Error( + `Final or correction release patch 33 and above must publish to the extended-stable npm dist-tag; got ${npmDistTag}.`, + ); + } + return parsed; +} + +export function resolveNpmPreflightSdkSelectors(packageVersion, npmDistTag) { + const parsed = parseReleaseVersion(packageVersion); + return parsed && + classifyReleaseTrain(parsed) === "stable" && + ["beta", "latest"].includes(npmDistTag) + ? ["beta", "latest"] + : [npmDistTag]; +} + +export function validateNpmPreflightDistTag({ manifest, npmDistTag }) { + if (SUPPORTED_DIST_TAGS.has(npmDistTag) && manifest?.npmDistTag === npmDistTag) { + return; + } + // Only the qualified package format proves both regular-release predecessors. + // Historical receipts and prerelease/extended-stable packages retain exact channels. + const selectors = manifest?.pluginSdkApi?.selectors; + if ( + manifest?.version === 3 && + typeof manifest.packageVersion === "string" && + resolveNpmPreflightSdkSelectors(manifest.packageVersion, manifest.npmDistTag).length === 2 && + ["beta", "latest"].includes(npmDistTag) && + manifest.pluginSdkApi?.schema === "openclaw.plugin-sdk-api-release-evidence-set/v1" && + selectors && + Object.keys(selectors).length === 2 && + ["beta", "latest"].every( + (selector) => selectors[selector]?.schema === "openclaw.plugin-sdk-api-release-evidence/v1", + ) + ) { + return; + } + throw new Error( + `npm preflight dist-tag mismatch: expected ${npmDistTag}, got ${manifest?.npmDistTag}`, + ); +} + +export function validateExtendedStableNpmReleaseRequest(request) { + const bypassExtendedStableGuard = request.bypassExtendedStableGuard ?? false; + requireExtendedStableBypassTag(request.npmDistTag, bypassExtendedStableGuard); + const shaPreflight = + request.preflightOnly === true && /^[0-9a-f]{40}$/iu.test(request.releaseTag); + if (shaPreflight) { + if (!/^[0-9a-f]{40}$/iu.test(request.checkoutSha)) { + throw new Error("Validation-only SHA preflight requires the full checked-out commit SHA."); + } + if (request.releaseTag.toLowerCase() !== request.checkoutSha.toLowerCase()) { + throw new Error("Validation-only SHA must match the checked-out commit exactly."); + } + } + const effectiveReleaseTag = shaPreflight ? `v${request.packageVersion}` : request.releaseTag; + const taggedVersion = effectiveReleaseTag.startsWith("v") + ? parseReleaseVersion(effectiveReleaseTag.slice(1)) + : null; + + if (request.npmDistTag !== "extended-stable") { + if (taggedVersion !== null) { + validateNpmPublishBoundary(taggedVersion.version, request.npmDistTag, { + bypassExtendedStableGuard, + }); + } else if (!SUPPORTED_DIST_TAGS.has(request.npmDistTag)) { + throw new Error(`Unsupported npm dist-tag "${request.npmDistTag}".`); + } + return { extendedStable: false }; + } + + if ( + taggedVersion === null || + effectiveReleaseTag !== `v${taggedVersion.version}` || + taggedVersion.channel !== "stable" + ) { + throw new Error( + "Extended-stable npm publication requires an exact final vYYYY.M.P release tag.", + ); + } + validateNpmPublishBoundary(taggedVersion.version, request.npmDistTag, { + bypassExtendedStableGuard, + }); + + const releaseVersion = taggedVersion.version; + const extendedStableBranch = `extended-stable/${taggedVersion.year}.${taggedVersion.month}.33`; + if (request.packageVersion !== releaseVersion) { + throw new Error( + `Extended-stable npm package version mismatch: expected ${releaseVersion}, got ${request.packageVersion}.`, + ); + } + + if (!shaPreflight) { + const expectedWorkflowRef = `refs/heads/${extendedStableBranch}`; + if (request.npmWorkflowRef !== expectedWorkflowRef) { + throw new Error( + `Extended-stable npm workflow ref mismatch: expected ${expectedWorkflowRef}, got ${request.npmWorkflowRef}.`, + ); + } + const shaValues = [request.checkoutSha, request.tagSha, request.extendedStableBranchSha]; + if (shaValues.some((sha) => !/^[0-9a-f]{40}$/iu.test(sha))) { + throw new Error("Extended-stable npm release identity requires full 40-character Git SHAs."); + } + if (new Set(shaValues.map((sha) => sha.toLowerCase())).size !== 1) { + throw new Error("Extended-stable npm checkout, tag, and branch tip SHAs must match exactly."); + } + } + + if (bypassExtendedStableGuard) { + return { + extendedStable: true, + releaseVersion, + extendedStableBranch, + bypassExtendedStableGuard: true, + }; + } + + const mainVersion = parseReleaseVersion(request.mainPackageVersion); + if ( + mainVersion === null || + mainVersion.channel !== "stable" || + mainVersion.correctionNumber !== undefined + ) { + throw new Error("Protected main package version must be an exact final YYYY.M.P version."); + } + const mainCalendarMonth = mainVersion.year * 12 + mainVersion.month; + const releaseCalendarMonth = taggedVersion.year * 12 + taggedVersion.month; + // Keep one active trailing-month line; advancing main another month retires the older line. + if (mainCalendarMonth - releaseCalendarMonth !== 1) { + const expectedYear = mainVersion.month === 1 ? mainVersion.year - 1 : mainVersion.year; + const expectedMonth = mainVersion.month === 1 ? 12 : mainVersion.month - 1; + throw new Error( + `Extended-stable publishes only the trailing completed month: protected main ${request.mainPackageVersion} allows ${expectedYear}.${expectedMonth}.PATCH, not ${releaseVersion}. Retire the older line or dispatch with BYPASS_EXTENDED_STABLE_GUARD for an explicitly approved exception.`, + ); + } + if (classifyReleaseTrain(mainVersion) !== "stable") { + throw new Error("Protected main must remain on a daily patch below 33."); + } + return { extendedStable: true, releaseVersion, extendedStableBranch }; +} + +export function validateExtendedStableRunIdentity({ + run, + kind, + npmDistTag, + expectedBranch, + expectedSha, + preflightRunId = "", + preflightRunAttempt = "", + fullReleaseRunId = "", + fullReleaseRunAttempt = "", + workflowPath = "", +}) { + const fullReleasePreflight = + kind === "preflight" && run.workflowName === "Full Release Validation"; + if ( + fullReleasePreflight && + (!/^[1-9][0-9]*$/u.test(preflightRunId) || + !/^[1-9][0-9]*$/u.test(preflightRunAttempt) || + preflightRunId !== fullReleaseRunId || + preflightRunAttempt !== fullReleaseRunAttempt || + String(run.databaseId) !== preflightRunId || + String(run.attempt) !== preflightRunAttempt || + workflowPath.split("@", 1)[0] !== ".github/workflows/full-release-validation.yml" || + run.status !== "completed") + ) { + throw new Error("FRV npm preflight must be the exact selected full release run and attempt."); + } + const expectedWorkflowName = + kind === "preflight" && !fullReleasePreflight + ? "OpenClaw NPM Release" + : kind === "plugin" + ? "Plugin NPM Release" + : "Full Release Validation"; + const checks = [ + ["workflowName", expectedWorkflowName], + ["event", "workflow_dispatch"], + ...(kind === "validation" || kind === "plugin" ? [["status", "completed"]] : []), + ["conclusion", "success"], + ...(kind === "plugin" && npmDistTag === "extended-stable" + ? [["displayTitle", `Plugin NPM Release [extended-stable] ${expectedSha}`]] + : []), + ]; + for (const [key, expected] of checks) { + if (run[key] !== expected) { + throw new Error( + `Referenced ${kind} run must have ${key}=${expected}, got ${run[key] ?? "<missing>"}.`, + ); + } + } + // FRV runs trusted tooling against a separately pinned release source; its + // qualified manifest, not the workflow head, binds that source SHA. + if ( + !fullReleasePreflight && + npmDistTag === "extended-stable" && + (run.headBranch !== expectedBranch || run.headSha !== expectedSha) + ) { + throw new Error( + `Referenced extended-stable ${kind} run must have headBranch=${expectedBranch} and headSha=${expectedSha}; got ${run.headBranch ?? "<missing>"} and ${run.headSha ?? "<missing>"}.`, + ); + } + return run; +} + +export function validateFullReleaseValidationManifest({ + manifest, + npmDistTag, + expectedWorkflowRef, + expectedSha, + expectedRunId, + expectedRunAttempt, +}) { + if (manifest.workflowName !== "Full Release Validation") { + throw new Error( + `Full release validation manifest workflow mismatch: ${manifest.workflowName ?? "<missing>"}.`, + ); + } + if (manifest.targetSha !== expectedSha) { + throw new Error( + `Full release validation target SHA mismatch: expected ${expectedSha}, got ${manifest.targetSha ?? "<missing>"}.`, + ); + } + if (expectedRunId !== undefined && String(manifest.runId) !== String(expectedRunId)) { + throw new Error( + `Full release validation run ID mismatch: expected ${expectedRunId}, got ${manifest.runId ?? "<missing>"}.`, + ); + } + if ( + expectedRunAttempt !== undefined && + String(manifest.runAttempt) !== String(expectedRunAttempt) + ) { + throw new Error( + `Full release validation run attempt mismatch: expected ${expectedRunAttempt}, got ${manifest.runAttempt ?? "<missing>"}.`, + ); + } + if (npmDistTag === "extended-stable" && manifest.workflowRef !== expectedWorkflowRef) { + throw new Error( + `Full release validation workflow ref mismatch: expected ${expectedWorkflowRef}, got ${manifest.workflowRef ?? "<missing>"}.`, + ); + } + return manifest; +} + +export function parsePriorExtendedStableSelector(stdout) { + let tags; + try { + tags = JSON.parse(stdout); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new Error(`npm dist-tags query returned invalid JSON: ${message}`, { + cause: error, + }); + } + if (tags === null || typeof tags !== "object" || Array.isArray(tags)) { + throw new Error("npm dist-tags query did not return a JSON object."); + } + if (!Object.hasOwn(tags, "extended-stable")) { + return "absent"; + } + if (typeof tags["extended-stable"] !== "string" || tags["extended-stable"].trim() === "") { + throw new Error("npm extended-stable dist-tag was not a non-empty version string."); + } + return tags["extended-stable"]; +} + +export function capturePriorExtendedStableSelector({ query }) { + const result = query(); + if (result.status !== 0) { + throw new Error(`npm dist-tags query failed with exit code ${result.status ?? "unknown"}.`); + } + return parsePriorExtendedStableSelector(result.stdout); +} + +export async function verifyExtendedStableRegistryReadback({ + expectedVersion, + query, + sleep, + attempts = 12, + delayMs = 10_000, +}) { + let exactVersion = "missing"; + let extendedStableSelector = "missing"; + for (let attempt = 1; attempt <= attempts; attempt += 1) { + const exactResult = await query(`openclaw@${expectedVersion}`); + const extendedStableResult = await query("openclaw@extended-stable"); + exactVersion = exactResult.status === 0 ? exactResult.stdout.trim() : "missing"; + extendedStableSelector = + extendedStableResult.status === 0 ? extendedStableResult.stdout.trim() : "missing"; + if (exactVersion === expectedVersion && extendedStableSelector === expectedVersion) { + return { exactVersion, extendedStableSelector, attemptsUsed: attempt }; + } + if (attempt < attempts) { + await sleep(delayMs); + } + } + throw new Error( + `npm registry did not converge to openclaw@${expectedVersion} and openclaw@extended-stable=${expectedVersion} after ${attempts} attempts (exact=${exactVersion}, extended-stable=${extendedStableSelector}).`, + ); +} + +export function extendedStableSelectorRepairCommand(expectedVersion) { + const normalizedVersion = (expectedVersion ?? "").replace(/^v/u, ""); + const parsed = parseReleaseVersion(normalizedVersion); + if (parsed === null || parsed.channel !== "stable" || parsed.correctionNumber !== undefined) { + throw new Error("Extended-stable selector repair requires an exact final YYYY.M.P version."); + } + return `npm dist-tag add openclaw@${parsed.version} extended-stable`; +} + +function git(args) { + return execFileSync("git", args, { encoding: "utf8" }).trim(); +} + +function packageVersionAt(ref) { + return JSON.parse(git(["show", `${ref}:package.json`])).version; +} + +function validateRequestFromRepository() { + const npmDistTag = process.env.RELEASE_NPM_DIST_TAG ?? ""; + const releaseTag = process.env.RELEASE_TAG ?? ""; + const npmWorkflowRef = process.env.NPM_WORKFLOW_REF ?? ""; + const preflightOnly = process.env.PREFLIGHT_ONLY === "true"; + const bypassExtendedStableGuard = parseExtendedStableGuardBypass( + process.env.BYPASS_EXTENDED_STABLE_GUARD ?? "", + ); + const packageVersion = JSON.parse(readFileSync("package.json", "utf8")).version; + if (npmDistTag !== "extended-stable") { + return validateExtendedStableNpmReleaseRequest({ + bypassExtendedStableGuard, + npmDistTag, + releaseTag, + npmWorkflowRef, + checkoutSha: "", + tagSha: "", + extendedStableBranchSha: "", + packageVersion, + mainPackageVersion: "", + }); + } + + const shaPreflight = preflightOnly && /^[0-9a-f]{40}$/iu.test(releaseTag); + const effectiveReleaseTag = shaPreflight ? `v${packageVersion}` : releaseTag; + const parsed = effectiveReleaseTag.startsWith("v") + ? parseReleaseVersion(effectiveReleaseTag.slice(1)) + : null; + if (!parsed || parsed.channel !== "stable" || parsed.correctionNumber !== undefined) { + return validateExtendedStableNpmReleaseRequest({ + npmDistTag, + bypassExtendedStableGuard, + releaseTag, + preflightOnly, + npmWorkflowRef, + checkoutSha: "", + tagSha: "", + extendedStableBranchSha: "", + packageVersion, + mainPackageVersion: "", + }); + } + const extendedStableBranch = `extended-stable/${parsed.year}.${parsed.month}.33`; + if (shaPreflight) { + if (!bypassExtendedStableGuard) { + execFileSync( + "git", + ["fetch", "--no-tags", "origin", "+refs/heads/main:refs/remotes/origin/main"], + { stdio: "inherit" }, + ); + } + const checkoutSha = git(["rev-parse", "HEAD"]); + return validateExtendedStableNpmReleaseRequest({ + npmDistTag, + bypassExtendedStableGuard, + releaseTag, + preflightOnly, + npmWorkflowRef, + checkoutSha, + tagSha: checkoutSha, + extendedStableBranchSha: "", + packageVersion, + mainPackageVersion: bypassExtendedStableGuard + ? "" + : packageVersionAt("refs/remotes/origin/main"), + }); + } + if (bypassExtendedStableGuard) { + execFileSync( + "git", + [ + "fetch", + "--no-tags", + "origin", + `+refs/heads/${extendedStableBranch}:refs/remotes/origin/${extendedStableBranch}`, + ], + { stdio: "inherit" }, + ); + execFileSync( + "git", + ["fetch", "--no-tags", "origin", `+refs/tags/${releaseTag}:refs/tags/${releaseTag}`], + { stdio: "inherit" }, + ); + return validateExtendedStableNpmReleaseRequest({ + npmDistTag, + bypassExtendedStableGuard, + releaseTag, + npmWorkflowRef, + checkoutSha: git(["rev-parse", "HEAD"]), + tagSha: git(["rev-parse", `${releaseTag}^{commit}`]), + extendedStableBranchSha: git(["rev-parse", `refs/remotes/origin/${extendedStableBranch}`]), + packageVersion, + mainPackageVersion: "", + }); + } + execFileSync( + "git", + [ + "fetch", + "--no-tags", + "origin", + `+refs/heads/${extendedStableBranch}:refs/remotes/origin/${extendedStableBranch}`, + "+refs/heads/main:refs/remotes/origin/main", + ], + { stdio: "inherit" }, + ); + execFileSync( + "git", + ["fetch", "--no-tags", "origin", `+refs/tags/${releaseTag}:refs/tags/${releaseTag}`], + { stdio: "inherit" }, + ); + return validateExtendedStableNpmReleaseRequest({ + npmDistTag, + bypassExtendedStableGuard, + releaseTag, + npmWorkflowRef, + checkoutSha: git(["rev-parse", "HEAD"]), + tagSha: git(["rev-parse", `${releaseTag}^{commit}`]), + extendedStableBranchSha: git(["rev-parse", `refs/remotes/origin/${extendedStableBranch}`]), + packageVersion, + mainPackageVersion: packageVersionAt("refs/remotes/origin/main"), + }); +} + +function appendOutput(values) { + const output = process.env.GITHUB_OUTPUT; + if (!output) { + throw new Error("GITHUB_OUTPUT is required."); + } + appendFileSync( + output, + Object.entries(values) + .map(([key, value]) => `${key}=${value}\n`) + .join(""), + ); +} + +async function main() { + const command = process.argv[2]; + if (command === "validate-request") { + const result = validateRequestFromRepository(); + console.log( + result.extendedStable + ? `Validated extended-stable npm release ${result.releaseVersion}.` + : "Validated regular npm release request.", + ); + return; + } + if (command === "publish-plan") { + const npmDistTag = process.env.REQUESTED_PUBLISH_TAG ?? ""; + const bypassExtendedStableGuard = parseExtendedStableGuardBypass( + process.env.BYPASS_EXTENDED_STABLE_GUARD ?? "", + ); + const parsed = validateNpmPublishBoundary(process.env.PACKAGE_VERSION ?? "", npmDistTag, { + bypassExtendedStableGuard, + }); + console.log(parsed.channel); + console.log(npmDistTag); + return; + } + if (command === "verify-run") { + const run = JSON.parse(readFileSync(0, "utf8")); + validateExtendedStableRunIdentity({ + run, + kind: process.env.RUN_KIND, + npmDistTag: process.env.RELEASE_NPM_DIST_TAG, + expectedBranch: process.env.EXPECTED_EXTENDED_STABLE_BRANCH, + expectedSha: process.env.EXPECTED_RELEASE_SHA, + preflightRunId: process.env.PREFLIGHT_RUN_ID, + preflightRunAttempt: process.env.PREFLIGHT_RUN_ATTEMPT, + fullReleaseRunId: process.env.FULL_RELEASE_VALIDATION_RUN_ID, + fullReleaseRunAttempt: process.env.FULL_RELEASE_VALIDATION_RUN_ATTEMPT, + workflowPath: process.env.RUN_WORKFLOW_PATH, + }); + console.log(`Verified referenced ${process.env.RUN_KIND} run.`); + return; + } + if (command === "verify-preflight-channel") { + validateNpmPreflightDistTag({ + manifest: JSON.parse(readFileSync(0, "utf8")), + npmDistTag: process.env.RELEASE_NPM_DIST_TAG, + }); + return; + } + if (command === "verify-manifest") { + const manifest = JSON.parse(readFileSync(process.env.MANIFEST_FILE, "utf8")); + validateFullReleaseValidationManifest({ + manifest, + npmDistTag: process.env.RELEASE_NPM_DIST_TAG, + expectedWorkflowRef: process.env.EXPECTED_WORKFLOW_REF, + expectedSha: process.env.EXPECTED_RELEASE_SHA, + expectedRunId: process.env.FULL_RELEASE_VALIDATION_RUN_ID, + expectedRunAttempt: process.env.FULL_RELEASE_VALIDATION_RUN_ATTEMPT, + }); + return; + } + if (command === "capture-selector") { + const previous = capturePriorExtendedStableSelector({ + query: () => + spawnSync("npm", ["view", "openclaw", "dist-tags", "--json"], { encoding: "utf8" }), + }); + appendOutput({ previous }); + return; + } + if (command === "verify-readback") { + const expectedVersion = (process.env.EXPECTED_VERSION ?? "").replace(/^v/u, ""); + const result = await verifyExtendedStableRegistryReadback({ + expectedVersion, + query: (target) => spawnSync("npm", ["view", target, "version"], { encoding: "utf8" }), + sleep: (delayMs) => + new Promise((resolve) => { + setTimeout(resolve, delayMs); + }), + }); + appendOutput({ + exact_version: result.exactVersion, + extended_stable_selector: result.extendedStableSelector, + }); + return; + } + if (command === "repair-command") { + console.log(extendedStableSelectorRepairCommand(process.env.EXPECTED_VERSION)); + return; + } + throw new Error(`Unknown extended-stable npm release command: ${command ?? "<missing>"}.`); +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { + try { + await main(); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + console.error(`openclaw-npm-extended-stable-release: ${message}`); + process.exit(1); + } +} diff --git a/scripts/openclaw-npm-postpublish-verify.ts b/scripts/openclaw-npm-postpublish-verify.ts new file mode 100644 index 0000000000000000000000000000000000000000..a790935bd2bc1481daa1472371461624246fc86e --- /dev/null +++ b/scripts/openclaw-npm-postpublish-verify.ts @@ -0,0 +1,1252 @@ +#!/usr/bin/env -S node --import tsx +// Openclaw Npm Postpublish Verify script supports OpenClaw repository automation. + +import { createHash } from "node:crypto"; +import { + existsSync, + lstatSync, + mkdtempSync, + opendirSync, + readdirSync, + readFileSync, + realpathSync, + rmSync, +} from "node:fs"; +import { builtinModules, createRequire } from "node:module"; +import { tmpdir } from "node:os"; +import { isAbsolute, join, posix as pathPosix, relative, win32 as pathWin32 } from "node:path"; +import { pathToFileURL } from "node:url"; +import { expectDefined } from "../packages/normalization-core/src/expect.js"; +import { ALWAYS_ALLOWED_RUNTIME_DIR_NAMES } from "../src/plugin-sdk/facade-activation-contract.ts"; +import { BUNDLED_RUNTIME_SIDECAR_PATHS } from "../src/plugins/runtime-sidecar-paths.ts"; +import { + WORKER_BUNDLE_ENTRY_PATH, + WORKER_BUNDLE_RSYNC_RECEIVER_PATH, +} from "../src/shared/worker-bundle-hash.js"; +import { readBoundedResponseText } from "./lib/bounded-response.mjs"; +import { listBundledPluginPackArtifacts } from "./lib/bundled-plugin-build-entries.mjs"; +import { formatErrorMessage } from "./lib/error-format.mts"; +import { verifyNpmRegistrySignatures } from "./lib/npm-registry-signatures.mjs"; +import { runNpmVerifyCommand } from "./lib/npm-verify-exec.ts"; +import { + comparePackageDistInventory, + PACKAGE_DIST_INVENTORY_RELATIVE_PATH, +} from "./lib/package-dist-inventory-contract.mts"; +import { collectPackageRootImports } from "./lib/package-root-imports.ts"; +import { + collectRuntimeDependencySpecs, + packageNameFromSpecifier, +} from "./lib/plugin-package-dependencies.mts"; +import { classifyReleaseTrain } from "./lib/release-version.mjs"; +import { + readRuntimeDependencyOwnership, + RUNTIME_DEPENDENCY_OWNERSHIP_RELATIVE_PATH, + type RuntimeDependencyOwnership, +} from "./lib/runtime-dependency-ownership-contract.mts"; +import { runInstalledWorkspaceBootstrapSmoke } from "./lib/workspace-bootstrap-smoke.mts"; +import { parseReleaseVersion, resolveNpmCommandInvocation } from "./openclaw-npm-release-check.ts"; +import { buildCmdExeCommandLine, resolveWindowsCmdExePath } from "./windows-cmd-helpers.mjs"; + +type InstalledPackageJson = { + name?: string; + version?: string; + dependencies?: Record<string, string>; + optionalDependencies?: Record<string, string>; +}; + +type InstalledBundledExtensionPackageJson = { + dependencies?: Record<string, string>; + optionalDependencies?: Record<string, string>; +}; + +type InstalledBundledExtensionManifestRecord = { + id: string; + manifest: InstalledBundledExtensionPackageJson; + path: string; +}; + +const MAX_BUNDLED_EXTENSION_MANIFEST_BYTES = 1024 * 1024; +const LEGACY_CONTEXT_ENGINE_UNRESOLVED_RUNTIME_MARKER = + "Failed to load legacy context engine runtime."; +// Package verification always covers the complete published inventory, even +// when the invoking build inherited a plugin-selection filter. +const PACKAGED_BUNDLED_PLUGIN_ARTIFACTS = new Set( + listBundledPluginPackArtifacts({ + env: { ...process.env, OPENCLAW_BUNDLED_PLUGIN_BUILD_IDS: undefined }, + }), +); +const PUBLISHED_BUNDLED_RUNTIME_SIDECAR_PATHS = BUNDLED_RUNTIME_SIDECAR_PATHS.filter( + (relativePath) => PACKAGED_BUNDLED_PLUGIN_ARTIFACTS.has(relativePath), +); +const NODE_BUILTIN_MODULES = new Set(builtinModules.map((name) => name.replace(/^node:/u, ""))); +const MAX_INSTALLED_ROOT_PACKAGE_JSON_BYTES = 1024 * 1024; +const MAX_INSTALLED_ROOT_DIST_JS_BYTES = 6 * 1024 * 1024; +const MAX_INSTALLED_WORKER_DEPLOY_DIST_JS_BYTES = 80 * 1024 * 1024; +// Keep the dependency scan bounded while allowing headroom for generated root chunks. +const MAX_INSTALLED_ROOT_DIST_JS_FILES = 10_000; +const ROOT_DIST_JAVASCRIPT_MODULE_FILE_RE = /\.(?:c|m)?js$/u; +// The ~69 MB self-contained worker needs extra headroom, but synchronous read/parse stays bounded. +const SELF_CONTAINED_WORKER_DEPLOY_DIST_PATHS = new Set([ + `worker/${WORKER_BUNDLE_ENTRY_PATH}`, + `worker/${WORKER_BUNDLE_RSYNC_RECEIVER_PATH}`, +]); +const OPTIONAL_OR_EXTERNALIZED_RUNTIME_IMPORTS = new Set([ + // Optional A2UI markdown renderer. The Canvas host bundle catches the missing + // package and falls back when the optional renderer is unavailable. + "@a2ui/markdown-it", + "@discordjs/opus", + "@lancedb/lancedb", + // Feishu/Lark remains a bundled plugin package. Root dist can retain orphaned + // lazy chunks from the plugin build even though dist/extensions/feishu is + // externalized from the root package scan. + "@larksuiteoapi/node-sdk", + // Discord remains an official external plugin. The root package can retain + // orphaned lazy chunks from the plugin build, but the plugin owns prism-media. + "prism-media", + "@matrix-org/matrix-sdk-crypto-nodejs", + "link-preview-js", + "matrix-js-sdk", + // Public plugin SDK contract helpers are intentionally test-only entrypoints. + // Consumers importing them run under their own Vitest dev dependency. + "vitest", +]); +const require = createRequire(import.meta.url); +const acorn = require("acorn") as typeof import("acorn"); + +type DistJavaScriptFileListResult = + | { files: string[]; limitExceeded: false } + | { files: string[]; limit: number; limitExceeded: true }; + +type InstalledRootDistJavaScriptReadResult = + | { error: string; ok: false } + | { ok: true; relativePath: string; source: string }; + +type PublishedInstallScenario = { + name: string; + installSpecs: string[]; + expectedVersion: string; +}; + +type OpenClawNpmPostpublishVerifyArgs = + | { + help: false; + version: string; + } + | { + help: true; + version: ""; + }; + +export function openClawNpmPostpublishVerifyUsage(): string { + return "Usage: node --import tsx scripts/openclaw-npm-postpublish-verify.ts <version>"; +} + +export function parseOpenClawNpmPostpublishVerifyArgs( + argv: readonly string[], +): OpenClawNpmPostpublishVerifyArgs { + const args = argv[0] === "--" ? argv.slice(1) : argv; + const version = args[0]?.trim() ?? ""; + if (version === "--help" || version === "-h") { + return { help: true, version: "" }; + } + if (!version) { + throw new Error(openClawNpmPostpublishVerifyUsage()); + } + if (version.startsWith("-")) { + throw new Error(`Unknown openclaw npm postpublish verifier option: ${version}`); + } + const extraArg = args[1]?.trim(); + if (args.length > 1) { + throw new Error(`Unexpected openclaw npm postpublish verifier argument: ${extraArg}`); + } + return { help: false, version }; +} + +export function buildPublishedInstallScenarios(version: string): PublishedInstallScenario[] { + const parsed = parseReleaseVersion(version); + if (parsed === null) { + throw new Error(`Unsupported release version "${version}".`); + } + + const exactSpec = `openclaw@${version}`; + const scenarios: PublishedInstallScenario[] = [ + { + name: "fresh-exact", + installSpecs: [exactSpec], + expectedVersion: version, + }, + ]; + + if (parsed.channel === "stable" && parsed.correctionNumber !== undefined) { + scenarios.push({ + name: "upgrade-from-base-stable", + installSpecs: [`openclaw@${parsed.baseVersion}`, exactSpec], + expectedVersion: version, + }); + } + + return scenarios; +} + +type NpmRegistryKey = { + key: string; + keyid: string; +}; + +type NpmRegistrySignature = { + keyid: string; + sig: string; +}; + +type NpmRegistryAttestation = { + bundle?: { + dsseEnvelope?: { + payload?: string; + }; + }; + predicateType?: string; +}; + +type NpmProvenanceVerificationPolicy = { + certificateIdentityURI: string; + certificateIssuer: string; +}; + +type VerifyNpmProvenanceBundle = ( + bundle: unknown, + policy: NpmProvenanceVerificationPolicy, +) => Promise<void>; + +type NpmProvenanceStatement = { + predicate?: { + buildDefinition?: { + externalParameters?: { + workflow?: { + path?: string; + ref?: string; + repository?: string; + }; + }; + resolvedDependencies?: Array<{ + digest?: { + gitCommit?: string; + }; + uri?: string; + }>; + }; + runDetails?: { + builder?: { + id?: string; + }; + }; + }; + subject?: Array<{ + digest?: Record<string, string>; + name?: string; + }>; +}; + +const NPM_PROVENANCE_PREDICATE_TYPE = "https://slsa.dev/provenance/v1"; +const NPM_PROVENANCE_REPOSITORY = "https://github.com/openclaw/openclaw"; +const NPM_PROVENANCE_WORKFLOW_PATH = ".github/workflows/openclaw-npm-release.yml"; +const NPM_PROVENANCE_CERTIFICATE_ISSUER = "https://token.actions.githubusercontent.com"; +const NPM_PROVENANCE_BUILDER_ID = "https://github.com/actions/runner/github-hosted"; +const NPM_REGISTRY_REQUEST_TIMEOUT_MS = 30_000; +const NPM_REGISTRY_RESPONSE_BODY_MAX_BYTES = 4 * 1024 * 1024; +const NPM_REGISTRY_PROVENANCE_ATTEMPTS = 30; +const NPM_REGISTRY_PROVENANCE_RETRY_MAX_DELAY_MS = 10_000; + +type FetchRegistryJsonOptions = { + fetchImpl?: typeof fetch; + maxBodyBytes?: number; + timeoutMs?: number; +}; + +function resolveNpmProvenanceVerificationPolicy( + statement: NpmProvenanceStatement, + version: string, + expectedWorkflow?: { + ref?: string; + sha?: string; + }, +): NpmProvenanceVerificationPolicy { + const parsedVersion = parseReleaseVersion(version); + if (parsedVersion === null) { + throw new Error(`Unsupported release version "${version}".`); + } + const workflow = statement.predicate?.buildDefinition?.externalParameters?.workflow; + const workflowRef = workflow?.ref; + const expectedReleaseRef = `refs/heads/release/${parsedVersion.baseVersion}`; + // A month's final patch >=33 releases stay on its canonical .33 maintenance branch. + const isExpectedExtendedStableRef = + classifyReleaseTrain(parsedVersion) === "extended-stable" && + workflowRef === `refs/heads/extended-stable/${parsedVersion.year}.${parsedVersion.month}.33`; + const protectedReleasePublishMatch = + /^refs\/tags\/release-publish\/([a-f0-9]{12})-[1-9][0-9]*$/u.exec(workflowRef ?? ""); + let protectedReleasePublishTrusted = false; + if (protectedReleasePublishMatch) { + const expectedRef = expectedWorkflow?.ref; + const expectedSha = expectedWorkflow?.sha; + if ( + expectedRef !== workflowRef || + !/^[a-f0-9]{40}$/u.test(expectedSha ?? "") || + expectedSha?.slice(0, 12) !== protectedReleasePublishMatch[1] + ) { + throw new Error( + "npm provenance SHA-pinned release-publish ref does not match the approved workflow ref and SHA.", + ); + } + const expectedDependencyUri = `git+${NPM_PROVENANCE_REPOSITORY}@${workflowRef}`; + protectedReleasePublishTrusted = + statement.predicate?.buildDefinition?.resolvedDependencies?.some( + (dependency) => + dependency.uri === expectedDependencyUri && dependency.digest?.gitCommit === expectedSha, + ) === true; + if (!protectedReleasePublishTrusted) { + throw new Error( + "npm provenance does not bind the approved SHA-pinned release-publish ref to its workflow revision.", + ); + } + } + const isTrustedRef = + workflowRef === "refs/heads/main" || + workflowRef === expectedReleaseRef || + isExpectedExtendedStableRef || + protectedReleasePublishTrusted || + (parsedVersion.channel === "alpha" && + /^refs\/heads\/tideclaw\/alpha\/[0-9]{4}-[0-9]{2}-[0-9]{2}-[0-9]{4}Z$/u.test( + workflowRef ?? "", + )); + + if ( + workflow?.repository !== NPM_PROVENANCE_REPOSITORY || + workflow?.path !== NPM_PROVENANCE_WORKFLOW_PATH || + !isTrustedRef || + statement.predicate?.runDetails?.builder?.id !== NPM_PROVENANCE_BUILDER_ID + ) { + throw new Error( + `npm provenance attestation does not bind ${version} to the trusted OpenClaw GitHub release workflow.`, + ); + } + + return { + certificateIssuer: NPM_PROVENANCE_CERTIFICATE_ISSUER, + certificateIdentityURI: `${NPM_PROVENANCE_REPOSITORY}/${NPM_PROVENANCE_WORKFLOW_PATH}@${workflowRef}`, + }; +} + +async function verifySigstoreNpmProvenanceBundle( + bundle: unknown, + policy: NpmProvenanceVerificationPolicy, +): Promise<void> { + const sigstore = require("sigstore") as { verify: VerifyNpmProvenanceBundle }; + await sigstore.verify(bundle, policy); +} + +export async function verifyNpmProvenanceAttestation(params: { + attestations: NpmRegistryAttestation[]; + expectedWorkflowRef?: string; + expectedWorkflowSha?: string; + integrity: string; + packageName: string; + verifyBundle?: VerifyNpmProvenanceBundle; + version: string; +}): Promise<void> { + const expectedSubject = `pkg:npm/${params.packageName}@${params.version}`; + const expectedSha512 = Buffer.from(params.integrity.slice("sha512-".length), "base64").toString( + "hex", + ); + const verifyBundle = params.verifyBundle ?? verifySigstoreNpmProvenanceBundle; + let verificationError: unknown; + let policyError: unknown; + + for (const attestation of params.attestations) { + if (attestation.predicateType !== NPM_PROVENANCE_PREDICATE_TYPE) { + continue; + } + const payload = attestation.bundle?.dsseEnvelope?.payload; + if (!payload) { + continue; + } + try { + const statement = JSON.parse( + Buffer.from(payload, "base64").toString("utf8"), + ) as NpmProvenanceStatement; + if ( + statement.subject?.some( + (subject) => + subject.name === expectedSubject && subject.digest?.sha512 === expectedSha512, + ) + ) { + let policy: NpmProvenanceVerificationPolicy; + try { + policy = resolveNpmProvenanceVerificationPolicy(statement, params.version, { + ref: params.expectedWorkflowRef, + sha: params.expectedWorkflowSha, + }); + } catch (error) { + policyError = error; + continue; + } + try { + await verifyBundle(attestation.bundle, policy); + return; + } catch (error) { + verificationError = error; + } + } + } catch { + // Try the remaining attestations before reporting the missing match. + } + } + + if (verificationError) { + throw new Error( + `npm provenance attestation failed Sigstore verification for ${params.packageName}@${params.version}: ${formatErrorMessage(verificationError)}`, + ); + } + + if (policyError instanceof Error) { + throw policyError; + } + if (policyError) { + throw new Error( + `npm provenance attestation policy evaluation failed for ${params.packageName}@${params.version}: ${formatErrorMessage(policyError)}`, + ); + } + + throw new Error( + `npm provenance attestation does not match ${params.packageName}@${params.version} and its registry integrity.`, + ); +} + +export function collectInstalledPackageErrors(params: { + additionalCompanionManifestRoots?: string[]; + expectedVersion: string; + installedVersion: string; + packageRoot: string; +}): string[] { + const errors: string[] = []; + const installedVersion = normalizeInstalledBinaryVersion(params.installedVersion); + + if (installedVersion !== params.expectedVersion) { + errors.push( + `installed package version mismatch: expected ${params.expectedVersion}, found ${params.installedVersion || "<missing>"}.`, + ); + } + + for (const relativePath of collectInstalledBundledRuntimeSidecarPaths(params.packageRoot)) { + if (!existsSync(join(params.packageRoot, relativePath))) { + errors.push(`installed package is missing required bundled runtime sidecar: ${relativePath}`); + } + } + + errors.push(...collectInstalledBundledExtensionManifestErrors(params.packageRoot)); + errors.push(...collectInstalledAlwaysAllowedRuntimeFacadeErrors(params.packageRoot)); + errors.push(...collectInstalledContextEngineRuntimeErrors(params.packageRoot)); + errors.push(...collectInstalledPluginSdkDeclarationErrors(params.packageRoot)); + errors.push( + ...collectInstalledRootDependencyManifestErrors( + params.packageRoot, + params.additionalCompanionManifestRoots, + ), + ); + + return [...new Set(errors)]; +} + +export function collectInstalledAlwaysAllowedRuntimeFacadeErrors(packageRoot: string): string[] { + const errors: string[] = []; + const activationRuntimePath = "dist/facade-activation-check.runtime.js"; + if (!existsSync(join(packageRoot, activationRuntimePath))) { + errors.push( + `installed package is missing required facade activation runtime: ${activationRuntimePath}`, + ); + } + for (const dirName of ALWAYS_ALLOWED_RUNTIME_DIR_NAMES) { + const relativePath = `dist/extensions/${dirName}/runtime-api.js`; + if (!existsSync(join(packageRoot, relativePath))) { + errors.push( + `installed package allows bundled runtime facade ${dirName}/runtime-api.js but is missing required runtime sidecar: ${relativePath}.`, + ); + } + } + return errors; +} + +function collectInstalledBundledExtensionIds(packageRoot: string): Set<string> { + const extensionsDir = join(packageRoot, "dist", "extensions"); + if (!existsSync(extensionsDir)) { + return new Set(); + } + const ids = new Set<string>(); + for (const entry of readdirSync(extensionsDir, { withFileTypes: true })) { + if (!entry.isDirectory()) { + continue; + } + if (existsSync(join(extensionsDir, entry.name, "package.json"))) { + ids.add(entry.name); + } + } + return ids; +} + +export function collectInstalledBundledRuntimeSidecarPaths(packageRoot: string): string[] { + const installedExtensionIds = collectInstalledBundledExtensionIds(packageRoot); + return PUBLISHED_BUNDLED_RUNTIME_SIDECAR_PATHS.filter((relativePath) => { + const match = /^dist\/extensions\/([^/]+)\//u.exec(relativePath); + return ( + match !== null && + installedExtensionIds.has(expectDefined(match[1], "bundled runtime extension id")) + ); + }); +} + +export function collectInstalledBundledExtensionManifestErrors(packageRoot: string): string[] { + return readBundledExtensionPackageJsons(packageRoot).errors; +} + +export function normalizeInstalledBinaryVersion(output: string): string { + const trimmed = output.trim(); + const versionMatch = /\b\d{4}\.\d{1,2}\.\d{1,2}(?:-\d+|-(?:alpha|beta)\.\d+)?\b/u.exec(trimmed); + return versionMatch?.[0] ?? trimmed; +} + +function listInstalledRootDistJavaScriptFiles(packageRoot: string): DistJavaScriptFileListResult { + const distDir = join(packageRoot, "dist"); + if (!existsSync(distDir)) { + return { files: [], limitExceeded: false }; + } + + const pending = [distDir]; + const files: string[] = []; + while (pending.length > 0) { + const currentDir = pending.pop(); + if (!currentDir) { + continue; + } + const dir = opendirSync(currentDir); + try { + while (true) { + const entry = dir.readSync(); + if (!entry) { + break; + } + + const entryPath = join(currentDir, entry.name); + const relativePath = relative(distDir, entryPath).replaceAll("\\", "/"); + if (relativePath === "extensions" || relativePath.startsWith("extensions/")) { + continue; + } + if (entry.isDirectory()) { + pending.push(entryPath); + continue; + } + if (entry.isFile() && ROOT_DIST_JAVASCRIPT_MODULE_FILE_RE.test(entry.name)) { + files.push(entryPath); + if (files.length > MAX_INSTALLED_ROOT_DIST_JS_FILES) { + return { + files, + limit: MAX_INSTALLED_ROOT_DIST_JS_FILES, + limitExceeded: true, + }; + } + } + } + } finally { + dir.closeSync(); + } + } + + return { files, limitExceeded: false }; +} + +function formatInstalledDistFileScanLimitError(scope: string, limit: number): string { + return `installed package ${scope} contains more than ${limit} JavaScript files; refusing to scan unbounded package contents.`; +} + +function readInstalledRootDistJavaScriptFile( + packageRoot: string, + filePath: string, +): InstalledRootDistJavaScriptReadResult { + const relativePath = relative(join(packageRoot, "dist"), filePath).replaceAll("\\", "/"); + const maxBytes = SELF_CONTAINED_WORKER_DEPLOY_DIST_PATHS.has(relativePath) + ? MAX_INSTALLED_WORKER_DEPLOY_DIST_JS_BYTES + : MAX_INSTALLED_ROOT_DIST_JS_BYTES; + const fileStat = lstatSync(filePath); + if (!fileStat.isFile() || fileStat.size > maxBytes) { + return { + error: `installed package root dist file '${relativePath}' is invalid or exceeds ${maxBytes} bytes.`, + ok: false, + }; + } + return { ok: true, relativePath, source: readFileSync(filePath, "utf8") }; +} + +export function collectInstalledContextEngineRuntimeErrors(packageRoot: string): string[] { + const distFiles = listInstalledRootDistJavaScriptFiles(packageRoot); + if (distFiles.limitExceeded) { + return [formatInstalledDistFileScanLimitError("root dist", distFiles.limit)]; + } + + // The legacy marker is a root runtime bundling contract; extension assets are plugin-owned. + for (const filePath of distFiles.files) { + const file = readInstalledRootDistJavaScriptFile(packageRoot, filePath); + if (!file.ok) { + return [file.error]; + } + if (file.source.includes(LEGACY_CONTEXT_ENGINE_UNRESOLVED_RUNTIME_MARKER)) { + return [ + "installed package includes unresolved legacy context engine runtime loader; rebuild with a bundler-traceable LegacyContextEngine import.", + ]; + } + } + return []; +} + +function collectInstalledPluginSdkDeclarationErrors(packageRoot: string): string[] { + const pluginSdkDistRoot = join(packageRoot, "dist", "plugin-sdk"); + const errors: string[] = []; + const forbiddenPrivateWorkspaceSpecifiers = ["@openclaw/llm-core"]; + + if (!existsSync(pluginSdkDistRoot)) { + return []; + } + + for (const entry of readdirSync(pluginSdkDistRoot, { withFileTypes: true })) { + if (!entry.isFile() || !entry.name.endsWith(".d.ts")) { + continue; + } + + const relativePath = `dist/plugin-sdk/${entry.name}`; + const content = readFileSync(join(pluginSdkDistRoot, entry.name), "utf8"); + for (const specifier of forbiddenPrivateWorkspaceSpecifiers) { + if (content.includes(`"${specifier}`) || content.includes(`'${specifier}`)) { + errors.push( + `installed package plugin SDK declaration '${relativePath}' references private workspace package ${specifier}.`, + ); + } + } + } + + return errors; +} + +type ParsedImportSpecifiersResult = + | { + ok: true; + imports: string[]; + } + | { ok: false; error: string }; + +function extractJavaScriptImportSpecifiers(source: string): ParsedImportSpecifiersResult { + try { + // Keep strict JavaScript validation: TypeScript accepts some invalid JS bindings/contexts. + acorn.parse(source, { + allowHashBang: true, + ecmaVersion: "latest", + sourceType: "module", + }); + return { ok: true, imports: collectPackageRootImports(source) }; + } catch (error) { + return { ok: false, error: formatErrorMessage(error) }; + } +} + +export function collectInstalledRootDependencyManifestErrors( + packageRoot: string, + additionalCompanionManifestRoots: string[] = [], +): string[] { + const packageJsonPath = join(packageRoot, "package.json"); + if (!existsSync(packageJsonPath)) { + return ["installed package is missing package.json."]; + } + const packageJsonStat = lstatSync(packageJsonPath); + if (!packageJsonStat.isFile() || packageJsonStat.size > MAX_INSTALLED_ROOT_PACKAGE_JSON_BYTES) { + return [ + `installed package.json is invalid or exceeds ${MAX_INSTALLED_ROOT_PACKAGE_JSON_BYTES} bytes.`, + ]; + } + let rootPackageJson: InstalledPackageJson; + try { + rootPackageJson = JSON.parse(readFileSync(packageJsonPath, "utf8")) as InstalledPackageJson; + } catch (error) { + return [`installed package.json could not be parsed: ${formatErrorMessage(error)}.`]; + } + const declaredRuntimeDeps = new Set([ + ...Object.keys(rootPackageJson.dependencies ?? {}), + ...Object.keys(rootPackageJson.optionalDependencies ?? {}), + ]); + const distFiles = listInstalledRootDistJavaScriptFiles(packageRoot); + if (distFiles.limitExceeded) { + return [formatInstalledDistFileScanLimitError("root dist", distFiles.limit)]; + } + const missingImporters = new Map<string, Set<string>>(); + const bundledExtensionRuntimeDependencyOwners = + collectBundledExtensionRuntimeDependencyOwners(packageRoot); + const companionManifestRoots = [ + join(packageRoot, "..", "@openclaw"), + ...additionalCompanionManifestRoots, + ]; + const companionManifestCache = new Map<string, InstalledPackageJson | null>(); + let runtimeDependencyOwnership: RuntimeDependencyOwnership | null; + try { + runtimeDependencyOwnership = readRuntimeDependencyOwnership(packageRoot); + } catch (error) { + return [ + `installed package runtime dependency ownership is invalid: ${RUNTIME_DEPENDENCY_OWNERSHIP_RELATIVE_PATH}: ${formatErrorMessage(error)}.`, + ]; + } + const importsByFile = new Map<string, string[]>(); + const extensionsByFile = new Map<string, string[]>(); + + for (const filePath of distFiles.files) { + const file = readInstalledRootDistJavaScriptFile(packageRoot, filePath); + if (!file.ok) { + return [file.error]; + } + const parsedSpecifiers = extractJavaScriptImportSpecifiers(file.source); + if (!parsedSpecifiers.ok) { + return [ + `installed package root dist file '${file.relativePath}' could not be parsed for runtime dependency verification: ${parsedSpecifiers.error}.`, + ]; + } + importsByFile.set(file.relativePath, parsedSpecifiers.imports); + const owners = runtimeDependencyOwnership?.chunks[file.relativePath]; + if (owners) { + if (owners.sha256 !== createHash("sha256").update(file.source).digest("hex")) { + return [ + `installed package runtime dependency ownership does not match '${file.relativePath}'; rebuild the package.`, + ]; + } + extensionsByFile.set(file.relativePath, owners.extensions); + } + } + + // Root imports from any build output override plugin evidence, including + // relative createRequire calls that are absent from the bundler's graph. + const pending = [...importsByFile.keys()].filter((file) => !extensionsByFile.has(file)); + const visited = new Set<string>(); + const distDir = importsByFile.size ? realpathSync(join(packageRoot, "dist")) : ""; + while (pending.length > 0) { + const file = pending.pop()!; + if (visited.has(file)) { + continue; + } + visited.add(file); + extensionsByFile.delete(file); + const resolveImport = createRequire(pathToFileURL(join(distDir, file))).resolve; + for (const specifier of importsByFile.get(file) ?? []) { + if (specifier.startsWith(".")) { + try { + const importedPath = resolveImport(specifier.replace(/[?#].*$/u, "")); + const importedFile = relative(distDir, importedPath).replaceAll("\\", "/"); + if (importsByFile.has(importedFile)) { + pending.push(importedFile); + } + } catch { + // Missing relative modules are reported by package closure checks. + } + } + } + } + + for (const [file, imports] of importsByFile) { + const extensions = extensionsByFile.get(file); + for (const specifier of imports) { + const dependencyName = packageNameFromSpecifier(specifier); + if ( + !dependencyName || + NODE_BUILTIN_MODULES.has(dependencyName) || + OPTIONAL_OR_EXTERNALIZED_RUNTIME_IMPORTS.has(dependencyName) || + declaredRuntimeDeps.has(dependencyName) || + (extensions?.length && + extensions.every( + (extensionId) => + bundledExtensionRuntimeDependencyOwners.get(dependencyName)?.has(extensionId) || + isInstalledCompanionExtensionOwnedRuntimeImport({ + dependencyName, + extensionId, + manifestRoots: companionManifestRoots, + manifestCache: companionManifestCache, + }), + )) + ) { + continue; + } + const importers = missingImporters.get(dependencyName) ?? new Set<string>(); + importers.add(file); + missingImporters.set(dependencyName, importers); + } + } + + return [...missingImporters.entries()] + .map(([dependencyName, importers]) => { + const importerList = [...importers].toSorted((left, right) => left.localeCompare(right)); + return `installed package root is missing declared runtime dependency '${dependencyName}' for dist importers: ${importerList.join(", ")}. Add it to package.json dependencies/optionalDependencies.`; + }) + .toSorted((left, right) => left.localeCompare(right)); +} + +function isInstalledCompanionExtensionOwnedRuntimeImport(params: { + dependencyName: string; + extensionId: string; + manifestRoots: string[]; + manifestCache: Map<string, InstalledPackageJson | null>; +}): boolean { + const extensionId = params.extensionId; + for (const manifestRoot of params.manifestRoots) { + const cacheKey = `${manifestRoot}\0${extensionId}`; + let manifest = params.manifestCache.get(cacheKey); + if (manifest !== undefined) { + if ( + manifest && + (Object.hasOwn(manifest.dependencies ?? {}, params.dependencyName) || + Object.hasOwn(manifest.optionalDependencies ?? {}, params.dependencyName)) + ) { + return true; + } + continue; + } + const manifestPath = join(manifestRoot, extensionId, "package.json"); + manifest = null; + if (existsSync(manifestPath)) { + const stat = lstatSync(manifestPath); + if (stat.isFile() && stat.size <= MAX_INSTALLED_ROOT_PACKAGE_JSON_BYTES) { + try { + const parsed = JSON.parse(readFileSync(manifestPath, "utf8")) as InstalledPackageJson; + if (parsed.name === `@openclaw/${extensionId}`) { + manifest = parsed; + } + } catch { + // Invalid companion manifests cannot authorize root runtime imports. + } + } + } + params.manifestCache.set(cacheKey, manifest); + if ( + manifest && + (Object.hasOwn(manifest.dependencies ?? {}, params.dependencyName) || + Object.hasOwn(manifest.optionalDependencies ?? {}, params.dependencyName)) + ) { + return true; + } + } + return false; +} + +function collectBundledExtensionRuntimeDependencyOwners( + packageRoot: string, +): Map<string, Set<string>> { + const ownersByDependency = new Map<string, Set<string>>(); + const { manifests } = readBundledExtensionPackageJsons(packageRoot); + for (const { id, manifest } of manifests) { + for (const dependencyName of collectRuntimeDependencySpecs(manifest).keys()) { + const owners = ownersByDependency.get(dependencyName) ?? new Set<string>(); + owners.add(id); + ownersByDependency.set(dependencyName, owners); + } + } + return ownersByDependency; +} + +export function resolveInstalledBinaryPath(prefixDir: string, platform = process.platform): string { + return platform === "win32" + ? pathWin32.join(prefixDir, "openclaw.cmd") + : pathPosix.join(prefixDir, "bin", "openclaw"); +} + +export function resolveInstalledBinaryCommandInvocation( + prefixDir: string, + args: string[], + params: { comSpec?: string; platform?: NodeJS.Platform } = {}, +): { + args: string[]; + command: string; + windowsVerbatimArguments?: boolean; +} { + const platform = params.platform ?? process.platform; + const binaryPath = resolveInstalledBinaryPath(prefixDir, platform); + if (platform === "win32") { + return { + command: params.comSpec ?? resolveWindowsCmdExePath(), + args: ["/d", "/s", "/c", buildCmdExeCommandLine(binaryPath, args)], + windowsVerbatimArguments: true, + }; + } + + return { + command: binaryPath, + args, + }; +} + +function readBundledExtensionPackageJsons(packageRoot: string): { + manifests: InstalledBundledExtensionManifestRecord[]; + errors: string[]; +} { + const extensionsDir = join(packageRoot, "dist", "extensions"); + const manifests: InstalledBundledExtensionManifestRecord[] = []; + const errors: string[] = []; + const inventoryPath = join(packageRoot, PACKAGE_DIST_INVENTORY_RELATIVE_PATH); + + try { + const inventory: unknown = JSON.parse(readFileSync(inventoryPath, "utf8")); + if (!Array.isArray(inventory) || inventory.some((entry) => typeof entry !== "string")) { + throw new Error("inventory must be an array of file paths"); + } + const inventoryParity = comparePackageDistInventory({ + files: PACKAGED_BUNDLED_PLUGIN_ARTIFACTS, + inventory: inventory as string[], + }); + errors.push( + ...inventoryParity.packagedFilesMissingFromInventory.map( + (relativePath) => + `installed bundled plugin artifact omitted from dist inventory: ${relativePath}.`, + ), + ); + } catch (error) { + errors.push( + `installed package dist inventory is missing or invalid: ${PACKAGE_DIST_INVENTORY_RELATIVE_PATH}: ${formatErrorMessage(error)}.`, + ); + } + + const installedArtifactParity = comparePackageDistInventory({ + files: [...PACKAGED_BUNDLED_PLUGIN_ARTIFACTS].filter((relativePath) => + existsSync(join(packageRoot, relativePath)), + ), + inventory: PACKAGED_BUNDLED_PLUGIN_ARTIFACTS, + }); + for (const relativePath of installedArtifactParity.inventoryEntriesMissingFromPackage) { + errors.push( + relativePath.endsWith("/package.json") + ? `installed bundled extension manifest missing: ${join(packageRoot, relativePath)}.` + : `installed bundled plugin artifact missing: ${relativePath}.`, + ); + } + + if (!existsSync(extensionsDir)) { + return { manifests, errors }; + } + + for (const entry of readdirSync(extensionsDir, { withFileTypes: true })) { + if (!entry.isDirectory()) { + continue; + } + + const extensionDirPath = join(extensionsDir, entry.name); + const packageJsonPath = join(extensionsDir, entry.name, "package.json"); + if (!existsSync(packageJsonPath)) { + continue; + } + + try { + const packageJsonStats = lstatSync(packageJsonPath); + if (!packageJsonStats.isFile()) { + throw new Error("manifest must be a regular file"); + } + if (packageJsonStats.size > MAX_BUNDLED_EXTENSION_MANIFEST_BYTES) { + throw new Error(`manifest exceeds ${MAX_BUNDLED_EXTENSION_MANIFEST_BYTES} bytes`); + } + + const realExtensionDirPath = realpathSync(extensionDirPath); + const realPackageJsonPath = realpathSync(packageJsonPath); + const relativeManifestPath = relative(realExtensionDirPath, realPackageJsonPath); + if ( + relativeManifestPath.length === 0 || + relativeManifestPath.startsWith("..") || + isAbsolute(relativeManifestPath) + ) { + throw new Error("manifest resolves outside the bundled extension directory"); + } + + manifests.push({ + id: entry.name, + manifest: JSON.parse( + readFileSync(realPackageJsonPath, "utf8"), + ) as InstalledBundledExtensionPackageJson, + path: realPackageJsonPath, + }); + } catch (error) { + errors.push( + `installed bundled extension manifest invalid: failed to parse ${packageJsonPath}: ${formatErrorMessage(error)}.`, + ); + } + } + + return { manifests, errors }; +} + +function npmExec(args: string[], cwd: string): string { + const invocation = resolveNpmCommandInvocation({ + npmArgs: args, + npmExecPath: process.env.npm_execpath, + nodeExecPath: process.execPath, + platform: process.platform, + }); + + return runNpmVerifyCommand(invocation, cwd); +} + +function resolveGlobalRoot(prefixDir: string, cwd: string): string { + return npmExec(["root", "-g", "--prefix", prefixDir], cwd); +} + +export function buildPublishedInstallCommandArgs(prefixDir: string, spec: string): string[] { + return ["install", "-g", "--prefix", prefixDir, spec, "--no-fund", "--no-audit"]; +} + +function installSpec(prefixDir: string, spec: string, cwd: string): void { + npmExec(buildPublishedInstallCommandArgs(prefixDir, spec), cwd); +} + +export async function fetchRegistryJson( + url: string, + options: FetchRegistryJsonOptions = {}, +): Promise<unknown> { + const timeoutMs = options.timeoutMs ?? NPM_REGISTRY_REQUEST_TIMEOUT_MS; + const maxBodyBytes = options.maxBodyBytes ?? NPM_REGISTRY_RESPONSE_BODY_MAX_BYTES; + const controller = new AbortController(); + const timeoutError = Object.assign( + new Error(`npm registry request timed out after ${timeoutMs}ms: ${url}`), + { code: "ETIMEDOUT" }, + ); + let timeout: ReturnType<typeof setTimeout> | undefined; + const timeoutPromise = new Promise<never>((_resolve, reject) => { + timeout = setTimeout(() => { + controller.abort(timeoutError); + reject(timeoutError); + }, timeoutMs); + timeout.unref?.(); + }); + try { + const response = await Promise.race([ + (options.fetchImpl ?? fetch)(url, { + headers: { + Accept: "application/json", + }, + redirect: "error", + signal: controller.signal, + }), + timeoutPromise, + ]); + if (!response.ok) { + throw new Error(`npm registry request failed (${response.status}): ${url}`); + } + return JSON.parse( + await readBoundedResponseText(response, `npm registry ${url}`, maxBodyBytes, { + signal: controller.signal, + timeoutPromise, + }), + ); + } finally { + clearTimeout(timeout); + } +} + +function isRetryableRegistryProvenanceError(error: unknown): boolean { + const message = error instanceof Error ? error.message : String(error); + return ( + /npm registry request failed \((?:404|408|425|429|5\d\d)\)/u.test(message) || + message.includes("npm registry metadata is incomplete") || + message.includes("npm registry provenance metadata is incomplete") || + message.includes("npm provenance attestation does not bind") || + /aborted|fetch failed|network|timeout|timed out/u.test(message) + ); +} + +export async function retryNpmRegistryProvenanceRead<T>( + read: () => Promise<T>, + options: { + attempts?: number; + delay?: (delayMs: number) => Promise<void>; + } = {}, +): Promise<T> { + const attempts = options.attempts ?? NPM_REGISTRY_PROVENANCE_ATTEMPTS; + const delay = + options.delay ?? + ((delayMs: number) => + new Promise<void>((resolveDelay) => { + setTimeout(resolveDelay, delayMs); + })); + let lastError: unknown; + + for (let attempt = 1; attempt <= attempts; attempt += 1) { + try { + return await read(); + } catch (error) { + lastError = error; + if (!isRetryableRegistryProvenanceError(error) || attempt === attempts) { + throw error; + } + await delay(Math.min(attempt * 1000, NPM_REGISTRY_PROVENANCE_RETRY_MAX_DELAY_MS)); + } + } + + throw lastError; +} + +async function verifyPublishedRegistryProvenanceOnce(version: string): Promise<void> { + const registry = new URL(process.env.npm_config_registry ?? "https://registry.npmjs.org"); + if (registry.protocol !== "https:") { + throw new Error(`npm registry must use HTTPS: ${registry}`); + } + if (!registry.pathname.endsWith("/")) { + registry.pathname = `${registry.pathname}/`; + } + const packageName = "openclaw"; + const packageDocument = (await fetchRegistryJson( + new URL( + `${encodeURIComponent(packageName)}/${encodeURIComponent(version)}`, + registry, + ).toString(), + )) as { + dist?: { + attestations?: { + provenance?: { + predicateType?: string; + }; + url?: string; + }; + integrity?: string; + signatures?: NpmRegistrySignature[]; + }; + }; + const keysDocument = (await fetchRegistryJson(new URL("-/npm/v1/keys", registry).toString())) as { + keys?: NpmRegistryKey[]; + }; + const integrity = packageDocument.dist?.integrity; + const signatures = packageDocument.dist?.signatures; + const provenance = packageDocument.dist?.attestations?.provenance; + const attestationUrl = packageDocument.dist?.attestations?.url; + if (!integrity || !signatures || !keysDocument.keys) { + throw new Error(`npm registry metadata is incomplete for ${packageName}@${version}.`); + } + if ( + provenance?.predicateType !== NPM_PROVENANCE_PREDICATE_TYPE || + typeof attestationUrl !== "string" || + attestationUrl.length === 0 + ) { + throw new Error( + `npm registry provenance metadata is incomplete for ${packageName}@${version}.`, + ); + } + const parsedAttestationUrl = new URL(attestationUrl); + const attestationPathPrefix = new URL("-/npm/v1/attestations/", registry).pathname; + if ( + parsedAttestationUrl.protocol !== "https:" || + parsedAttestationUrl.origin !== registry.origin || + !parsedAttestationUrl.pathname.startsWith(attestationPathPrefix) + ) { + throw new Error( + `npm registry returned an untrusted provenance attestation URL for ${packageName}@${version}.`, + ); + } + + verifyNpmRegistrySignatures({ + packageName, + version, + integrity, + signatures, + keys: keysDocument.keys, + }); + const attestationDocument = (await fetchRegistryJson(parsedAttestationUrl.toString())) as { + attestations?: NpmRegistryAttestation[]; + }; + const attestations = attestationDocument.attestations ?? []; + if (attestations.length === 0) { + throw new Error( + `npm registry provenance metadata is incomplete for ${packageName}@${version}.`, + ); + } + await verifyNpmProvenanceAttestation({ + packageName, + version, + integrity, + attestations, + expectedWorkflowRef: process.env.OPENCLAW_NPM_EXPECTED_WORKFLOW_REF, + expectedWorkflowSha: process.env.OPENCLAW_NPM_EXPECTED_WORKFLOW_SHA, + }); + console.log( + `openclaw-npm-postpublish-verify: registry signature and provenance attestation verified (${version})`, + ); +} + +async function verifyPublishedRegistryProvenance(version: string): Promise<void> { + await retryNpmRegistryProvenanceRead(() => verifyPublishedRegistryProvenanceOnce(version)); +} + +function readInstalledBinaryVersion(prefixDir: string, cwd: string): string { + const invocation = resolveInstalledBinaryCommandInvocation(prefixDir, ["--version"]); + return runNpmVerifyCommand(invocation, cwd); +} + +function verifyScenario(version: string, scenario: PublishedInstallScenario): void { + const workingDir = mkdtempSync(join(tmpdir(), `openclaw-postpublish-${scenario.name}.`)); + const prefixDir = join(workingDir, "prefix"); + + try { + for (const spec of scenario.installSpecs) { + installSpec(prefixDir, spec, workingDir); + } + + const globalRoot = resolveGlobalRoot(prefixDir, workingDir); + const packageRoot = join(globalRoot, "openclaw"); + const pkg = JSON.parse( + readFileSync(join(packageRoot, "package.json"), "utf8"), + ) as InstalledPackageJson; + const errors = collectInstalledPackageErrors({ + expectedVersion: scenario.expectedVersion, + installedVersion: pkg.version?.trim() ?? "", + packageRoot, + }); + const installedBinaryVersion = readInstalledBinaryVersion(prefixDir, workingDir); + + if (normalizeInstalledBinaryVersion(installedBinaryVersion) !== scenario.expectedVersion) { + errors.push( + `installed openclaw binary version mismatch: expected ${scenario.expectedVersion}, found ${installedBinaryVersion || "<missing>"}.`, + ); + } + + if (errors.length === 0) { + runInstalledWorkspaceBootstrapSmoke({ packageRoot }); + } + + if (errors.length > 0) { + throw new Error(`${scenario.name} failed:\n- ${errors.join("\n- ")}`); + } + + console.log(`openclaw-npm-postpublish-verify: ${scenario.name} OK (${version})`); + } finally { + rmSync(workingDir, { force: true, recursive: true }); + } +} + +async function main(argv = process.argv.slice(2)): Promise<void> { + const args = parseOpenClawNpmPostpublishVerifyArgs(argv); + if (args.help) { + console.log(openClawNpmPostpublishVerifyUsage()); + return; + } + + const { version } = args; + const scenarios = buildPublishedInstallScenarios(version); + await verifyPublishedRegistryProvenance(version); + for (const scenario of scenarios) { + verifyScenario(version, scenario); + } + + console.log( + `openclaw-npm-postpublish-verify: verified published npm install paths for ${version}.`, + ); +} + +const entrypoint = process.argv[1] ? pathToFileURL(process.argv[1]).href : null; +if (entrypoint !== null && import.meta.url === entrypoint) { + try { + await main(); + } catch (error) { + console.error(`openclaw-npm-postpublish-verify: ${formatErrorMessage(error)}`); + process.exitCode = 1; + } +} diff --git a/scripts/openclaw-npm-publish.sh b/scripts/openclaw-npm-publish.sh new file mode 100644 index 0000000000000000000000000000000000000000..eb0eb4f8f27d7fbec086cf4a23e41c2dbf4fdaaa --- /dev/null +++ b/scripts/openclaw-npm-publish.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash + +set -euo pipefail + +usage() { + echo "usage: bash scripts/openclaw-npm-publish.sh --publish [package.tgz]" +} + +if [[ "${1:-}" == "--help" || "${1:-}" == "-h" ]]; then + usage + exit 0 +fi + +if [[ "${1:-}" != "--publish" ]]; then + usage >&2 + exit 2 +fi +shift + +publish_target="" +if [[ "${1:-}" == "--" ]]; then + shift +fi +if [[ "$#" -gt 0 ]]; then + case "$1" in + -*) echo "error: unexpected npm publish target option: $1" >&2; exit 2 ;; + *) publish_target="$1"; shift ;; + esac +fi +if [[ "$#" -gt 0 ]]; then + echo "error: unexpected npm publish argument: $1" >&2 + exit 2 +fi + +if [[ -n "${publish_target}" && -f "${publish_target}" ]]; then + case "${publish_target}" in + /*|./*|../*) ;; + *) publish_target="./${publish_target}" ;; + esac +fi + +package_version="$(node -p "require('./package.json').version")" +if [[ -n "${publish_target}" ]]; then + if [[ ! -f "${publish_target}" ]]; then + echo "error: npm publish tarball not found: ${publish_target}" >&2 + exit 2 + fi + if ! tarball_package_json="$(tar -xOf "${publish_target}" package/package.json)"; then + echo "error: npm publish tarball is missing a readable package/package.json: ${publish_target}" >&2 + exit 2 + fi + if ! tarball_package_version="$(printf '%s' "${tarball_package_json}" | node -e ' + let input = ""; + process.stdin.on("data", (chunk) => { input += chunk; }); + process.stdin.on("end", () => { + const pkg = JSON.parse(input); + if (!pkg || typeof pkg !== "object" || Array.isArray(pkg) || typeof pkg.version !== "string" || pkg.version.trim() === "") { + throw new Error("package/package.json must contain a nonempty string version"); + } + process.stdout.write(pkg.version.trim()); + }); + ')"; then + echo "error: npm publish tarball package/package.json is malformed or has no valid version: ${publish_target}" >&2 + exit 2 + fi + if [[ "${tarball_package_version}" != "${package_version}" ]]; then + echo "error: npm publish tarball version mismatch: expected ${package_version}, got ${tarball_package_version}" >&2 + exit 2 + fi +fi + +publish_plan="$( + PACKAGE_VERSION="${package_version}" REQUESTED_PUBLISH_TAG="${OPENCLAW_NPM_PUBLISH_TAG:-}" \ + BYPASS_EXTENDED_STABLE_GUARD="${BYPASS_EXTENDED_STABLE_GUARD:-}" \ + node scripts/openclaw-npm-extended-stable-release.mjs publish-plan +)" + +release_channel="${publish_plan%%$'\n'*}" +publish_tag="${publish_plan#*$'\n'}" +publish_cmd=(npm publish) +if [[ -n "${publish_target}" ]]; then + publish_cmd+=("${publish_target}") +fi +publish_cmd+=(--access public --tag "${publish_tag}" --provenance) + +echo "Resolved package version: ${package_version}" +echo "Resolved release channel: ${release_channel}" +echo "Resolved publish tag: ${publish_tag}" +echo "Publish auth: GitHub OIDC trusted publishing" +if [[ -n "${publish_target}" ]]; then + echo "Resolved publish target: ${publish_target}" +fi + +printf 'Publish command:' +printf ' %q' "${publish_cmd[@]}" +printf '\n' + +"${publish_cmd[@]}" diff --git a/scripts/openclaw-npm-release-check.ts b/scripts/openclaw-npm-release-check.ts new file mode 100644 index 0000000000000000000000000000000000000000..a5d8859e4035360bf9d339ee9002cd4bde56f5a3 --- /dev/null +++ b/scripts/openclaw-npm-release-check.ts @@ -0,0 +1,705 @@ +#!/usr/bin/env -S node --import tsx +// Openclaw Npm Release Check script supports OpenClaw repository automation. + +import { execFileSync } from "node:child_process"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; +import { resolveNpmJsonEntries } from "./lib/npm-json-output.mts"; +import { resolveNpmDistTagMirrorAuth as resolveNpmDistTagMirrorAuthBase } from "./lib/npm-publish-plan.mjs"; +import { readPositiveEnvInt } from "./lib/numeric-options.mjs"; +import { + PACKAGE_DIST_INVENTORY_RELATIVE_PATH, + writePackageDistInventory, +} from "./lib/package-dist-inventory.ts"; +import { collectForbiddenPackedPathErrors } from "./lib/packed-cargo-policy.mts"; +import { isRecord } from "./lib/record-shared.mjs"; +import { + compareReleaseVersions as compareReleaseVersionsBase, + collectReleaseVersionFloorErrors as collectReleaseVersionFloorErrorsBase, + parseReleaseVersion as parseReleaseVersionBase, +} from "./lib/release-version.mjs"; +import { WORKSPACE_TEMPLATE_PACK_PATHS } from "./lib/workspace-bootstrap-smoke.mts"; +import { buildCmdExeCommandLine, resolveWindowsCmdExePath } from "./windows-cmd-helpers.mjs"; + +type PackageJson = { + name?: string; + version?: string; + description?: string; + license?: string; + repository?: { url?: string } | string; + bin?: Record<string, string>; + dependencies?: Record<string, string>; + optionalDependencies?: Record<string, string>; + peerDependencies?: Record<string, string>; + peerDependenciesMeta?: Record<string, { optional?: boolean }>; +}; + +type ParsedReleaseTag = { + version: string; + packageVersion: string; + baseVersion: string; + channel: "stable" | "alpha" | "beta"; + correctionNumber?: number; +}; + +type ParsedReleaseVersion = { + alphaNumber?: number; + baseVersion: string; + betaNumber?: number; + channel: "stable" | "alpha" | "beta"; + correctionNumber?: number; + month: number; + patch: number; + version: string; + year: number; +}; + +type NpmPublishPlan = { + channel: "stable" | "alpha" | "beta"; + publishTag: "latest" | "alpha" | "beta"; + mirrorDistTags: ("latest" | "alpha" | "beta")[]; +}; + +type NpmDistTagMirrorAuth = { + hasAuth: boolean; + source: "node-auth-token" | "npm-token" | "none"; +}; +const EXPECTED_REPOSITORY_URL = "https://github.com/openclaw/openclaw"; +const FS_SAFE_PACKAGE = "@openclaw/fs-safe"; +const REQUIRED_PACKED_PATHS = [ + PACKAGE_DIST_INVENTORY_RELATIVE_PATH, + "dist/control-ui/index.html", + ...WORKSPACE_TEMPLATE_PACK_PATHS, +]; +const CONTROL_UI_ASSET_PREFIX = "dist/control-ui/assets/"; +const FORBIDDEN_PRIVATE_QA_CONTENT_MARKERS = [ + "//#region extensions/qa-lab/", + "qa-channel/runtime-api.js", + "qa-channel.js", + "qa-channel-protocol.js", + "qa-lab/cli.js", + "qa-lab/runtime-api.js", +] as const; +const FORBIDDEN_PRIVATE_QA_CONTENT_SCAN_PREFIXES = ["dist/"] as const; +const PACKED_TEST_CARGO_DIRECTORY_SEGMENTS = new Set([ + "__snapshots__", + "__tests__", + "test", + "tests", +]); +const PACKED_TEST_CARGO_FILE_RE = /(?:^|\/)[^/]+\.(?:test|spec)\.(?:[cm]?[jt]sx?)$/u; +const NPM_PACK_MAX_BUFFER_BYTES = 64 * 1024 * 1024; +const DEFAULT_RELEASE_CHECK_COMMAND_TIMEOUT_MS = 10 * 60 * 1000; +const skipPackValidationEnv = "OPENCLAW_NPM_RELEASE_SKIP_PACK_CHECK"; + +type ReleaseCheckCommandInvocation = { + command: string; + args: string[]; + windowsVerbatimArguments?: boolean; +}; + +function normalizePackedPath(packedPath: string): string { + return packedPath.replace(/\\/g, "/"); +} +function isNodeModulesPackageRoot(segments: string[], index: number): boolean { + const parent = segments[index - 1]; + if (parent === "node_modules") { + return true; + } + return parent !== undefined && parent.startsWith("@") && segments[index - 2] === "node_modules"; +} + +function pathContainsPackedTestCargo(packedPath: string): boolean { + const normalizedPath = normalizePackedPath(packedPath); + // Root docs ship Markdown reference material; topic directories such as + // "test" are not runtime test cargo. Dependency fixtures remain disallowed. + if (normalizedPath.startsWith("docs/") && normalizedPath.endsWith(".md")) { + return false; + } + if (PACKED_TEST_CARGO_FILE_RE.test(normalizedPath)) { + return true; + } + const segments = normalizedPath.split("/").filter(Boolean); + return segments.some( + (segment, index) => + index < segments.length - 1 && + PACKED_TEST_CARGO_DIRECTORY_SEGMENTS.has(segment) && + !isNodeModulesPackageRoot(segments, index), + ); +} + +function normalizeRepoUrl(value: unknown): string { + if (typeof value !== "string") { + return ""; + } + + return value + .trim() + .replace(/^git\+/, "") + .replace(/\.git$/i, "") + .replace(/\/+$/, ""); +} + +function isLocalDependencySpec(value: string | undefined): boolean { + return /^(?:file|link|workspace):/u.test(value ?? ""); +} + +export function parseReleaseVersion(version: string): ParsedReleaseVersion | null { + return parseReleaseVersionBase(version); +} + +export function compareReleaseVersions(left: string, right: string): number | null { + return compareReleaseVersionsBase(left, right); +} + +export function resolveNpmPublishPlan( + version: string, + _currentBetaVersion?: string | null, + requestedPublishTag?: "latest" | "alpha" | "beta" | null, +): NpmPublishPlan { + const parsedVersion = parseReleaseVersion(version); + if (parsedVersion === null) { + throw new Error(`Unsupported release version "${version}".`); + } + + const publishTag = + requestedPublishTag?.trim() === "latest" + ? "latest" + : requestedPublishTag?.trim() === "alpha" + ? "alpha" + : "beta"; + + if (parsedVersion.channel === "alpha") { + if (publishTag !== "alpha") { + throw new Error("Alpha prereleases must publish to the alpha dist-tag."); + } + return { + channel: "alpha", + publishTag: "alpha", + mirrorDistTags: [], + }; + } + + if (parsedVersion.channel === "beta") { + if (publishTag !== "beta") { + throw new Error("Beta prereleases must publish to the beta dist-tag."); + } + return { + channel: "beta", + publishTag: "beta", + mirrorDistTags: [], + }; + } + + return { + channel: "stable", + publishTag, + mirrorDistTags: [], + }; +} + +export function resolveNpmDistTagMirrorAuth(params?: { + nodeAuthToken?: string | null; + npmToken?: string | null; +}): NpmDistTagMirrorAuth { + const nodeAuthToken = + params && "nodeAuthToken" in params ? params.nodeAuthToken : process.env.NODE_AUTH_TOKEN; + const npmToken = params && "npmToken" in params ? params.npmToken : process.env.NPM_TOKEN; + return resolveNpmDistTagMirrorAuthBase({ + nodeAuthToken, + npmToken, + }) as NpmDistTagMirrorAuth; +} + +export function shouldSkipPackedTarballValidation(env = process.env): boolean { + const raw = env[skipPackValidationEnv]; + if (!raw) { + return false; + } + return !/^(0|false)$/i.test(raw); +} + +export function parseReleaseTagVersion(version: string): ParsedReleaseTag | null { + const trimmed = version.trim(); + if (!trimmed) { + return null; + } + + const parsedVersion = parseReleaseVersion(trimmed); + if (parsedVersion !== null) { + return { + version: trimmed, + packageVersion: parsedVersion.version, + baseVersion: parsedVersion.baseVersion, + channel: parsedVersion.channel, + correctionNumber: parsedVersion.correctionNumber, + }; + } + + return null; +} + +export function resolveNpmReleaseCheckCommandTimeoutMs( + env: NodeJS.ProcessEnv = process.env, +): number { + return readPositiveEnvInt( + "OPENCLAW_NPM_RELEASE_CHECK_COMMAND_TIMEOUT_MS", + env, + DEFAULT_RELEASE_CHECK_COMMAND_TIMEOUT_MS, + ); +} + +export function runNpmReleaseCheckCommand( + invocation: ReleaseCheckCommandInvocation, + options: { + cwd?: string; + encoding?: BufferEncoding; + env?: NodeJS.ProcessEnv; + maxBuffer?: number; + stdio: "ignore" | ["ignore", "pipe", "pipe"]; + timeoutMs?: number; + }, +): string { + const env = options.env ?? process.env; + const execOptions = { + cwd: options.cwd, + encoding: options.encoding, + env, + killSignal: "SIGKILL", + maxBuffer: options.maxBuffer ?? NPM_PACK_MAX_BUFFER_BYTES, + stdio: options.stdio, + timeout: options.timeoutMs ?? resolveNpmReleaseCheckCommandTimeoutMs(env), + windowsVerbatimArguments: invocation.windowsVerbatimArguments, + } as Parameters<typeof execFileSync>[2] & { windowsVerbatimArguments?: boolean }; + const output = execFileSync(invocation.command, invocation.args, execOptions) as + | Buffer + | string + | null; + if (output == null) { + return ""; + } + return typeof output === "string" ? output : output.toString("utf8"); +} + +export function collectReleasePackageMetadataErrors(pkg: PackageJson): string[] { + const actualRepositoryUrl = normalizeRepoUrl( + typeof pkg.repository === "string" ? pkg.repository : pkg.repository?.url, + ); + const errors: string[] = []; + + if (pkg.name !== "openclaw") { + errors.push(`package.json name must be "openclaw"; found "${pkg.name ?? ""}".`); + } + if (!pkg.description?.trim()) { + errors.push("package.json description must be non-empty."); + } + if (pkg.license !== "MIT") { + errors.push(`package.json license must be "MIT"; found "${pkg.license ?? ""}".`); + } + if (actualRepositoryUrl !== EXPECTED_REPOSITORY_URL) { + errors.push( + `package.json repository.url must resolve to ${EXPECTED_REPOSITORY_URL}; found ${ + actualRepositoryUrl || "<missing>" + }.`, + ); + } + if (pkg.bin?.openclaw !== "openclaw.mjs") { + errors.push( + `package.json bin.openclaw must be "openclaw.mjs"; found "${pkg.bin?.openclaw ?? ""}".`, + ); + } + if (isLocalDependencySpec(pkg.dependencies?.[FS_SAFE_PACKAGE])) { + errors.push( + `package.json dependencies["${FS_SAFE_PACKAGE}"] must use a published semver range before npm release; found "${pkg.dependencies?.[FS_SAFE_PACKAGE]}".`, + ); + } + + return errors; +} + +export function collectReleaseTagErrors(params: { + packageVersion: string; + releaseTag: string; + releaseSha?: string; + releaseMainRef?: string; +}): string[] { + const errors: string[] = []; + const releaseTag = params.releaseTag.trim(); + const packageVersion = params.packageVersion.trim(); + + const parsedVersion = parseReleaseVersion(packageVersion); + if (parsedVersion === null) { + errors.push( + `package.json version must match YYYY.M.PATCH, YYYY.M.PATCH-N, YYYY.M.PATCH-alpha.N, or YYYY.M.PATCH-beta.N; found "${packageVersion || "<missing>"}".`, + ); + } else { + errors.push(...collectReleaseVersionFloorErrorsBase(parsedVersion)); + } + + if (!releaseTag.startsWith("v")) { + errors.push(`Release tag must start with "v"; found "${releaseTag || "<missing>"}".`); + } + + const tagVersion = releaseTag.startsWith("v") ? releaseTag.slice(1) : releaseTag; + const parsedTag = parseReleaseTagVersion(tagVersion); + if (parsedTag === null) { + errors.push( + `Release tag must match vYYYY.M.PATCH, vYYYY.M.PATCH-alpha.N, vYYYY.M.PATCH-beta.N, or fallback correction tag vYYYY.M.PATCH-N; found "${releaseTag || "<missing>"}".`, + ); + } + + const expectedTag = packageVersion ? `v${packageVersion}` : "<missing>"; + const matchesExpectedTag = + parsedTag !== null && + parsedVersion !== null && + parsedTag.channel === parsedVersion.channel && + (parsedTag.packageVersion === parsedVersion.version || + (parsedVersion.channel === "stable" && + parsedVersion.correctionNumber === undefined && + parsedTag.correctionNumber !== undefined && + parsedTag.baseVersion === parsedVersion.baseVersion)); + if (!matchesExpectedTag) { + errors.push( + `Release tag ${releaseTag || "<missing>"} does not match package.json version ${ + packageVersion || "<missing>" + }; expected ${ + parsedVersion?.channel === "stable" && parsedVersion.correctionNumber === undefined + ? `${expectedTag} or ${expectedTag}-N` + : expectedTag + }.`, + ); + } + + if (params.releaseSha?.trim() && params.releaseMainRef?.trim()) { + try { + runNpmReleaseCheckCommand( + { + command: "git", + args: ["merge-base", "--is-ancestor", params.releaseSha, params.releaseMainRef], + }, + { stdio: "ignore" }, + ); + } catch { + errors.push( + `Tagged commit ${params.releaseSha} is not contained in ${params.releaseMainRef}.`, + ); + } + } + + return errors; +} + +function loadPackageJson(): PackageJson { + return JSON.parse(readFileSync("package.json", "utf8")) as PackageJson; +} + +function isNpmExecPath(value: string): boolean { + return /^npm(?:-cli)?(?:\.(?:c?js|cmd|exe))?$/.test(portableBasename(value).toLowerCase()); +} + +function portableBasename(value: string): string { + return value.split(/[/\\]/u).at(-1) ?? value; +} + +type NpmCommandInvocation = { + command: string; + args: string[]; + windowsVerbatimArguments?: boolean; +}; + +export function resolveNpmCommandInvocation( + params: { + comSpec?: string; + npmArgs?: string[]; + npmExecPath?: string; + nodeExecPath?: string; + platform?: NodeJS.Platform; + } = {}, +): NpmCommandInvocation { + const npmArgs = params.npmArgs ?? []; + const npmExecPath = params.npmExecPath ?? process.env.npm_execpath; + const nodeExecPath = params.nodeExecPath ?? process.execPath; + const platform = params.platform ?? process.platform; + + if (typeof npmExecPath === "string" && npmExecPath.length > 0 && isNpmExecPath(npmExecPath)) { + const name = portableBasename(npmExecPath).toLowerCase(); + if (platform === "win32" && (name.endsWith(".cmd") || name.endsWith(".bat"))) { + return { + command: params.comSpec ?? resolveWindowsCmdExePath(), + args: ["/d", "/s", "/c", buildCmdExeCommandLine(npmExecPath, npmArgs)], + windowsVerbatimArguments: true, + }; + } + if (platform === "win32" && name.endsWith(".exe")) { + return { command: npmExecPath, args: npmArgs }; + } + if (platform === "win32" && name === "npm") { + return { + command: params.comSpec ?? resolveWindowsCmdExePath(), + args: ["/d", "/s", "/c", buildCmdExeCommandLine(`${npmExecPath}.cmd`, npmArgs)], + windowsVerbatimArguments: true, + }; + } + if (name.endsWith(".js") || name.endsWith(".cjs") || name.endsWith(".mjs")) { + return { command: nodeExecPath, args: [npmExecPath, ...npmArgs] }; + } + return { command: npmExecPath, args: npmArgs }; + } + + if (platform === "win32") { + return { + command: params.comSpec ?? resolveWindowsCmdExePath(), + args: ["/d", "/s", "/c", buildCmdExeCommandLine("npm.cmd", npmArgs)], + windowsVerbatimArguments: true, + }; + } + + return { command: "npm", args: npmArgs }; +} + +function runNpmCommand(args: string[]): string { + const invocation = resolveNpmCommandInvocation({ npmArgs: args }); + return runNpmReleaseCheckCommand(invocation, { + encoding: "utf8", + maxBuffer: NPM_PACK_MAX_BUFFER_BYTES, + stdio: ["ignore", "pipe", "pipe"], + }); +} + +export type NpmPackResult = { + filename?: string; + files?: { path: string }[]; + unpackedSize?: number; +}; + +type ExecFailure = Error & { + stderr?: string | Uint8Array; + stdout?: string | Uint8Array; +}; + +function toTrimmedUtf8(value: string | Uint8Array | undefined): string { + if (typeof value === "string") { + return value.trim(); + } + if (value instanceof Uint8Array) { + return new TextDecoder().decode(value).trim(); + } + return ""; +} + +function describeExecFailure(error: unknown): string { + if (!(error instanceof Error)) { + return String(error); + } + const withStreams = error as ExecFailure; + const details: string[] = [error.message]; + const stderr = toTrimmedUtf8(withStreams.stderr); + const stdout = toTrimmedUtf8(withStreams.stdout); + if (stderr) { + details.push(`stderr: ${stderr}`); + } + if (stdout) { + details.push(`stdout: ${stdout}`); + } + return details.join(" | "); +} + +export function parseNpmPackJsonOutput(stdout: string): NpmPackResult[] | null { + const trimmed = stdout.trim(); + if (!trimmed) { + return null; + } + + const candidates = [trimmed]; + const trailingJsonStart = Math.max(trimmed.lastIndexOf("\n["), trimmed.lastIndexOf("\n{")); + if (trailingJsonStart !== -1) { + candidates.push(trimmed.slice(trailingJsonStart + 1).trim()); + } + + for (const candidate of candidates) { + try { + const parsed = JSON.parse(candidate) as unknown; + const entries = resolveNpmJsonEntries(parsed); + if ( + entries.length > 0 && + entries.every( + (entry): entry is NpmPackResult => + isRecord(entry) && + (entry.filename === undefined || typeof entry.filename === "string") && + (entry.unpackedSize === undefined || typeof entry.unpackedSize === "number") && + (entry.files === undefined || + (Array.isArray(entry.files) && + entry.files.every((file) => isRecord(file) && typeof file.path === "string"))), + ) + ) { + return entries; + } + } catch { + // Try the next candidate. npm lifecycle output can prepend non-JSON logs. + } + } + + return null; +} + +export function collectControlUiPackErrors(paths: Iterable<string>): string[] { + const packedPaths = new Set(paths); + const assetPaths = [...packedPaths].filter((path) => path.startsWith(CONTROL_UI_ASSET_PREFIX)); + const errors: string[] = []; + + for (const requiredPath of REQUIRED_PACKED_PATHS) { + if (!packedPaths.has(requiredPath)) { + errors.push( + `npm package is missing required path "${requiredPath}". Ensure UI assets are built and included before publish.`, + ); + } + } + + if (assetPaths.length === 0) { + errors.push( + `npm package is missing Control UI asset payload under "${CONTROL_UI_ASSET_PREFIX}". Refuse release when the dashboard tarball would be empty.`, + ); + } + + return errors; +} + +function collectPackedTarballErrors(): string[] { + const errors: string[] = []; + let stdout; + try { + stdout = runNpmCommand(["pack", "--json", "--dry-run", "--ignore-scripts"]); + } catch (error) { + const message = describeExecFailure(error); + errors.push( + `Failed to inspect npm tarball contents via \`npm pack --json --dry-run --ignore-scripts\`: ${message}`, + ); + return errors; + } + + const packResults = parseNpmPackJsonOutput(stdout); + if (!packResults) { + errors.push("Failed to parse JSON output from `npm pack --json --dry-run --ignore-scripts`."); + return errors; + } + const firstResult = packResults[0]; + if (!firstResult || !Array.isArray(firstResult.files)) { + errors.push( + "`npm pack --json --dry-run --ignore-scripts` did not return a files list to validate.", + ); + return errors; + } + + const packedPaths = new Set( + firstResult.files + .map((entry) => entry.path) + .filter((path): path is string => typeof path === "string" && path.length > 0), + ); + + return [ + ...collectControlUiPackErrors(packedPaths), + ...collectForbiddenPackedPathErrors(packedPaths), + ...collectForbiddenPackedContentErrors(packedPaths), + ...collectPackedTestCargoErrors(packedPaths), + ]; +} + +function collectNpmLockErrors(): string[] { + try { + runNpmReleaseCheckCommand( + { command: process.execPath, args: ["scripts/generate-npm-package-lock.mjs"] }, + { + cwd: process.cwd(), + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }, + ); + return []; + } catch (error) { + return [`npm package-lock validation failed: ${describeExecFailure(error)}`]; + } +} + +export { collectForbiddenPackedPathErrors } from "./lib/packed-cargo-policy.mts"; + +export function collectForbiddenPackedContentErrors( + paths: Iterable<string>, + rootDir = process.cwd(), +): string[] { + const textPathPattern = /\.(?:[cm]?js|d\.ts|json|md|mjs|cjs)$/u; + const errors: string[] = []; + for (const packedPath of paths) { + if ( + !FORBIDDEN_PRIVATE_QA_CONTENT_SCAN_PREFIXES.some((prefix) => packedPath.startsWith(prefix)) + ) { + continue; + } + if (!textPathPattern.test(packedPath)) { + continue; + } + let content: string; + try { + content = readFileSync(pathToFileURL(join(rootDir, packedPath)), "utf8"); + } catch { + continue; + } + const matchedMarker = FORBIDDEN_PRIVATE_QA_CONTENT_MARKERS.find((marker) => + content.includes(marker), + ); + if (!matchedMarker) { + continue; + } + errors.push( + `npm package must not include private QA lab marker "${matchedMarker}" in "${packedPath}".`, + ); + } + return errors.toSorted((left, right) => left.localeCompare(right)); +} + +export function collectPackedTestCargoErrors(paths: Iterable<string>): string[] { + const errors: string[] = []; + for (const packedPath of paths) { + if (!pathContainsPackedTestCargo(packedPath)) { + continue; + } + errors.push(`npm package must not include test cargo "${packedPath}".`); + } + return errors.toSorted((left, right) => left.localeCompare(right)); +} + +async function main(): Promise<number> { + const pkg = loadPackageJson(); + const skipPackValidation = shouldSkipPackedTarballValidation(); + const metadataErrors = collectReleasePackageMetadataErrors(pkg); + const tagErrors = collectReleaseTagErrors({ + packageVersion: pkg.version ?? "", + releaseTag: process.env.RELEASE_TAG ?? "", + releaseSha: process.env.RELEASE_SHA, + releaseMainRef: process.env.RELEASE_MAIN_REF, + }); + if (!skipPackValidation) { + await writePackageDistInventory(process.cwd()); + } + const npmLockErrors = skipPackValidation ? [] : collectNpmLockErrors(); + const tarballErrors = skipPackValidation ? [] : collectPackedTarballErrors(); + const errors = [...metadataErrors, ...tagErrors, ...npmLockErrors, ...tarballErrors]; + + if (errors.length > 0) { + for (const error of errors) { + console.error(`openclaw-npm-release-check: ${error}`); + } + return 1; + } + + const parsedVersion = parseReleaseVersion(pkg.version ?? ""); + const channel = parsedVersion?.channel ?? "unknown"; + console.log( + `openclaw-npm-release-check: validated ${channel} release ${pkg.version} (monthly patch version${skipPackValidation ? "; metadata-only" : ""}).`, + ); + return 0; +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { + process.exit(await main()); +} diff --git a/scripts/openclaw-performance-crabbox.sh b/scripts/openclaw-performance-crabbox.sh new file mode 100644 index 0000000000000000000000000000000000000000..4b68785653dfc0f4f4f6f07cbb8932cbec7b7f94 --- /dev/null +++ b/scripts/openclaw-performance-crabbox.sh @@ -0,0 +1,926 @@ +#!/usr/bin/env bash +set -euo pipefail + +readonly SUT_USER="openclaw-sut" +readonly RUNNER_USER="openclaw-bench" +readonly NODE_VERSION="24.19.0" +readonly NODE_SHA256="14b342e71204f811bde6153be8e04b62aef63c236fef92b55f9c83154b409647" +readonly PNPM_VERSION="11.15.1" +readonly OCM_BINARY="/opt/openclaw-performance/ocm" +readonly CRABBOX_COMMIT="8ba71f913bbe57285ae29af45ef0d8ec6712477d" +readonly MAX_ARTIFACT_FILES=256 +readonly MAX_ARTIFACT_BYTES=250000000 +readonly MAX_ARTIFACT_FILE_BYTES=50000000 +VERIFY_TMP="" + +build_helpers() { + local version output=".github/crabbox/performance-control/helpers" + [[ ! -e "$output" && ! -L "$output" ]] || die "helper output must be fresh" + version="$(node -p 'require("./package.json").devDependencies.esbuild')" + [[ "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || die "esbuild must be pinned" + npm exec --yes --package="esbuild@$version" -- esbuild \ + scripts/lib/kova-report-selector.mjs scripts/lib/kova-workflow-evidence.mts \ + scripts/lib/kova-report-gate.mts scripts/kova-ci-summary.mts \ + scripts/bench-cli-startup.ts scripts/openclaw-performance-source-summary.mts \ + --bundle --platform=node --format=esm --target=node24 --outbase=scripts \ + --outdir="$output" --out-extension:.js=.mjs \ + --alias:@openclaw/normalization-core/record-coerce=./packages/normalization-core/src/record-coerce.ts + [[ "$(find "$output" -type f | LC_ALL=C sort)" == "$(printf '%s\n' \ + "$output/bench-cli-startup.mjs" "$output/kova-ci-summary.mjs" \ + "$output/lib/kova-report-gate.mjs" "$output/lib/kova-report-selector.mjs" \ + "$output/lib/kova-workflow-evidence.mjs" "$output/openclaw-performance-source-summary.mjs")" ]] || + die "unexpected helper bundle set" +} + +die() { + printf 'openclaw-performance-crabbox: %s\n' "$*" >&2 + exit 1 +} + +require_sha() { + [[ "$2" =~ ^[0-9a-f]{40}$ ]] || die "$1 must be a 40-character lowercase SHA" +} + +require_scalar() { + [[ -n "$2" && ${#2} -le 256 && "$2" != *$'\n'* && "$2" != *$'\r'* ]] || + die "$1 must be a single line of at most 256 characters" +} + +file_size() { + stat -c %s "$1" 2>/dev/null || stat -f %z "$1" +} + +file_sha256() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$1" | cut -d' ' -f1 + else + shasum -a 256 "$1" | cut -d' ' -f1 + fi +} + +as_sut() { + local uid + uid="$(id -u "$SUT_USER")" + runuser -u "$SUT_USER" -- /usr/bin/env -C "/home/${SUT_USER}" -i \ + HOME="/home/${SUT_USER}" \ + XDG_CACHE_HOME="/home/${SUT_USER}/.cache" \ + XDG_RUNTIME_DIR="/run/user/${uid}" \ + DBUS_SESSION_BUS_ADDRESS="unix:path=/run/user/${uid}/bus" \ + PATH="/home/${SUT_USER}/.local/node_modules/.bin:/opt/node-v${NODE_VERSION}/bin:/opt/openclaw-performance:/usr/bin:/bin" \ + GIT_CONFIG_GLOBAL=/dev/null \ + GIT_CONFIG_SYSTEM=/dev/null \ + GIT_TERMINAL_PROMPT=0 \ + CI=1 \ + OPENCLAW_SKIP_CHANNELS=1 \ + OPENCLAW_SKIP_CRON=1 \ + "$@" +} + +as_runner() { + runuser -u "$RUNNER_USER" -- /usr/bin/env -C "/home/${RUNNER_USER}" -i \ + HOME="/home/${RUNNER_USER}" \ + XDG_CACHE_HOME="/home/${RUNNER_USER}/.cache" \ + PATH="/opt/node-v${NODE_VERSION}/bin:/usr/bin:/bin" \ + GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_SYSTEM=/dev/null GIT_TERMINAL_PROMPT=0 CI=1 \ + "$@" +} + +candidate_environment() { + printf '%s\n' \ + "HOME=/home/${SUT_USER}" \ + "XDG_CACHE_HOME=/home/${SUT_USER}/.cache" \ + "XDG_RUNTIME_DIR=/run/user/$(id -u "$SUT_USER")" \ + "DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/$(id -u "$SUT_USER")/bus" \ + "PATH=/home/${SUT_USER}/.local/node_modules/.bin:/opt/node-v${NODE_VERSION}/bin:/opt/openclaw-performance:/usr/bin:/bin" \ + GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_SYSTEM=/dev/null GIT_TERMINAL_PROMPT=0 \ + CI=1 OPENCLAW_SKIP_CHANNELS=1 OPENCLAW_SKIP_CRON=1 +} + +as_candidate() { + local entry env_args=() + while IFS= read -r entry; do env_args+=("$entry"); done < <(candidate_environment) + /usr/bin/sudo -n -u "$SUT_USER" -- /usr/bin/env -C "/home/${SUT_USER}" -i "${env_args[@]}" "$@" +} + +candidate_transport() { + candidate_environment | jq -Rn --arg binary "$1" --arg user "$SUT_USER" ' + [inputs | capture("^(?<key>[^=]+)=(?<value>.*)$")] | from_entries | + {prefix:["/usr/bin/sudo","-n","-u",$user,"--"],binary:$binary,env:.,cwd:.HOME}' +} + +clone_exact() { + local repository="$1" sha="$2" destination="$3" owner="$4" + local executor=as_sut + [[ "$owner" != "$RUNNER_USER" ]] || executor=as_runner + install -d -m 0700 -o "$owner" -g "$owner" "$destination" + "$executor" /usr/bin/git -C "$destination" init -b main + "$executor" /usr/bin/git -C "$destination" remote add origin "https://github.com/${repository}.git" + "$executor" /usr/bin/git -C "$destination" fetch --filter=blob:none --depth=1 origin "$sha" + "$executor" /usr/bin/git -C "$destination" checkout --detach FETCH_HEAD + [[ "$("$executor" /usr/bin/git -C "$destination" rev-parse HEAD)" == "$sha" ]] || + die "${repository} checkout drifted" + [[ "$("$executor" /usr/bin/git -C "$destination" remote get-url origin)" == "https://github.com/${repository}.git" ]] || + die "${repository} origin changed" +} + +install_toolchain() { + export DEBIAN_FRONTEND=noninteractive + apt-get update -qq + apt-get install -y -qq ca-certificates curl git iptables jq procps sudo tar xz-utils >/dev/null + + local node_root="/opt/node-v${NODE_VERSION}" node_archive="/tmp/node.tar.xz" + curl -fsSL --proto '=https' --tlsv1.2 --max-time 180 \ + "https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-x64.tar.xz" \ + -o "$node_archive" + echo "${NODE_SHA256} ${node_archive}" | sha256sum -c - + rm -rf "$node_root" + mkdir -p "$node_root" + tar -xJf "$node_archive" -C "$node_root" --strip-components=1 + [[ "$("$node_root/bin/node" --version)" == "v${NODE_VERSION}" ]] || die "Node version mismatch" + + local payload="$PWD/.github/crabbox/performance-control" + [[ -f "$payload/ocm" && ! -L "$payload/ocm" ]] || die "trusted OCM build is missing" + (cd "$payload" && sha256sum -c ocm.sha256) + install -D -o root -g root -m 0755 "$payload/ocm" "$OCM_BINARY" + "$OCM_BINARY" env artifact export --help >/dev/null || + die "OCM build does not support bounded artifact export" +} + +prepare_metadata_denial() { + local user="$1" executor="$2" uid + uid="$(id -u "$user")" + local firewall destination curl_bin curl_version family url http_code probe_status + for firewall in iptables ip6tables; do + command -v "$firewall" >/dev/null || die "metadata protection requires ${firewall}" + done + curl_bin="$(command -v curl)" || die "metadata protection requires curl" + [[ "$curl_bin" == /* && -x "$curl_bin" ]] || die "metadata curl is not an executable path" + curl_version="$("$executor" "$curl_bin" -q --version)" || die "metadata curl is unavailable to $user" + grep -Eq '^Protocols:.*[[:space:]]http([[:space:]]|$)' <<< "$curl_version" && + grep -Eq '^Features:.*[[:space:]]IPv6([[:space:]]|$)' <<< "$curl_version" || + die "metadata curl requires HTTP and IPv6 support" + + for destination in 169.254.169.254/32 169.254.170.2/32 fd00:ec2::254/128; do + firewall=iptables + [[ "$destination" != fd00:* ]] || firewall=ip6tables + "$firewall" -I OUTPUT -m owner --uid-owner "$uid" -d "$destination" -j REJECT || + die "metadata reject rule insertion failed" + "$firewall" -C OUTPUT -m owner --uid-owner "$uid" -d "$destination" -j REJECT || + die "metadata reject rule verification failed" + done + for family in 4 6; do + url='http://169.254.169.254/latest/meta-data/' + [[ "$family" != 6 ]] || url='http://[fd00:ec2::254]/latest/meta-data/' + probe_status=0 + # Preserve trailing bytes: only an exact curl 000 with connection failure is evidence. + http_code="$( + status=0 + "$executor" "$curl_bin" -q "-${family}" --noproxy '*' --silent \ + --connect-timeout 1 --max-time 2 --output /dev/null --write-out '%{http_code}' \ + "$url" 2>/dev/null || status=$? + printf '.' + exit "$status" + )" || probe_status=$? + [[ "$probe_status" == 7 && "$http_code" == "000." ]] || + die "$user IPv${family} metadata denial was not verified" + done +} + +prepare_sut() { + ! id "$SUT_USER" >/dev/null 2>&1 || die "dedicated lease already has ${SUT_USER}" + useradd --create-home --shell /bin/bash --user-group "$SUT_USER" + local uid + uid="$(id -u "$SUT_USER")" + + install -d -m 0700 -o "$SUT_USER" -g "$SUT_USER" "/home/${SUT_USER}/.cache" + [[ -z "$(find "/home/${SUT_USER}/.cache" -mindepth 1 -print -quit)" ]] || + die "SUT cache is not empty" + for credential_path in .aws .config/gh .gitconfig .npmrc; do + [[ ! -e "/home/${SUT_USER}/${credential_path}" ]] || + die "SUT home unexpectedly contains ${credential_path}" + done + if as_sut sudo -n true >/dev/null 2>&1; then + die "SUT unexpectedly has sudo" + fi + prepare_metadata_denial "$SUT_USER" as_sut + + local dirty_env + dirty_env="$(as_sut env | grep -E '^(ACTIONS_|AWS_|CRABBOX_|GITHUB_|RUNNER_)' || true)" + [[ -z "$dirty_env" ]] || die "SUT inherited control-plane environment" + + loginctl enable-linger "$SUT_USER" + systemctl start "user@${uid}.service" + [[ -S "/run/user/${uid}/systemd/private" ]] || die "SUT systemd user session is unavailable" +} + +prepare_runner() { + ! id "$RUNNER_USER" >/dev/null 2>&1 || die "dedicated lease already has ${RUNNER_USER}" + useradd --create-home --shell /bin/bash --user-group "$RUNNER_USER" + chmod 0700 "/home/${RUNNER_USER}" "/home/${SUT_USER}" + install -d -m 0700 -o "$RUNNER_USER" -g "$RUNNER_USER" \ + "/home/${RUNNER_USER}/.cache" "/home/${RUNNER_USER}/results" + prepare_metadata_denial "$RUNNER_USER" as_runner + # The trusted runner can become only B. Neither benchmark identity can become root. + printf 'Defaults:%s !use_pty\n%s ALL=(%s) NOPASSWD: /usr/bin/env\n' \ + "$RUNNER_USER" "$RUNNER_USER" "$SUT_USER" > /etc/sudoers.d/openclaw-performance + chmod 0440 /etc/sudoers.d/openclaw-performance + visudo -cf /etc/sudoers.d/openclaw-performance + if as_runner sudo -n -u root /usr/bin/env -i /usr/bin/true >/dev/null 2>&1; then + die "runner unexpectedly has root authority" + fi + [[ "$(as_runner "$0" __candidate-id)" == "$(id -u "$SUT_USER")" ]] || + die "runner cannot select the SUT identity" + for path in /home "/home/${RUNNER_USER}" "/home/${RUNNER_USER}/results" \ + /opt "/opt/node-v${NODE_VERSION}" /usr/local/libexec; do + if as_sut test -w "$path"; then die "SUT can replace runner ancestor $path"; fi + done +} + +prepare_candidate() { + cd "$1/openclaw" + npm --prefix "$HOME/.local" install --no-audit --no-fund "pnpm@${PNPM_VERSION}" + export PATH="$HOME/.local/node_modules/.bin:$PATH" + pnpm install --frozen-lockfile +} + +run_sut() { + local lane="$1" root="$2" profile="$3" repeat="$4" contract="$5" + local include_filters="$6" expected_entries="$7" fail_on_regression="$8" + local helpers="$9" model="${10}" require_instrumented="${11}" results="${12}" + local kova="${13}" admitted="${14}" + local openclaw="$root/openclaw" + local report_dir="$results/.artifacts/kova/reports/$lane" + local bundle_dir="$results/.artifacts/kova/bundles/$lane" + local summary_dir="$results/.artifacts/kova/summaries" + + if [[ "$lane" == "source" ]]; then + cd "$openclaw" + local source_dir="$openclaw/.artifacts/openclaw-performance/source/mock-provider" + mkdir -p "$source_dir/mock-hello" + if ! node -e "const fs=require('node:fs'); const scripts=require('./package.json').scripts||{}; const extensionProbe=['scripts/profile-extension-memory.mts','scripts/profile-extension-memory.mjs'].some((entry)=>fs.existsSync(entry)); process.exit(scripts['test:gateway:cpu-scenarios'] && scripts['test:extensions:memory'] && scripts.openclaw && fs.existsSync('openclaw.mjs') && extensionProbe ? 0 : 1)"; then + printf 'Source probes skipped: required probe entry points are unavailable in this tested ref.\n' + return + fi + local supported_startup_cases startup_case + local startup_case_args=() + supported_startup_cases="$( + node --import tsx scripts/bench-gateway-startup.ts --help | + sed -n 's/^ \([[:alnum:]_-][[:alnum:]_-]*\) (.*/\1/p' + )" + for startup_case in default skipChannels preparedRuntimeCatalogStall preparedRuntimeScaleOne preparedRuntimeScaleMany oneInternalHook allInternalHooks fiftyPlugins fiftyStartupLazyPlugins; do + if grep -Fxq "$startup_case" <<< "$supported_startup_cases"; then + startup_case_args+=(--startup-case "$startup_case") + fi + done + [[ " ${startup_case_args[*]} " == *" --startup-case default "* ]] || + die "target startup benchmark did not advertise its required default case" + pnpm test:gateway:cpu-scenarios \ + --output-dir "$source_dir/gateway-cpu" --runs "$repeat" --warmup 1 --skip-qa \ + "${startup_case_args[@]}" + pnpm test:extensions:memory -- --json "$source_dir/extension-memory.json" + local run_index run_dir + for ((run_index = 1; run_index <= repeat; run_index++)); do + run_dir=".artifacts/openclaw-performance/source/mock-provider/mock-hello/run-$(printf '%03d' "$run_index")" + pnpm openclaw qa suite --provider-mode mock-openai --model "mock-openai/$model" \ + --concurrency 1 --output-dir "$run_dir" --scenario channel-chat-baseline + done + if node -e "const fs=require('node:fs'); const scripts=require('./package.json').scripts||{}; process.exit(scripts['test:sqlite:perf:smoke'] && fs.existsSync('scripts/bench-sqlite-state.ts') ? 0 : 1)"; then + pnpm test:sqlite:perf:smoke + cp .artifacts/sqlite-perf/smoke.json "$source_dir/sqlite-perf-smoke.json" + else + echo "SQLite state smoke probe is unavailable in this tested ref; continuing with the remaining source probes." + fi + return + fi + + cd "$results" + npm --prefix "$kova" ci --ignore-scripts --no-audit --no-fund + export KOVA_HOME="$HOME/.kova" + kova() { node "$kova/bin/kova.mjs" "$@"; } + if [[ "$admitted" == true ]]; then + export KOVA_OCM_TRANSPORT_JSON + KOVA_OCM_TRANSPORT_JSON="$(candidate_transport "$OCM_BINARY")" + fi + mkdir -p "$report_dir" "$bundle_dir" "$summary_dir" + + local timeout_ms=300000 + [[ "$profile" == release ]] && timeout_ms=900000 + local plan_json="$results/.artifacts/kova/plans/$lane.json" + mkdir -p "$(dirname "$plan_json")" + kova matrix plan \ + --profile "$profile" --target "local-build:$openclaw" --include "$include_filters" \ + --parallel 1 --repeat "$repeat" --json > "$plan_json" + [[ "$admitted" != true ]] || validate_plan "$plan_json" "$profile" "$include_filters" "$expected_entries" + local args=( + matrix run --profile "$profile" --target "local-build:$openclaw" --include "$include_filters" + --parallel 1 --repeat "$repeat" --auth mock --timeout-ms "$timeout_ms" + --report-dir "$report_dir" --execute --json + ) + [[ "$lane" != "mock-deep-profile" ]] || args+=(--deep-profile) + [[ "$fail_on_regression" != true || "$admitted" != true ]] || args+=(--gate) + local status=0 bundle_status=0 + KOVA_OPENCLAW_CONFIG_CONTRACT="$contract" KOVA_SCENARIO_TIMEOUT_MS="$timeout_ms" \ + kova "${args[@]}" > "$report_dir/$lane.log" 2>&1 || status=$? + local report + if [[ "$admitted" == true ]]; then + report="$(node "$helpers/lib/kova-report-selector.mjs" --report-dir "$report_dir")" + else + # Custom Kova and its selection are diagnostics, never the trusted evaluator. + local candidate reports=() + for candidate in "$report_dir"/*.json; do + [[ "$candidate" == *.summary.json ]] || reports+=("$candidate") + done + ((${#reports[@]} == 1)) || die "custom Kova must emit one diagnostic report" + report="${reports[0]}" + fi + kova report bundle "$report" \ + --output-dir "$bundle_dir" --json > "$bundle_dir/bundle.json" || bundle_status=$? + ((bundle_status == 0)) || return "$bundle_status" + if [[ "$admitted" == true ]]; then + # Only A can attest a completed matrix phase after setup and bundling succeeded. + printf '%s\n' "$status" > "$results/.artifacts/kova/$lane-matrix-exit" + return 0 + fi + return "$status" +} + +validate_plan() { + node --input-type=module - "$@" <<'NODE' +import fs from "node:fs"; +const [file, profile, include, expected] = process.argv.slice(2); +const plan = JSON.parse(fs.readFileSync(file, "utf8")); +const filters = include.split(","); +if (!Array.isArray(plan.controls?.include) || + plan.controls.include.length !== filters.length || + plan.controls.include.some((filter, index) => filter !== filters[index])) { + throw new Error("Kova plan did not preserve the requested include filters"); +} +if (profile === "release") { + if (!Array.isArray(plan.entries)) throw new Error("Kova release plan did not contain entries"); + const actual = plan.entries.map((entry) => { + if (entry.status !== "SELECTED" || !entry.scenario?.id || !entry.state?.id) { + throw new Error("Kova release plan contained an invalid selected entry"); + } + return `${entry.scenario.id}:${entry.state.id}`; + }).sort(); + const required = expected.split(",").sort(); + if (actual.length !== required.length || actual.some((entry, index) => entry !== required[index])) { + throw new Error("Kova release plan entries did not match the required lane coverage"); + } +} +NODE +} + +validate_kova() { + local lane="$1" root="$2" profile="$3" repeat="$4" include_filters="$5" expected_entries="$6" + local fail_on_regression="$7" helpers="$8" model="$9" require_instrumented="${10}" + local results="${11}" status="${12}" admitted="${13}" + local report_dir="$results/.artifacts/kova/reports/$lane" + local bundle_dir="$results/.artifacts/kova/bundles/$lane" + local summary_dir="$results/.artifacts/kova/summaries" + local plan_json="$results/.artifacts/kova/plans/$lane.json" openclaw="$root/openclaw" + validate_plan "$plan_json" "$profile" "$include_filters" "$expected_entries" + local report + report="$(node "$helpers/lib/kova-report-selector.mjs" --report-dir "$report_dir")" + local evidence_status=0 summary_status=0 effective_status="$status" + node "$helpers/lib/kova-workflow-evidence.mjs" \ + --plan "$plan_json" --report "$report" --profile "$profile" \ + --target "local-build:$openclaw" --repeat "$repeat" --include "$include_filters" \ + --auth mock --model "$model" || evidence_status=$? + if [[ "$admitted" == true && "$evidence_status" == 0 && "$fail_on_regression" == true && "$status" != 0 ]]; then + local gate_args=("$report") + [[ "$require_instrumented" != true ]] || gate_args+=(--require-instrumented-performance-contract) + if node "$helpers/lib/kova-report-gate.mjs" "${gate_args[@]}"; then + effective_status=0 + fi + fi + node "$helpers/kova-ci-summary.mjs" --report "$report" \ + --output "$summary_dir/$lane.md" --lane "$lane" || summary_status=$? + node --input-type=module - "$report" <<'NODE' +import fs from "node:fs"; +const metadata = fs.lstatSync(process.argv[2]); +if (!metadata.isFile() || metadata.size > 50000000) throw new Error("invalid diagnostic report file"); +NODE + ((evidence_status == 0 && summary_status == 0)) || + die "Kova evidence, bundle, or summary validation failed" + [[ -s "$bundle_dir/bundle.json" && -s "$summary_dir/$lane.md" ]] || + die "Kova bundle or summary evidence is missing" + if [[ "$admitted" != true ]]; then + [[ "$fail_on_regression" != true ]] || die "custom Kova cannot authorize a performance gate" + fi + [[ "$fail_on_regression" != true ]] || return "$effective_status" +} + +build_source_performance() { + if [[ -f scripts/build-all.mts ]] && + node --import tsx scripts/build-all.mts --help | grep -Fxq ' sourcePerformance'; then + OPENCLAW_BUILD_PRIVATE_QA=1 node --import tsx scripts/build-all.mts sourcePerformance + elif [[ -f scripts/build-all.mjs ]] && + node scripts/build-all.mjs --help | grep -Fxq ' sourcePerformance'; then + OPENCLAW_BUILD_PRIVATE_QA=1 node scripts/build-all.mjs sourcePerformance + else + pnpm build + fi +} + +source_cli_probes() ( + local openclaw="$1" source_dir="$2" repeat="$3" helpers="$4" supported="$5" + [[ "$supported" == true || "$supported" == false ]] || die "invalid CLI capability" + mkdir -p "$source_dir" + printf '{"supported":%s,"entry":"openclaw.mjs"}\n' "$supported" > "$source_dir/cli-capability.json" + if [[ "$supported" == false ]]; then + printf '# OpenClaw Source Performance\n\nTrusted CLI measurement unsupported: the pinned target has no openclaw.mjs entry.\n' > "$source_dir/index.md" + return + fi + local gateway_home gateway_readiness_home gateway_port gateway_token gateway_pid="" + gateway_home="$(as_candidate mktemp -d)" + gateway_readiness_home="$(as_candidate mktemp -d)" + cleanup_gateway() { + if [[ -n "${gateway_pid:-}" ]] && kill -0 "$gateway_pid" 2>/dev/null; then + kill "$gateway_pid" 2>/dev/null || true + wait "$gateway_pid" 2>/dev/null || true + fi + as_candidate rm -rf -- "${gateway_home:-}" "${gateway_readiness_home:-}" + } + trap cleanup_gateway EXIT + gateway_port="$(node -e "const net=require('node:net'); const s=net.createServer(); s.listen(0,'127.0.0.1',()=>{ console.log(s.address().port); s.close(); });")" + gateway_token="$(node -e "console.log(require('node:crypto').randomBytes(32).toString('hex'))")" + local gateway_state="$gateway_home/.openclaw" gateway_config="$gateway_home/.openclaw/openclaw.json" + local readiness_state="$gateway_readiness_home/.openclaw" readiness_config="$gateway_readiness_home/.openclaw/openclaw.json" + local gateway_log="$source_dir/cli-gateway.log" readiness_log="$source_dir/cli-gateway-readiness.log" + as_candidate mkdir -p "$gateway_state" "$readiness_state" + local catalog_refresh_config="" + if as_candidate grep -q 'catalogRefresh:' "$openclaw/src/config/zod-schema.core.ts"; then + catalog_refresh_config='"models": { "catalogRefresh": { "enabled": false } },' + fi + as_candidate tee "$gateway_config" >/dev/null <<EOF +{ + "agents": { "defaults": { "heartbeat": { "every": "0m" } } }, + "browser": { "enabled": false }, + ${catalog_refresh_config} + "update": { "checkOnStart": false }, + "gateway": { + "mode": "local", "port": ${gateway_port}, "bind": "loopback", + "auth": { "mode": "token" }, "controlUi": { "enabled": false }, + "tailscale": { "mode": "off" } + }, + "plugins": { "enabled": true, "entries": { "browser": { "enabled": false } } } +} +EOF + as_candidate cp "$gateway_config" "$readiness_config" + as_candidate env OPENCLAW_GATEWAY_TOKEN="$gateway_token" OPENCLAW_HOME="$gateway_home" \ + OPENCLAW_STATE_DIR="$gateway_state" OPENCLAW_CONFIG_PATH="$gateway_config" \ + OPENCLAW_GATEWAY_PORT="$gateway_port" OPENCLAW_SKIP_CHANNELS=1 OPENCLAW_SKIP_CRON=1 \ + node "$openclaw/dist/entry.js" gateway run --bind loopback --port "$gateway_port" --auth token --allow-unconfigured --force \ + > "$gateway_log" 2>&1 & + gateway_pid="$!" + local deadline=$((SECONDS + 120)) remaining probe_timeout + while true; do + remaining=$((deadline - SECONDS)) + ((remaining > 0)) || die "timed out waiting for gateway HTTP health" + probe_timeout="$remaining" + ((probe_timeout <= 5)) || probe_timeout=5 + if curl -fsS --connect-timeout 2 --max-time "$probe_timeout" "http://127.0.0.1:$gateway_port/healthz" >/dev/null; then + break + fi + kill -0 "$gateway_pid" 2>/dev/null || die "gateway exited before HTTP health" + sleep 1 + done + while true; do + remaining=$((deadline - SECONDS)) + ((remaining > 0)) || die "timed out waiting for gateway WebSocket health" + if as_candidate env OPENCLAW_GATEWAY_TOKEN="$gateway_token" OPENCLAW_HOME="$gateway_readiness_home" \ + OPENCLAW_STATE_DIR="$readiness_state" OPENCLAW_CONFIG_PATH="$readiness_config" \ + node "$openclaw/dist/entry.js" gateway health --port "$gateway_port" --timeout "$((remaining * 1000))" \ + --json > "$readiness_log" 2>&1; then + break + fi + kill -0 "$gateway_pid" 2>/dev/null || die "gateway exited before WebSocket health" + ((SECONDS >= deadline)) || sleep 1 + done + OPENCLAW_BENCH_TRANSPORT_JSON="$(candidate_transport "/opt/node-v${NODE_VERSION}/bin/node")" \ + OPENCLAW_GATEWAY_TOKEN="$gateway_token" OPENCLAW_HOME="$gateway_home" \ + OPENCLAW_STATE_DIR="$gateway_state" OPENCLAW_CONFIG_PATH="$gateway_config" \ + OPENCLAW_GATEWAY_PORT="$gateway_port" \ + node "$helpers/bench-cli-startup.mjs" --entry "$openclaw/openclaw.mjs" \ + --case gatewayHealthJsonWarmState --case gatewayHealthJsonFreshState \ + --case configGetGatewayPort --runs "$repeat" --warmup 1 --output "$source_dir/cli-startup.json" +) + +collect_diagnostics() { + local source="$1" destination="$2" subtree="$3" + local file relative size total=0 count=0 + [[ -d "$source/$subtree" && ! -L "$source/$subtree" ]] || die "diagnostic subtree is missing" + # B has been stopped. Copy only bounded regular bytes, never a candidate-selected destination. + while IFS= read -r -d '' file; do + [[ ! -L "$file" && ( -d "$file" || -f "$file" ) ]] || die "diagnostic links or special files are forbidden" + [[ "$(as_sut /usr/bin/realpath "$file")" == "$file" ]] || die "diagnostic ancestor is redirected" + [[ -f "$file" ]] || continue + relative="${file#"$source/"}" + [[ "$relative" =~ ^\.artifacts/[A-Za-z0-9._/-]+$ && "$relative" != *"/../"* ]] || + die "unsafe diagnostic path" + case "$relative" in + .artifacts/openclaw-performance/source/mock-provider/cli-* | \ + .artifacts/openclaw-performance/source/mock-provider/index.md) + die "candidate diagnostics cannot supply runner measurement files" ;; + esac + [[ "$(as_sut /usr/bin/stat -c %h "$file")" == 1 ]] || die "diagnostic hardlinks are forbidden" + size="$(as_sut /usr/bin/stat -c %s "$file")" + ((size <= MAX_ARTIFACT_FILE_BYTES)) || die "diagnostic file exceeds byte limit" + count=$((count + 1)) total=$((total + size)) + ((count <= MAX_ARTIFACT_FILES && total <= MAX_ARTIFACT_BYTES)) || die "diagnostics exceed payload limit" + [[ ! -e "$destination/$relative" && ! -L "$destination/$relative" ]] || + die "diagnostics cannot replace runner output" + install -D -o "$RUNNER_USER" -g "$RUNNER_USER" -m 0600 "$file" "$destination/$relative" + done < <(as_sut /usr/bin/find "$source/$subtree" -print0) + # install -D creates root-owned ancestors; hand them only to A, after B is absent. + chown -R "$RUNNER_USER:$RUNNER_USER" "$destination/.artifacts" +} + +quiesce_sut() { + local uid deadline + uid="$(id -u "$SUT_USER")" + loginctl disable-linger "$SUT_USER" + systemctl stop "user@${uid}.service" + pkill -KILL -u "$uid" 2>/dev/null || true + deadline=$((SECONDS + 20)) + while pgrep -u "$uid" >/dev/null 2>&1; do + ((SECONDS < deadline)) || die "SUT processes survived termination" + sleep 1 + done +} + +write_payload() { + local lane="$1" root="$2" control_workspace="$3" tested_ref="$4" + local openclaw_sha="$5" kova_sha="$6" workflow_sha="$7" + local run_id="$8" run_attempt="$9" crabbox_commit="${10}" crabbox_version="${11}" + local started_at="${12}" finished_at="${13}" + local profile="${14}" repeat="${15}" contract="${16}" include_filters="${17}" + local fail_on_regression="${18}" workload_status="${19}" + local output="$control_workspace/.artifacts/performance-crabbox/$lane" + local manifest="$output/artifacts.jsonl" payload="$output/payload.tar.gz" + local paths=() + + case "$lane" in + mock-provider | mock-deep-profile) + paths=( + ".artifacts/kova/plans/$lane.json" + ".artifacts/kova/reports/$lane" + ".artifacts/kova/bundles/$lane" + ".artifacts/kova/summaries/$lane.md" + ) + ;; + source) paths=(".artifacts/openclaw-performance/source") ;; + *) die "unsupported payload lane $lane" ;; + esac + + [[ -d "$output" && ! -L "$output" ]] || die "collector export directory was not prepared" + : > "$manifest" + local file_count=0 total_bytes=0 path file rel size sha + for path in "${paths[@]}"; do + [[ -e "$root/$path" ]] || die "missing artifact path $path" + while IFS= read -r -d '' file; do + [[ ! -L "$file" ]] || die "artifact symlinks are forbidden" + rel="${file#"$root/"}" + [[ "$rel" =~ ^\.artifacts/[A-Za-z0-9._/-]+$ && "$rel" != *"/../"* ]] || + die "unsafe artifact path" + [[ "$(as_runner /usr/bin/realpath "$file")" == "$file" ]] || die "artifact symlink ancestors are forbidden" + size="$(as_runner /usr/bin/stat -c %s "$file")" + ((size > 0 && size <= MAX_ARTIFACT_FILE_BYTES)) || die "artifact size is out of bounds: $rel" + sha="$(as_runner /usr/bin/sha256sum "$file" | cut -d' ' -f1)" + jq -cn --arg path "$rel" --argjson size "$size" --arg sha256 "$sha" \ + '{path:$path,size:$size,sha256:$sha256}' >> "$manifest" + file_count=$((file_count + 1)) + total_bytes=$((total_bytes + size)) + ((file_count <= MAX_ARTIFACT_FILES && total_bytes <= MAX_ARTIFACT_BYTES)) || + die "artifact payload is too large" + done < <(as_runner /usr/bin/find "$root/$path" -type f -print0 | sort -z) + done + ((file_count > 0 && file_count <= MAX_ARTIFACT_FILES)) || die "artifact file count is out of bounds" + ((total_bytes <= MAX_ARTIFACT_BYTES)) || die "artifact payload is too large" + jq -sr 'sort_by(.path)' "$manifest" > "$output/artifacts.json" + jq -jr '.[] | .path + "\u0000"' "$output/artifacts.json" | + as_runner /usr/bin/tar --dereference -C "$root" -czf - --null --verbatim-files-from -T - > "$payload" + + jq -n \ + --arg lane "$lane" --arg testedRef "$tested_ref" \ + --arg openclawSha "$openclaw_sha" --arg kovaSha "$kova_sha" \ + --arg workflowSha "$workflow_sha" --arg runId "$run_id" --arg runAttempt "$run_attempt" \ + --arg crabboxCommit "$crabbox_commit" --arg crabboxVersion "$crabbox_version" \ + --arg startedAt "$started_at" --arg finishedAt "$finished_at" \ + --arg profile "$profile" --arg repeat "$repeat" --arg contract "$contract" \ + --arg includeFilters "$include_filters" --arg failOnRegression "$fail_on_regression" \ + --argjson exitCode "$workload_status" \ + --slurpfile artifacts "$output/artifacts.json" \ + '{ + schemaVersion:1,lane:$lane,testedRef:$testedRef,openclawSha:$openclawSha,kovaSha:$kovaSha, + workflow:{sha:$workflowSha,runId:$runId,runAttempt:$runAttempt}, + crabbox:{commit:$crabboxCommit,version:$crabboxVersion}, + command:{ + name:$lane, + argv:["profile="+$profile,"repeat="+$repeat,"contract="+$contract, + "include="+$includeFilters,"failOnRegression="+$failOnRegression], + exitCode:$exitCode,startedAt:$startedAt,finishedAt:$finishedAt + }, + isolation:{ + sutUser:"openclaw-sut",trustedHarnessRootOwned:true,noSudo:true, + imdsBlocked:true,environmentClean:true,cachesEmptyBefore:true, + tailscaleRequested:false,tailscaleMetadataAbsent:true + }, + artifacts:$artifacts[0], + lease:{provider:"aws",market:"on-demand",cleanupPolicy:"always"} + }' > "$output/remote-evidence.json" + rm -f "$manifest" "$output/artifacts.json" + chmod 0644 "$payload" "$output/remote-evidence.json" +} + +remote_main() { + (($# == 17)) || die "remote mode requires 17 arguments" + local lane="$1" openclaw_sha="$2" kova_sha="$3" workflow_sha="$4" tested_ref="$5" + local profile="$6" repeat="$7" contract="$8" include_filters="$9" + local expected_entries="${10}" fail_on_regression="${11}" run_id="${12}" run_attempt="${13}" + local crabbox_version="${14}" + local model="${15}" require_instrumented="${16}" + local admitted_kova_sha="${17}" + case "$lane" in + source | mock-provider | mock-deep-profile) ;; + *) die "unsupported lane $lane" ;; + esac + require_sha openclaw_sha "$openclaw_sha" + require_sha kova_sha "$kova_sha" + require_sha workflow_sha "$workflow_sha" + require_sha admitted_kova_sha "$admitted_kova_sha" + require_scalar tested_ref "$tested_ref" + require_scalar crabbox_version "$crabbox_version" + require_scalar model "$model" + [[ "$require_instrumented" == true || "$require_instrumented" == false ]] || + die "instrumented contract requirement must be boolean" + [[ "$repeat" =~ ^[1-9][0-9]*$ ]] || die "repeat must be positive" + + if ((EUID != 0)); then + local self_sha root_script control_workspace + self_sha="$(sha256sum "$0" | cut -d' ' -f1)" + root_script="/usr/local/libexec/openclaw-performance-${self_sha}.sh" + control_workspace="$(dirname "$(dirname "$(dirname "$(realpath "$0")")")")" + [[ -d "$control_workspace/.crabbox/scripts" ]] || die "Crabbox workspace is invalid" + [[ "${CRABBOX_LEASE_ID:-}" =~ ^cbx_[0-9a-f]{12}$ && + "$control_workspace" == "/work/crabbox/$CRABBOX_LEASE_ID/openclaw" ]] || + die "expected dedicated raw workspace" + local hydration="$HOME/.crabbox/actions/$CRABBOX_LEASE_ID.env" + [[ ! -e "$hydration" && ! -L "$hydration" ]] || die "hydrated workspace is forbidden" + # The SSH collector must own each private ancestor before root writes the payload. + local path status=0 + for path in .artifacts .artifacts/performance-crabbox ".artifacts/performance-crabbox/$lane"; do + [[ ! -L "$control_workspace/$path" ]] || die "collector export path is a symlink" + install -d -m 0700 "$control_workspace/$path" + done + [[ ! -e "$control_workspace/.artifacts/performance-crabbox/$lane/workload-result.json" ]] || + die "workload receipt already exists" + sudo /usr/bin/env -i PATH=/usr/sbin:/usr/bin:/sbin:/bin /bin/bash -c \ + 'install -D -o root -g root -m 0755 "$1" "$2"; workspace=$3; shift 3; cd "$workspace"; exec "$0" "$@"' \ + "$root_script" "$0" "$root_script" "$control_workspace" remote "$@" || status=$? + # Always terminate candidate output with the collector's own result, including null on failure. + local receipt="$control_workspace/.artifacts/performance-crabbox/$lane/workload-result.json" result=null + if [[ -f "$receipt" && ! -L "$receipt" && + "$(stat -c '%u:%g:%a' "$receipt")" == "0:0:644" && + ! -e "$hydration" && ! -L "$hydration" ]]; then + result="$(jq -c --arg workspace "$control_workspace" --arg runId "$CRABBOX_RUN_ID" \ + '. + {workspace:$workspace,runId:$runId,noHydration:true}' "$receipt")" + fi + printf 'performance-result %s\n' "$result" + return "$status" + fi + [[ "$0" == /usr/local/libexec/openclaw-performance-*.sh ]] || die "root harness is not installed" + [[ "$(stat -c '%U:%G:%a' "$0")" == "root:root:755" ]] || die "root harness ownership is invalid" + local installed_hash="${0##*/openclaw-performance-}" + installed_hash="${installed_hash%.sh}" + [[ "$(sha256sum "$0" | cut -d' ' -f1)" == "$installed_hash" ]] || die "root harness hash is invalid" + + local control_workspace="$PWD" root="/srv/openclaw-performance" started_at finished_at status + rm -rf "$root" + install -d -m 0755 "$root" + install_toolchain + local helpers="/usr/local/libexec/openclaw-performance-helpers" + local helper + for helper in lib/kova-report-selector.mjs lib/kova-workflow-evidence.mjs \ + lib/kova-report-gate.mjs kova-ci-summary.mjs bench-cli-startup.mjs \ + openclaw-performance-source-summary.mjs; do + install -D -o root -g root -m 0644 \ + "$control_workspace/.github/crabbox/performance-control/helpers/$helper" "$helpers/$helper" + done + prepare_sut + prepare_runner + clone_exact openclaw/openclaw "$openclaw_sha" "$root/openclaw" "$SUT_USER" + local admitted=false kova="$root/kova" executor=as_sut + local results="/home/${RUNNER_USER}/results" workload_results="$root/openclaw" + if [[ "$kova_sha" == "$admitted_kova_sha" ]]; then + admitted=true kova="/home/${RUNNER_USER}/kova" executor=as_runner + workload_results="$results" + fi + clone_exact openclaw/Kova "$kova_sha" "$kova" "$([[ "$admitted" == true ]] && echo "$RUNNER_USER" || echo "$SUT_USER")" + + local source_cli_supported=false source_cli_entry + if [[ "$lane" == source ]]; then + # Capture immutable entry availability before any candidate lifecycle code runs. + source_cli_entry="$(as_sut /usr/bin/git -C "$root/openclaw" ls-tree --name-only "$openclaw_sha" -- openclaw.mjs)" + case "$source_cli_entry" in + openclaw.mjs) source_cli_supported=true ;; + "") ;; + *) die "unexpected CLI entry inventory" ;; + esac + fi + started_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + set +e + ( + set -e + as_sut "$0" __prepare "$root" + if [[ "$lane" == source ]]; then + [[ "$source_cli_supported" != true ]] || as_sut "$0" __build "$root/openclaw" + source_status=0 + as_sut "$0" __sut "$lane" "$root" "$profile" "$repeat" "$contract" "$include_filters" \ + "$expected_entries" false "$helpers" "$model" "$require_instrumented" "$workload_results" "$kova" false || source_status=$? + as_runner "$0" __source-cli "$root/openclaw" \ + "$results/.artifacts/openclaw-performance/source/mock-provider" "$repeat" "$helpers" "$source_cli_supported" + exit "$source_status" + else + as_sut "$0" __build "$root/openclaw" + "$executor" "$0" __sut "$lane" "$root" "$profile" "$repeat" "$contract" "$include_filters" \ + "$expected_entries" "$fail_on_regression" "$helpers" "$model" "$require_instrumented" "$workload_results" "$kova" "$admitted" + fi + ) + status=$? + set -e + finished_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + quiesce_sut + [[ "$(as_sut /usr/bin/git -C "$root/openclaw" rev-parse HEAD)" == "$openclaw_sha" ]] || + die "OpenClaw HEAD changed during SUT execution" + [[ "$("$executor" /usr/bin/git -C "$kova" rev-parse HEAD)" == "$kova_sha" ]] || + die "Kova HEAD changed during SUT execution" + + if [[ "$lane" == source ]]; then + local source_status + set +e + ( + set -e + collect_diagnostics "$root/openclaw" "$results" ".artifacts/openclaw-performance/source" + if [[ "$source_cli_supported" == true ]]; then + as_runner node "$helpers/openclaw-performance-source-summary.mjs" \ + --source-dir "$results/.artifacts/openclaw-performance/source/mock-provider" \ + --output "$results/.artifacts/openclaw-performance/source/mock-provider/index.md" + fi + ) + source_status=$? + set -e + if ((status == 0)); then status="$source_status"; fi + elif [[ "$admitted" != true ]]; then + local finalization_status + set +e + ( + set -e + collect_diagnostics "$root/openclaw" "$results" ".artifacts/kova" + local validation_status=0 + as_runner "$0" __validate-kova "$lane" "$root" "$profile" "$repeat" "$include_filters" \ + "$expected_entries" "$fail_on_regression" "$helpers" "$model" "$require_instrumented" \ + "$results" "$status" "$admitted" || validation_status=$? + as_runner /bin/sh -eu -c ' + mkdir -p "$(dirname "$1")" + printf "\nCustom Kova: candidate-produced diagnostics only; not gate evidence.\n" >> "$1" + ' sh "$results/.artifacts/kova/summaries/$lane.md" + exit "$validation_status" + ) + finalization_status=$? + set -e + if ((status == 0)); then status="$finalization_status"; fi + else + local validation_status=0 matrix_status="$status" + if [[ "$status" == 0 ]]; then + local matrix_receipt="$results/.artifacts/kova/$lane-matrix-exit" matrix_receipt_size + matrix_receipt_size="$(as_runner /usr/bin/stat -c %s "$matrix_receipt")" + ((matrix_receipt_size >= 2 && matrix_receipt_size <= 4)) || die "invalid matrix phase receipt size" + matrix_status="$(as_runner /usr/bin/cat "$matrix_receipt")" + [[ "$matrix_status" =~ ^(0|[1-9][0-9]{0,2})$ && "$matrix_status" -le 255 ]] || + die "invalid matrix phase exit" + fi + as_runner "$0" __validate-kova "$lane" "$root" "$profile" "$repeat" "$include_filters" \ + "$expected_entries" "$fail_on_regression" "$helpers" "$model" "$require_instrumented" \ + "$results" "$matrix_status" "$admitted" || validation_status=$? + if ((status == 0)); then status="$validation_status"; fi + fi + local export_status + set +e + ( + set -e + write_payload "$lane" "$results" "$control_workspace" "$tested_ref" "$openclaw_sha" "$kova_sha" \ + "$workflow_sha" "$run_id" "$run_attempt" "$CRABBOX_COMMIT" "$crabbox_version" \ + "$started_at" "$finished_at" "$profile" "$repeat" "$contract" "$include_filters" \ + "$fail_on_regression" "$status" + ) + export_status=$? + set -e + local receipt="$control_workspace/.artifacts/performance-crabbox/$lane/workload-result.json" + jq -n --argjson exitCode "$status" --argjson exportExitCode "$export_status" \ + --arg startedAt "$started_at" --arg finishedAt "$finished_at" \ + '{exitCode:$exitCode,exportExitCode:$exportExitCode,startedAt:$startedAt,finishedAt:$finishedAt}' > "$receipt" + chmod 0644 "$receipt" + ((status == 0)) || return "$status" + return "$export_status" +} + +verify_payload() { + (($# == 7)) || die "verify mode requires lane, timing, lease, evidence, payload, output, and expectations" + local lane="$1" timing="$2" lease_id="$3" evidence="$4" payload="$5" output="$6" + local expected="$7" + local tmp + tmp="$(mktemp -d)" + VERIFY_TMP="$tmp" + trap 'rm -rf -- "$VERIFY_TMP"' EXIT + + jq -e --arg id "$lease_id" --slurpfile expected "$expected" \ + '.provider == "aws" and .leaseId == $id and . == $expected[0].timing and + $expected[0].stopped == true' "$timing" >/dev/null || + die "Crabbox timing did not bind the expected lease" + jq -e --arg lane "$lane" --slurpfile expected "$expected" \ + '. as $e | $expected[0] as $x | + .schemaVersion == 1 and .lane == $lane and + all(["testedRef","openclawSha","kovaSha","workflow","crabbox","command"][]; $e[.] == $x[.]) and + (.artifacts | length > 0 and length <= 256) and + (.artifacts | map(.path) | length == (unique | length)) and + all(.artifacts[]; (.size > 0 and .size <= 50000000) and + (.path | test("^\\.artifacts/[A-Za-z0-9._/-]+$")) and + (.path | split("/") | all(. != ".." and . != "."))) and + ([.artifacts[].size] | add <= 250000000)' \ + "$evidence" >/dev/null || die "remote evidence is invalid" + + tar -tzf "$payload" > "$tmp/tar-paths" + grep -Ev '^\.artifacts/[A-Za-z0-9._/-]+$' "$tmp/tar-paths" > "$tmp/unsafe" || true + [[ ! -s "$tmp/unsafe" ]] || die "payload contains unsafe paths" + jq -r '.artifacts[].path' "$evidence" > "$tmp/evidence-paths" + diff -u "$tmp/evidence-paths" "$tmp/tar-paths" + tar -xzf "$payload" -C "$tmp" + + while IFS=$'\t' read -r path size sha; do + [[ -f "$tmp/$path" && ! -L "$tmp/$path" ]] || die "payload file missing: $path" + [[ "$(file_size "$tmp/$path")" == "$size" ]] || die "payload size mismatch: $path" + [[ "$(file_sha256 "$tmp/$path")" == "$sha" ]] || + die "payload hash mismatch: $path" + done < <(jq -r '.artifacts[] | [.path,.size,.sha256] | @tsv' "$evidence") + + mkdir -p "$(dirname "$output")" .artifacts + cp -R "$tmp/.artifacts/." .artifacts/ + jq --arg leaseId "$lease_id" \ + '.lease += {id:$leaseId,stopped:true,stopError:""}' "$evidence" > "$output" + jq -e --arg lane "$lane" --arg id "$lease_id" \ + '.schemaVersion == 1 and .lane == $lane and .lease.id == $id and .lease.stopped == true and + .lease.stopError == ""' "$output" >/dev/null || + die "final evidence is invalid" +} + +confirm_stop() { + (($# == 2)) || die "confirm-stop requires Crabbox path and lease id" + [[ -x "$1" && "$2" =~ ^cbx_[0-9a-f]{12}$ ]] || die "invalid explicit stop request" + "$1" stop --provider aws --id "$2" >/dev/null 2>&1 +} + +case "${1:-}" in + build-helpers) + build_helpers + ;; + remote) + shift + remote_main "$@" + ;; + __sut) + shift + run_sut "$@" + ;; + __prepare) + shift + prepare_candidate "$@" + ;; + __build) + cd "$2" + build_source_performance + ;; + __source-cli) + shift + source_cli_probes "$@" + ;; + __candidate-id) + as_candidate id -u + ;; + __validate-kova) + shift + validate_kova "$@" + ;; + verify) + shift + verify_payload "$@" + ;; + confirm-stop) + shift + confirm_stop "$@" + ;; + *) + die "usage: $0 build-helpers|remote|verify|confirm-stop ..." + ;; +esac diff --git a/scripts/openclaw-performance-source-summary.mts b/scripts/openclaw-performance-source-summary.mts new file mode 100644 index 0000000000000000000000000000000000000000..56d0e7b8364f317298fed76209c6767658107b65 --- /dev/null +++ b/scripts/openclaw-performance-source-summary.mts @@ -0,0 +1,1018 @@ +#!/usr/bin/env node + +// Summarizes OpenClaw performance source fixtures for reports. +import fs from "node:fs"; +import path from "node:path"; +import process from "node:process"; +import { pathToFileURL } from "node:url"; +import { isRecord } from "@openclaw/normalization-core/record-coerce"; +import { requireOptionArgument } from "./lib/arg-utils.mts"; +import { + assertCompatibleCliStartupExecutionModes, + assertCompatibleCliStartupMemoryMetrics, + cliStartupExecutionMode, + cliStartupMemoryMetric, +} from "./lib/cli-startup-memory-contract.mts"; +import { isStartupTraceDuration } from "./lib/gateway-startup-trace-ranking.js"; +import { collectSqliteQueryPlanEvidence } from "./lib/sqlite-query-plan-evidence.js"; + +type JsonObject = { [key: string]: JsonValue }; +type JsonValue = boolean | number | string | null | JsonObject | JsonValue[]; +type RequiredOption = { required?: boolean }; + +function isJsonValue(value: unknown): value is JsonValue { + if (value === null || ["boolean", "number", "string"].includes(typeof value)) { + return true; + } + if (Array.isArray(value)) { + return value.every(isJsonValue); + } + return isRecord(value) && Object.values(value).every(isJsonValue); +} + +function parseJson(source: string): JsonValue { + const value: unknown = JSON.parse(source); + if (!isJsonValue(value)) { + throw new Error("parsed value is not JSON"); + } + return value; +} + +function isJsonObject(value: JsonValue | undefined): value is JsonObject { + return isRecord(value); +} + +function valueAt(value: JsonValue | undefined, ...keys: string[]): JsonValue | undefined { + let current = value; + for (const key of keys) { + current = isJsonObject(current) ? current[key] : undefined; + } + return current; +} + +function objectArray(value: JsonValue | undefined): JsonObject[] { + return Array.isArray(value) ? value.filter(isJsonObject) : []; +} + +function stringArray(value: JsonValue | undefined): string[] { + return Array.isArray(value) + ? value.filter((entry): entry is string => typeof entry === "string") + : []; +} + +function hasControlCharacters(value: string): boolean { + for (const character of value) { + const codePoint = character.codePointAt(0); + if (codePoint !== undefined && (codePoint <= 31 || codePoint === 127)) { + return true; + } + } + return false; +} + +export function parseArgs(argv: string[]) { + const options: Record<"baselineSourceDir" | "sourceDir" | "output", string | null> = { + baselineSourceDir: null, + sourceDir: null, + output: null, + }; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index] ?? ""; + const readValue = () => { + const value = requireOptionArgument(argv, index, arg); + index += 1; + return value; + }; + switch (arg) { + case "--source-dir": + options.sourceDir = path.resolve(readValue()); + break; + case "--baseline-source-dir": + options.baselineSourceDir = path.resolve(readValue()); + break; + case "--output": + options.output = path.resolve(readValue()); + break; + case "--help": + printHelp(); + process.exit(0); + break; + default: + throw new Error(`Unknown argument: ${arg}`); + } + } + if (!options.sourceDir) { + throw new Error("--source-dir is required"); + } + return { ...options, sourceDir: options.sourceDir }; +} + +function printHelp() { + console.log(`Usage: node --import tsx scripts/openclaw-performance-source-summary.mts --source-dir <dir> [--baseline-source-dir <dir>] [--output <summary.md>] + +Summarizes OpenClaw-native performance probe artifacts for CI reports.`); +} + +function readJsonIfExists(filePath: string): JsonValue { + if (!fs.existsSync(filePath)) { + return null; + } + return parseJson(fs.readFileSync(filePath, "utf8")); +} + +function readRequiredJson(filePath: string, label: string): JsonValue { + if (!fs.existsSync(filePath)) { + throw new Error(`[source-performance] missing required ${label}: ${filePath}`); + } + return parseJson(fs.readFileSync(filePath, "utf8")); +} + +function finiteNumber(value: unknown): value is number { + return typeof value === "number" && Number.isFinite(value); +} + +function formatMs(value: unknown) { + return finiteNumber(value) ? `${value.toFixed(1)}ms` : "n/a"; +} + +function formatMb(value: unknown, unit: "MB" | "MiB" = "MB") { + return finiteNumber(value) ? `${value.toFixed(1)}${unit}` : "n/a"; +} + +function formatBytesAsMb(value: unknown) { + return finiteNumber(value) ? formatMb(value / 1024 / 1024) : "n/a"; +} + +function formatRatio(value: unknown) { + return finiteNumber(value) ? value.toFixed(3) : "n/a"; +} + +function metric(stats: JsonValue | undefined, key: "p50" | "p95" = "p50") { + const value = valueAt(stats, key); + return typeof value === "number" ? value : null; +} + +function percentDelta(before: unknown, after: unknown) { + if (typeof before !== "number" || typeof after !== "number") { + return null; + } + if (before === 0) { + return after === 0 ? 0 : null; + } + return ((after - before) / before) * 100; +} + +function formatDeltaMb(before: unknown, after: unknown, unit: "MB" | "MiB" = "MB") { + if (typeof before !== "number" || typeof after !== "number") { + return "n/a"; + } + const delta = after - before; + const percent = percentDelta(before, after); + const sign = delta > 0 ? "+" : ""; + const percentText = percent == null ? "new" : `${percent > 0 ? "+" : ""}${percent.toFixed(1)}%`; + return `${sign}${formatMb(delta, unit)} (${percentText})`; +} + +function memoryRisk(before: unknown, after: unknown) { + const percent = percentDelta(before, after); + const delta = typeof before === "number" && typeof after === "number" ? after - before : null; + if (percent == null || delta == null) { + return "n/a"; + } + if (percent >= 20 && delta >= 10) { + return "watch"; + } + if (percent <= -10 && delta <= -10) { + return "improved"; + } + return "stable"; +} + +function escapeCell(value: unknown) { + return String(value).replaceAll("|", "\\|"); +} + +function table(headers: string[], rows: unknown[][]) { + if (rows.length === 0) { + return ["No data.", ""]; + } + return [ + `| ${headers.join(" | ")} |`, + `| ${headers.map(() => "---").join(" | ")} |`, + ...rows.map((row) => `| ${row.map((cell) => escapeCell(cell)).join(" | ")} |`), + "", + ]; +} + +function validateMockHelloSummary(summary: JsonValue, filePath: string) { + const total = valueAt(summary, "counts", "total"); + const passed = valueAt(summary, "counts", "passed"); + const failed = valueAt(summary, "counts", "failed"); + const scenarios = objectArray(valueAt(summary, "scenarios")); + if ( + !isNonNegativeInteger(total) || + !isNonNegativeInteger(passed) || + !isNonNegativeInteger(failed) || + total <= 0 || + failed !== 0 || + passed !== total + ) { + throw new Error(`[source-performance] invalid mock hello summary counts: ${filePath}`); + } + if (scenarios.length !== total) { + throw new Error(`[source-performance] invalid mock hello scenario evidence: ${filePath}`); + } + const passedScenarios = scenarios.filter((scenario) => scenario.status === "pass").length; + const failedScenarios = scenarios.filter((scenario) => scenario.status === "fail").length; + const invalidScenario = scenarios.find((scenario) => { + const status = scenario.status; + return typeof status !== "string" || !["pass", "fail", "skip"].includes(status); + }); + if (invalidScenario || passedScenarios !== passed || failedScenarios !== failed) { + throw new Error(`[source-performance] invalid mock hello scenario evidence: ${filePath}`); + } + const requiredMetrics = [ + "wallMs", + "gatewayCpuCoreRatio", + "gatewayProcessRssStartBytes", + "gatewayProcessRssEndBytes", + "gatewayProcessRssDeltaBytes", + ]; + const missingMetric = requiredMetrics.find( + (key) => !finiteNumber(valueAt(summary, "metrics", key)), + ); + if (missingMetric) { + throw new Error(`[source-performance] missing mock hello metric ${missingMetric}: ${filePath}`); + } +} + +function isNonNegativeInteger(value: unknown): value is number { + return typeof value === "number" && Number.isInteger(value) && value >= 0; +} + +function loadMockHelloSummaries(sourceDir: string, { required = false }: RequiredOption = {}) { + const root = path.join(sourceDir, "mock-hello"); + if (!fs.existsSync(root)) { + if (required) { + throw new Error(`[source-performance] missing required mock hello directory: ${root}`); + } + return []; + } + const summaries = fs + .readdirSync(root, { withFileTypes: true }) + .filter((entry) => entry.isDirectory()) + .map((entry) => ({ + id: entry.name, + summaryPath: path.join(root, entry.name, "qa-suite-summary.json"), + })) + .filter((entry) => fs.existsSync(entry.summaryPath)) + .map((entry) => ({ + id: entry.id, + summary: parseJson(fs.readFileSync(entry.summaryPath, "utf8")), + summaryPath: entry.summaryPath, + })) + .toSorted((a, b) => a.id.localeCompare(b.id)); + if (required && summaries.length === 0) { + throw new Error(`[source-performance] missing required mock hello summaries: ${root}`); + } + if (required) { + for (const entry of summaries) { + validateMockHelloSummary(entry.summary, entry.summaryPath); + } + } + return summaries.map(({ id, summary }) => ({ id, summary })); +} + +type MockHelloEntry = ReturnType<typeof loadMockHelloSummaries>[number]; + +function validateStartupArtifact(startup: JsonValue, filePath: string) { + const results = objectArray(valueAt(startup, "results")); + if (results.length === 0) { + throw new Error(`[source-performance] missing gateway startup results: ${filePath}`); + } + for (const result of results) { + if ( + !finiteNumber(valueAt(result, "summary", "readyzMs", "p50")) || + !finiteNumber(valueAt(result, "summary", "maxRssMb", "p95")) || + !finiteNumber(valueAt(result, "summary", "cpuCoreRatio", "p95")) + ) { + const resultId = typeof result.id === "string" ? result.id : "unknown"; + throw new Error( + `[source-performance] incomplete gateway startup metrics for ${resultId}: ${filePath}`, + ); + } + } +} + +function validateCliArtifact(cli: JsonValue, filePath: string) { + cliStartupExecutionMode(valueAt(cli, "primary")); + cliStartupMemoryMetric(valueAt(cli, "primary")); + const cases = objectArray(valueAt(cli, "primary", "cases")); + if (cases.length === 0) { + throw new Error(`[source-performance] missing CLI startup cases: ${filePath}`); + } + for (const entry of cases) { + if ( + !finiteNumber(valueAt(entry, "summary", "durationMs", "p50")) || + !finiteNumber(valueAt(entry, "summary", "maxRssMb", "p95")) + ) { + const entryId = typeof entry.id === "string" ? entry.id : "unknown"; + throw new Error( + `[source-performance] incomplete CLI startup metrics for ${entryId}: ${filePath}`, + ); + } + } +} + +function validateExtensionMemoryArtifact(extensionMemory: JsonValue, filePath: string) { + const rows = objectArray(valueAt(extensionMemory, "topByDeltaMb")); + if (rows.length === 0) { + throw new Error(`[source-performance] missing extension memory rows: ${filePath}`); + } + if ( + !finiteNumber(valueAt(extensionMemory, "baseline", "maxRssMb")) || + !finiteNumber(valueAt(extensionMemory, "combined", "maxRssMb")) + ) { + throw new Error(`[source-performance] incomplete extension memory context: ${filePath}`); + } + for (const entry of rows) { + if (!finiteNumber(entry.maxRssMb) || !finiteNumber(entry.deltaFromBaselineMb)) { + const entryDir = typeof entry.dir === "string" ? entry.dir : "unknown"; + throw new Error( + `[source-performance] incomplete extension memory metrics for ${entryDir}: ${filePath}`, + ); + } + } +} + +function validateSqlitePerfCommon(sqlitePerf: JsonValue, filePath: string) { + if (valueAt(sqlitePerf, "profile") !== "smoke") { + throw new Error(`[source-performance] invalid SQLite perf profile: ${filePath}`); + } + if (valueAt(sqlitePerf, "integrity", "state") !== "ok") { + throw new Error(`[source-performance] SQLite integrity check did not pass: ${filePath}`); + } + const agentIntegrity = valueAt(sqlitePerf, "integrity", "agent"); + if ( + !Array.isArray(agentIntegrity) || + agentIntegrity.length === 0 || + agentIntegrity.some((entry) => entry !== "ok") + ) { + throw new Error(`[source-performance] SQLite agent integrity check did not pass: ${filePath}`); + } + const stateRows = valueAt(sqlitePerf, "rows", "stateRows"); + const agentRows = valueAt(sqlitePerf, "rows", "agentCacheEntries"); + const totalMs = valueAt(sqlitePerf, "timingsMs", "total"); + const stateWalBefore = valueAt(sqlitePerf, "walBytes", "stateBefore"); + const queries = objectArray(valueAt(sqlitePerf, "queries")); + if ( + !isNonNegativeInteger(stateRows) || + stateRows <= 0 || + !isNonNegativeInteger(agentRows) || + agentRows <= 0 || + !finiteNumber(totalMs) || + totalMs < 0 || + !finiteNumber(stateWalBefore) || + stateWalBefore < 0 || + valueAt(sqlitePerf, "walBytes", "stateAfter") !== 0 || + queries.length === 0 + ) { + throw new Error(`[source-performance] incomplete SQLite perf metrics: ${filePath}`); + } + return queries; +} + +function isNormalizedStringArray( + value: JsonValue | undefined, + { allowEmpty = true, unique = false } = {}, +) { + if (!Array.isArray(value) || (!allowEmpty && value.length === 0)) { + return false; + } + const strings = value.filter((entry): entry is string => typeof entry === "string"); + return ( + strings.length === value.length && + strings.every( + (entry) => entry.length > 0 && entry.trim() === entry && !hasControlCharacters(entry), + ) && + (!unique || new Set(strings).size === strings.length) + ); +} + +function validateLegacySqlitePerfArtifact(sqlitePerf: JsonValue, filePath: string) { + const queries = validateSqlitePerfCommon(sqlitePerf, filePath); + for (const entry of queries) { + if (!finiteNumber(entry.p50Ms) || !finiteNumber(entry.p95Ms) || !finiteNumber(entry.rows)) { + throw new Error(`[source-performance] incomplete SQLite query metrics: ${filePath}`); + } + } +} + +function validateSqlitePerfV2Artifact(sqlitePerf: JsonValue, filePath: string) { + const queries = validateSqlitePerfCommon(sqlitePerf, filePath); + const rawQueries = valueAt(sqlitePerf, "queries"); + const sqliteVersion = valueAt(sqlitePerf, "versions", "sqlite"); + const stateSchemaVersion = valueAt(sqlitePerf, "versions", "stateSchema"); + const agentSchemaVersion = valueAt(sqlitePerf, "versions", "agentSchema"); + if ( + typeof sqliteVersion !== "string" || + sqliteVersion.length === 0 || + sqliteVersion.trim() !== sqliteVersion || + hasControlCharacters(sqliteVersion) || + !isNonNegativeInteger(stateSchemaVersion) || + stateSchemaVersion <= 0 || + !isNonNegativeInteger(agentSchemaVersion) || + agentSchemaVersion <= 0 || + !Array.isArray(rawQueries) || + rawQueries.length !== queries.length + ) { + throw new Error(`[source-performance] invalid SQLite run metadata: ${filePath}`); + } + + const ids = new Set<string>(); + for (const entry of queries) { + const id = entry.id; + if ( + typeof id !== "string" || + id.length === 0 || + id.trim() !== id || + hasControlCharacters(id) || + ids.has(id) + ) { + throw new Error(`[source-performance] invalid SQLite scenario ID: ${filePath}`); + } + ids.add(id); + + if ( + (entry.database !== "state" && entry.database !== "agent") || + typeof entry.sql !== "string" || + entry.sql.trim().length === 0 || + !isNonNegativeInteger(entry.rows) || + !isNonNegativeInteger(entry.runs) || + entry.runs <= 0 || + !finiteNumber(entry.p50Ms) || + entry.p50Ms < 0 || + !finiteNumber(entry.p95Ms) || + entry.p95Ms < entry.p50Ms + ) { + throw new Error(`[source-performance] invalid SQLite scenario metrics: ${filePath}`); + } + + const plan = isJsonObject(entry.plan) ? entry.plan : undefined; + if ( + !plan || + !isNormalizedStringArray(plan.raw, { allowEmpty: false }) || + !isNormalizedStringArray(plan.indexes, { unique: true }) || + !isNormalizedStringArray(plan.fullTableScans) || + !isNormalizedStringArray(plan.tempSorts) + ) { + throw new Error(`[source-performance] invalid SQLite scenario plan: ${filePath}`); + } + const expectedPlan = collectSqliteQueryPlanEvidence(plan.raw as string[]); + if ( + JSON.stringify(plan.indexes) !== JSON.stringify(expectedPlan.indexes) || + JSON.stringify(plan.fullTableScans) !== JSON.stringify(expectedPlan.fullTableScans) || + JSON.stringify(plan.tempSorts) !== JSON.stringify(expectedPlan.tempSorts) + ) { + throw new Error(`[source-performance] invalid SQLite scenario plan: ${filePath}`); + } + } +} + +function validateSqlitePerfArtifact(sqlitePerf: JsonValue, filePath: string) { + const schemaVersion = valueAt(sqlitePerf, "schemaVersion"); + if (schemaVersion === undefined) { + validateLegacySqlitePerfArtifact(sqlitePerf, filePath); + return; + } + if (schemaVersion !== 2) { + throw new Error(`[source-performance] unsupported SQLite perf schema version: ${filePath}`); + } + validateSqlitePerfV2Artifact(sqlitePerf, filePath); +} + +function validateGatewaySummaryArtifact(gatewaySummary: JsonValue, filePath: string) { + if (!Array.isArray(valueAt(gatewaySummary, "observations"))) { + throw new Error(`[source-performance] missing gateway observation summary: ${filePath}`); + } +} + +function loadSourceArtifacts(sourceDir: string | null, { required = false }: RequiredOption = {}) { + if (!sourceDir || !fs.existsSync(sourceDir)) { + if (required) { + throw new Error(`[source-performance] missing required source dir: ${sourceDir}`); + } + return null; + } + const stat = fs.statSync(sourceDir); + if (!stat.isDirectory()) { + throw new Error(`[source-performance] source path is not a directory: ${sourceDir}`); + } + const startupPath = path.join(sourceDir, "gateway-cpu", "gateway-startup-bench.json"); + const cliPath = path.join(sourceDir, "cli-startup.json"); + const extensionMemoryPath = path.join(sourceDir, "extension-memory.json"); + const sqlitePerfPath = path.join(sourceDir, "sqlite-perf-smoke.json"); + const artifacts = { + startup: required + ? readRequiredJson(startupPath, "gateway startup artifact") + : readJsonIfExists(startupPath), + cli: required ? readRequiredJson(cliPath, "CLI startup artifact") : readJsonIfExists(cliPath), + extensionMemory: required + ? readRequiredJson(extensionMemoryPath, "extension memory artifact") + : readJsonIfExists(extensionMemoryPath), + sqlitePerf: readJsonIfExists(sqlitePerfPath), + mockHelloSummaries: loadMockHelloSummaries(sourceDir, { required }), + }; + if (required) { + validateStartupArtifact(artifacts.startup, startupPath); + validateCliArtifact(artifacts.cli, cliPath); + validateExtensionMemoryArtifact(artifacts.extensionMemory, extensionMemoryPath); + } + if (artifacts.sqlitePerf) { + validateSqlitePerfArtifact(artifacts.sqlitePerf, sqlitePerfPath); + } + return artifacts; +} + +type SourceArtifacts = NonNullable<ReturnType<typeof loadSourceArtifacts>>; + +function buildStartupRows(startup: JsonValue) { + return objectArray(valueAt(startup, "results")).map((result) => [ + result.id ?? "unknown", + result.name ?? result.id ?? "unknown", + formatMs(metric(valueAt(result, "summary", "readyzMs"))), + formatMs(metric(valueAt(result, "summary", "readyzMs"), "p95")), + formatMs(metric(valueAt(result, "summary", "healthzMs"))), + formatMs(metric(valueAt(result, "summary", "httpListenLogMs"))), + formatMs(metric(valueAt(result, "summary", "gatewayReadyLogMs"))), + formatMs(metric(valueAt(result, "summary", "firstOutputMs"))), + formatMb(metric(valueAt(result, "summary", "maxRssMb"), "p95")), + formatRatio(metric(valueAt(result, "summary", "cpuCoreRatio"), "p95")), + ]); +} + +function buildTraceRows(startup: JsonValue) { + const rows: unknown[][] = []; + for (const result of objectArray(valueAt(startup, "results"))) { + const trace = valueAt(result, "summary", "startupTrace"); + const traceEntries = (isJsonObject(trace) ? Object.entries(trace) : []) + .filter(([name, stats]) => isStartupTraceDuration(name) && metric(stats) !== null) + .toSorted((a, b) => (metric(b[1]) ?? 0) - (metric(a[1]) ?? 0)) + .slice(0, 5); + for (const [name, stats] of traceEntries) { + rows.push([ + result.id ?? "unknown", + name, + formatMs(metric(stats)), + formatMs(metric(stats, "p95")), + ]); + } + } + return rows; +} + +function buildMockHelloRows(summaries: MockHelloEntry[]) { + return summaries.map(({ id, summary }) => { + const failed = valueAt(summary, "counts", "failed"); + const countInfo = valueAt(summary, "counts"); + const status = typeof failed === "number" && failed > 0 ? "fail" : "pass"; + const passed = isJsonObject(countInfo) ? countInfo.passed : undefined; + const total = isJsonObject(countInfo) ? countInfo.total : undefined; + const counts = + isNonNegativeInteger(passed) && isNonNegativeInteger(total) ? `${passed}/${total}` : "n/a"; + return [ + id, + status, + counts, + formatMs(valueAt(summary, "metrics", "wallMs")), + formatRatio(valueAt(summary, "metrics", "gatewayCpuCoreRatio")), + formatBytesAsMb(valueAt(summary, "metrics", "gatewayProcessRssStartBytes")), + formatBytesAsMb(valueAt(summary, "metrics", "gatewayProcessRssEndBytes")), + formatBytesAsMb(valueAt(summary, "metrics", "gatewayProcessRssDeltaBytes")), + valueAt(summary, "run", "primaryModel") ?? "n/a", + ]; + }); +} + +function buildCliRows(cli: JsonValue) { + return objectArray(valueAt(cli, "primary", "cases")).map((commandCase) => [ + commandCase.id ?? "unknown", + commandCase.name ?? commandCase.id ?? "unknown", + formatMs(valueAt(commandCase, "summary", "durationMs", "p50")), + formatMs(valueAt(commandCase, "summary", "durationMs", "p95")), + formatMb(valueAt(commandCase, "summary", "maxRssMb", "p95"), "MiB"), + formatExitSummary(valueAt(commandCase, "summary", "exitSummary")), + ]); +} + +function buildStartupMemoryDeltaRows(current: JsonValue, baseline: JsonValue) { + const baselineById = new Map( + objectArray(valueAt(baseline, "results")).map((result) => [result.id, result]), + ); + return objectArray(valueAt(current, "results")) + .map((result) => { + const before = baselineById.get(result.id); + if (!before) { + return null; + } + const beforeRss = metric(valueAt(before, "summary", "maxRssMb"), "p95"); + const afterRss = metric(valueAt(result, "summary", "maxRssMb"), "p95"); + const beforeReadyHeap = metric( + valueAt(before, "summary", "startupTrace", "memory.ready.heapUsedMb"), + "p95", + ); + const afterReadyHeap = metric( + valueAt(result, "summary", "startupTrace", "memory.ready.heapUsedMb"), + "p95", + ); + return [ + "gateway boot", + result.id ?? "unknown", + formatMb(beforeRss), + formatMb(afterRss), + formatDeltaMb(beforeRss, afterRss), + formatDeltaMb(beforeReadyHeap, afterReadyHeap), + memoryRisk(beforeRss, afterRss), + ]; + }) + .filter((row) => row !== null); +} + +function buildCliMemoryDeltaRows(current: JsonValue, baseline: JsonValue) { + if (!current || !baseline) { + return []; + } + assertCompatibleCliStartupExecutionModes( + valueAt(baseline, "primary"), + valueAt(current, "primary"), + ); + assertCompatibleCliStartupMemoryMetrics( + valueAt(baseline, "primary"), + valueAt(current, "primary"), + ); + const baselineById = new Map( + objectArray(valueAt(baseline, "primary", "cases")).map((entry) => [entry.id, entry]), + ); + return objectArray(valueAt(current, "primary", "cases")) + .map((entry) => { + const before = baselineById.get(entry.id); + if (!before) { + return null; + } + const beforeRss = metric(valueAt(before, "summary", "maxRssMb"), "p95"); + const afterRss = metric(valueAt(entry, "summary", "maxRssMb"), "p95"); + return [ + "cli", + entry.id ?? "unknown", + formatMb(beforeRss, "MiB"), + formatMb(afterRss, "MiB"), + formatDeltaMb(beforeRss, afterRss, "MiB"), + "n/a", + memoryRisk(beforeRss, afterRss), + ]; + }) + .filter((row) => row !== null); +} + +function average(values: unknown[]) { + const numeric = values.filter(finiteNumber); + if (numeric.length === 0) { + return null; + } + return numeric.reduce((sum, value) => sum + value, 0) / numeric.length; +} + +function buildMockHelloMemoryDeltaRows(current: MockHelloEntry[], baseline: MockHelloEntry[]) { + const beforeDeltaBytes = average( + baseline.map((entry) => valueAt(entry.summary, "metrics", "gatewayProcessRssDeltaBytes")), + ); + const afterDeltaBytes = average( + current.map((entry) => valueAt(entry.summary, "metrics", "gatewayProcessRssDeltaBytes")), + ); + if (beforeDeltaBytes == null || afterDeltaBytes == null) { + return []; + } + const beforeDelta = beforeDeltaBytes / 1024 / 1024; + const afterDelta = afterDeltaBytes / 1024 / 1024; + return [ + [ + "mock hello", + "gateway RSS delta avg", + formatMb(beforeDelta), + formatMb(afterDelta), + formatDeltaMb(beforeDelta, afterDelta), + "n/a", + memoryRisk(beforeDelta, afterDelta), + ], + ]; +} + +function buildExtensionMemoryRows(extensionMemory: JsonValue) { + return objectArray(valueAt(extensionMemory, "topByDeltaMb")) + .slice(0, 10) + .map((entry) => [ + entry.dir ?? "unknown", + formatMb(entry.maxRssMb), + formatMb(entry.deltaFromBaselineMb), + entry.status ?? "unknown", + ]); +} + +function buildExtensionMemoryContextRows(extensionMemory: JsonValue) { + const baselineMb = valueAt(extensionMemory, "baseline", "maxRssMb"); + const combinedMb = valueAt(extensionMemory, "combined", "maxRssMb"); + const totalEntries = valueAt(extensionMemory, "counts", "totalEntries"); + return [ + [ + "empty Node process", + formatMb(baselineMb), + formatMb(0), + valueAt(extensionMemory, "baseline", "status") ?? "unknown", + ], + [ + finiteNumber(totalEntries) + ? `all ${totalEntries} bundled plugins` + : "all selected bundled plugins", + formatMb(combinedMb), + finiteNumber(baselineMb) && finiteNumber(combinedMb) + ? formatMb(combinedMb - baselineMb) + : "n/a", + valueAt(extensionMemory, "combined", "status") ?? "unknown", + ], + ]; +} + +function sqliteArtifactFormat(sqlitePerf: JsonValue) { + return valueAt(sqlitePerf, "schemaVersion") === 2 ? "v2" : "legacy"; +} + +function buildSqliteRunRows(current: JsonValue, baseline: JsonValue) { + const rows: unknown[][] = []; + for (const [label, artifact] of [ + ["current", current], + ["baseline", baseline], + ] as const) { + if (!artifact) { + continue; + } + rows.push([ + label, + sqliteArtifactFormat(artifact), + valueAt(artifact, "profile") ?? "unknown", + valueAt(artifact, "versions", "sqlite") ?? "n/a", + valueAt(artifact, "versions", "stateSchema") ?? "n/a", + valueAt(artifact, "versions", "agentSchema") ?? "n/a", + valueAt(artifact, "rows", "stateRows") ?? "n/a", + valueAt(artifact, "rows", "agentCacheEntries") ?? "n/a", + valueAt(artifact, "integrity", "state") ?? "n/a", + formatBytesAsMb(valueAt(artifact, "walBytes", "stateBefore")), + formatBytesAsMb(valueAt(artifact, "walBytes", "stateAfter")), + formatMs(valueAt(artifact, "timingsMs", "total")), + ]); + } + return rows; +} + +function formatPercentDelta(before: unknown, after: unknown) { + const delta = percentDelta(before, after); + if (delta == null) { + return "n/a"; + } + return `${delta > 0 ? "+" : ""}${delta.toFixed(1)}%`; +} + +function formatSqlitePlan(plan: JsonValue | undefined) { + if (!isJsonObject(plan)) { + return "not reported"; + } + const indexes = stringArray(plan.indexes); + const fullTableScans = stringArray(plan.fullTableScans); + const tempSorts = stringArray(plan.tempSorts); + return [ + `indexes: ${indexes.length > 0 ? indexes.join(", ") : "none"}`, + `full scans: ${fullTableScans.length > 0 ? fullTableScans.join(", ") : "none"}`, + `temp sorts: ${tempSorts.length > 0 ? tempSorts.join(", ") : "none"}`, + ].join("; "); +} + +function buildSqliteScenarioRows(current: JsonValue, baseline: JsonValue) { + if (!current) { + return []; + } + const currentIsV2 = valueAt(current, "schemaVersion") === 2; + const baselineById = + valueAt(baseline, "schemaVersion") === 2 + ? new Map(objectArray(valueAt(baseline, "queries")).map((entry) => [entry.id, entry])) + : new Map<JsonValue | undefined, JsonObject>(); + + return objectArray(valueAt(current, "queries")).map((entry, index) => { + const id = currentIsV2 ? entry.id : `legacy query ${index + 1}`; + const before = currentIsV2 ? baselineById.get(entry.id) : undefined; + const comparable = + before !== undefined && + before.database === entry.database && + before.sql === entry.sql && + before.rows === entry.rows && + before.runs === entry.runs; + return [ + id ?? "unknown", + currentIsV2 ? (entry.database ?? "unknown") : "unknown", + entry.rows ?? "n/a", + currentIsV2 ? (entry.runs ?? "n/a") : "n/a", + formatMs(entry.p50Ms), + formatMs(entry.p95Ms), + before?.rows ?? "n/a", + before?.runs ?? "n/a", + formatMs(before?.p95Ms), + comparable + ? formatPercentDelta(before.p95Ms, entry.p95Ms) + : before + ? "n/a (workload differs)" + : "n/a", + formatSqlitePlan(entry.plan), + ]; + }); +} + +function buildMemoryDeltaRows(current: SourceArtifacts, baseline: SourceArtifacts | null) { + if (!baseline) { + return []; + } + return [ + ...buildStartupMemoryDeltaRows(current.startup, baseline.startup), + ...buildCliMemoryDeltaRows(current.cli, baseline.cli), + ...buildMockHelloMemoryDeltaRows(current.mockHelloSummaries, baseline.mockHelloSummaries), + ]; +} + +function formatExitSummary(value: unknown) { + if (typeof value !== "string" || !value) { + return "n/a"; + } + return value.replaceAll(/\b(code:(?:null|-?\d+)|signal:[^,\s]+)x(\d+)\b/g, "$1 x$2"); +} + +function buildObservationRows(summary: JsonValue) { + return objectArray(valueAt(summary, "observations")).map((observation) => [ + observation.kind ?? "unknown", + observation.id ?? "unknown", + formatRatio(observation.cpuCoreRatio ?? observation.cpuCoreRatioMax), + formatMs(observation.wallMs ?? observation.wallMsMax), + ]); +} + +export function buildMarkdown(sourceDir: string, baselineSourceDir: string | null) { + const current = loadSourceArtifacts(sourceDir, { required: true }); + const baseline = loadSourceArtifacts(baselineSourceDir ?? null); + const gatewaySummaryPath = path.join(sourceDir, "gateway-cpu", "summary.json"); + const gatewaySummary = readRequiredJson(gatewaySummaryPath, "gateway observation summary"); + if (!current) { + throw new Error(`[source-performance] missing required source dir: ${sourceDir}`); + } + validateGatewaySummaryArtifact(gatewaySummary, gatewaySummaryPath); + const memoryDeltaRows = buildMemoryDeltaRows(current, baseline); + + const lines = [ + "# OpenClaw Source Performance", + "", + `Generated: ${new Date().toISOString()}`, + "", + "## Gateway Boot", + "", + ...table( + [ + "case", + "name", + "readyz p50", + "readyz p95", + "healthz p50", + "http listen p50", + "gateway ready p50", + "first output p50", + "RSS p95", + "CPU core p95", + ], + buildStartupRows(current.startup), + ), + "## Memory Trend", + "", + baseline + ? "Compared with the latest published mock-provider source probe for this tested ref." + : "No published source baseline was available for this tested ref.", + "", + ...table( + [ + "surface", + "case", + "baseline RSS p95", + "current RSS p95", + "RSS delta", + "heap delta", + "state", + ], + memoryDeltaRows, + ), + "## Bundled Plugin Import Memory", + "", + "Per-plugin rows are isolated cold imports and are not additive. The combined row measures all selected bundled-plugin entrypoints in one process.", + "", + ...table( + ["measurement", "max RSS", "delta from empty process", "status"], + buildExtensionMemoryContextRows(current.extensionMemory), + ), + ...table( + ["plugin", "isolated max RSS", "isolated delta from empty process", "status"], + buildExtensionMemoryRows(current.extensionMemory), + ), + "## Startup Hotspots", + "", + ...table(["case", "phase", "p50", "p95"], buildTraceRows(current.startup)), + "## Fake Model Hello Loops", + "", + ...table( + [ + "run", + "status", + "pass", + "wall", + "gateway CPU core", + "RSS start", + "RSS end", + "RSS delta", + "model", + ], + buildMockHelloRows(current.mockHelloSummaries), + ), + "## CLI Against Booted Gateway", + "", + `RSS metric: ${cliStartupMemoryMetric(valueAt(current.cli, "primary"))}; values are MiB.`, + "", + ...table( + ["case", "command", "duration p50", "duration p95", "RSS p95", "exits"], + buildCliRows(current.cli), + ), + "## SQLite State Smoke", + "", + ...table( + [ + "run", + "format", + "profile", + "SQLite", + "state schema", + "agent schema", + "state rows", + "agent rows", + "integrity", + "WAL before", + "WAL after", + "total", + ], + buildSqliteRunRows(current.sqlitePerf, baseline?.sqlitePerf ?? null), + ), + ...table( + [ + "scenario", + "database", + "rows", + "runs", + "p50", + "p95", + "baseline rows", + "baseline runs", + "baseline p95", + "delta", + "plan/index", + ], + buildSqliteScenarioRows(current.sqlitePerf, baseline?.sqlitePerf ?? null), + ), + "## Observations", + "", + ...table(["kind", "id", "CPU core", "wall"], buildObservationRows(gatewaySummary)), + ]; + + return `${lines.join("\n")}\n`; +} + +async function main() { + const options = parseArgs(process.argv.slice(2)); + const markdown = buildMarkdown(options.sourceDir, options.baselineSourceDir); + if (options.output) { + fs.mkdirSync(path.dirname(options.output), { recursive: true }); + fs.writeFileSync(options.output, markdown, "utf8"); + } else { + process.stdout.write(markdown); + } +} + +function isCliEntry() { + const cliArg = process.argv[1]; + return cliArg ? import.meta.url === pathToFileURL(cliArg).href : false; +} + +if (isCliEntry()) { + main().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + }); +} diff --git a/scripts/openclaw-postpack.mjs b/scripts/openclaw-postpack.mjs new file mode 100644 index 0000000000000000000000000000000000000000..93c7690941740740776ca3c5117f2e5e433707d0 --- /dev/null +++ b/scripts/openclaw-postpack.mjs @@ -0,0 +1,34 @@ +#!/usr/bin/env node + +import { rm } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { + LEGACY_PACKAGE_INSTALL_GUARD_RELATIVE_PATH, + PACKAGE_LIFECYCLE_PENDING_RELATIVE_PATH, +} from "./lib/package-lifecycle-marker.mjs"; +// Restores every source artifact temporarily rewritten for npm packaging. +import { restorePackageChangelog } from "./package-changelog.mjs"; +import { restorePackageDocsMap } from "./package-docs-map.mjs"; +import { restorePackageManifest } from "./package-manifest.mjs"; + +export async function restorePrepackArtifacts(cwd = process.cwd()) { + await restorePackageChangelog(cwd); + await restorePackageManifest(cwd); + await Promise.all( + [PACKAGE_LIFECYCLE_PENDING_RELATIVE_PATH, LEGACY_PACKAGE_INSTALL_GUARD_RELATIVE_PATH].map( + (relativePath) => rm(path.join(cwd, relativePath), { force: true }), + ), + ); + // Release the lifecycle receipt only after every other source mutation settles. + await restorePackageDocsMap(cwd); +} + +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + try { + await restorePrepackArtifacts(); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + } +} diff --git a/scripts/openclaw-release-ready.mjs b/scripts/openclaw-release-ready.mjs new file mode 100644 index 0000000000000000000000000000000000000000..66dff67290d60bf925c7d3d30961bb622e8d068e --- /dev/null +++ b/scripts/openclaw-release-ready.mjs @@ -0,0 +1,861 @@ +#!/usr/bin/env node + +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { + appendFileSync, + mkdirSync, + mkdtempSync, + readFileSync, + renameSync, + writeFileSync, +} from "node:fs"; +import { join, resolve } from "node:path"; +import { setTimeout as delay } from "node:timers/promises"; +import { pathToFileURL } from "node:url"; +import { parseArgs, isDeepStrictEqual } from "node:util"; +import { + preparedClawHubArtifactName, + resolvePreparedClawHubMatrix, + restorePreparedClawHubPackage, +} from "./clawhub-prepared-artifact.mjs"; +import { + downloadActionsArtifactArchive, + inspectActionsArtifactZipWithPolicy, + readBoundedRegularFile, +} from "./lib/actions-artifact-archive.mjs"; +import { isRecord } from "./lib/record-shared.mjs"; +import { classifyReleaseTrain, parseReleaseVersion } from "./lib/release-version.mjs"; +import { + consumePreparedNpmPackage, + downloadPreparedNpmRelease, + preparedNpmArtifactName, +} from "./plugin-npm-prepared-release.mjs"; +import { runReleaseToolingGh, verifyReleaseToolingIdentity } from "./release-tooling-identity.mjs"; + +const REPOSITORY = "openclaw/openclaw"; +const PREPARE_WORKFLOW = ".github/workflows/openclaw-release-prepare.yml"; +const PUBLISH_WORKFLOW = ".github/workflows/openclaw-release-publish.yml"; +const READY_FILE = "release-ready.json"; +const MAX_MANIFEST_BYTES = 1024 * 1024; +const SHA = /^[a-f0-9]{40}$/u; +const INPUTS = new Set([ + "tag", + "preflight_run_id", + "plugin_sdk_api_acknowledgement", + "full_release_validation_run_id", + "full_release_validation_run_attempt", + "windows_node_tag", + "windows_node_installer_digests", + "npm_telegram_run_id", + "openclaw_npm_resume_run_id", + "npm_dist_tag", + "plugin_publish_scope", + "plugins", + "publish_openclaw_npm", + "publish_docker_only", + "release_profile", + "release_evidence_mode", + "wait_for_clawhub", +]); + +function requireValue(condition, message) { + if (!condition) { + throw new Error(message); + } +} + +function api(path, args = []) { + return JSON.parse(runReleaseToolingGh(["api", `repos/${REPOSITORY}/${path}`, ...args])); +} + +function git(args, cwd = ".") { + return execFileSync("git", args, { + cwd, + encoding: "utf8", + timeout: 60_000, + maxBuffer: MAX_MANIFEST_BYTES, + }).trim(); +} + +function output(name, value) { + if (process.env.GITHUB_OUTPUT) { + appendFileSync( + process.env.GITHUB_OUTPUT, + `${name}=${typeof value === "string" ? value : JSON.stringify(value)}\n`, + ); + } +} + +function writeJson(path, value) { + const text = `${JSON.stringify(value, null, 2)}\n`; + requireValue(Buffer.byteLength(text) <= MAX_MANIFEST_BYTES, "Release request exceeds its limit."); + writeFileSync(path, text, { flag: "wx", mode: 0o600 }); +} + +function replaceJson(path, value) { + const nextPath = path.replace(/\.json$/u, ".next.json"); + writeJson(nextPath, value); + renameSync(nextPath, path); +} + +function readJson(path) { + return JSON.parse( + readBoundedRegularFile(path, { label: path, maxBytes: MAX_MANIFEST_BYTES }).toString("utf8"), + ); +} + +/** @returns {Record<string, string>} */ +export function validateReleaseButtonInputs(value) { + requireValue( + isRecord(value) && Object.keys(value).every((key) => INPUTS.has(key)), + "Prepared release contains unsupported publication inputs.", + ); + const inputs = Object.fromEntries( + Object.entries(value).map(([key, entry]) => { + requireValue( + (typeof entry === "string" || typeof entry === "boolean") && + !/[\r\n\0]/u.test(String(entry)), + `Invalid release input: ${key}.`, + ); + return [key, String(entry)]; + }), + ); + const version = parseReleaseVersion(inputs.tag?.slice(1) ?? ""); + requireValue( + version !== null && inputs.tag === `v${version.version}`, + "Prepared release requires an exact version tag.", + ); + requireValue( + ["beta", "stable"].includes(classifyReleaseTrain(version)) && + inputs.npm_dist_tag !== "extended-stable", + "Alpha and extended-stable releases retain their existing owner workflows.", + ); + const expectedChannel = version.channel === "stable" ? ["beta", "latest"] : [version.channel]; + requireValue( + expectedChannel.includes(inputs.npm_dist_tag), + "Prepared release npm channel differs from its tag.", + ); + for (const name of [ + "preflight_run_id", + "full_release_validation_run_id", + "full_release_validation_run_attempt", + ]) { + requireValue( + /^[1-9][0-9]*$/u.test(inputs[name] ?? ""), + `Prepared release requires exact ${name}.`, + ); + } + requireValue( + (inputs.plugin_publish_scope ?? "all-publishable") === "all-publishable" && !inputs.plugins, + "The release button prepares the complete publishable plugin roster; use the existing publisher for selected repairs.", + ); + requireValue( + (inputs.publish_openclaw_npm ?? "true") === "true" && + (inputs.publish_docker_only ?? "false") === "false" && + (inputs.release_evidence_mode ?? "full-release-validation") === "full-release-validation", + "The release button requires full release validation and core npm publication.", + ); + return { + ...inputs, + plugin_publish_scope: "all-publishable", + publish_openclaw_npm: "true", + publish_docker_only: "false", + release_evidence_mode: "full-release-validation", + wait_for_clawhub: "true", + }; +} + +function toolingIdentity(environment = process.env) { + requireValue( + environment.GITHUB_REPOSITORY === REPOSITORY, + "Release preparation must run in the canonical repository.", + ); + const identity = verifyReleaseToolingIdentity({ + repository: REPOSITORY, + workflowRef: environment.GITHUB_REF_NAME, + workflowFullRef: environment.GITHUB_REF, + workflowSha: environment.GITHUB_WORKFLOW_SHA, + }); + requireValue( + identity.route === "protected-tag", + "Prepared publication requires a frozen protected release-publish tag.", + ); + return { ref: identity.ref, fullRef: identity.fullRef, sha: identity.sha }; +} + +function producer(run, workflowPath, tooling) { + const [path, fullRef] = String(run.path).split("@"); + requireValue( + run.repository?.full_name === REPOSITORY && + path === workflowPath && + (fullRef === undefined || fullRef === tooling.fullRef) && + run.event === "workflow_dispatch" && + run.head_branch === tooling.ref && + run.head_sha === tooling.sha && + Number.isSafeInteger(run.id) && + run.id > 0 && + Number.isSafeInteger(run.run_attempt) && + run.run_attempt > 0, + "Release producer workflow identity mismatch.", + ); + return { + repository: REPOSITORY, + runId: run.id, + runAttempt: run.run_attempt, + workflowPath, + workflowEvent: run.event, + workflowHeadBranch: run.head_branch, + workflowSha: run.head_sha, + }; +} + +export function readyArtifactName(sourceSha, runId, runAttempt) { + requireValue( + SHA.test(sourceSha) && + Number.isSafeInteger(runId) && + runId > 0 && + Number.isSafeInteger(runAttempt) && + runAttempt > 0, + "Invalid prepared release artifact identity.", + ); + return `release-ready-${sourceSha.slice(0, 12)}-${runId}-${runAttempt}`; +} + +function artifactFor(run, workflow, tooling, name) { + const identity = producer(run, workflow, tooling); + const artifacts = []; + for (let page = 1; page <= 20; page++) { + const result = api(`actions/runs/${run.id}/artifacts?per_page=100&page=${page}`); + requireValue(result.total_count <= 2000, "Prepared release artifact list exceeds its limit."); + artifacts.push(...result.artifacts); + if (artifacts.length >= result.total_count) { + break; + } + } + const matches = artifacts.filter((artifact) => artifact.name === name); + requireValue(matches.length === 1, `Expected one exact prepared artifact: ${name}.`); + const [artifact] = matches; + requireValue( + artifact.expired === false && + artifact.workflow_run?.id === run.id && + artifact.workflow_run?.head_sha === tooling.sha && + /^sha256:[a-f0-9]{64}$/u.test(artifact.digest ?? ""), + "Prepared artifact is expired or differs from its producer.", + ); + return { + ...identity, + artifactId: artifact.id, + artifactName: artifact.name, + artifactDigest: artifact.digest, + artifactSizeBytes: artifact.size_in_bytes, + }; +} + +async function waitForRun(runId, attempt, workflow, tooling) { + const deadline = Date.now() + 110 * 60_000; + while (Date.now() < deadline) { + const run = api(`actions/runs/${runId}`); + producer(run, workflow, tooling); + requireValue( + run.id === runId && run.run_attempt === attempt, + "Release producer attempt changed; retain the original evidence and select an exact new attempt explicitly.", + ); + if (run.status === "completed") { + requireValue( + run.conclusion === "success", + `${workflow} run ${runId}/${attempt} ended ${run.conclusion}; ${ + workflow === PUBLISH_WORKFLOW + ? "inspect child outcomes for owner recovery, then start a new button run with the same readiness receipt and openclaw_npm_resume_run_id when core npm succeeded. A published core version without a successful child requires core-owner reconciliation. This button remains bound to its original parent attempt." + : "rerun all jobs of this linked non-publishing child to produce a complete same-attempt package set, then rerun only the outer Verify and seal prepared publication job. Do not repeat preparation dispatch." + }`, + ); + return run; + } + await delay(15_000); + } + throw new Error( + `Run ${runId}/${attempt} is still pending; resume verification, not publication.`, + ); +} + +function dispatch(workflow, inputs, tooling) { + verifyReleaseToolingIdentity({ + repository: REPOSITORY, + workflowRef: tooling.ref, + workflowFullRef: tooling.fullRef, + workflowSha: tooling.sha, + }); + const args = [ + "--method", + "POST", + "-H", + "X-GitHub-Api-Version: 2026-03-10", + "-f", + `ref=${tooling.ref}`, + ]; + for (const [key, value] of Object.entries(inputs)) { + args.push("-f", `inputs[${key}]=${value}`); + } + // Dispatch is a mutation: never retry an uncertain response. A new request + // could create a second publisher even when the first response was lost. + const result = api(`actions/workflows/${workflow}/dispatches`, args); + requireValue( + Number.isSafeInteger(result.workflow_run_id) && result.workflow_run_id > 0, + "Dispatch response did not identify its run. Inspect Actions before dispatching again.", + ); + return result; +} + +function reportDispatch(workflow, runId) { + const link = `https://github.com/${REPOSITORY}/actions/runs/${runId}`; + console.log(`${workflow}: ${link}`); + if (process.env.GITHUB_STEP_SUMMARY) { + appendFileSync(process.env.GITHUB_STEP_SUMMARY, `- ${workflow}: ${link}\n`); + } +} + +export function validateReadyRelease(value, expected) { + requireValue( + isRecord(value) && + value.schema === "openclaw.release-ready/v1" && + value.repository === REPOSITORY && + SHA.test(value.sourceSha) && + value.sourceSha === expected.sourceSha && + isDeepStrictEqual(value.tooling, expected.tooling), + "Prepared release source or tooling identity mismatch.", + ); + const inputs = validateReleaseButtonInputs(value.inputs); + requireValue( + isDeepStrictEqual(inputs, value.inputs) && + isRecord(value.plugins) && + Object.keys(value.plugins).toSorted().join(",") === "clawhub,npm", + "Prepared release input or package handoff is invalid.", + ); + for (const [target, path] of [ + ["npm", ".github/workflows/plugin-npm-release.yml"], + ["clawhub", ".github/workflows/plugin-clawhub-release.yml"], + ]) { + const descriptor = value.plugins[target]; + requireValue( + descriptor?.repository === REPOSITORY && + descriptor.workflowSha === value.tooling.sha && + descriptor.workflowHeadBranch === value.tooling.ref && + descriptor.workflowPath === path, + `Prepared ${target} producer differs from the release tooling.`, + ); + } + return value; +} + +export async function loadPreparedPlugins({ + plugins, + inputs, + sourceSha, + toolingSha, + sourceRoot, + outputDir, + token, + fetchImpl, +}) { + requireValue( + isRecord(plugins) && Object.keys(plugins).toSorted().join(",") === "clawhub,npm", + "Both npm and ClawHub preparation artifacts are required.", + ); + mkdirSync(outputDir, { recursive: true }); + const publisherPolicySha256 = createHash("sha256") + .update(readFileSync(new URL("../.github/workflows/plugin-npm-release.yml", import.meta.url))) + .digest("hex"); + const common = { token, fetchImpl, sourceRoot, selectionMode: "all-publishable", plugins: "" }; + const results = await Promise.allSettled([ + downloadPreparedNpmRelease({ + ...common, + artifact: plugins.npm, + sourceSha, + workflowSha: toolingSha, + repository: REPOSITORY, + npmDistTag: inputs.npm_dist_tag === "extended-stable" ? "extended-stable" : "default", + publisherPolicySha256, + archivePath: join(outputDir, `npm-${plugins.npm.artifactId}.zip`), + }), + resolvePreparedClawHubMatrix({ + ...common, + descriptor: plugins.clawhub, + candidateSha: sourceSha, + toolingSha, + archivePath: join(outputDir, `clawhub-${plugins.clawhub.artifactId}.zip`), + }), + ]); + const failure = results.find((result) => result.status === "rejected"); + if (failure) { + throw failure.reason; + } + const [npm, clawhub] = results.map((result) => result.value); + return { + npm, + clawhub: { packages: clawhub.map((entry) => entry.prepared) }, + publisherPolicySha256, + }; +} + +export async function prefetchPreparedPlugins(options) { + const manifests = await loadPreparedPlugins(options); + const { plugins, inputs, sourceSha, toolingSha, sourceRoot, outputDir, token, fetchImpl } = + options; + const admissionDir = mkdtempSync(join(outputDir, "verified-")); + const tasks = [ + ...manifests.npm.packages.map( + (entry) => () => + consumePreparedNpmPackage({ + package: entry, + repository: REPOSITORY, + sourceSha, + workflowSha: toolingSha, + npmDistTag: manifests.npm.npmDistTag, + publisherPolicySha256: manifests.publisherPolicySha256, + sourcePackageJson: join(sourceRoot, entry.packageDir, "package.json"), + cacheDir: join(outputDir, "npm"), + outputDir: join(admissionDir, `npm-${entry.extensionId}`), + token, + fetchImpl, + }), + ), + ...manifests.clawhub.packages.map( + (entry) => () => + restorePreparedClawHubPackage({ + descriptor: plugins.clawhub, + toolingSha, + entry, + outputDir: join(admissionDir, `clawhub-${entry.packageDir.split("/").at(-1)}`), + archivePath: join(outputDir, `clawhub-${entry.artifactId}.zip`), + token, + fetchImpl, + }), + ), + ]; + // Drain the current batch before failing; no publication can race a transfer + // still being verified, and complete archives remain available for re-entry. + for (let index = 0; index < tasks.length; index += 4) { + const results = await Promise.allSettled(tasks.slice(index, index + 4).map((task) => task())); + const failure = results.find((result) => result.status === "rejected"); + if (failure) { + throw failure.reason; + } + } + writeJson(join(admissionDir, "verified.json"), { + sourceSha, + toolingSha, + npm: manifests.npm.packages.length, + clawhub: manifests.clawhub.packages.length, + }); + return { ...manifests, inputs }; +} + +async function readReady(descriptor, directory, tooling, token) { + requireValue( + descriptor?.workflowPath === PREPARE_WORKFLOW && + descriptor.repository === REPOSITORY && + descriptor.workflowSha === tooling.sha && + descriptor.workflowHeadBranch === tooling.ref, + "Release readiness must come from the exact protected preparation workflow.", + ); + const downloaded = await downloadActionsArtifactArchive({ + token, + expected: { ...descriptor, runStatePolicy: "completed-success" }, + maxArchiveBytes: MAX_MANIFEST_BYTES, + archivePath: join(directory, `${descriptor.artifactId}.zip`), + }); + const files = inspectActionsArtifactZipWithPolicy(downloaded.archiveBytes, { + expectedEntries: [READY_FILE], + maxArchiveBytes: MAX_MANIFEST_BYTES, + maxExpandedBytes: MAX_MANIFEST_BYTES, + maxEntryBytes: () => MAX_MANIFEST_BYTES, + }); + const value = JSON.parse(files.get(READY_FILE).toString("utf8")); + requireValue( + descriptor.artifactName === + readyArtifactName(value.sourceSha, descriptor.runId, descriptor.runAttempt), + "Release readiness artifact name mismatch.", + ); + const sourceSha = api(`commits/${encodeURIComponent(value.inputs?.tag)}`, ["--jq", "{sha}"]).sha; + return validateReadyRelease(value, { sourceSha, tooling }); +} + +function publicationInputs(ready, resumeRunId) { + const effectiveResumeRunId = resumeRunId || ready.inputs.openclaw_npm_resume_run_id || ""; + requireValue( + effectiveResumeRunId === "" || + (/^[1-9][0-9]*$/u.test(effectiveResumeRunId) && + Number.isSafeInteger(Number(effectiveResumeRunId))), + "openclaw_npm_resume_run_id must be a positive safe integer.", + ); + return { + ...ready.inputs, + ...(resumeRunId ? { openclaw_npm_resume_run_id: resumeRunId } : {}), + prepared_plugins: JSON.stringify(ready.plugins), + }; +} + +async function readPublicationRequest(path, directory, tooling, token) { + const request = readJson(path); + requireValue( + isRecord(request) && + request.schema === "openclaw.release-dispatch/v1" && + request.repository === REPOSITORY && + request.workflowPath === PUBLISH_WORKFLOW && + request.workflowEvent === "workflow_dispatch" && + request.state === "acknowledged" && + Number.isSafeInteger(request.button?.runId) && + request.button.runId > 0 && + request.button.runAttempt === 1 && + request.expectedReleaseRunAttempt === 1 && + Number.isSafeInteger(request.releaseRunId) && + request.releaseRunId > 0 && + request.releaseRunAttempt === 1 && + SHA.test(request.sourceSha) && + isDeepStrictEqual(request.tooling, tooling) && + (request.openclawNpmResumeRunId === null || + typeof request.openclawNpmResumeRunId === "string") && + isDeepStrictEqual(request.producer, { + repository: REPOSITORY, + runId: request.releaseRunId, + runAttempt: request.releaseRunAttempt, + workflowPath: PUBLISH_WORKFLOW, + workflowEvent: "workflow_dispatch", + workflowHeadBranch: tooling.ref, + workflowSha: tooling.sha, + }), + "Publication request is unknown, unverified, malformed, or inconsistent; do not redispatch.", + ); + const ready = await readReady(request.preparedArtifact, directory, tooling, token); + const inputs = publicationInputs(ready, request.openclawNpmResumeRunId ?? ""); + requireValue( + ready.sourceSha === request.sourceSha && + isDeepStrictEqual(inputs, request.inputs) && + request.openclawNpmResumeRunId === (inputs.openclaw_npm_resume_run_id ?? null), + "Publication request differs from the frozen readiness inputs.", + ); + return request; +} + +async function main() { + const { positionals, values } = parseArgs({ + allowPositionals: true, + options: { + inputs: { type: "string" }, + request: { type: "string" }, + artifact: { type: "string" }, + output: { type: "string" }, + "openclaw-npm-resume-run-id": { type: "string" }, + "source-root": { type: "string", default: "." }, + }, + }); + const [operation] = positionals; + const directory = resolve(values.output ?? ".artifacts/release-ready"); + mkdirSync(directory, { recursive: true }); + const tooling = toolingIdentity(); + const token = process.env.GH_TOKEN; + if (operation === "dispatch-prepare") { + requireValue( + values.request !== undefined || process.env.GITHUB_RUN_ATTEMPT === "1", + "Preparation dispatch already ran; recover failed linked non-publishing child runs with Re-run all jobs, then rerun only the outer seal. Do not repeat this dispatch.", + ); + const inputs = validateReleaseButtonInputs(readJson(values.inputs)); + const sourceSha = api(`commits/${encodeURIComponent(inputs.tag)}`, ["--jq", "{sha}"]).sha; + requireValue(SHA.test(sourceSha), "Release tag does not resolve to a commit."); + const packageJson = JSON.parse(git(["show", `${sourceSha}:package.json`])); + requireValue( + inputs.tag === `v${packageJson.version}`, + "Release tag and source package version differ.", + ); + const request = { + schema: "openclaw.release-ready/v1", + repository: REPOSITORY, + sourceSha, + tooling, + inputs, + npmRunId: null, + clawhubRunId: null, + }; + const requestPath = join(directory, "request.json"); + if (values.request !== undefined) { + const recovered = readJson(values.request); + requireValue( + isRecord(recovered) && + isDeepStrictEqual({ ...recovered, npmRunId: null, clawhubRunId: null }, request), + "Preparation recovery differs from the frozen source, tooling, or publication inputs.", + ); + for (const [key, workflow] of [ + ["npmRunId", ".github/workflows/plugin-npm-release.yml"], + ["clawhubRunId", ".github/workflows/plugin-clawhub-release.yml"], + ]) { + requireValue( + Number.isSafeInteger(recovered[key]) && recovered[key] > 0, + `Preparation recovery requires an explicitly identified ${key}; inspect Actions before selecting it.`, + ); + producer(api(`actions/runs/${recovered[key]}`), workflow, tooling); + request[key] = recovered[key]; + } + // Adoption is read-only. The seal still proves each selected run's source, + // complete package roster, producer attempt, and exact qualified bytes. + } + const { authenticateFullReleaseValidationEvidence } = + await import("./validate-full-release-validation-evidence.mjs"); + await authenticateFullReleaseValidationEvidence({ + run: api( + `actions/runs/${inputs.full_release_validation_run_id}/attempts/${inputs.full_release_validation_run_attempt}`, + ), + expectedRepository: REPOSITORY, + expectedRunId: inputs.full_release_validation_run_id, + expectedRunAttempt: inputs.full_release_validation_run_attempt, + expectedTargetSha: sourceSha, + expectedReleaseTag: inputs.tag, + expectedTrustedWorkflowFullRef: tooling.fullRef, + expectedTrustedWorkflowSha: tooling.sha, + expectedPublicationSelection: { + route: "prepared", + npmDistTag: inputs.npm_dist_tag, + publishOpenclawNpm: true, + pluginPublishScope: "all-publishable", + plugins: [], + ...(inputs.windows_node_tag + ? { + windowsNodeTag: inputs.windows_node_tag, + windowsNodeInstallerDigests: JSON.parse(inputs.windows_node_installer_digests ?? ""), + } + : {}), + }, + verifierSourceSha: tooling.sha, + verifierSourceContent: readFileSync(new URL("./release-ci-summary.mjs", import.meta.url)), + isTrustedMainAncestor: (sha) => { + const comparison = api(`compare/${sha}...main`, ["--jq", "{status,merge_base_commit}"]); + return ( + ["ahead", "identical"].includes(comparison.status) && + comparison.merge_base_commit?.sha === sha + ); + }, + }); + writeJson(requestPath, request); + if (values.request === undefined) { + // Persist before either mutation, then after each acknowledged run. A null + // ID means unconfirmed, never proof that a lost dispatch created no run. + request.npmRunId = dispatch( + "plugin-npm-release.yml", + { + ref: sourceSha, + publish_scope: "all-publishable", + npm_dist_tag: inputs.npm_dist_tag === "extended-stable" ? "extended-stable" : "default", + preflight_only: "true", + trusted_publisher_preflight: "false", + }, + tooling, + ).workflow_run_id; + replaceJson(requestPath, request); + reportDispatch("plugin-npm-release.yml", request.npmRunId); + request.clawhubRunId = dispatch( + "plugin-clawhub-release.yml", + { ref: sourceSha, publish_scope: "all-publishable", dry_run: "true" }, + tooling, + ).workflow_run_id; + replaceJson(requestPath, request); + reportDispatch("plugin-clawhub-release.yml", request.clawhubRunId); + } + output("npm_run_id", String(request.npmRunId)); + output("clawhub_run_id", String(request.clawhubRunId)); + output("request", request); + output("source_sha", sourceSha); + return; + } + if (operation === "seal") { + const request = readJson(values.request); + requireValue( + isDeepStrictEqual(request.tooling, tooling), + "Preparation tooling moved before sealing.", + ); + // Retrying a failed seal job may follow an explicit rerun of the original + // producers. Resolve each current attempt once, never fall back to an older success. + const npmAttempt = api(`actions/runs/${request.npmRunId}`).run_attempt; + const clawhubAttempt = api(`actions/runs/${request.clawhubRunId}`).run_attempt; + const [npmRun, clawhubRun] = await Promise.all([ + waitForRun(request.npmRunId, npmAttempt, ".github/workflows/plugin-npm-release.yml", tooling), + waitForRun( + request.clawhubRunId, + clawhubAttempt, + ".github/workflows/plugin-clawhub-release.yml", + tooling, + ), + ]); + const npmProducer = producer(npmRun, ".github/workflows/plugin-npm-release.yml", tooling); + const clawhubProducer = producer( + clawhubRun, + ".github/workflows/plugin-clawhub-release.yml", + tooling, + ); + const plugins = { + npm: artifactFor( + npmRun, + npmProducer.workflowPath, + tooling, + preparedNpmArtifactName(request.sourceSha, npmProducer), + ), + clawhub: artifactFor( + clawhubRun, + clawhubProducer.workflowPath, + tooling, + preparedClawHubArtifactName(request.sourceSha, clawhubProducer), + ), + }; + const ready = validateReadyRelease( + { + schema: request.schema, + repository: REPOSITORY, + inputs: request.inputs, + sourceSha: request.sourceSha, + tooling, + plugins, + }, + request, + ); + await prefetchPreparedPlugins({ + plugins, + inputs: ready.inputs, + sourceSha: ready.sourceSha, + toolingSha: tooling.sha, + sourceRoot: resolve(values["source-root"]), + outputDir: join(directory, "packages"), + token, + }); + writeJson(join(directory, READY_FILE), ready); + output( + "artifact_name", + readyArtifactName( + ready.sourceSha, + Number(process.env.GITHUB_RUN_ID), + Number(process.env.GITHUB_RUN_ATTEMPT), + ), + ); + return; + } + if (operation === "dispatch-publish") { + requireValue( + process.env.GITHUB_RUN_ATTEMPT === "1", + "Publication may already have been dispatched. Rerun failed verification jobs, not the successful dispatch job.", + ); + const resumeRunId = values["openclaw-npm-resume-run-id"] ?? ""; + requireValue( + resumeRunId === "" || + (/^[1-9][0-9]*$/u.test(resumeRunId) && Number.isSafeInteger(Number(resumeRunId))), + "openclaw_npm_resume_run_id must be a positive safe integer.", + ); + const descriptor = readJson(values.artifact); + const ready = await readReady(descriptor, directory, tooling, token); + // Recovery changes only the existing owner's resume selector. The parent + // verifies its authority and canonical core bytes before starting writers. + const inputs = publicationInputs(ready, resumeRunId); + const request = { + schema: "openclaw.release-dispatch/v1", + repository: REPOSITORY, + workflowPath: PUBLISH_WORKFLOW, + workflowEvent: "workflow_dispatch", + state: "unknown", + button: { + runId: Number(process.env.GITHUB_RUN_ID), + runAttempt: Number(process.env.GITHUB_RUN_ATTEMPT), + }, + expectedReleaseRunAttempt: 1, + releaseRunId: null, + releaseRunAttempt: null, + producer: null, + sourceSha: ready.sourceSha, + tooling, + preparedArtifact: descriptor, + inputs, + openclawNpmResumeRunId: inputs.openclaw_npm_resume_run_id ?? null, + }; + requireValue( + Number.isSafeInteger(request.button.runId) && request.button.runId > 0, + "Publication requires an exact initiating button run.", + ); + const requestPath = join(directory, "dispatch.json"); + // A retained unknown is intentional: a crash even before POST cannot prove + // non-execution. Existing intent never authorizes another mutation. + writeJson(requestPath, request); + try { + request.releaseRunId = dispatch( + "openclaw-release-publish.yml", + inputs, + tooling, + ).workflow_run_id; + request.state = "unverified"; + replaceJson(requestPath, request); + const run = api(`actions/runs/${request.releaseRunId}`); + const observed = producer(run, PUBLISH_WORKFLOW, tooling); + requireValue( + observed.runId === request.releaseRunId && + observed.runAttempt === request.expectedReleaseRunAttempt, + "Publication acknowledgement run or attempt differs from the original dispatch.", + ); + request.producer = observed; + request.releaseRunAttempt = observed.runAttempt; + request.state = "acknowledged"; + replaceJson(requestPath, request); + reportDispatch("openclaw-release-publish.yml", request.releaseRunId); + console.log( + `Retained request: ${requestPath}; button ${request.button.runId}/${request.button.runAttempt}. Verify only: node scripts/openclaw-release-ready.mjs verify --request ${JSON.stringify(requestPath)}`, + ); + output("release_run_id", String(request.releaseRunId)); + output("source_sha", ready.sourceSha); + output("release_tag", ready.inputs.tag); + output("npm_dist_tag", ready.inputs.npm_dist_tag); + output("request", request); + } catch (error) { + console.error( + `Retained request: ${requestPath}; button ${request.button.runId}/${request.button.runAttempt}; publisher ${request.releaseRunId ?? "unconfirmed"}. Dispatch or handoff outcome unknown; do not redispatch. Inspect the retained request and any dispatch.next.json before manual reconciliation.`, + ); + throw error; + } + return; + } + if (operation === "verify" || operation === "validate-request") { + const request = await readPublicationRequest(values.request, directory, tooling, token); + if (operation === "validate-request") { + const run = api(`actions/runs/${request.releaseRunId}`); + const observed = producer(run, PUBLISH_WORKFLOW, tooling); + requireValue( + isDeepStrictEqual(observed, request.producer) && + run.status === "completed" && + run.conclusion === "success", + "Publication request no longer identifies a successful exact publisher.", + ); + return; + } + const run = await waitForRun( + request.releaseRunId, + request.releaseRunAttempt, + PUBLISH_WORKFLOW, + tooling, + ); + const { verifyClawHubPostpublish } = await import("./clawhub-postpublish.mjs"); + await verifyClawHubPostpublish({ + event: { workflow_run: run }, + verifierSha: tooling.sha, + token, + outputDir: join(directory, "clawhub-public-verification"), + }); + output("verified", "true"); + return; + } + throw new Error( + "Expected dispatch-prepare, seal, dispatch-publish, verify, or validate-request.", + ); +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { + try { + await main(); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + console.error("[openclaw-release-ready] FAILED (exit 1)"); + process.exitCode = 1; + } +} diff --git a/scripts/opengrep-github-sarif.mjs b/scripts/opengrep-github-sarif.mjs new file mode 100644 index 0000000000000000000000000000000000000000..12e8f7bd680e222f5e4adaed39fe2a93839a0688 --- /dev/null +++ b/scripts/opengrep-github-sarif.mjs @@ -0,0 +1,42 @@ +import { readFileSync } from "node:fs"; + +try { + const [inputPath, ...extraArgs] = process.argv.slice(2); + if (!inputPath || extraArgs.length > 0) { + throw new Error("Usage: node scripts/opengrep-github-sarif.mjs <raw.sarif>"); + } + const report = JSON.parse(readFileSync(inputPath, "utf8")); + if (report.version !== "2.1.0" || !Array.isArray(report.runs)) { + throw new Error("Expected a SARIF 2.1.0 report with runs"); + } + let omitted = 0; + for (const run of report.runs) { + if (run.results == null) { + continue; + } + const resultCount = run.results.length; + run.results = run.results.filter((result) => { + // Opengrep 1.27.1 emits inSource without status for ignored matches; + // GitHub ignores SARIF suppressions. Keep unknown or disputed decisions. + const suppressions = result?.suppressions; + return !( + Array.isArray(suppressions) && + suppressions.length > 0 && + suppressions.every( + (suppression) => + suppression?.kind === "inSource" && + (suppression.status === undefined || suppression.status === "accepted"), + ) + ); + }); + omitted += resultCount - run.results.length; + } + console.error( + `[opengrep-github-sarif] Omitted ${omitted} accepted in-source suppression(s); raw audit: ${inputPath}`, + ); + process.stdout.write(`${JSON.stringify(report, null, 2)}\n`); +} catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + console.error("[opengrep-github-sarif] FAILED (exit 1)"); + process.exitCode = 1; +} diff --git a/scripts/oxlint-boundary-guards.mjs b/scripts/oxlint-boundary-guards.mjs new file mode 100644 index 0000000000000000000000000000000000000000..9fd5b167e3666abf5824bb8a983e90a1496bda41 --- /dev/null +++ b/scripts/oxlint-boundary-guards.mjs @@ -0,0 +1,600 @@ +import { + BOUNDARY_GUARD_FIXTURE_ROOT, + CHAINED_ASSERTION_EXCLUDED_ROOTS, + TYPE_ASSERTION_PRODUCTION_ROOTS, + TYPE_ASSERTION_TEST_FILE_SUFFIXES, + isSkippedTypeAssertionTestPath, + pathMatchesTypeAssertionRoot, +} from "./lib/type-assertion-guard-scope.mjs"; + +const EXPRESSION_WRAPPER_RE = + /^(?:ChainExpression|ParenthesizedExpression|TSAsExpression|TSNonNullExpression|TSTypeAssertion)$/; + +function unwrapExpression(node) { + let current = node; + while (EXPRESSION_WRAPPER_RE.test(current.type)) { + current = current.expression; + } + return current; +} + +function restrictedCallRule({ allowedFiles = [], message, objects, property, roots }) { + return { + create(context) { + const filename = context.physicalFilename.replaceAll("\\", "/"); + const cwd = context.cwd.replaceAll("\\", "/"); + const repoPath = filename.startsWith(`${cwd}/`) ? filename.slice(cwd.length + 1) : filename; + if ( + !filename.endsWith(".ts") || + !roots.some((root) => pathMatchesTypeAssertionRoot(repoPath, root)) || + TYPE_ASSERTION_TEST_FILE_SUFFIXES.some((suffix) => filename.endsWith(suffix)) || + allowedFiles.includes(repoPath) + ) { + return {}; + } + return { + CallExpression(node) { + const callee = unwrapExpression(node.callee); + if ( + callee.type !== "MemberExpression" || + callee.computed || + callee.property.type !== "Identifier" || + callee.property.name !== property + ) { + return; + } + const receiver = unwrapExpression(callee.object); + if (objects && (receiver.type !== "Identifier" || !objects.includes(receiver.name))) { + return; + } + context.report({ message, node: node.callee }); + }, + }; + }, + }; +} + +// Adapted from dmmulroy/anti-slop@446268e5d15baa968eaec669ff65358d36ae6259, MIT. +function isTypeAssertionExpression(node) { + return node.type === "TSAsExpression" || node.type === "TSTypeAssertion"; +} + +function isConstAssertion(node) { + const { typeAnnotation } = node; + return ( + typeAnnotation.type === "TSTypeReference" && + typeAnnotation.typeName.type === "Identifier" && + typeAnnotation.typeName.name === "const" + ); +} + +function isOutermostAssertionInChain(node) { + let current = node; + let parent = node.parent; + + while (parent.type === "ParenthesizedExpression" && parent.expression === current) { + current = parent; + parent = parent.parent; + } + + return !isTypeAssertionExpression(parent) || parent.expression !== current; +} + +function isForbiddenAssertionChain(node) { + let assertionCount = 0; + let hasNonConstAssertion = false; + let current = node; + + while (isTypeAssertionExpression(current)) { + assertionCount += 1; + hasNonConstAssertion ||= !isConstAssertion(current); + current = unwrapExpressionParentheses(current.expression); + } + + return assertionCount > 1 && hasNonConstAssertion; +} + +function noChainedTypeAssertionsRule({ excludedRoots = [], roots }) { + return { + meta: { + type: "problem", + docs: { + description: + "Disallow chained TypeScript as and angle-bracket assertions, including parenthesized chains.", + }, + messages: { + chained: + "This assertion chain discards type evidence. Keep the original precise type, or parse untrusted input at its boundary before narrowing it.", + }, + }, + create(context) { + const filename = context.physicalFilename.replaceAll("\\", "/"); + const cwd = context.cwd.replaceAll("\\", "/"); + const repoPath = filename.startsWith(`${cwd}/`) ? filename.slice(cwd.length + 1) : filename; + if ( + !roots.some((root) => pathMatchesTypeAssertionRoot(repoPath, root)) || + excludedRoots.some((root) => pathMatchesTypeAssertionRoot(repoPath, root)) || + isSkippedTypeAssertionTestPath(repoPath) + ) { + return {}; + } + + const checkTypeAssertion = (node) => { + if (!isOutermostAssertionInChain(node) || !isForbiddenAssertionChain(node)) { + return; + } + context.report({ node, messageId: "chained" }); + }; + + return { + TSAsExpression: checkTypeAssertion, + TSTypeAssertion: checkTypeAssertion, + }; + }, + }; +} + +// Adapted from dmmulroy/anti-slop, MIT. +const FUNCTION_BOUNDARY_TYPES = new Set([ + "ArrowFunctionExpression", + "FunctionDeclaration", + "FunctionExpression", + "TSDeclareFunction", + "TSEmptyBodyFunctionExpression", +]); +const MAX_TRANSPARENT_ALIAS_DEPTH = 32; + +function unwrapExpressionParentheses(expression) { + let current = expression; + while (current.type === "ParenthesizedExpression") { + current = current.expression; + } + return current; +} + +function unwrapTypeParentheses(type) { + let current = type; + while (current.type === "TSParenthesizedType") { + current = current.typeAnnotation; + } + return current; +} + +function typeReferenceName(type) { + return type.typeName.type === "Identifier" ? type.typeName.name : null; +} + +function isUnknownOrAnyType(type) { + const unwrapped = unwrapTypeParentheses(type); + return unwrapped.type === "TSUnknownKeyword" || unwrapped.type === "TSAnyKeyword"; +} + +function isBroadRecordKeyType(type) { + const unwrapped = unwrapTypeParentheses(type); + if ( + unwrapped.type === "TSStringKeyword" || + unwrapped.type === "TSNumberKeyword" || + unwrapped.type === "TSSymbolKeyword" + ) { + return true; + } + if (unwrapped.type === "TSUnionType") { + return unwrapped.types.every(isBroadRecordKeyType); + } + return unwrapped.type === "TSTypeReference" && typeReferenceName(unwrapped) === "PropertyKey"; +} + +function isBroadRecordType(type) { + const unwrapped = unwrapTypeParentheses(type); + + if (unwrapped.type === "TSTypeReference") { + if (typeReferenceName(unwrapped) === "Readonly") { + const [inner] = unwrapped.typeArguments?.params ?? []; + return inner !== undefined && isBroadRecordType(inner); + } + + if (typeReferenceName(unwrapped) !== "Record") { + return false; + } + const parameters = unwrapped.typeArguments?.params ?? []; + return ( + parameters.length === 2 && + parameters[0] !== undefined && + parameters[1] !== undefined && + isBroadRecordKeyType(parameters[0]) && + isUnknownOrAnyType(parameters[1]) + ); + } + + if (unwrapped.type !== "TSTypeLiteral" || unwrapped.members.length !== 1) { + return false; + } + const [member] = unwrapped.members; + const [parameter] = member?.type === "TSIndexSignature" ? member.parameters : []; + return ( + member?.type === "TSIndexSignature" && + member.parameters.length === 1 && + parameter !== undefined && + isBroadRecordKeyType(parameter.typeAnnotation.typeAnnotation) && + isUnknownOrAnyType(member.typeAnnotation.typeAnnotation) + ); +} + +function broadTypeKind(type) { + const unwrapped = unwrapTypeParentheses(type); + if (unwrapped.type === "TSUnknownKeyword" || unwrapped.type === "TSAnyKeyword") { + return "top"; + } + if (unwrapped.type === "TSObjectKeyword") { + return "object"; + } + return isBroadRecordType(unwrapped) ? "record" : null; +} + +function assertedExpression(node) { + return unwrapExpressionParentheses(node.expression); +} + +function assertedIdentifier(node) { + let expression = assertedExpression(node); + while (expression.type === "TSAsExpression" || expression.type === "TSTypeAssertion") { + expression = assertedExpression(expression); + } + return expression.type === "Identifier" ? expression : null; +} + +function isNestedAssertion(node) { + let parent = node.parent; + while (parent?.type === "ParenthesizedExpression") { + parent = parent.parent; + } + return ( + (parent?.type === "TSAsExpression" || parent?.type === "TSTypeAssertion") && + assertedExpression(parent) === node + ); +} + +function assertionFromExpression(expression) { + const unwrapped = unwrapExpressionParentheses(expression); + return unwrapped.type === "TSAsExpression" || unwrapped.type === "TSTypeAssertion" + ? unwrapped + : null; +} + +function normalizedTypeText(sourceText, type) { + return sourceText.slice(type.start, type.end).replaceAll(/\s+/gu, ""); +} + +function typesHaveSameSyntax(sourceText, left, right) { + return ( + left !== null && + normalizedTypeText(sourceText, unwrapTypeParentheses(left)) === + normalizedTypeText(sourceText, unwrapTypeParentheses(right)) + ); +} + +function isDefinitelyObjectType(type) { + const unwrapped = unwrapTypeParentheses(type); + switch (unwrapped.type) { + case "TSArrayType": + case "TSConstructorType": + case "TSFunctionType": + case "TSMappedType": + case "TSObjectKeyword": + case "TSTupleType": + return true; + case "TSTypeLiteral": + return unwrapped.members.length > 0; + case "TSIntersectionType": + return unwrapped.types.every(isDefinitelyObjectType); + case "TSTypeOperator": + return unwrapped.operator === "readonly" && isDefinitelyObjectType(unwrapped.typeAnnotation); + default: + return false; + } +} + +function isDefinitelyNarrowerRecordType(type) { + const unwrapped = unwrapTypeParentheses(type); + if (unwrapped.type === "TSTypeLiteral") { + return unwrapped.members.some((member) => member.type !== "TSIndexSignature"); + } + + if (unwrapped.type !== "TSTypeReference") { + return false; + } + if (typeReferenceName(unwrapped) === "Readonly") { + const [inner] = unwrapped.typeArguments?.params ?? []; + return inner !== undefined && isDefinitelyNarrowerRecordType(inner); + } + if (typeReferenceName(unwrapped) !== "Record") { + return false; + } + + const parameters = unwrapped.typeArguments?.params ?? []; + return ( + parameters.length === 2 && parameters[1] !== undefined && !isUnknownOrAnyType(parameters[1]) + ); +} + +function functionBoundary(node) { + let current = node.parent; + while (current !== null && current.type !== "Program") { + if (FUNCTION_BOUNDARY_TYPES.has(current.type)) { + return current; + } + current = current.parent; + } + return null; +} + +function resolvedVariableForIdentifier(scopes, identifier) { + for (const scope of scopes) { + const reference = scope.references.find( + (candidate) => + candidate.identifier.start === identifier.start && + candidate.identifier.end === identifier.end, + ); + if (reference !== undefined) { + return reference.resolved; + } + } + return null; +} + +function variableDeclarator(variable) { + for (const definition of variable.defs) { + if (definition.type === "Variable" && definition.node.type === "VariableDeclarator") { + return definition.node; + } + } + return null; +} + +function knownValueEvidence(expression, scopes, boundary, visitedVariables) { + const unwrapped = unwrapExpressionParentheses(expression); + + if (unwrapped.type === "TSAsExpression" || unwrapped.type === "TSTypeAssertion") { + if (broadTypeKind(unwrapped.typeAnnotation) !== null) { + return null; + } + return { type: unwrapped.typeAnnotation }; + } + + if (unwrapped.type === "Literal" || unwrapped.type === "TemplateLiteral") { + return { type: null }; + } + + if ( + unwrapped.type === "ArrayExpression" || + unwrapped.type === "ArrowFunctionExpression" || + unwrapped.type === "ClassExpression" || + unwrapped.type === "FunctionExpression" || + unwrapped.type === "NewExpression" || + unwrapped.type === "ObjectExpression" + ) { + return { type: null }; + } + + if (unwrapped.type !== "Identifier") { + return null; + } + const variable = resolvedVariableForIdentifier(scopes, unwrapped); + if (variable === null || visitedVariables.has(variable)) { + return null; + } + + const annotatedIdentifier = variable.identifiers.find( + (identifier) => identifier.typeAnnotation !== null && identifier.typeAnnotation !== undefined, + ); + const annotation = annotatedIdentifier?.typeAnnotation?.typeAnnotation; + if (annotation !== undefined && annotatedIdentifier !== undefined) { + if (functionBoundary(annotatedIdentifier) !== boundary || broadTypeKind(annotation) !== null) { + return null; + } + return { type: annotation }; + } + + const declarator = variableDeclarator(variable); + if ( + declarator === null || + declarator.parent.type !== "VariableDeclaration" || + declarator.parent.kind !== "const" || + declarator.init === null || + variable.references.some((reference) => reference.isWrite() && !reference.init) || + functionBoundary(declarator) !== boundary + ) { + return null; + } + + return knownValueEvidence( + declarator.init, + scopes, + boundary, + new Set([...visitedVariables, variable]), + ); +} + +function widenedBinding(variable, scopes) { + const declarator = variableDeclarator(variable); + if ( + declarator === null || + declarator.parent.type !== "VariableDeclaration" || + declarator.parent.kind !== "const" || + declarator.id.type !== "Identifier" || + declarator.init === null || + variable.references.some((reference) => reference.isWrite() && !reference.init) + ) { + return null; + } + + const boundary = functionBoundary(declarator); + const declaredType = declarator.id.typeAnnotation?.typeAnnotation; + const initializerAssertion = assertionFromExpression(declarator.init); + const initializerBroadKind = + initializerAssertion === null ? null : broadTypeKind(initializerAssertion.typeAnnotation); + const declaredBroadKind = declaredType === undefined ? null : broadTypeKind(declaredType); + const broadKind = declaredBroadKind ?? initializerBroadKind; + if (broadKind === null) { + return null; + } + + const originalExpression = + initializerAssertion !== null && initializerBroadKind !== null + ? assertedExpression(initializerAssertion) + : declarator.init; + const evidence = knownValueEvidence(originalExpression, scopes, boundary, new Set([variable])); + return evidence === null ? null : { broadKind, evidence, declaredAt: declarator.end, boundary }; +} + +function resolveWidenedBinding(variable, scopes, boundary, assertedAt) { + const visitedVariables = new Set(); + let current = variable; + for (let depth = 0; depth < MAX_TRANSPARENT_ALIAS_DEPTH; depth += 1) { + if (visitedVariables.has(current)) { + return null; + } + visitedVariables.add(current); + + const widened = widenedBinding(current, scopes); + if (widened !== null) { + return widened; + } + + const declarator = variableDeclarator(current); + if ( + declarator === null || + declarator.parent.type !== "VariableDeclaration" || + declarator.parent.kind !== "const" || + declarator.id.type !== "Identifier" || + (declarator.id.typeAnnotation !== null && declarator.id.typeAnnotation !== undefined) || + declarator.init === null || + declarator.end >= assertedAt || + current.references.some((reference) => reference.isWrite() && !reference.init) || + functionBoundary(declarator) !== boundary + ) { + return null; + } + + const initializer = unwrapExpressionParentheses(declarator.init); + if (initializer.type !== "Identifier") { + return null; + } + current = resolvedVariableForIdentifier(scopes, initializer); + if (current === null) { + return null; + } + } + return null; +} + +function assertionIsNarrower(sourceText, broadKind, evidence, assertedType) { + if (broadTypeKind(assertedType) !== null) { + return false; + } + if (broadKind === "top") { + return true; + } + if (typesHaveSameSyntax(sourceText, evidence.type, assertedType)) { + return true; + } + if (broadKind === "object") { + return isDefinitelyObjectType(assertedType); + } + return isDefinitelyNarrowerRecordType(assertedType); +} + +function noWidenThenAssertRule({ roots }) { + return { + meta: { + type: "problem", + docs: { + description: + "Disallow local const flows that explicitly widen a known value before asserting the widened binding to a narrower type.", + }, + messages: { + widenThenAssert: + 'Binding "{{name}}" discards type evidence and later recreates it with an assertion. Keep the precise type from initialization through use; parse boundary input once.', + }, + }, + create(context) { + const filename = context.physicalFilename.replaceAll("\\", "/"); + const cwd = context.cwd.replaceAll("\\", "/"); + const repoPath = filename.startsWith(`${cwd}/`) ? filename.slice(cwd.length + 1) : filename; + if (!roots.some((root) => repoPath === root || repoPath.startsWith(`${root}/`))) { + return {}; + } + + let scopes = []; + const checkAssertion = (node) => { + if (isNestedAssertion(node)) { + return; + } + const expression = assertedIdentifier(node); + if (expression === null) { + return; + } + + const variable = resolvedVariableForIdentifier(scopes, expression); + if (variable === null) { + return; + } + const boundary = functionBoundary(node); + const widened = resolveWidenedBinding(variable, scopes, boundary, node.start); + if ( + widened === null || + node.start <= widened.declaredAt || + boundary !== widened.boundary || + !assertionIsNarrower( + context.sourceCode.text, + widened.broadKind, + widened.evidence, + node.typeAnnotation, + ) + ) { + return; + } + + context.report({ + node, + messageId: "widenThenAssert", + data: { name: expression.name }, + }); + }; + + return { + Program() { + scopes = context.sourceCode.scopeManager.scopes; + }, + TSAsExpression: checkAssertion, + TSTypeAssertion: checkAssertion, + }; + }, + }; +} + +export default { + meta: { name: "openclaw-boundaries" }, + rules: { + "no-raw-window-open-call": restrictedCallRule({ + allowedFiles: ["ui/src/lib/editor-links.ts", "ui/src/lib/open-external-url.ts"], + roots: ["ui/src", "test/fixtures/oxlint-boundary-guards"], + property: "open", + objects: ["window", "globalThis"], + message: "Use openExternalUrlSafe(...) from ui/src/lib/open-external-url.ts instead.", + }), + "no-register-http-handler-call": restrictedCallRule({ + roots: ["src", "extensions", "test/fixtures/oxlint-boundary-guards"], + property: "registerHttpHandler", + message: + "Use registerHttpRoute({ path, auth, match, handler }) and registerPluginHttpRoute for dynamic webhook paths.", + }), + "no-widen-then-assert": noWidenThenAssertRule({ + roots: [...TYPE_ASSERTION_PRODUCTION_ROOTS, BOUNDARY_GUARD_FIXTURE_ROOT], + }), + "no-chained-type-assertions": noChainedTypeAssertionsRule({ + roots: [...TYPE_ASSERTION_PRODUCTION_ROOTS, BOUNDARY_GUARD_FIXTURE_ROOT], + excludedRoots: CHAINED_ASSERTION_EXCLUDED_ROOTS, + }), + }, +}; diff --git a/scripts/package-mac-app.sh b/scripts/package-mac-app.sh new file mode 100644 index 0000000000000000000000000000000000000000..123916714410a5181750d53ef8d05fe1f4e49c4a --- /dev/null +++ b/scripts/package-mac-app.sh @@ -0,0 +1,604 @@ +#!/bin/bash +set -euo pipefail + +# Build and bundle OpenClaw with its matching private worker runtime. +# Outputs to dist/OpenClaw.app + +ROOT_DIR="$(cd "$(dirname "$0")/.." && pwd)" +source "$ROOT_DIR/scripts/lib/plistbuddy.sh" +source "$ROOT_DIR/scripts/lib/swift-toolchain.sh" +source "$ROOT_DIR/scripts/lib/build-metadata.sh" +source "$ROOT_DIR/scripts/lib/mac-app-bundle.sh" +DEFAULT_APP_ROOT="$ROOT_DIR/dist/OpenClaw.app" +APP_ROOT="${OPENCLAW_PACKAGE_APP_ROOT:-$DEFAULT_APP_ROOT}" +case "$APP_ROOT" in + "$ROOT_DIR/dist/"*) ;; + *) + echo "ERROR: OPENCLAW_PACKAGE_APP_ROOT must stay under $ROOT_DIR/dist" >&2 + exit 1 + ;; +esac +APP_DESTINATION="$APP_ROOT" +APP_STAGE_DIR="" +SWIFT_BUILD_PID="" +SWIFT_BUILD_RESULTS="" +BUILD_ROOT="$ROOT_DIR/apps/macos/.build" +PRODUCT="OpenClaw" +MLX_TTS_HELPER_PRODUCT="openclaw-mlx-tts" +MLX_TTS_HELPER_ROOT="$ROOT_DIR/apps/macos-mlx-tts" +MLX_TTS_HELPER_BUILD_ROOT="$MLX_TTS_HELPER_ROOT/.build" +BUNDLE_ID="${BUNDLE_ID:-ai.openclaw.mac.debug}" +PKG_VERSION="$(cd "$ROOT_DIR" && node -p "require('./package.json').version" 2>/dev/null || echo "0.0.0")" +BUILD_CONFIG="${BUILD_CONFIG:-debug}" +SIGNING_VARIANT="${OPENCLAW_MAC_SIGNING_VARIANT:-standard}" +case "$SIGNING_VARIANT" in + standard | elevation-host) ;; + *) + echo "ERROR: Unknown OPENCLAW_MAC_SIGNING_VARIANT value: $SIGNING_VARIANT (use standard|elevation-host)" >&2 + exit 1 + ;; +esac +# OPENCLAW_SKIP_MLX_TTS=1 packages the app without the local MLX voice helper. +# The helper pulls in the full mlx-swift Metal shader stack, which some beta +# Xcode toolchains cannot compile (flaky `metal` diagnostics), needlessly +# blocking unrelated dev/proof builds. Release builds must always ship the +# helper (notarization verifies it), so refuse the skip there instead of +# producing a silently incomplete release bundle. +SKIP_MLX_TTS="${OPENCLAW_SKIP_MLX_TTS:-0}" +if [[ "$SKIP_MLX_TTS" == "1" && "$BUILD_CONFIG" == "release" ]]; then + echo "ERROR: OPENCLAW_SKIP_MLX_TTS is not allowed for release builds; the MLX voice helper must ship in release." >&2 + exit 1 +fi +BUILD_TS="$(openclaw_resolve_build_timestamp)" +if [[ "$BUILD_CONFIG" == "release" ]]; then + OPENCLAW_REQUIRE_BUILD_METADATA=1 +fi +BUILD_GIT_COMMIT="$(openclaw_resolve_git_commit "$ROOT_DIR")" +if [[ "$BUILD_CONFIG" == "release" ]]; then + /bin/bash "$ROOT_DIR/scripts/apple-release-source-check.sh" \ + --root "$ROOT_DIR" \ + --expected-commit "$BUILD_GIT_COMMIT" +fi +export OPENCLAW_BUILD_TIMESTAMP="$BUILD_TS" +if openclaw_is_full_git_commit "$BUILD_GIT_COMMIT"; then + export GIT_COMMIT="$BUILD_GIT_COMMIT" +else + unset GIT_COMMIT +fi +GIT_BUILD_NUMBER=$(cd "$ROOT_DIR" && git rev-list --count HEAD 2>/dev/null || echo "0") +APP_VERSION="${APP_VERSION:-$PKG_VERSION}" +APP_BUILD="${APP_BUILD:-}" +if [[ -n "${BUILD_ARCHS:-}" ]]; then + BUILD_ARCHS_VALUE="${BUILD_ARCHS}" +elif [[ "$BUILD_CONFIG" == "release" ]]; then + # Release packaging should be universal unless explicitly overridden. + BUILD_ARCHS_VALUE="all" +else + BUILD_ARCHS_VALUE="$(uname -m)" +fi +if [[ "${BUILD_ARCHS_VALUE}" == "all" ]]; then + BUILD_ARCHS_VALUE="arm64 x86_64" +fi +IFS=' ' read -r -a BUILD_ARCHS <<< "$BUILD_ARCHS_VALUE" +PRIMARY_ARCH="${BUILD_ARCHS[0]}" +SPARKLE_PUBLIC_ED_KEY="${SPARKLE_PUBLIC_ED_KEY:-AGCY8w5vHirVfGGDGc8Szc5iuOqupZSh9pMj/Qs67XI=}" +SPARKLE_FEED_URL="${SPARKLE_FEED_URL:-https://raw.githubusercontent.com/openclaw/openclaw/main/appcast.xml}" +AUTO_CHECKS=true +if [[ "$BUNDLE_ID" == *.debug ]]; then + SPARKLE_FEED_URL="" + AUTO_CHECKS=false +fi + +resolve_peekaboo_source_commit() { + local resolved_file="$ROOT_DIR/apps/macos/Package.resolved" + local revision + revision="$(/usr/bin/python3 - "$resolved_file" <<'PY' +import json +from pathlib import Path +import re +import sys + +resolved_file = Path(sys.argv[1]) +try: + resolved = json.loads(resolved_file.read_text()) +except (OSError, json.JSONDecodeError) as error: + raise SystemExit(f"ERROR: Could not parse Peekaboo source revision from {resolved_file}: {error}") + +pins = resolved.get("pins") if isinstance(resolved, dict) else None +if not isinstance(pins, list): + raise SystemExit(f"ERROR: Expected a pins array in {resolved_file}") + +peekaboo_pins = [pin for pin in pins if isinstance(pin, dict) and pin.get("identity") == "peekaboo"] +if len(peekaboo_pins) != 1: + raise SystemExit(f"ERROR: Expected exactly one 'peekaboo' pin in {resolved_file}; found {len(peekaboo_pins)}") + +state = peekaboo_pins[0].get("state") +revision = state.get("revision") if isinstance(state, dict) else None +if not isinstance(revision, str) or re.fullmatch(r"[0-9a-f]{40}", revision) is None: + raise SystemExit( + f"ERROR: Peekaboo pin in {resolved_file} must have an exact 40-character lowercase hexadecimal revision" + ) + +print(revision, end="") +PY + )" + local expected="${OPENCLAW_EXPECTED_PEEKABOO_SOURCE_COMMIT:-}" + if [[ -n "$expected" && ! "$expected" =~ ^[0-9a-f]{40}$ ]]; then + echo "ERROR: OPENCLAW_EXPECTED_PEEKABOO_SOURCE_COMMIT must be a full lowercase 40-character SHA" >&2 + return 1 + fi + if [[ -n "$expected" && "$revision" != "$expected" ]]; then + echo "ERROR: Peekaboo pin '$revision' does not match requested release source '$expected'" >&2 + return 1 + fi + printf '%s' "$revision" +} + +sparkle_canonical_build_from_version() { + (cd "$ROOT_DIR" && node --import tsx "$ROOT_DIR/scripts/sparkle-build.ts" canonical-build "$1") +} + +source "$ROOT_DIR/scripts/lib/mac-swift-build.sh" + +cleanup_package_build() { + if [[ -n "$SWIFT_BUILD_RESULTS" && ! -f "$SWIFT_BUILD_RESULTS/cleanup-complete" ]]; then + echo "ERROR: Swift cleanup was not verified; retaining $APP_STAGE_DIR for inspection" >&2 + return + fi + [[ -z "$APP_STAGE_DIR" ]] || rm -rf "$APP_STAGE_DIR" +} + +interrupt_package_build() { + local signal="$1" code="$2" + if [[ -n "$SWIFT_BUILD_PID" ]]; then + kill -"$signal" "$SWIFT_BUILD_PID" 2>/dev/null || true + wait "$SWIFT_BUILD_PID" || true + SWIFT_BUILD_PID="" + fi + exit "$code" +} + +trap cleanup_package_build EXIT +trap 'interrupt_package_build INT 130' INT +trap 'interrupt_package_build TERM 143' TERM +trap 'interrupt_package_build HUP 129' HUP + +PNPM_CMD=() + +resolve_pnpm_cmd() { + if command -v corepack >/dev/null 2>&1 && (cd "$ROOT_DIR" && corepack pnpm --version >/dev/null 2>&1); then + PNPM_CMD=(corepack pnpm) + return 0 + fi + + if command -v pnpm >/dev/null 2>&1; then + PNPM_CMD=(pnpm) + return 0 + fi + + echo "ERROR: pnpm is not on PATH and corepack pnpm is unavailable. Install pnpm or run with Node/Corepack on PATH." >&2 + exit 1 +} + +run_pnpm() { + if [[ "${#PNPM_CMD[@]}" -eq 0 ]]; then + resolve_pnpm_cmd + fi + (cd "$ROOT_DIR" && "${PNPM_CMD[@]}" "$@") +} + +merge_framework_machos() { + local primary="$1" + local dest="$2" + shift 2 + local others=("$@") + + archs_for() { + /usr/bin/lipo -info "$1" | /usr/bin/sed -E 's/.*are: //; s/.*architecture: //' + } + + arch_in_list() { + local needle="$1" + shift + for item in "$@"; do + if [[ "$item" == "$needle" ]]; then + return 0 + fi + done + return 1 + } + + while IFS= read -r -d '' file; do + if /usr/bin/file "$file" | /usr/bin/grep "Mach-O" >/dev/null; then + local rel="${file#"$primary"/}" + local primary_archs + primary_archs=$(archs_for "$file") + IFS=' ' read -r -a primary_arch_array <<< "$primary_archs" + + local missing_files=() + local tmp_dir + tmp_dir=$(mktemp -d) + for fw in "${others[@]}"; do + local other_file="$fw/$rel" + if [[ ! -f "$other_file" ]]; then + echo "ERROR: Missing $rel in $fw" >&2 + rm -rf "$tmp_dir" + exit 1 + fi + if /usr/bin/file "$other_file" | /usr/bin/grep "Mach-O" >/dev/null; then + local other_archs + other_archs=$(archs_for "$other_file") + IFS=' ' read -r -a other_arch_array <<< "$other_archs" + for arch in "${other_arch_array[@]}"; do + if ! arch_in_list "$arch" "${primary_arch_array[@]}"; then + local thin_file="$tmp_dir/${rel//\//_}-$arch" + /usr/bin/lipo -thin "$arch" "$other_file" -output "$thin_file" + missing_files+=("$thin_file") + primary_arch_array+=("$arch") + fi + done + fi + done + + if [[ "${#missing_files[@]}" -gt 0 ]]; then + /usr/bin/lipo -create "$file" "${missing_files[@]}" -output "$dest/$rel" + fi + rm -rf "$tmp_dir" + fi + done < <(find "$primary" -type f -print0) +} + +PEEKABOO_SOURCE_COMMIT="$(resolve_peekaboo_source_commit)" +PEEKABOO_LOCKED_SOURCE_COMMIT="$PEEKABOO_SOURCE_COMMIT" + +require_swift_toolchain + +if [[ "${SKIP_PNPM_INSTALL:-0}" != "1" ]]; then + echo "📦 Ensuring deps (pnpm install --frozen-lockfile)" + run_pnpm install --frozen-lockfile --config.node-linker=hoisted +else + echo "📦 Skipping pnpm install (SKIP_PNPM_INSTALL=1)" +fi + +if [[ -z "${APP_BUILD:-}" ]]; then + APP_BUILD="$GIT_BUILD_NUMBER" + if [[ "$APP_VERSION" =~ ^[0-9]{4}\.[0-9]{1,2}\.[0-9]{1,2}([.-].*)?$ ]]; then + CANONICAL_BUILD="$(sparkle_canonical_build_from_version "$APP_VERSION")" || { + echo "ERROR: Failed to derive canonical Sparkle APP_BUILD from APP_VERSION '$APP_VERSION'." >&2 + exit 1 + } + if [[ "$CANONICAL_BUILD" =~ ^[0-9]+$ ]] && (( CANONICAL_BUILD > APP_BUILD )); then + APP_BUILD="$CANONICAL_BUILD" + fi + fi +fi + +if [[ "$AUTO_CHECKS" == "true" && ! "$APP_BUILD" =~ ^[0-9]+$ ]]; then + echo "ERROR: APP_BUILD must be numeric for Sparkle compare (CFBundleVersion). Got: $APP_BUILD" >&2 + exit 1 +fi + +if [[ "${SKIP_TSC:-0}" == "1" ]]; then + echo "📦 SKIP_TSC no longer skips the app's private runtime; using the content-checked build cache" +fi +echo "📦 Building JS (pnpm build)" +run_pnpm build + +node - "$ROOT_DIR/dist/build-info.json" "$APP_VERSION" "$BUILD_GIT_COMMIT" "$BUILD_TS" <<'NODE' +const fs = require("node:fs"); +const [file, version, commit, builtAt] = process.argv.slice(2); +const actual = JSON.parse(fs.readFileSync(file, "utf8")); +if (actual.version !== version || actual.commit !== commit || actual.builtAt !== builtAt || !actual.buildId) { + throw new Error("JavaScript build provenance does not match this app. Rebuild from matching package inputs."); +} +NODE + +node "$ROOT_DIR/scripts/prepare-apple-mermaid.mjs" + +# pnpm build owns the Control UI and content-checked build stamps as well. +# Private Swift and worker staging must stay outside the published dist tree. +mkdir -p "$(dirname "$APP_DESTINATION")" "$ROOT_DIR/.artifacts" +APP_STAGE_DIR="$(mktemp -d "$ROOT_DIR/.artifacts/.openclaw-package.XXXXXX")" +APP_ROOT="$APP_STAGE_DIR/OpenClaw.app" + +echo "🔨 Building $PRODUCT ($BUILD_CONFIG) [${BUILD_ARCHS[*]}]" +SWIFT_BUILD_RESULTS="$APP_STAGE_DIR/swift-builds" +node "$ROOT_DIR/scripts/build-mac-swift.mts" "$ROOT_DIR" "$BUILD_CONFIG" \ + "$PEEKABOO_LOCKED_SOURCE_COMMIT" "$SKIP_MLX_TTS" "$SWIFT_BUILD_RESULTS" "${BUILD_ARCHS[@]}" & +SWIFT_BUILD_PID=$! +if wait "$SWIFT_BUILD_PID"; then + SWIFT_BUILD_PID="" + PEEKABOO_SOURCE_COMMIT="$PEEKABOO_LOCKED_SOURCE_COMMIT" +else + build_status=$? + SWIFT_BUILD_PID="" + exit "$build_status" +fi + +BIN_PRIMARY="$(bin_for_arch "$PRIMARY_ARCH")" +echo "pkg: binary $BIN_PRIMARY" >&2 +echo "📦 Assembling replacement app bundle" +mkdir -p "$APP_ROOT/Contents/MacOS" +mkdir -p "$APP_ROOT/Contents/Resources" +mkdir -p "$APP_ROOT/Contents/Frameworks" + +echo "📄 Copying Info.plist template" +INFO_PLIST_SRC="$ROOT_DIR/apps/macos/Sources/OpenClaw/Resources/Info.plist" +if [ ! -f "$INFO_PLIST_SRC" ]; then + echo "ERROR: Info.plist template missing at $INFO_PLIST_SRC" >&2 + exit 1 +fi +cp "$INFO_PLIST_SRC" "$APP_ROOT/Contents/Info.plist" +PORT_GUARDIAN_STORAGE_VERSION="$(plist_print_required "$APP_ROOT/Contents/Info.plist" OpenClawPortGuardianStorageVersion)" +if [[ ! "$PORT_GUARDIAN_STORAGE_VERSION" =~ ^[1-9][0-9]*$ ]]; then + echo "ERROR: OpenClawPortGuardianStorageVersion must be a positive integer." >&2 + exit 1 +fi +plist_set_string_required "$APP_ROOT/Contents/Info.plist" CFBundleIdentifier "$BUNDLE_ID" +plist_set_string_required "$APP_ROOT/Contents/Info.plist" CFBundleShortVersionString "$APP_VERSION" +plist_set_string_required "$APP_ROOT/Contents/Info.plist" CFBundleVersion "$APP_BUILD" +plist_set_string_required "$APP_ROOT/Contents/Info.plist" OpenClawBuildTimestamp "$BUILD_TS" +plist_set_string_required "$APP_ROOT/Contents/Info.plist" OpenClawGitCommit "$BUILD_GIT_COMMIT" +WORKER_BUILD_ID="$(node -e 'console.log(require(process.argv[1]).buildId)' "$ROOT_DIR/dist/build-info.json")" +plist_set_or_add_string "$APP_ROOT/Contents/Info.plist" OpenClawWorkerBuildID "$WORKER_BUILD_ID" +plist_set_string_required "$APP_ROOT/Contents/Info.plist" PeekabooSourceCommit "$PEEKABOO_SOURCE_COMMIT" +if [[ "$BUILD_CONFIG" == "release" ]]; then + EMBEDDED_GIT_COMMIT="$(plist_print_required "$APP_ROOT/Contents/Info.plist" OpenClawGitCommit)" + BRIDGE_SOURCE_COMMIT="$(plist_print_required "$APP_ROOT/Contents/Info.plist" PeekabooSourceCommit)" + if [[ "$EMBEDDED_GIT_COMMIT" != "$BUILD_GIT_COMMIT" ]]; then + echo "ERROR: Release app OpenClaw source mismatch: OpenClawGitCommit='$EMBEDDED_GIT_COMMIT', expected='$BUILD_GIT_COMMIT'." >&2 + exit 1 + fi + if [[ "$BRIDGE_SOURCE_COMMIT" != "$PEEKABOO_SOURCE_COMMIT" ]]; then + echo "ERROR: Release app Peekaboo source mismatch: PeekabooSourceCommit='$BRIDGE_SOURCE_COMMIT', expected='$PEEKABOO_SOURCE_COMMIT'." >&2 + exit 1 + fi +fi +plist_set_or_add_string "$APP_ROOT/Contents/Info.plist" SUFeedURL "$SPARKLE_FEED_URL" +plist_set_or_add_string "$APP_ROOT/Contents/Info.plist" SUPublicEDKey "$SPARKLE_PUBLIC_ED_KEY" +plist_set_or_add_bool "$APP_ROOT/Contents/Info.plist" SUEnableAutomaticChecks "$AUTO_CHECKS" + +echo "🚚 Copying binary" +cp "$BIN_PRIMARY" "$APP_ROOT/Contents/MacOS/OpenClaw" +if [[ "${#BUILD_ARCHS[@]}" -gt 1 ]]; then + BIN_INPUTS=() + for arch in "${BUILD_ARCHS[@]}"; do + BIN_INPUTS+=("$(bin_for_arch "$arch")") + done + /usr/bin/lipo -create "${BIN_INPUTS[@]}" -output "$APP_ROOT/Contents/MacOS/OpenClaw" +fi +chmod +x "$APP_ROOT/Contents/MacOS/OpenClaw" +# SwiftPM outputs ad-hoc signed binaries; strip the signature before install_name_tool to avoid warnings. +/usr/bin/codesign --remove-signature "$APP_ROOT/Contents/MacOS/OpenClaw" 2>/dev/null || true + +echo "🚚 Copying macOS control CLI" +cp "$(mac_cli_bin_for_arch "$PRIMARY_ARCH")" "$APP_ROOT/Contents/MacOS/openclaw-mac" +if [[ "${#BUILD_ARCHS[@]}" -gt 1 ]]; then + MAC_CLI_BIN_INPUTS=() + for arch in "${BUILD_ARCHS[@]}"; do + MAC_CLI_BIN_INPUTS+=("$(mac_cli_bin_for_arch "$arch")") + done + /usr/bin/lipo -create "${MAC_CLI_BIN_INPUTS[@]}" -output "$APP_ROOT/Contents/MacOS/openclaw-mac" +fi +chmod +x "$APP_ROOT/Contents/MacOS/openclaw-mac" +/usr/bin/codesign --remove-signature "$APP_ROOT/Contents/MacOS/openclaw-mac" 2>/dev/null || true + +if [[ "$SKIP_MLX_TTS" == "1" ]]; then + echo "🔇 Skipping MLX TTS helper copy (OPENCLAW_SKIP_MLX_TTS=1) — bundle omits Contents/MacOS/$MLX_TTS_HELPER_PRODUCT" +else + echo "🚚 Copying MLX TTS helper" + cp "$(helper_bin_for_arch "$PRIMARY_ARCH")" "$APP_ROOT/Contents/MacOS/$MLX_TTS_HELPER_PRODUCT" + if [[ "${#BUILD_ARCHS[@]}" -gt 1 ]]; then + HELPER_BIN_INPUTS=() + for arch in "${BUILD_ARCHS[@]}"; do + HELPER_BIN_INPUTS+=("$(helper_bin_for_arch "$arch")") + done + /usr/bin/lipo -create "${HELPER_BIN_INPUTS[@]}" -output "$APP_ROOT/Contents/MacOS/$MLX_TTS_HELPER_PRODUCT" + fi + chmod +x "$APP_ROOT/Contents/MacOS/$MLX_TTS_HELPER_PRODUCT" + /usr/bin/codesign --remove-signature "$APP_ROOT/Contents/MacOS/$MLX_TTS_HELPER_PRODUCT" 2>/dev/null || true +fi + +SPARKLE_FRAMEWORK_PRIMARY="$(sparkle_framework_for_arch "$PRIMARY_ARCH")" +if [ -d "$SPARKLE_FRAMEWORK_PRIMARY" ]; then + echo "✨ Embedding Sparkle.framework" + cp -R "$SPARKLE_FRAMEWORK_PRIMARY" "$APP_ROOT/Contents/Frameworks/" + if [[ "${#BUILD_ARCHS[@]}" -gt 1 ]]; then + OTHER_FRAMEWORKS=() + for arch in "${BUILD_ARCHS[@]}"; do + if [[ "$arch" == "$PRIMARY_ARCH" ]]; then + continue + fi + OTHER_FRAMEWORKS+=("$(sparkle_framework_for_arch "$arch")") + done + merge_framework_machos "$SPARKLE_FRAMEWORK_PRIMARY" "$APP_ROOT/Contents/Frameworks/Sparkle.framework" "${OTHER_FRAMEWORKS[@]}" + fi + chmod -R a+rX "$APP_ROOT/Contents/Frameworks/Sparkle.framework" +fi + +echo "📦 Copying Swift 6.2 compatibility libraries" +SWIFT_COMPAT_LIB="$(xcode-select -p)/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift-6.2/macosx/libswiftCompatibilitySpan.dylib" +if [ -f "$SWIFT_COMPAT_LIB" ]; then + cp "$SWIFT_COMPAT_LIB" "$APP_ROOT/Contents/Frameworks/" + chmod +x "$APP_ROOT/Contents/Frameworks/libswiftCompatibilitySpan.dylib" +elif [[ "$BUILD_CONFIG" == "release" ]]; then + echo "ERROR: Swift compatibility library not found at $SWIFT_COMPAT_LIB" >&2 + exit 1 +else + echo "WARN: Swift compatibility library not found at $SWIFT_COMPAT_LIB (continuing)" >&2 +fi + +echo "🖼 Compiling app icon" +xcrun actool "$ROOT_DIR/apps/macos/Icon.icon" \ + --compile "$APP_ROOT/Contents/Resources" \ + --output-format human-readable-text --notices --warnings --errors \ + --output-partial-info-plist "$APP_STAGE_DIR/icon.plist" \ + --app-icon Icon --include-all-app-icons --enable-on-demand-resources NO \ + --development-region en --target-device mac \ + --minimum-deployment-target "$(plist_print_required "$APP_ROOT/Contents/Info.plist" LSMinimumSystemVersion)" \ + --platform macosx +mv "$APP_ROOT/Contents/Resources/Icon.icns" "$APP_ROOT/Contents/Resources/OpenClaw.icns" +cp -R "$ROOT_DIR/apps/macos/Sources/OpenClaw/Resources/AppIcons" "$APP_ROOT/Contents/Resources/AppIcons" + +echo "📦 Copying device model resources" +rm -rf "$APP_ROOT/Contents/Resources/DeviceModels" +cp -R "$ROOT_DIR/apps/macos/Sources/OpenClaw/Resources/DeviceModels" "$APP_ROOT/Contents/Resources/DeviceModels" + +echo "📦 Copying provider icon resources" +PROVIDER_ICONS_SRC="$ROOT_DIR/apps/macos/Sources/OpenClaw/Resources/ProviderIcons" +if [ ! -d "$PROVIDER_ICONS_SRC" ]; then + echo "ERROR: Provider icon resources missing at $PROVIDER_ICONS_SRC" >&2 + exit 1 +fi +rm -rf "$APP_ROOT/Contents/Resources/ProviderIcons" +cp -R "$PROVIDER_ICONS_SRC" "$APP_ROOT/Contents/Resources/ProviderIcons" + +if [[ "$SIGNING_VARIANT" == "elevation-host" ]]; then + echo "🖥 Omitting embedded CUA driver from elevation-host package" +else + echo "🖥 Staging embedded CUA driver" + "$ROOT_DIR/scripts/stage-cua-driver-macos.sh" "$APP_ROOT/Contents/Resources/cua-driver" +fi + +echo "📦 Staging browser sign-in helper" +for arch in "${BUILD_ARCHS[@]}"; do + /bin/bash "$ROOT_DIR/scripts/stage-cloudflared-macos.sh" "$arch" "$APP_ROOT/Contents/Resources/cloudflared" +done + +echo "📦 Copying CLI installer" +INSTALL_CLI_SRC="$ROOT_DIR/scripts/install-cli.sh" +if [ ! -f "$INSTALL_CLI_SRC" ]; then + echo "ERROR: CLI installer missing at $INSTALL_CLI_SRC" >&2 + exit 1 +fi +cp "$INSTALL_CLI_SRC" "$APP_ROOT/Contents/Resources/install-cli.sh" +chmod 0644 "$APP_ROOT/Contents/Resources/install-cli.sh" + +echo "📦 Provisioning the matching private node worker [${BUILD_ARCHS[*]}]" +/bin/bash "$ROOT_DIR/scripts/stage-mac-node-worker.sh" "$APP_ROOT/Contents/Resources/node-worker" "${BUILD_ARCHS[@]}" + +echo "🌐 Copying app localizations" +node --import tsx "$ROOT_DIR/scripts/apple-app-i18n.ts" compile-macos \ + --output "$APP_ROOT/Contents/Resources" + +echo "📦 Copying Control UI assets" +CONTROL_UI_SRC="$ROOT_DIR/dist/control-ui" +CONTROL_UI_DEST="$APP_ROOT/Contents/Resources/control-ui" +if [ -d "$CONTROL_UI_SRC" ] && [ -f "$CONTROL_UI_SRC/index.html" ]; then + rm -rf "$CONTROL_UI_DEST" + cp -R "$CONTROL_UI_SRC" "$CONTROL_UI_DEST" +else + echo "ERROR: Control UI assets missing at $CONTROL_UI_SRC. Run pnpm ui:build first." >&2 + exit 1 +fi + +echo "📦 Copying SwiftPM resource bundles" +SWIFTPM_BUILD_PRODUCTS=("$(build_path_for_arch "$PRIMARY_ARCH")/$BUILD_CONFIG") +if [[ "$SKIP_MLX_TTS" != "1" ]]; then + SWIFTPM_BUILD_PRODUCTS+=("$(helper_products_for_arch "$PRIMARY_ARCH")") +fi +# Main app and helper dependencies share the signed Resources directory. +# MLX loads its compiled Metal library from its resource bundle there. +for build_products in "${SWIFTPM_BUILD_PRODUCTS[@]}"; do + for resource_bundle_src in "$build_products"/*.bundle; do + [[ -d "$resource_bundle_src" ]] || continue + resource_bundle="${resource_bundle_src##*/}" + rm -rf "$APP_ROOT/Contents/Resources/$resource_bundle" + cp -R "$resource_bundle_src" "$APP_ROOT/Contents/Resources/$resource_bundle" + done +done +REQUIRED_SWIFTPM_RESOURCE_BUNDLES=( + "GRDB_GRDB.bundle" + "KeyboardShortcuts_KeyboardShortcuts.bundle" + "OpenClaw_OpenClaw.bundle" + "OpenClawKit_OpenClawKit.bundle" + "OpenClawKit_OpenClawChatUI.bundle" + "SwiftMath_SwiftMath.bundle" +) +for resource_bundle in "${REQUIRED_SWIFTPM_RESOURCE_BUNDLES[@]}"; do + if [[ ! -d "$APP_ROOT/Contents/Resources/$resource_bundle" ]]; then + echo "ERROR: Required SwiftPM resource bundle not found at $APP_ROOT/Contents/Resources/$resource_bundle" >&2 + exit 1 + fi +done +if [[ "$SKIP_MLX_TTS" != "1" && ! -f "$APP_ROOT/Contents/Resources/mlx-swift_Cmlx.bundle/Contents/Resources/default.metallib" ]]; then + echo "ERROR: Required MLX shaders not found at $APP_ROOT/Contents/Resources/mlx-swift_Cmlx.bundle/Contents/Resources/default.metallib" >&2 + exit 1 +fi + +running_packaged_app_pids() { + command -v pgrep >/dev/null 2>&1 || return 0 + local app_binary="$APP_DESTINATION/Contents/MacOS/OpenClaw" + local pid + pgrep -x "$PRODUCT" 2>/dev/null | while IFS= read -r pid; do + [[ "$pid" =~ ^[0-9]+$ ]] || continue + if command -v lsof >/dev/null 2>&1 && + lsof -a -p "$pid" -d txt -Fn 2>/dev/null | sed 's/^n//' | grep -Fx "$app_binary" >/dev/null; then + printf '%s\n' "$pid" + continue + fi + local command_line + command_line="$(ps -p "$pid" -o command= 2>/dev/null || true)" + if [[ "$command_line" == "$app_binary" || "$command_line" == "$app_binary "* ]]; then + printf '%s\n' "$pid" + fi + done +} + +stop_packaged_app_if_running() { + local pids=() + local pid + while IFS= read -r pid; do + [[ -n "$pid" ]] && pids+=("$pid") + done < <(running_packaged_app_pids) + if [[ "${#pids[@]}" -eq 0 ]]; then + return 0 + fi + + echo "⏹ Stopping packaged OpenClaw bundle (${pids[*]})" + kill "${pids[@]}" 2>/dev/null || true + for _ in $(seq 1 40); do + local alive=0 + for pid in "${pids[@]}"; do + if kill -0 "$pid" 2>/dev/null; then + alive=1 + fi + done + [[ "$alive" == "0" ]] && return 0 + sleep 0.25 + done + kill -KILL "${pids[@]}" 2>/dev/null || true + for _ in $(seq 1 20); do + local alive=0 + for pid in "${pids[@]}"; do + if kill -0 "$pid" 2>/dev/null; then + alive=1 + fi + done + [[ "$alive" == "0" ]] && return 0 + sleep 0.1 + done + echo "ERROR: Packaged OpenClaw bundle did not exit: ${pids[*]}" >&2 + return 1 +} + +if [[ -n "${SIGN_IDENTITY:-}" ]]; then + echo "🔏 Signing bundle with explicit SIGN_IDENTITY" +else + echo "🔏 Signing bundle (auto-selecting signing identity)" +fi +"$ROOT_DIR/scripts/codesign-mac-app.sh" "$APP_ROOT" +codesign --verify --deep --strict "$APP_ROOT" +for arch in "${BUILD_ARCHS[@]}"; do + env -i HOME="$APP_STAGE_DIR" PATH="/usr/bin:/bin:/usr/sbin:/sbin" TMPDIR="${TMPDIR:-/tmp}" \ + "$APP_ROOT/Contents/Resources/node-worker/$arch/bin/node" \ + "$ROOT_DIR/scripts/verify-mac-node-worker.mjs" \ + "$APP_ROOT/Contents/Resources/node-worker/$arch" "$ROOT_DIR/dist/build-info.json" +done +codesign --verify --deep --strict "$APP_ROOT" + +# Nothing touches the previous app until build, provisioning and signing pass. +stop_packaged_app_if_running +replace_mac_app_bundle "$APP_ROOT" "$APP_DESTINATION" + +echo "✅ Bundle ready at $APP_DESTINATION" diff --git a/scripts/package-manifest.d.mts b/scripts/package-manifest.d.mts new file mode 100644 index 0000000000000000000000000000000000000000..b604f32574484296eae68c3bbdd5ea33f572a957 --- /dev/null +++ b/scripts/package-manifest.d.mts @@ -0,0 +1,2 @@ +export function restorePackageManifest(cwd?: string): Promise<boolean>; +export function preparePackageManifest(cwd?: string): Promise<boolean>; diff --git a/scripts/package-openclaw-for-docker.mts b/scripts/package-openclaw-for-docker.mts new file mode 100644 index 0000000000000000000000000000000000000000..45d99f5e66b31567d7bbf81b2df404370d11ccd8 --- /dev/null +++ b/scripts/package-openclaw-for-docker.mts @@ -0,0 +1,1153 @@ +#!/usr/bin/env node +// Builds the canonical OpenClaw package artifact used by Docker E2E. +import { spawn } from "node:child_process"; +import { closeSync, openSync } from "node:fs"; +import fs from "node:fs/promises"; +import { createRequire } from "node:module"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { + booleanFlag, + parseFlagArgs, + stringFlag, + stringListFlag, +} from "./lib/arg-utils.runtime.mjs"; +import { DOCKER_SELECTED_PLUGIN_BUILD_IDS_ENV } from "./lib/bundled-plugin-build-entries.mjs"; +import { toErrorObject } from "./lib/error-format.mts"; +import { terminateManagedChild } from "./lib/managed-child-process.mts"; +import { resolveNpmJsonEntries } from "./lib/npm-json-output.mts"; +import { assertRealOutputRoot } from "./lib/output-root-guard.mjs"; +import { + LEGACY_PACKAGE_INSTALL_GUARD_RELATIVE_PATH, + PACKAGE_LIFECYCLE_PENDING_RELATIVE_PATH, +} from "./lib/package-lifecycle-marker.mjs"; +import { isRecord } from "./lib/record-shared.mjs"; +import { resolveNpmRunner } from "./npm-runner.mts"; +import { preparePackageChangelog, restorePackageChangelog } from "./package-changelog.mjs"; +import { validateBundledPackageDependencyAlignment } from "./package-source-dependencies.mjs"; +import { resolvePnpmRunner } from "./pnpm-runner.mts"; + +const ROOT_DIR = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const DEFAULT_PACKAGE_BUILD_TIMEOUT_MS = 45 * 60 * 1000; +const DEFAULT_PACKAGE_INVENTORY_TIMEOUT_MS = 5 * 60 * 1000; +const DEFAULT_PACKAGE_PACK_TIMEOUT_MS = 5 * 60 * 1000; +const DEFAULT_PACKAGE_TARBALL_CHECK_TIMEOUT_MS = 5 * 60 * 1000; +const DEFAULT_TIMEOUT_KILL_AFTER_MS = 5_000; +const PROCESS_GROUP_EXIT_POLL_MS = 25; +const POST_FORCE_KILL_WAIT_MS = 1_000; +const DEFAULT_CAPTURED_STDOUT_MAX_BYTES = 1024 * 1024; +const MAX_TIMER_TIMEOUT_MS = 2_147_000_000; +const AI_RUNTIME_PACKAGE = "@openclaw/ai"; +const AI_RUNTIME_BACKUP_DIR = ".openclaw-ai-package-backup"; + +type KillChild = (signal: NodeJS.Signals) => void; +type RunOptions = { + captureStdout?: boolean; + env?: NodeJS.ProcessEnv; + killAfterMs?: unknown; + maxCapturedStdoutBytes?: number; + stdinFilePath?: string; + stdoutFilePath?: string; + timeoutMs?: unknown; +}; +type CommandRunnerOptions = { + env?: NodeJS.ProcessEnv; + timeoutMs?: number; +}; +type CommandRunner = ( + command: string, + args: string[], + cwd: string, + options: CommandRunnerOptions, +) => Promise<unknown>; +type RunImpl = ( + command: string, + args: string[], + cwd: string, + options: RunOptions, +) => Promise<string>; +type DocsMapLifecycle = { + preparePackageDocsMap: (cwd: string) => Promise<unknown>; + restorePackageDocsMap: (cwd: string) => Promise<unknown>; +}; +type PackageManifestLifecycle = { + preparePackageManifest: (cwd: string) => Promise<unknown>; + restorePackageManifest: (cwd: string) => Promise<unknown>; +}; +type PackageOptions = RunOptions & { + bundlePlugins?: string[]; + allowUnreleasedChangelog?: unknown; + extractAiRuntime?: (tarballPath: string, destination: string) => Promise<unknown>; + normalizeTarballModes?: (tarballPath: string) => Promise<unknown>; + outputName?: string; + packJsonPath?: string; + pnpmPack?: boolean; + prepareBundledAiRuntime?: typeof prepareBundledAiRuntimePackage; + prepareChangelog?: (cwd: string) => Promise<unknown>; + prepareDocsMap?: (cwd: string) => Promise<unknown>; + prepareManifest?: (cwd: string) => Promise<unknown>; + restoreChangelog?: (cwd: string) => Promise<unknown>; + restoreDocsMap?: (cwd: string) => Promise<unknown>; + restoreManifest?: (cwd: string) => Promise<unknown>; + runCaptureImpl?: RunImpl; + runImpl?: CommandRunner; +}; +type MutableJsonRecord = Record<string, unknown>; + +function isDocsMapLifecycle(value: unknown): value is DocsMapLifecycle { + return ( + isRecord(value) && + typeof value.preparePackageDocsMap === "function" && + typeof value.restorePackageDocsMap === "function" + ); +} + +function isPackageManifestLifecycle(value: unknown): value is PackageManifestLifecycle { + return ( + isRecord(value) && + typeof value.preparePackageManifest === "function" && + typeof value.restorePackageManifest === "function" + ); +} + +function hasErrorCode(error: unknown, code: string) { + return isRecord(error) && error.code === code; +} + +const ACTIVE_CHILD_KILLERS = new Set<KillChild>(); +const PACKAGE_BUILD_PLUGIN_SELECTION_ENV_NAMES = [ + "OPENCLAW_EXTENSIONS", + "OPENCLAW_DOCKER_BUILD_EXTENSIONS", + DOCKER_SELECTED_PLUGIN_BUILD_IDS_ENV, + // Public package builds must not inherit a smoke lane's private QA entrypoints. + "OPENCLAW_BUILD_PRIVATE_QA", +]; +const SIGNAL_EXIT_CODES = { + SIGHUP: 129, + SIGINT: 130, + SIGTERM: 143, +} satisfies Partial<Record<NodeJS.Signals, number>>; +let forwardedSignalExitCode: number | undefined; + +class ForwardedSignalExitError extends Error { + exitCode: number; + + constructor(exitCode: number) { + super(`forwarded signal requested exit ${exitCode}`); + this.exitCode = exitCode; + } +} + +for (const signal of Object.keys(SIGNAL_EXIT_CODES) as Array<keyof typeof SIGNAL_EXIT_CODES>) { + process.on(signal, () => { + forwardedSignalExitCode ??= SIGNAL_EXIT_CODES[signal]; + if (ACTIVE_CHILD_KILLERS.size === 0) { + process.exit(forwardedSignalExitCode); + } + for (const killChild of ACTIVE_CHILD_KILLERS) { + killChild(signal); + } + setTimeout(() => { + for (const killChild of ACTIVE_CHILD_KILLERS) { + killChild("SIGKILL"); + } + process.exit(forwardedSignalExitCode); + }, DEFAULT_TIMEOUT_KILL_AFTER_MS); + }); +} + +function resolveTimeoutMs(envName: string, defaultValue: number) { + const raw = process.env[envName]; + if (raw === undefined || raw === "") { + return defaultValue; + } + if (!/^[0-9]+$/u.test(raw)) { + throw new Error(`${envName} must be a positive timeout in milliseconds`); + } + const parsed = Number(raw); + if (!Number.isSafeInteger(parsed) || parsed <= 0) { + throw new Error(`${envName} must be a positive timeout in milliseconds`); + } + return parsed; +} + +function numericTimerValueMs(valueMs: unknown) { + const value = Number(valueMs); + return Number.isFinite(value) ? Math.floor(value) : undefined; +} + +function resolvePackageBuildTimeoutMs( + valueMs: unknown, + fallbackMs: unknown = MAX_TIMER_TIMEOUT_MS, +) { + const value = numericTimerValueMs(valueMs) ?? numericTimerValueMs(fallbackMs); + return Math.min(Math.max(value ?? MAX_TIMER_TIMEOUT_MS, 1), MAX_TIMER_TIMEOUT_MS); +} + +function resolveOptionalTimerTimeoutMs(valueMs: unknown) { + if (valueMs === undefined) { + return undefined; + } + return resolvePackageBuildTimeoutMs(valueMs, 1); +} + +function validateOutputName(value: string) { + if (!/^[A-Za-z0-9][A-Za-z0-9._-]*\.t(?:ar\.)?gz$/u.test(value)) { + throw new Error(`--output-name must be a tarball filename, not a path: ${value}`); + } +} + +function resolvePackedOpenClawFileName(value: string) { + const filename = value.trim(); + if ( + !filename.endsWith(".tgz") || + (!filename.startsWith("openclaw-") && + !filename.includes(":") && + !filename.includes("/") && + !filename.includes("\\")) + ) { + return ""; + } + if ( + !/^openclaw-[A-Za-z0-9._-]+\.tgz$/u.test(filename) || + filename.includes("\0") || + filename !== path.basename(filename) || + filename !== path.win32.basename(filename) + ) { + throw new Error(`npm pack reported unsafe OpenClaw tarball filename: ${filename}`); + } + return filename; +} + +export function parseArgs(argv: string[]) { + const options = parseFlagArgs( + argv, + { + bundlePlugins: [] as string[], + allowUnreleasedChangelog: false, + outputDir: "", + outputName: "", + packJson: "", + pnpmPack: false, + skipBuild: false, + sourceDir: ROOT_DIR, + }, + [ + booleanFlag("--allow-unreleased-changelog", "allowUnreleasedChangelog"), + stringListFlag("--bundle-plugin", "bundlePlugins", { rejectShortOptions: true }), + stringFlag("--output-dir", "outputDir", { rejectShortOptions: true }), + stringFlag("--output-name", "outputName", { rejectShortOptions: true }), + stringFlag("--pack-json", "packJson", { rejectShortOptions: true }), + booleanFlag("--pnpm-pack", "pnpmPack"), + booleanFlag("--skip-build", "skipBuild"), + stringFlag("--source-dir", "sourceDir", { rejectShortOptions: true }), + ], + { + ignoreDoubleDash: false, + onUnhandledArg(arg: string) { + throw new Error(`unknown argument: ${arg}`); + }, + }, + ); + if (options.outputName) { + validateOutputName(options.outputName); + } + if (options.packJson && options.pnpmPack) { + throw new Error("--pack-json cannot be combined with --pnpm-pack"); + } + return options; +} + +function run(command: string, args: string[], cwd: string, options: RunOptions = {}) { + const setupError = + options.captureStdout && options.stdoutFilePath + ? new Error("captureStdout and stdoutFilePath cannot be combined") + : forwardedSignalExitCode && new ForwardedSignalExitError(forwardedSignalExitCode); + if (setupError) { + return Promise.reject(setupError); + } + return new Promise<string>((resolve, reject) => { + const resolvedTimeoutMs = resolveOptionalTimerTimeoutMs(options.timeoutMs); + const resolvedKillAfterMs = resolvePackageBuildTimeoutMs( + options.killAfterMs, + DEFAULT_TIMEOUT_KILL_AFTER_MS, + ); + const useProcessGroup = process.platform !== "win32"; + const env = options.env ?? process.env; + // POSIX callers own PATH; retain that selection while supporting Corepack-only builders. + const npmExecPath = process.platform === "win32" ? env.npm_execpath : ""; + const invocation: { + args: string[]; + command: string; + env?: NodeJS.ProcessEnv; + shell: boolean; + windowsVerbatimArguments?: boolean; + } = + command === "pnpm" + ? resolvePnpmRunner({ cwd, env, npmExecPath, pnpmArgs: args }) + : process.platform === "win32" && command === "npm" + ? resolveNpmRunner({ env, npmArgs: args }) + : { args, command, shell: false }; + let stdinFd: number | undefined; + let stdoutFd: number | undefined; + let child: ReturnType<typeof spawn>; + try { + stdinFd = options.stdinFilePath ? openSync(options.stdinFilePath, "r") : undefined; + stdoutFd = options.stdoutFilePath ? openSync(options.stdoutFilePath, "wx") : undefined; + child = spawn(invocation.command, invocation.args, { + cwd, + stdio: [stdinFd ?? "ignore", stdoutFd ?? "pipe", "pipe"], + env: invocation.env ?? env, + detached: useProcessGroup, + shell: invocation.shell, + windowsVerbatimArguments: invocation.windowsVerbatimArguments, + }); + } finally { + if (stdinFd !== undefined) { + closeSync(stdinFd); + } + if (stdoutFd !== undefined) { + closeSync(stdoutFd); + } + } + let timedOut = false; + let outputLimitExceeded = false; + let stdout = ""; + let stdoutBytes = 0; + let settled = false; + let forceKillTimeout: ReturnType<typeof setTimeout> | undefined; + const maxCapturedStdoutBytes = Math.max( + 1, + options.maxCapturedStdoutBytes ?? DEFAULT_CAPTURED_STDOUT_MAX_BYTES, + ); + const finish = (error: unknown, value = ""): void => { + if (settled) { + return; + } + settled = true; + if (timeout) { + clearTimeout(timeout); + } + ACTIVE_CHILD_KILLERS.delete(killChild); + if (forwardedSignalExitCode !== undefined && ACTIVE_CHILD_KILLERS.size === 0) { + process.exit(forwardedSignalExitCode); + } + if (error) { + reject(toErrorObject(error, "Non-Error rejection")); + return; + } + resolve(value); + }; + const killChild: KillChild = (signal) => { + terminateManagedChild(child, signal); + }; + const processGroupAlive = () => { + if (!useProcessGroup || !child.pid) { + return false; + } + try { + process.kill(-child.pid, 0); + return true; + } catch (error) { + return error instanceof Error && "code" in error && error.code === "EPERM"; + } + }; + const waitForProcessGroupExit = async (timeoutMs: number): Promise<boolean> => { + const deadlineAt = Date.now() + timeoutMs; + while (Date.now() < deadlineAt) { + if (!processGroupAlive()) { + return true; + } + await new Promise((resolvePoll) => { + setTimeout(resolvePoll, PROCESS_GROUP_EXIT_POLL_MS); + }); + } + return !processGroupAlive(); + }; + const terminateChild = (): void => { + killChild("SIGTERM"); + forceKillTimeout = setTimeout(() => { + forceKillTimeout = undefined; + if (settled && !processGroupAlive()) { + return; + } + killChild("SIGKILL"); + }, resolvedKillAfterMs); + forceKillTimeout.unref?.(); + }; + ACTIVE_CHILD_KILLERS.add(killChild); + const timeout = + resolvedTimeoutMs === undefined + ? undefined + : setTimeout(() => { + timedOut = true; + terminateChild(); + }, resolvedTimeoutMs); + timeout?.unref?.(); + const finishAfterTeardown = async (error: unknown, value = ""): Promise<void> => { + if (processGroupAlive()) { + await waitForProcessGroupExit(resolvedKillAfterMs); + } + if (processGroupAlive()) { + killChild("SIGKILL"); + await waitForProcessGroupExit(POST_FORCE_KILL_WAIT_MS); + } + finish(error, value); + }; + if (options.captureStdout) { + child.stdout?.on("data", (chunk) => { + if (outputLimitExceeded) { + return; + } + const chunkText = String(chunk); + const chunkBytes = Buffer.byteLength(chunkText); + if (stdoutBytes + chunkBytes > maxCapturedStdoutBytes) { + outputLimitExceeded = true; + terminateChild(); + return; + } + stdout += chunkText; + stdoutBytes += chunkBytes; + }); + } else if (!options.stdoutFilePath) { + child.stdout?.pipe(process.stderr, { end: false }); + } + child.stderr?.pipe(process.stderr, { end: false }); + child.on("error", (error) => finish(error)); + child.on("close", (status, signal) => { + if (timedOut) { + void finishAfterTeardown( + new Error(`${command} ${args.join(" ")} timed out after ${resolvedTimeoutMs}ms`), + ); + return; + } + if (outputLimitExceeded) { + void finishAfterTeardown( + new Error( + `${command} ${args.join(" ")} exceeded captured stdout limit (${maxCapturedStdoutBytes} bytes)`, + ), + ); + return; + } + if (status === 0) { + finish(undefined, stdout); + return; + } + finish(new Error(`${command} ${args.join(" ")} failed with ${status ?? signal}`)); + }); + }); +} + +export async function buildPackageArtifacts( + sourceDir: string, + packageOptions: PackageOptions = {}, +) { + const runImpl = packageOptions.runImpl ?? run; + const buildEnv: NodeJS.ProcessEnv = { + ...process.env, + OPENCLAW_BUILD_ALL_NO_PNPM: "1", + OPENCLAW_RUN_NODE_SKIP_DTS_BUILD: "0", + }; + for (const envName of PACKAGE_BUILD_PLUGIN_SELECTION_ENV_NAMES) { + delete buildEnv[envName]; + } + if (packageOptions.bundlePlugins?.length) { + // Default frozen-ref harnesses must load without source-only composition dependencies. + const { resolvePackageBundledPlugins } = await import("./lib/package-bundled-plugins.mts"); + const selectedPlugins = resolvePackageBundledPlugins(sourceDir, packageOptions.bundlePlugins); + buildEnv[DOCKER_SELECTED_PLUGIN_BUILD_IDS_ENV] = selectedPlugins.map(({ id }) => id).join(","); + } + const timeoutMs = resolveTimeoutMs( + "OPENCLAW_DOCKER_PACKAGE_BUILD_TIMEOUT_MS", + DEFAULT_PACKAGE_BUILD_TIMEOUT_MS, + ); + const distDir = path.join(sourceDir, "dist"); + assertRealOutputRoot(distDir); + console.error("==> Cleaning OpenClaw package artifacts"); + await fs.rm(distDir, { force: true, recursive: true }); + + // Frozen sources own their build entrypoint and may predate clean:dist. + console.error("==> Building OpenClaw package artifacts"); + await runImpl("pnpm", ["run", "build"], sourceDir, { env: buildEnv, timeoutMs }); +} + +async function runCapture(command: string, args: string[], cwd: string, options: RunOptions = {}) { + return await run(command, args, cwd, { ...options, captureStdout: !options.stdoutFilePath }); +} + +export { run as runCommandForTest, runCapture as runCaptureForTest }; + +async function newestOpenClawTarball(outputDir: string, packOutput: string) { + let fromOutput = ""; + for (const line of packOutput.split(/\r?\n/u)) { + const filename = resolvePackedOpenClawFileName(line); + if (filename) { + fromOutput = filename; + } + } + if (fromOutput) { + return path.join(outputDir, fromOutput); + } + + const entries = await fs.readdir(outputDir); + const packed = entries + .filter((entry) => { + try { + return resolvePackedOpenClawFileName(entry) === entry; + } catch { + return false; + } + }) + .toSorted() + .at(-1); + if (!packed) { + throw new Error(`missing packed OpenClaw tarball in ${outputDir}`); + } + return path.join(outputDir, packed); +} + +async function writePackJson( + packOutput: string, + tarball: string, + packJsonPath: string, + sourceDir: string, +) { + let parsed; + try { + parsed = JSON.parse(packOutput); + } catch (error) { + throw new Error("npm pack --json output was not valid JSON", { cause: error }); + } + const entries = resolveNpmJsonEntries(parsed); + if ( + entries.length === 0 || + entries.some((entry) => !entry || typeof entry !== "object" || Array.isArray(entry)) + ) { + throw new Error("npm pack --json output did not contain package results"); + } + const filename = path.basename(tarball); + for (const entry of entries) { + if ( + entry && + typeof entry === "object" && + "filename" in entry && + typeof entry.filename === "string" + ) { + (entry as MutableJsonRecord).filename = filename; + } + } + const target = path.resolve(sourceDir, packJsonPath); + await fs.mkdir(path.dirname(target), { recursive: true }); + await fs.writeFile(target, `${JSON.stringify(entries, null, 2)}\n`); +} + +async function cleanPackedOpenClawTarballs(outputDir: string) { + let entries: string[]; + try { + entries = await fs.readdir(outputDir); + } catch (error) { + if (error instanceof Error && "code" in error && error.code === "ENOENT") { + entries = []; + } else { + throw error; + } + } + await Promise.all( + entries + .filter((entry) => { + try { + return resolvePackedOpenClawFileName(entry) === entry; + } catch { + return false; + } + }) + .map((entry) => fs.rm(path.join(outputDir, entry), { force: true })), + ); +} + +function isPackedAiRuntimeTarball(filename: string) { + return /^openclaw-ai-[A-Za-z0-9._-]+\.tgz$/u.test(filename); +} + +function runPackageTar( + operation: "extract" | "create", + tarballPath: string, + directory: string, + args: string[] = [], +) { + // Frozen-source harnesses use system tar. Stream archives to avoid drive-as-host + // parsing without changing cwd/PATH for tar or gzip; -C uses forward slashes + // because Windows directory separators can become backslash escapes. + const creating = operation === "create"; + const flags = creating ? ["--no-xattrs", "-czf"] : ["-xzf"]; + return run( + "tar", + [...flags, "-", "-C", directory.replaceAll(path.sep, "/"), ...args], + process.cwd(), + { + ...(creating + ? { + stdoutFilePath: tarballPath, + // BSD tar has separate PAX xattr and AppleDouble (._*) metadata paths. + env: { ...process.env, COPYFILE_DISABLE: "1" }, + } + : { stdinFilePath: tarballPath }), + timeoutMs: resolveTimeoutMs( + "OPENCLAW_DOCKER_PACKAGE_PACK_TIMEOUT_MS", + DEFAULT_PACKAGE_PACK_TIMEOUT_MS, + ), + }, + ); +} + +export async function prepareBundledAiRuntimePackage( + sourceDir: string, + outputDir: string, + runCaptureImpl: RunImpl = runCapture, + packageOptions: PackageOptions = {}, +) { + const packageJsonPath = path.join(sourceDir, "package.json"); + const aiRuntimePackageJsonPath = path.join(sourceDir, "packages", "ai", "package.json"); + const aiRuntimeSourceDir = path.dirname(aiRuntimePackageJsonPath); + const aiRuntimePath = path.join(sourceDir, "node_modules", "@openclaw", "ai"); + const aiRuntimeBackupPath = path.join( + sourceDir, + "node_modules", + "@openclaw", + AI_RUNTIME_BACKUP_DIR, + ); + const extractAiRuntime = + packageOptions.extractAiRuntime ?? + ((tarballPath: string, destination: string) => + runPackageTar("extract", tarballPath, destination, ["--strip-components=1"])); + const prepareManifest = packageOptions.prepareManifest ?? (async () => false); + const restoreManifest = packageOptions.restoreManifest ?? (async () => false); + const originalPackageJson = await fs.readFile(packageJsonPath, "utf8"); + let packageJson: MutableJsonRecord & { + bundleDependencies?: unknown; + dependencies?: Record<string, unknown>; + }; + try { + packageJson = JSON.parse(originalPackageJson) as typeof packageJson; + } catch (error) { + throw new Error(`failed to parse ${packageJsonPath}`, { cause: error }); + } + const aiRuntimeDependency = packageJson.dependencies?.[AI_RUNTIME_PACKAGE]; + let hasAiRuntimeWorkspace = false; + try { + await fs.access(aiRuntimePackageJsonPath); + hasAiRuntimeWorkspace = true; + } catch (error) { + if (!hasErrorCode(error, "ENOENT")) { + throw error; + } + } + // Release checks can package refs from before the AI runtime was split into a workspace package. + if (!hasAiRuntimeWorkspace && aiRuntimeDependency === undefined) { + return async () => {}; + } + if (!hasAiRuntimeWorkspace) { + throw new Error("@openclaw/ai dependency requires the packages/ai workspace"); + } + if (typeof aiRuntimeDependency !== "string") { + throw new Error("root package.json must declare @openclaw/ai as a dependency"); + } + + try { + await fs.access(aiRuntimeBackupPath); + throw new Error(`refusing to overwrite existing ${aiRuntimeBackupPath}`); + } catch (error) { + if (!hasErrorCode(error, "ENOENT")) { + throw error; + } + } + + let packedAiTarballs: string[] = []; + let packageJsonChanged = false; + let originalAiRuntimeMoved = false; + let stagedAiRuntimeCreated = false; + const cleanup = async (): Promise<void> => { + let cleanupError: unknown; + const attempt = async (action: () => Promise<unknown>): Promise<void> => { + try { + await action(); + } catch (error) { + cleanupError ??= error; + } + }; + if (packageJsonChanged) { + await attempt(async () => await fs.writeFile(packageJsonPath, originalPackageJson)); + } + if (stagedAiRuntimeCreated) { + await attempt(async () => await fs.rm(aiRuntimePath, { force: true, recursive: true })); + } + if (originalAiRuntimeMoved) { + await attempt(async () => await fs.rename(aiRuntimeBackupPath, aiRuntimePath)); + } + await attempt(async () => { + await Promise.all(packedAiTarballs.map((filename) => fs.rm(filename, { force: true }))); + }); + packageJsonChanged = false; + stagedAiRuntimeCreated = false; + originalAiRuntimeMoved = false; + packedAiTarballs = []; + if (cleanupError) { + throw toErrorObject(cleanupError, "Package cleanup failed."); + } + }; + + try { + let packError: Error | undefined; + await prepareManifest(aiRuntimeSourceDir); + try { + await runCaptureImpl( + "pnpm", + [ + "--dir", + "packages/ai", + "pack", + "--loglevel=error", + "--use-stderr", + "--pack-destination", + outputDir, + ], + sourceDir, + { + timeoutMs: resolveTimeoutMs( + "OPENCLAW_DOCKER_PACKAGE_PACK_TIMEOUT_MS", + DEFAULT_PACKAGE_PACK_TIMEOUT_MS, + ), + }, + ); + } catch (error) { + packError = toErrorObject(error, "AI runtime package failed."); + } + try { + await restoreManifest(aiRuntimeSourceDir); + } catch (restoreError) { + throw packError ? packagePreparationRestoreError(packError, restoreError) : restoreError; + } + if (packError) { + throw packError; + } + packedAiTarballs = (await fs.readdir(outputDir)) + .filter(isPackedAiRuntimeTarball) + .map((filename) => path.join(outputDir, filename)); + if (packedAiTarballs.length !== 1) { + throw new Error( + `expected one packed @openclaw/ai tarball in ${outputDir}, found ${packedAiTarballs.length}`, + ); + } + + try { + await fs.lstat(aiRuntimePath); + await fs.rename(aiRuntimePath, aiRuntimeBackupPath); + originalAiRuntimeMoved = true; + } catch (error) { + if (!hasErrorCode(error, "ENOENT")) { + throw error; + } + } + await fs.mkdir(aiRuntimePath, { recursive: true }); + stagedAiRuntimeCreated = true; + await extractAiRuntime(packedAiTarballs[0]!, aiRuntimePath); + + const stagedPackageJsonPath = path.join(aiRuntimePath, "package.json"); + const stagedPackageJson = JSON.parse( + await fs.readFile(stagedPackageJsonPath, "utf8"), + ) as MutableJsonRecord & { + dependencies?: Record<string, unknown>; + version?: unknown; + }; + if (typeof stagedPackageJson.version !== "string" || !stagedPackageJson.version) { + throw new Error("packed @openclaw/ai package must declare a version"); + } + const alignedDependencies = validateBundledPackageDependencyAlignment({ + bundledDependencies: stagedPackageJson.dependencies, + bundledPackageLabel: "packed @openclaw/ai", + rootDependencies: packageJson.dependencies, + }); + for (const [name, version] of alignedDependencies) { + packageJson.dependencies![name] = version; + } + // Root owns these exact dependencies. Removing them from the staged copy keeps npm from + // recursively bundling duplicate packages alongside the one private workspace runtime. + delete stagedPackageJson.dependencies; + await fs.writeFile(stagedPackageJsonPath, `${JSON.stringify(stagedPackageJson, null, 2)}\n`); + + packageJson.dependencies![AI_RUNTIME_PACKAGE] = stagedPackageJson.version; + const bundleDependencies = packageJson.bundleDependencies ?? []; + if (!Array.isArray(bundleDependencies)) { + throw new Error("root package.json bundleDependencies must be an array when present"); + } + packageJson.bundleDependencies = [...new Set([...bundleDependencies, AI_RUNTIME_PACKAGE])]; + packageJsonChanged = true; + await fs.writeFile(packageJsonPath, `${JSON.stringify(packageJson, null, 2)}\n`); + return cleanup; + } catch (error) { + await cleanup(); + throw error; + } +} + +async function normalizeOpenClawTarballModes(tarballPath: string) { + // npm can retain restrictive source read bits. Normalize those for non-root + // installers while preserving the archive's executable intent. pnpm derives + // that intent from bin and publishConfig.executableFiles, not source modes. + const stageDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-package-modes-")); + try { + await runPackageTar("extract", tarballPath, stageDir); + let stagedFileCount = 0; + const normalizeStagedModes = async (dir: string): Promise<void> => { + for (const entry of await fs.readdir(dir, { withFileTypes: true })) { + const entryPath = path.join(dir, entry.name); + if (entry.isDirectory()) { + await fs.chmod(entryPath, 0o755); + await normalizeStagedModes(entryPath); + } else if (entry.isFile()) { + // Umask masking on extraction only clears group/other bits, so the + // owner exec bit still says whether the packed entry was executable. + const executable = ((await fs.stat(entryPath)).mode & 0o100) !== 0; + await fs.chmod(entryPath, executable ? 0o755 : 0o644); + stagedFileCount += 1; + } + } + }; + await normalizeStagedModes(stageDir); + if (stagedFileCount === 0) { + throw new Error(`packed OpenClaw tarball has no file entries: ${tarballPath}`); + } + const stageRootEntries = await fs.readdir(stageDir); + const normalizedPath = `${tarballPath}.modes-tmp`; + await fs.rm(normalizedPath, { force: true }); + await runPackageTar("create", normalizedPath, stageDir, stageRootEntries); + await fs.rename(normalizedPath, tarballPath); + } finally { + await fs.rm(stageDir, { force: true, recursive: true }); + } +} + +async function restorePackageSourceArtifacts( + sourceDir: string, + restoreDocsMap: (cwd: string) => Promise<unknown>, + restoreManifest: (cwd: string) => Promise<unknown>, + restoreChangelog: (cwd: string) => Promise<unknown>, +) { + await restoreChangelog(sourceDir); + await restoreManifest(sourceDir); + await Promise.all( + [PACKAGE_LIFECYCLE_PENDING_RELATIVE_PATH, LEGACY_PACKAGE_INSTALL_GUARD_RELATIVE_PATH].map( + (relativePath) => fs.rm(path.join(sourceDir, relativePath), { force: true }), + ), + ); + // Release the lifecycle receipt only after every other source mutation settles. + await restoreDocsMap(sourceDir); +} + +async function loadSourcePackageLifecycle( + sourceDir: string, + moduleName: string, + validate: (value: unknown) => boolean, +) { + const modulePath = path.join(sourceDir, "scripts", moduleName); + try { + await fs.access(modulePath); + } catch (error) { + if (hasErrorCode(error, "ENOENT")) { + return null; + } + throw error; + } + const lifecycle: unknown = await import(pathToFileURL(modulePath).href); + if (!validate(lifecycle)) { + throw new Error(`source package lifecycle is invalid: ${modulePath}`); + } + return lifecycle; +} + +function packagePreparationRestoreError(error: unknown, restoreError: unknown) { + return new AggregateError([error, restoreError], "Package operation and cleanup both failed.", { + cause: error, + }); +} + +export async function packOpenClawPackageForDocker( + sourcePath: string, + outputPath: string, + packageOptions: PackageOptions = {}, +) { + const runCaptureImpl = packageOptions.runCaptureImpl ?? runCapture; + const prepareChangelog = + packageOptions.prepareChangelog ?? + ((cwd: string) => + preparePackageChangelog(cwd, { + allowUnreleased: packageOptions.allowUnreleasedChangelog, + })); + const restoreChangelog = packageOptions.restoreChangelog ?? restorePackageChangelog; + // Frozen refs own their package contents. Only refs carrying this lifecycle ship a generated map. + const sourceDocsMapLifecycle = + packageOptions.prepareDocsMap && packageOptions.restoreDocsMap + ? null + : ((await loadSourcePackageLifecycle( + sourcePath, + "package-docs-map.mjs", + isDocsMapLifecycle, + )) as DocsMapLifecycle | null); + const prepareDocsMap = + packageOptions.prepareDocsMap ?? + sourceDocsMapLifecycle?.preparePackageDocsMap ?? + (async () => false); + const restoreDocsMap = + packageOptions.restoreDocsMap ?? + sourceDocsMapLifecycle?.restorePackageDocsMap ?? + (async () => false); + const sourceManifestLifecycle = + packageOptions.prepareManifest && packageOptions.restoreManifest + ? null + : ((await loadSourcePackageLifecycle( + sourcePath, + "package-manifest.mjs", + isPackageManifestLifecycle, + )) as PackageManifestLifecycle | null); + const prepareManifest = + packageOptions.prepareManifest ?? + sourceManifestLifecycle?.preparePackageManifest ?? + (async () => false); + const restoreManifest = + packageOptions.restoreManifest ?? + sourceManifestLifecycle?.restorePackageManifest ?? + (async () => false); + const prepareBundledAiRuntime = + packageOptions.prepareBundledAiRuntime ?? prepareBundledAiRuntimePackage; + const packTool = packageOptions.pnpmPack ? "pnpm" : "npm"; + if (packageOptions.packJsonPath && packageOptions.pnpmPack) { + throw new Error("packJsonPath cannot be combined with pnpmPack"); + } + console.error("==> Packing OpenClaw package"); + // This receipt is the package lifecycle lock; acquire it before touching CHANGELOG.md. + await prepareDocsMap(sourcePath); + const deferSignalExit: KillChild = () => {}; + ACTIVE_CHILD_KILLERS.add(deferSignalExit); + const releaseSignalExit = () => { + ACTIVE_CHILD_KILLERS.delete(deferSignalExit); + if (forwardedSignalExitCode !== undefined) { + throw new ForwardedSignalExitError(forwardedSignalExitCode); + } + }; + try { + console.error("==> Writing OpenClaw package inventory"); + await writePackageInventoryForDocker(sourcePath, packageOptions.runImpl ?? run); + + await prepareManifest(sourcePath); + await prepareChangelog(sourcePath); + } catch (error) { + try { + await restorePackageSourceArtifacts( + sourcePath, + restoreDocsMap, + restoreManifest, + restoreChangelog, + ); + } catch (restoreError) { + releaseSignalExit(); + throw packagePreparationRestoreError(error, restoreError); + } + releaseSignalExit(); + throw error; + } + let packOutput = ""; + let packageError: unknown; + let packReceiptDir: string | undefined; + try { + let cleanupBundledAiRuntime = async () => {}; + let cleanupBundledPlugins = async () => {}; + try { + await cleanPackedOpenClawTarballs(outputPath); + if (packageOptions.bundlePlugins?.length) { + const { preparePackageBundledPlugins } = await import("./lib/package-bundled-plugins.mts"); + cleanupBundledPlugins = await preparePackageBundledPlugins( + sourcePath, + packageOptions.bundlePlugins, + ); + } + cleanupBundledAiRuntime = await prepareBundledAiRuntime( + sourcePath, + outputPath, + runCaptureImpl, + { + prepareManifest, + restoreManifest, + }, + ); + // AI staging materializes the bundled tree; pack must not inherit the + // source workspace's isolated linker setting for that prepared bundle. + const packArgs = [ + "pack", + "--silent", + ...(packTool === "pnpm" + ? ["--config.ignore-scripts=true", "--config.node-linker=hoisted"] + : ["--ignore-scripts"]), + "--pack-destination", + outputPath, + ...(packTool === "npm" ? ["--json=false"] : []), + ]; + packOutput = await runCaptureImpl(packTool, packArgs, sourcePath, { + timeoutMs: resolveTimeoutMs( + "OPENCLAW_DOCKER_PACKAGE_PACK_TIMEOUT_MS", + DEFAULT_PACKAGE_PACK_TIMEOUT_MS, + ), + }); + } finally { + try { + await cleanupBundledAiRuntime(); + } finally { + try { + await cleanupBundledPlugins(); + } finally { + await restorePackageSourceArtifacts( + sourcePath, + restoreDocsMap, + restoreManifest, + restoreChangelog, + ); + } + } + } + // Scan the emptied pnpm destination instead of trusting its absolute-path output. + let tarball = await newestOpenClawTarball( + outputPath, + packageOptions.pnpmPack ? "" : packOutput, + ); + if (packageOptions.outputName) { + const target = path.join(outputPath, packageOptions.outputName); + if (target !== tarball) { + await fs.rm(target, { force: true }); + await fs.rename(tarball, target); + tarball = target; + } + } + await (packageOptions.normalizeTarballModes ?? normalizeOpenClawTarballModes)(tarball); + if (packageOptions.packJsonPath) { + // npm's original receipt predates normalization. Inspect the finished bytes; + // dry-run preserves the archive while npm owns hashes, modes, and inventory. + packReceiptDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-npm-pack-receipt-")); + const packReceiptPath = path.join(packReceiptDir, "pack.json"); + await runCaptureImpl( + "npm", + ["pack", tarball, "--dry-run", "--json", "--ignore-scripts", "--offline", "--silent"], + sourcePath, + { + stdoutFilePath: packReceiptPath, + timeoutMs: resolveTimeoutMs( + "OPENCLAW_DOCKER_PACKAGE_PACK_TIMEOUT_MS", + DEFAULT_PACKAGE_PACK_TIMEOUT_MS, + ), + }, + ); + await writePackJson( + await fs.readFile(packReceiptPath, "utf8"), + tarball, + packageOptions.packJsonPath, + sourcePath, + ); + } + return tarball; + } catch (error) { + packageError = error; + throw error; + } finally { + try { + if (packReceiptDir) { + try { + await fs.rm(packReceiptDir, { force: true, recursive: true }); + } catch (cleanupError) { + // oxlint-disable-next-line eslint/no-unsafe-finally -- Preserve primary and cleanup failures. + throw packageError + ? packagePreparationRestoreError(packageError, cleanupError) + : cleanupError; + } + } + } finally { + releaseSignalExit(); + } + } +} + +export async function writePackageInventoryForDocker( + sourceDir: string, + runImpl: CommandRunner = run, +) { + // Frozen release refs own their inventory shape; run their writer instead of importing current-main helpers. + // Resolve the loader from that checkout too: the workflow harness may install production deps only. + const sourceRequire = createRequire(path.join(sourceDir, "package.json")); + const tsxModuleUrl = pathToFileURL(sourceRequire.resolve("tsx")).href; + await runImpl( + "node", + ["--import", tsxModuleUrl, path.join(sourceDir, "scripts/write-package-dist-inventory.ts")], + sourceDir, + { + timeoutMs: resolveTimeoutMs( + "OPENCLAW_DOCKER_PACKAGE_INVENTORY_TIMEOUT_MS", + DEFAULT_PACKAGE_INVENTORY_TIMEOUT_MS, + ), + }, + ); +} + +async function main() { + const options = parseArgs(process.argv.slice(2)); + const sourceDir = path.resolve(ROOT_DIR, options.sourceDir || ROOT_DIR); + const outputDir = path.resolve( + ROOT_DIR, + options.outputDir || path.join(".artifacts", "docker-e2e-package"), + ); + await fs.mkdir(outputDir, { recursive: true }); + + if (!options.skipBuild) { + await buildPackageArtifacts(sourceDir, { bundlePlugins: options.bundlePlugins }); + } + + const tarball = await packOpenClawPackageForDocker(sourceDir, outputDir, { + bundlePlugins: options.bundlePlugins, + allowUnreleasedChangelog: options.allowUnreleasedChangelog, + outputName: options.outputName, + packJsonPath: options.packJson, + pnpmPack: options.pnpmPack, + }); + + console.error("==> Checking OpenClaw package tarball"); + const checkStartedAt = Date.now(); + await run( + "node", + [ + path.join(ROOT_DIR, "scripts/check-openclaw-package-tarball.mjs"), + "--require-bundled-workspace-deps", + tarball, + ], + sourceDir, + { + timeoutMs: resolveTimeoutMs( + "OPENCLAW_DOCKER_PACKAGE_TARBALL_CHECK_TIMEOUT_MS", + DEFAULT_PACKAGE_TARBALL_CHECK_TIMEOUT_MS, + ), + }, + ); + console.error( + `==> OpenClaw package tarball check finished in ${Math.round((Date.now() - checkStartedAt) / 1000)}s`, + ); + + process.stdout.write(`${tarball}\n`); +} + +if ( + process.argv[1] && + (await fs.realpath(process.argv[1])) === (await fs.realpath(fileURLToPath(import.meta.url))) +) { + await main().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : String(error)); + const exitCode = + error && typeof error === "object" && "exitCode" in error ? error.exitCode : undefined; + process.exit(typeof exitCode === "number" && Number.isInteger(exitCode) ? exitCode : 1); + }); +} diff --git a/scripts/package-source-dependencies.d.mts b/scripts/package-source-dependencies.d.mts new file mode 100644 index 0000000000000000000000000000000000000000..5c68f0f36d01b56cd606f86023781d509548e2a1 --- /dev/null +++ b/scripts/package-source-dependencies.d.mts @@ -0,0 +1,6 @@ +export function validateBundledPackageDependencyAlignment(params: { + bundledDependencies?: unknown; + bundledPackageLabel: string; + rootDependencies?: unknown; + rootPackageLabel?: string; +}): Array<[string, string]>; diff --git a/scripts/package-source-preflight.d.mts b/scripts/package-source-preflight.d.mts new file mode 100644 index 0000000000000000000000000000000000000000..c147cd65bdf6cfa0aa33aab6c8a50e717ee73b78 --- /dev/null +++ b/scripts/package-source-preflight.d.mts @@ -0,0 +1,18 @@ +type PackageSourcePreflightOptions = { + allowUnreleasedChangelog?: boolean; +}; + +export function validatePackageSource(params: { + aiManifestContent: string | null; + allowUnreleasedChangelog?: boolean; + changelogContent: string; + rootManifestContent: string; +}): string; +export function validatePackageSourceRef( + ref: string, + options?: PackageSourcePreflightOptions, +): string; +export function validatePackageSourceDir( + sourceDir: string, + options?: PackageSourcePreflightOptions, +): string; diff --git a/scripts/plan-targeted-docker-lane-groups.mjs b/scripts/plan-targeted-docker-lane-groups.mjs new file mode 100644 index 0000000000000000000000000000000000000000..a89ce896711926c96099a60e9184947e5566d55d --- /dev/null +++ b/scripts/plan-targeted-docker-lane-groups.mjs @@ -0,0 +1,217 @@ +// Plans grouped targeted Docker lane matrix entries without installed dependencies. +import { fileURLToPath } from "node:url"; +import { parsePositiveInt } from "./lib/numeric-options.mjs"; +import { + CUSTOM_PLUGIN_SIBLINGS_BASELINE, + normalizeUpgradeSurvivorBaselineSpec, + parseUpgradeSurvivorBaselineSpecs, + parseUpgradeSurvivorScenarios, + supportsUpgradeSurvivorScenarioAtBaseline, +} from "./lib/upgrade-survivor-policy.mjs"; + +const BASELINE_SHARDED_LANES = new Set(["published-upgrade-survivor", "update-migration"]); +const SURVIVOR_SCENARIOS_PER_GROUP = 3; + +function splitTokens(raw) { + return [ + ...new Set( + String(raw ?? "") + .split(/[,\s]+/u) + .map((token) => token.trim()) + .filter(Boolean), + ), + ]; +} + +function sanitizeLabel(value) { + return ( + String(value) + .replace(/^openclaw@/u, "") + .replace(/[^A-Za-z0-9._-]+/g, "-") + .replace(/^-+|-+$/g, "") || "targeted" + ); +} + +/** + * Groups selected Docker lanes and expands sharded upgrade-survivor baselines. + * + * @param {{ + * groupSize?: number | string; + * lanes?: string; + * upgradeSurvivorBaseline?: string; + * upgradeSurvivorBaselineScope?: "all-scenarios" | "legacy-operator-state"; + * upgradeSurvivorBaselines?: string; + * upgradeSurvivorScenarios?: string; + * }} [options] + * @returns {{ + * docker_lanes: string; + * label: string; + * published_upgrade_survivor_baselines?: string; + * published_upgrade_survivor_scenarios?: string; + * timeout_minutes?: number; + * }[]} + */ +export function planTargetedDockerLaneGroups({ + groupSize = 1, + lanes, + upgradeSurvivorBaseline, + upgradeSurvivorBaselineScope = "all-scenarios", + upgradeSurvivorBaselines = "", + upgradeSurvivorScenarios = "", +} = {}) { + const selectedLanes = splitTokens(lanes); + if (selectedLanes.length === 0) { + throw new Error("docker_lanes is required when planning targeted Docker lane groups."); + } + + const parsedGroupSize = parsePositiveInt(groupSize, "groupSize"); + if (!["all-scenarios", "legacy-operator-state"].includes(upgradeSurvivorBaselineScope)) { + throw new Error("Unknown upgrade survivor baseline scope."); + } + const baselineSpecs = parseUpgradeSurvivorBaselineSpecs(upgradeSurvivorBaselines); + const hasExpandedSurvivorScenarios = splitTokens(upgradeSurvivorScenarios).length > 0; + const survivorScenarios = selectedLanes.some((lane) => BASELINE_SHARDED_LANES.has(lane)) + ? parseUpgradeSurvivorScenarios(upgradeSurvivorScenarios) + : []; + let pairedScenarios; + if ( + upgradeSurvivorBaselineScope === "legacy-operator-state" && + selectedLanes.some((lane) => BASELINE_SHARDED_LANES.has(lane)) + ) { + const predecessor = normalizeUpgradeSurvivorBaselineSpec(upgradeSurvivorBaseline); + if (!predecessor || !/^openclaw@\d{4}\.\d+\.\d+(?:-\d+)?$/u.test(predecessor)) { + throw new Error("Supported-line pairing requires an exact published predecessor."); + } + if (baselineSpecs.length === 0) { + throw new Error("Supported-line pairing requires resolved baselines."); + } + const requested = survivorScenarios.length > 0 ? survivorScenarios : ["base"]; + pairedScenarios = new Map(baselineSpecs.map((baseline) => [baseline, []])); + for (const scenario of requested) { + // Keep the reported first-hop driver even when source and published + // packages share a version and generic baseline resolution omits it. + const baselines = + scenario === "custom-plugin-siblings" + ? [CUSTOM_PLUGIN_SIBLINGS_BASELINE] + : scenario === "legacy-operator-state" + ? baselineSpecs + : [predecessor]; + for (const baseline of baselines) { + if (!supportsUpgradeSurvivorScenarioAtBaseline(scenario, baseline)) { + continue; + } + const scenarios = pairedScenarios.get(baseline) ?? []; + scenarios.push(scenario); + pairedScenarios.set(baseline, scenarios); + } + } + } + const groups = []; + let pendingLanes = []; + + const addGroup = (group) => { + const groupLanes = splitTokens(group.docker_lanes); + if ( + hasExpandedSurvivorScenarios && + groupLanes.some((lane) => BASELINE_SHARDED_LANES.has(lane)) + ) { + group.timeout_minutes = 90; + } + groups.push(group); + }; + + const flushPending = () => { + if (pendingLanes.length === 0) { + return; + } + const first = sanitizeLabel(pendingLanes[0]); + const last = sanitizeLabel(pendingLanes[pendingLanes.length - 1]); + const label = pendingLanes.length === 1 ? first : `${first}--${last}`; + addGroup({ docker_lanes: pendingLanes.join(" "), label }); + pendingLanes = []; + }; + + for (const lane of selectedLanes) { + if (BASELINE_SHARDED_LANES.has(lane) && pairedScenarios) { + flushPending(); + for (const [baseline, scenarios] of pairedScenarios) { + const label = `${sanitizeLabel(lane)}-${sanitizeLabel(baseline)}`; + for (let offset = 0; offset < scenarios.length; offset += SURVIVOR_SCENARIOS_PER_GROUP) { + addGroup({ + docker_lanes: lane, + label: + scenarios.length > SURVIVOR_SCENARIOS_PER_GROUP + ? `${label}-scenarios-${offset / SURVIVOR_SCENARIOS_PER_GROUP + 1}` + : label, + published_upgrade_survivor_baselines: baseline, + published_upgrade_survivor_scenarios: scenarios + .slice(offset, offset + SURVIVOR_SCENARIOS_PER_GROUP) + .join(" "), + }); + } + } + continue; + } + if ( + BASELINE_SHARDED_LANES.has(lane) && + survivorScenarios.length > SURVIVOR_SCENARIOS_PER_GROUP + ) { + flushPending(); + for (const baselineSpec of baselineSpecs.length > 0 ? baselineSpecs : [undefined]) { + // Filter at the policy owner before partitioning so old baselines cannot + // receive a shard containing only scenarios they never supported. + const scenarios = survivorScenarios.filter((scenario) => + supportsUpgradeSurvivorScenarioAtBaseline(scenario, baselineSpec), + ); + const label = [lane, baselineSpec].filter(Boolean).map(sanitizeLabel).join("-"); + for (let offset = 0; offset < scenarios.length; offset += SURVIVOR_SCENARIOS_PER_GROUP) { + addGroup({ + docker_lanes: lane, + label: `${label}-scenarios-${offset / SURVIVOR_SCENARIOS_PER_GROUP + 1}`, + ...(baselineSpec ? { published_upgrade_survivor_baselines: baselineSpec } : {}), + published_upgrade_survivor_scenarios: scenarios + .slice(offset, offset + SURVIVOR_SCENARIOS_PER_GROUP) + .join(" "), + }); + } + } + continue; + } + if (BASELINE_SHARDED_LANES.has(lane) && baselineSpecs.length > 1) { + flushPending(); + for (const baselineSpec of baselineSpecs) { + addGroup({ + docker_lanes: lane, + label: `${sanitizeLabel(lane)}-${sanitizeLabel(baselineSpec)}`, + published_upgrade_survivor_baselines: baselineSpec, + }); + } + continue; + } + + pendingLanes.push(lane); + if (pendingLanes.length >= parsedGroupSize) { + flushPending(); + } + } + + flushPending(); + return groups; +} + +const isMain = process.argv[1] ? fileURLToPath(import.meta.url) === process.argv[1] : false; + +if (isMain) { + process.stdout.write( + JSON.stringify( + planTargetedDockerLaneGroups({ + groupSize: process.env.GROUP_SIZE, + lanes: process.env.LANES, + upgradeSurvivorBaseline: process.env.OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPEC, + upgradeSurvivorBaselineScope: process.env.OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SCOPE, + upgradeSurvivorBaselines: process.env.OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPECS, + upgradeSurvivorScenarios: process.env.OPENCLAW_UPGRADE_SURVIVOR_SCENARIOS, + }), + ), + ); +} diff --git a/scripts/plugin-clawhub-release-check.ts b/scripts/plugin-clawhub-release-check.ts new file mode 100644 index 0000000000000000000000000000000000000000..d103b8dbed791daac21172db4a29e87bf1b755df --- /dev/null +++ b/scripts/plugin-clawhub-release-check.ts @@ -0,0 +1,76 @@ +#!/usr/bin/env -S node --import tsx +// Plugin Clawhub Release Check script supports OpenClaw repository automation. + +import { pathToFileURL } from "node:url"; +import { + assertPluginReleaseDependencyFreshness, + collectClawHubPublishablePluginPackages, + collectClawHubVersionGateErrors, + assertPluginReleaseVersionFloors, + parsePluginReleaseArgs, + resolveSelectedClawHubPublishablePluginPackages, +} from "./lib/plugin-clawhub-release.ts"; +import type { NpmLatestVersionResolver } from "./lib/plugin-npm-release.ts"; + +export async function runPluginClawHubReleaseCheck( + argv: string[], + options: { + rootDir?: string; + resolveLatestVersion?: NpmLatestVersionResolver; + } = {}, +) { + const { selection, selectionMode, baseRef, headRef } = parsePluginReleaseArgs(argv); + const rootDir = options.rootDir ?? "."; + const publishable = collectClawHubPublishablePluginPackages(rootDir, { + packageNames: + selectionMode === "all-publishable" || selection.length === 0 ? undefined : selection, + }); + const gitRange = baseRef && headRef ? { baseRef, headRef } : undefined; + const selected = resolveSelectedClawHubPublishablePluginPackages({ + plugins: publishable, + selection, + selectionMode, + gitRange, + rootDir, + }); + + if (selectionMode !== undefined || selection.length > 0) { + assertPluginReleaseVersionFloors(selected, "plugin-clawhub-release-check"); + } + assertPluginReleaseDependencyFreshness( + selected, + "plugin-clawhub-release-check", + options.resolveLatestVersion, + ); + + if (gitRange) { + const errors = collectClawHubVersionGateErrors({ + plugins: publishable, + gitRange, + rootDir, + }); + if (errors.length > 0) { + throw new Error( + `plugin-clawhub-release-check: version bumps required before ClawHub publish:\n${errors + .map((error) => ` - ${error}`) + .join("\n")}`, + ); + } + } + + console.log("plugin-clawhub-release-check: publishable plugin metadata looks OK."); + if (gitRange && selected.length === 0) { + console.log( + ` - no publishable plugin package changes detected between ${gitRange.baseRef} and ${gitRange.headRef}`, + ); + } + for (const plugin of selected) { + console.log( + ` - ${plugin.packageName}@${plugin.version} (${plugin.channel}, ${plugin.extensionId})`, + ); + } +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { + await runPluginClawHubReleaseCheck(process.argv.slice(2)); +} diff --git a/scripts/plugin-npm-publish.sh b/scripts/plugin-npm-publish.sh new file mode 100644 index 0000000000000000000000000000000000000000..fb1bc99eb52b4aecb1e4d752c50c8be3ce61cd05 --- /dev/null +++ b/scripts/plugin-npm-publish.sh @@ -0,0 +1,285 @@ +#!/usr/bin/env bash +# Bash 5.3+ can deadlock writing heredoc pipes on macOS before the reader starts. +if [[ ${OSTYPE:-} == darwin* && $BASH != /bin/bash ]] && ((BASH_VERSINFO[0] > 5 || (BASH_VERSINFO[0] == 5 && BASH_VERSINFO[1] >= 3))); then + exec /bin/bash "$0" "$@" +fi + +set -euo pipefail + +usage() { + echo "usage: bash scripts/plugin-npm-publish.sh [--repo-root <dir>] [--dry-run|--pack|--pack-dry-run|--publish] <package-dir>" +} + +if [[ "${1:-}" == "--help" || "${1:-}" == "-h" ]]; then + usage + exit 0 +fi + +tooling_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +repo_root="${tooling_root}" +if [[ "${1:-}" == "--repo-root" ]]; then + if [[ -z "${2:-}" || "${2:-}" == -* ]]; then + echo "--repo-root requires a directory" >&2 + exit 2 + fi + repo_root="$(cd "$2" && pwd)" || { + echo "repository root is not a directory: $2" >&2 + exit 2 + } + shift 2 +fi + +mode="${1:-}" +if [[ "${mode}" != "--dry-run" && "${mode}" != "--pack" && "${mode}" != "--pack-dry-run" && "${mode}" != "--publish" ]]; then + usage >&2 + exit 2 +fi +shift + +if [[ "${1:-}" == "--" ]]; then + shift +fi +package_dir="" +if [[ "$#" -gt 0 ]]; then + case "$1" in + -*) echo "unexpected plugin npm package-dir option: $1" >&2; exit 2 ;; + /*) package_dir="$1"; shift ;; + *) package_dir="${repo_root}/$1"; shift ;; + esac +fi +if [[ -z "${package_dir}" ]]; then + echo "missing package dir" >&2 + exit 2 +fi +if [[ "$#" -gt 0 ]]; then + echo "unexpected plugin npm publish argument: $1" >&2 + exit 2 +fi +if [[ "${mode}" == "--pack" && -z "${OPENCLAW_PLUGIN_NPM_PACK_OUTPUT_DIR:-}" ]]; then + echo "--pack requires OPENCLAW_PLUGIN_NPM_PACK_OUTPUT_DIR" >&2 + exit 2 +fi + +cd "${tooling_root}" +package_name="$(node -e 'const pkg = require(require("node:path").resolve(process.argv[1], "package.json")); console.log(pkg.name)' "${package_dir}")" +package_version="$(node -e 'const pkg = require(require("node:path").resolve(process.argv[1], "package.json")); console.log(pkg.version)' "${package_dir}")" +current_beta_version="$(npm view "${package_name}" dist-tags.beta 2>/dev/null || true)" +log() { + if [[ "${mode}" == "--pack" || "${mode}" == "--pack-dry-run" ]]; then + printf '%s\n' "$*" >&2 + else + printf '%s\n' "$*" + fi +} +publish_plan_output="$( + PACKAGE_VERSION="${package_version}" CURRENT_BETA_VERSION="${current_beta_version}" PUBLISH_MODE="${mode}" node --input-type=module <<'EOF' +import { + resolveNpmDistTagMirrorAuth, + resolveNpmPublishPlan, + shouldRequireNpmDistTagMirrorAuth, +} from "./scripts/lib/npm-publish-plan.mjs"; + +const plan = resolveNpmPublishPlan( + process.env.PACKAGE_VERSION ?? "", + process.env.CURRENT_BETA_VERSION, + process.env.OPENCLAW_PLUGIN_NPM_PUBLISH_TAG, +); +const auth = resolveNpmDistTagMirrorAuth({ + nodeAuthToken: process.env.NODE_AUTH_TOKEN, + npmToken: process.env.NPM_TOKEN, +}); +const shouldRequireMirrorAuth = shouldRequireNpmDistTagMirrorAuth({ + mode: process.env.PUBLISH_MODE === "--publish" ? "--publish" : "--dry-run", + mirrorDistTags: plan.mirrorDistTags, + hasAuth: auth.hasAuth, +}); +console.log(plan.channel); +console.log(plan.publishTag); +console.log(plan.mirrorDistTags.join(",")); +console.log(auth.source); +console.log(shouldRequireMirrorAuth ? "required" : "optional"); +EOF +)" +release_channel="$(printf '%s\n' "${publish_plan_output}" | sed -n '1p')" +publish_tag="$(printf '%s\n' "${publish_plan_output}" | sed -n '2p')" +mirror_dist_tags_csv="$(printf '%s\n' "${publish_plan_output}" | sed -n '3p')" +mirror_auth_source="$(printf '%s\n' "${publish_plan_output}" | sed -n '4p')" +mirror_auth_requirement="$(printf '%s\n' "${publish_plan_output}" | sed -n '5p')" +mirror_auth_source="${mirror_auth_source:-none}" +mirror_auth_requirement="${mirror_auth_requirement:-optional}" +publish_cmd=(npm publish --access public --tag "${publish_tag}") +if [[ "${OPENCLAW_NPM_PUBLISH_PROVENANCE:-1}" != "0" && "${OPENCLAW_NPM_PUBLISH_PROVENANCE:-1}" != "false" ]]; then + publish_cmd+=(--provenance) +fi + +log "Resolved package dir: ${package_dir}" +log "Resolved repository root: ${repo_root}" +log "Resolved package name: ${package_name}" +log "Resolved package version: ${package_version}" +log "Current beta dist-tag: ${current_beta_version:-<missing>}" +log "Resolved release channel: ${release_channel}" +log "Resolved publish tag: ${publish_tag}" +log "Resolved mirror dist-tags: ${mirror_dist_tags_csv:-<none>}" +log "Mirror dist-tag auth source: ${mirror_auth_source}" +log "Mirror dist-tag auth requirement: ${mirror_auth_requirement}" + +build_package_runtime() { + if [[ "${OPENCLAW_PLUGIN_NPM_RUNTIME_BUILD:-1}" == "0" || "${OPENCLAW_PLUGIN_NPM_RUNTIME_BUILD:-1}" == "false" ]]; then + log "Package-local runtime build: skipped" + return + fi + log "Package-local runtime build: ${package_dir}" + ( + cd "${repo_root}" + node "${tooling_root}/scripts/lib/plugin-npm-runtime-build.mjs" "${package_dir}" >&2 + ) +} + +check_package_npm_lock() { + log "Package-local npm package-lock check: ${package_dir}" + ( + cd "${repo_root}" + OPENCLAW_NPM_PACKAGE_LOCK_REPO_ROOT="${repo_root}" \ + node "${tooling_root}/scripts/generate-npm-package-lock.mjs" \ + --package-dir "${package_dir}" >&2 + ) +} + +mirror_auth_token="" +case "${mirror_auth_source}" in + node-auth-token) + mirror_auth_token="${NODE_AUTH_TOKEN:-}" + ;; + npm-token) + mirror_auth_token="${NPM_TOKEN:-}" + ;; +esac +publish_auth_token="${mirror_auth_token}" +publish_auth_source="${mirror_auth_source}" +if [[ "${OPENCLAW_NPM_PUBLISH_AUTH_MODE:-}" == "trusted-publisher" ]]; then + publish_auth_token="" + publish_auth_source="trusted-publisher" +fi +publish_provenance="without provenance" +if [[ " ${publish_cmd[*]} " == *" --provenance "* ]]; then + publish_provenance="with provenance" +fi +if [[ -n "${publish_auth_token}" ]]; then + log "Publish auth: ${publish_auth_source} ${publish_provenance}" +else + log "Publish auth: GitHub OIDC trusted publishing" +fi + +if [[ "${mirror_auth_requirement}" == "required" && -z "${mirror_auth_token}" ]]; then + echo "npm dist-tag mirroring requires explicit npm auth via NODE_AUTH_TOKEN or NPM_TOKEN." >&2 + echo "Refusing publish before npm latest/beta promotion can diverge." >&2 + exit 1 +fi + +verify_release_tooling_identity() { + if [[ "${OPENCLAW_RELEASE_TOOLING_IDENTITY_REQUIRED:-}" != "true" ]]; then + return 0 + fi + identity_args=( + verify + --repository "${OPENCLAW_RELEASE_TOOLING_REPOSITORY:-}" + --workflow-ref "${OPENCLAW_RELEASE_TOOLING_REF:-}" + --workflow-full-ref "${OPENCLAW_RELEASE_TOOLING_FULL_REF:-}" + --workflow-sha "${OPENCLAW_RELEASE_TOOLING_SHA:-}" + --release-publish-run-id "${OPENCLAW_RELEASE_PUBLISH_RUN_ID:-}" + --release-publish-run-attempt "${OPENCLAW_RELEASE_PUBLISH_RUN_ATTEMPT:-}" + --release-publish-ref "${OPENCLAW_RELEASE_PUBLISH_REF:-}" + --release-publish-full-ref "${OPENCLAW_RELEASE_PUBLISH_FULL_REF:-}" + --release-publish-parent-state-policy "${OPENCLAW_RELEASE_PUBLISH_PARENT_STATE_POLICY:-}" + ) + if [[ "${OPENCLAW_RELEASE_TOOLING_ALLOW_PREVALIDATED_REF:-}" == "true" ]]; then + identity_args+=(--allow-prevalidated-ref) + fi + node "${tooling_root}/scripts/release-tooling-identity.mjs" "${identity_args[@]}" +} + +if [[ "${mode}" == "--pack" || "${mode}" == "--pack-dry-run" ]]; then + { + printf 'Publish command:' + printf ' %q' "${publish_cmd[@]}" + printf '\n' + } >&2 +else + printf 'Publish command:' + printf ' %q' "${publish_cmd[@]}" + printf '\n' +fi + +if [[ "${mode}" == "--dry-run" ]]; then + exit 0 +fi + +build_package_runtime +check_package_npm_lock + +if [[ "${mode}" == "--pack" || "${mode}" == "--pack-dry-run" ]]; then + pack_args=(npm pack --json --ignore-scripts) + if [[ "${mode}" == "--pack-dry-run" ]]; then + pack_args+=(--dry-run) + else + mkdir -p "${OPENCLAW_PLUGIN_NPM_PACK_OUTPUT_DIR}" + pack_output_dir="$(cd "${OPENCLAW_PLUGIN_NPM_PACK_OUTPUT_DIR}" && pwd)" + pack_args+=(--pack-destination "${pack_output_dir}") + fi + ( + cd "${repo_root}" + OPENCLAW_NPM_PACKAGE_LOCK_REPO_ROOT="${repo_root}" \ + OPENCLAW_PLUGIN_NPM_BUNDLE_DEPENDENCIES=1 \ + node "${tooling_root}/scripts/lib/plugin-npm-package-manifest.mjs" \ + --run "${package_dir}" -- "${pack_args[@]}" + ) + exit 0 +fi + +( + cleanup_files=() + trap 'rm -f ${cleanup_files[@]+"${cleanup_files[@]}"}' EXIT + run_with_manifest_overlay() { + ( + cd "${repo_root}" + OPENCLAW_NPM_PACKAGE_LOCK_REPO_ROOT="${repo_root}" \ + OPENCLAW_PLUGIN_NPM_BUNDLE_DEPENDENCIES=1 \ + node "${tooling_root}/scripts/lib/plugin-npm-package-manifest.mjs" \ + --run "${package_dir}" -- "$@" + ) + } + publish_userconfig="" + if [[ -n "${publish_auth_token}" ]]; then + publish_userconfig="$(mktemp)" + cleanup_files+=("${publish_userconfig}") + chmod 0600 "${publish_userconfig}" + printf '%s\n' "//registry.npmjs.org/:_authToken=${publish_auth_token}" > "${publish_userconfig}" + fi + verify_release_tooling_identity + if [[ -n "${publish_auth_token}" ]]; then + NPM_CONFIG_USERCONFIG="${publish_userconfig}" run_with_manifest_overlay "${publish_cmd[@]}" + else + run_with_manifest_overlay "${publish_cmd[@]}" + fi + + if [[ -n "${mirror_dist_tags_csv}" ]]; then + mirror_userconfig="$(mktemp)" + cleanup_files+=("${mirror_userconfig}") + chmod 0600 "${mirror_userconfig}" + printf '%s\n' "//registry.npmjs.org/:_authToken=${mirror_auth_token}" > "${mirror_userconfig}" + + IFS=',' read -r -a mirror_dist_tags <<< "${mirror_dist_tags_csv}" + for dist_tag in "${mirror_dist_tags[@]}"; do + [[ -n "${dist_tag}" ]] || continue + echo "Mirroring ${package_name}@${package_version} onto dist-tag ${dist_tag}" + verify_release_tooling_identity + if ! NPM_CONFIG_USERCONFIG="${mirror_userconfig}" \ + npm dist-tag add "${package_name}@${package_version}" "${dist_tag}"; then + if [[ "${mirror_auth_requirement}" == "required" ]]; then + exit 1 + fi + echo "Warning: optional npm dist-tag mirror failed for ${package_name}@${package_version} -> ${dist_tag}; published package remains live." >&2 + fi + done + fi +) diff --git a/scripts/plugin-npm-release-check.ts b/scripts/plugin-npm-release-check.ts new file mode 100644 index 0000000000000000000000000000000000000000..5ed00f3adc9d6cc69de4560ca635e0bcef41ada9 --- /dev/null +++ b/scripts/plugin-npm-release-check.ts @@ -0,0 +1,71 @@ +#!/usr/bin/env -S node --import tsx +// Plugin Npm Release Check script supports OpenClaw repository automation. + +import { pathToFileURL } from "node:url"; +import { + assertPluginReleaseDependencyFreshness, + assertPluginReleaseVersionFloors, + collectPluginNpmGitRangeSelection, + collectPublishablePluginPackages, + parsePluginNpmReleaseArgs, + resolveChangedPublishablePluginPackages, + resolveSelectedPublishablePluginPackages, +} from "./lib/plugin-npm-release.ts"; + +function runPluginNpmReleaseCheck(argv: string[]) { + const { selection, selectionMode, npmDistTag, baseRef, headRef } = + parsePluginNpmReleaseArgs(argv); + const gitRangeSelection = + baseRef && headRef + ? collectPluginNpmGitRangeSelection({ + gitRange: { baseRef, headRef }, + }) + : undefined; + const publishable = collectPublishablePluginPackages(".", { + extensionIds: + selectionMode === "all-publishable" || + !gitRangeSelection || + gitRangeSelection.authorityChanged + ? undefined + : gitRangeSelection.changedExtensionIds, + packageNames: selection.length > 0 ? selection : undefined, + npmDistTag, + }); + const selected = + selectionMode === "all-publishable" + ? publishable + : selection.length > 0 + ? resolveSelectedPublishablePluginPackages({ + plugins: publishable, + selection, + }) + : gitRangeSelection + ? gitRangeSelection.authorityChanged + ? publishable + : resolveChangedPublishablePluginPackages({ + plugins: publishable, + changedExtensionIds: gitRangeSelection.changedExtensionIds, + }) + : publishable; + + if (selectionMode !== undefined || selection.length > 0) { + assertPluginReleaseVersionFloors(selected, "plugin-npm-release-check"); + } + assertPluginReleaseDependencyFreshness(selected, "plugin-npm-release-check"); + + console.log("plugin-npm-release-check: publishable plugin metadata looks OK."); + if (baseRef && headRef && selected.length === 0) { + console.log( + ` - no publishable plugin package changes detected between ${baseRef} and ${headRef}`, + ); + } + for (const plugin of selected) { + console.log( + ` - ${plugin.packageName}@${plugin.version} (${plugin.channel}, ${plugin.extensionId})`, + ); + } +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { + runPluginNpmReleaseCheck(process.argv.slice(2)); +} diff --git a/scripts/plugin-publication-artifact.mjs b/scripts/plugin-publication-artifact.mjs new file mode 100644 index 0000000000000000000000000000000000000000..9337536c7b73ba3175bfe26ae05521620e5de061 --- /dev/null +++ b/scripts/plugin-publication-artifact.mjs @@ -0,0 +1,1290 @@ +#!/usr/bin/env node + +import { createHash } from "node:crypto"; +import { lstatSync, mkdirSync, readdirSync, statSync, writeFileSync } from "node:fs"; +import { basename, join, resolve } from "node:path"; +import { pathToFileURL } from "node:url"; +import { gunzipSync, inflateRawSync } from "node:zlib"; +import { + downloadActionsArtifactArchive, + describeActionsArtifactFiles, + inspectActionsArtifactZip, + inspectActionsArtifactZipWithPolicy, + readBoundedRegularFile, + readPublicationArtifactArchive, + sha256Digest, + validateActionsArtifactBinding, + validateActionsArtifactProducerJob, +} from "./lib/actions-artifact-archive.mjs"; +import { parseStrictBooleanArg } from "./lib/arg-utils.runtime.mjs"; +import { resolveNpmPublishPlan } from "./lib/npm-publish-plan.mjs"; + +export { + downloadActionsArtifactArchive, + describeActionsArtifactFiles, + inspectActionsArtifactZip, + inspectActionsArtifactZipWithPolicy, + readBoundedRegularFile, + readPublicationArtifactArchive, + validateActionsArtifactBinding, + validateActionsArtifactProducerJob, +}; + +const MANIFEST_FILENAME = "plugin-publication-manifest.json"; +const MANIFEST_SCHEMA = "openclaw.plugin-publication-artifact/v1"; +const TAR_BLOCK_BYTES = 512; +const TAR_END_MARKER_BYTES = TAR_BLOCK_BYTES * 2; +const TAR_USTAR_MAGIC = Buffer.from("ustar\0", "ascii"); +const TAR_USTAR_VERSION = Buffer.from("00", "ascii"); +const MAX_ARCHIVE_BYTES = 256 * 1024 * 1024; +const MAX_EXPANDED_BYTES = 512 * 1024 * 1024; +const MAX_MANIFEST_BYTES = 4 * 1024 * 1024; +const MAX_PLUGIN_MANIFEST_BYTES = 2 * 1024 * 1024; +// Bundled SDKs can exceed 10k files; byte, path, and expansion caps remain authoritative. +const MAX_TAR_ENTRIES = 20_000; +const MAX_TAR_PATH_BYTES = 4 * 1024 * 1024; +const MAX_TAR_TOTAL_FILE_BYTES = 512 * 1024 * 1024; +export const CLAWHUB_PUBLICATION_TAR_LIMITS = Object.freeze({ + maxArchiveBytes: 120 * 1024 * 1024, + maxEntries: 10_000, + maxEntryBytes: 50 * 1024 * 1024, + maxExpandedBytes: 64 * 1024 * 1024, + maxPathBytes: 4 * 1024 * 1024, + maxTotalFileBytes: 50 * 1024 * 1024, +}); +const SHA_RE = /^[0-9a-f]{40}$/u; +const SHA256_RE = /^[0-9a-f]{64}$/u; +const ARTIFACT_DIGEST_RE = /^sha256:[0-9a-f]{64}$/u; +const ARTIFACT_NAME_RE = /^[A-Za-z0-9][A-Za-z0-9_.-]*$/u; +const PACKAGE_NAME_RE = /^(?:@[a-z0-9][a-z0-9._-]*\/)?[a-z0-9][a-z0-9._-]*$/u; +const VERSION_RE = + /^[0-9]{4}\.[1-9][0-9]*\.[1-9][0-9]*(?:-(?:alpha|beta)\.[1-9][0-9]*|-[1-9][0-9]*)?$/u; +const NPM_ROUTE_POLICIES = new Map([ + ["npm-oidc", { authMode: "trusted-publisher", capability: "trusted-publisher" }], + ["npm-token-bootstrap", { authMode: "token-bootstrap", capability: "first-publication" }], + [ + "npm-token-placeholder-recovery", + { authMode: "token-bootstrap", capability: "placeholder-recovery" }, + ], + ["npm-mirror", { authMode: "release-token", capability: "dist-tag-mirror" }], + ["npm-tag-repair", { authMode: "release-token", capability: "dist-tag-repair" }], + ["npm-readback", { authMode: "none", capability: "registry-readback" }], +]); +const ROUTES = new Set([ + ...NPM_ROUTE_POLICIES.keys(), + "clawhub-token-release", + "clawhub-token-bootstrap", + "clawhub-readback", +]); +const NPM_TAGS = new Set(["latest", "alpha", "beta", "extended-stable"]); +const CLAWHUB_TAGS = new Set(["latest", "alpha", "beta"]); +const META_PACKAGE = "@openclaw/meta-provider"; +const META_PACKAGE_DIR = "extensions/meta"; + +function sha256(bytes) { + return createHash("sha256").update(bytes).digest("hex"); +} + +function npmIntegrity(bytes) { + return `sha512-${createHash("sha512").update(bytes).digest("base64")}`; +} + +function npmShasum(bytes) { + return createHash("sha1").update(bytes).digest("hex"); +} + +function compareCodeUnits(left, right) { + return left < right ? -1 : left > right ? 1 : 0; +} + +function assertString(value, label) { + if (typeof value !== "string" || value.trim() !== value || value.length === 0) { + throw new Error(`${label} must be a non-empty trimmed string.`); + } + return value; +} + +function assertPositiveInteger(value, label) { + if (!Number.isSafeInteger(value) || value <= 0) { + throw new Error(`${label} must be a safe positive integer.`); + } + return value; +} + +function hasControlCharacters(value) { + for (const character of value) { + const codePoint = character.codePointAt(0); + if (codePoint <= 0x1f || codePoint === 0x7f) { + return true; + } + } + return false; +} + +function normalizeManualOverrideReason(value) { + if (value === undefined || value === null || value === "") { + return null; + } + const reason = assertString(value, "manual override reason"); + if (reason.length > 500 || hasControlCharacters(reason)) { + throw new Error( + "Manual override reason must be at most 500 characters and contain no control characters.", + ); + } + return reason; +} + +function assertSafePackageDir(value) { + const packageDir = assertString(value, "package dir"); + if (!/^extensions\/[a-z0-9][a-z0-9._-]*$/u.test(packageDir) || packageDir.includes("\\")) { + throw new Error(`Unsafe plugin package dir: ${packageDir}`); + } + return packageDir; +} + +function assertSafeArtifactName(value) { + const artifactName = assertString(value, "artifact name"); + if (!ARTIFACT_NAME_RE.test(artifactName)) { + throw new Error(`Unsafe plugin publication artifact name: ${artifactName}`); + } + return artifactName; +} + +function assertSafeArchivePath(value, label) { + const raw = assertString(value, label); + if ( + raw.startsWith("/") || + raw.includes("\\") || + raw.includes("\0") || + raw.normalize("NFC") !== raw || + hasControlCharacters(raw) + ) { + throw new Error(`Unsafe ${label}: ${JSON.stringify(raw)}`); + } + const withoutTrailingSlash = raw.endsWith("/") ? raw.slice(0, -1) : raw; + const parts = withoutTrailingSlash.split("/"); + if ( + withoutTrailingSlash.length === 0 || + parts.some((part) => part.length === 0 || part === "." || part === "..") + ) { + throw new Error(`Unsafe ${label}: ${JSON.stringify(raw)}`); + } + return withoutTrailingSlash; +} + +function normalizePublicationReason(value) { + const reason = assertString(value, "publication reason"); + if (reason.length > 500 || hasControlCharacters(reason)) { + throw new Error( + "Publication reason must be at most 500 characters and contain no control characters.", + ); + } + return reason; +} + +function normalizePublisherPolicy(value) { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error("Publisher policy must be an object."); + } + const keys = Object.keys(value).toSorted(); + if (JSON.stringify(keys) !== JSON.stringify(["policyId", "schema", "sha256"])) { + throw new Error("Publisher policy must contain exactly schema, policyId, and sha256."); + } + const schema = assertString(value.schema, "publisher policy schema"); + const policyId = assertString(value.policyId, "publisher policy id"); + const policySha256 = assertString(value.sha256, "publisher policy SHA-256"); + if ( + schema.length > 200 || + policyId.length > 200 || + hasControlCharacters(schema) || + hasControlCharacters(policyId) || + !SHA256_RE.test(policySha256) + ) { + throw new Error("Publisher policy identity is invalid."); + } + return { policyId, schema, sha256: policySha256 }; +} + +function boundedTarLimit(value, fallback, label) { + if (value === undefined) { + return fallback; + } + if (!Number.isSafeInteger(value) || value <= 0 || value > fallback) { + throw new Error(`${label} must be a positive safe integer no larger than ${fallback}.`); + } + return value; +} + +function normalizeTarInspectionOptions(options = {}) { + const maxArchiveBytes = boundedTarLimit( + options.maxArchiveBytes, + MAX_ARCHIVE_BYTES, + "Plugin tarball byte limit", + ); + const maxExpandedBytes = boundedTarLimit( + options.maxExpandedBytes, + MAX_EXPANDED_BYTES, + "Plugin tarball expanded-byte limit", + ); + const maxEntryBytes = boundedTarLimit( + options.maxEntryBytes, + maxExpandedBytes, + "Plugin tarball per-entry byte limit", + ); + const maxTotalFileBytes = boundedTarLimit( + options.maxTotalFileBytes, + Math.min(MAX_TAR_TOTAL_FILE_BYTES, maxExpandedBytes), + "Plugin tarball total-file byte limit", + ); + const maxEntries = boundedTarLimit( + options.maxEntries, + MAX_TAR_ENTRIES, + "Plugin tarball entry-count limit", + ); + const maxPathBytes = boundedTarLimit( + options.maxPathBytes, + MAX_TAR_PATH_BYTES, + "Plugin tarball path-byte limit", + ); + return { + maxArchiveBytes, + maxEntries, + maxEntryBytes, + maxExpandedBytes, + maxPathBytes, + maxTotalFileBytes, + }; +} + +function decodeTarString(bytes, label = "tar string field") { + const nul = bytes.indexOf(0); + if (nul !== -1 && bytes.subarray(nul + 1).some((byte) => byte !== 0)) { + throw new Error(`${label} has non-zero bytes after its NUL terminator.`); + } + const value = bytes.subarray(0, nul === -1 ? bytes.length : nul); + try { + return new TextDecoder("utf-8", { fatal: true }).decode(value); + } catch { + throw new Error(`${label} is not valid UTF-8.`); + } +} + +function decodeConsumerTarPathField(bytes, label) { + const raw = decodeTarString(bytes, label); + const consumerValue = raw.trim(); + if (raw !== consumerValue) { + throw new Error(`${label} changes under the pinned ClawHub path normalization.`); + } + return consumerValue; +} + +function normalizeConsumerTarPath(value, options = {}) { + const rawPath = options.directory === true && value.endsWith("/") ? value.slice(0, -1) : value; + const normalized = rawPath.replaceAll("\\", "/").replace(/^\.\/+/u, ""); + const segments = normalized.split("/").filter(Boolean); + const consumerPath = segments.join("/"); + if ( + consumerPath !== rawPath || + segments.length === 0 || + segments.some((segment) => segment === "." || segment === "..") + ) { + throw new Error( + `Tar entry path changes under the pinned ClawHub normalization: ${JSON.stringify(value)}.`, + ); + } + return assertSafeArchivePath(consumerPath, "tar entry path"); +} + +function parseCanonicalTarNumber(bytes, label, options = {}) { + const { allowEmpty = false } = options; + if ((bytes[0] & 0x80) !== 0) { + throw new Error(`${label} must not use base-256 encoding.`); + } + if (bytes.every((byte) => byte === 0)) { + if (allowEmpty) { + return 0; + } + throw new Error(`${label} must not be empty.`); + } + const raw = bytes.toString("ascii"); + const canonicalDigits = new RegExp(`^[0-7]{${bytes.length - 2}}$`, "u"); + const canonical = canonicalDigits.test(raw.slice(0, -2)) && raw.endsWith(" \0"); + if (!canonical) { + throw new Error(`${label} is not canonically encoded.`); + } + const octal = raw.slice(0, -2); + const value = Number.parseInt(octal, 8); + if (!Number.isSafeInteger(value) || value < 0) { + throw new Error(`Invalid ${label}: ${JSON.stringify(octal)}`); + } + return value; +} + +function verifyTarChecksum(header) { + const expected = parseCanonicalTarNumber(header.subarray(148, 156), "tar checksum"); + let actual = 0; + for (let index = 0; index < header.length; index += 1) { + actual += index >= 148 && index < 156 ? 0x20 : header[index]; + } + if (actual !== expected) { + throw new Error(`Tar header checksum mismatch: expected ${expected}, got ${actual}.`); + } +} + +function verifyCanonicalTarHeader(header) { + if ( + !header.subarray(257, 263).equals(TAR_USTAR_MAGIC) || + !header.subarray(263, 265).equals(TAR_USTAR_VERSION) + ) { + throw new Error("Plugin tarball entries must use canonical POSIX USTAR headers."); + } + decodeTarString(header.subarray(0, 100), "tar entry name"); + parseCanonicalTarNumber(header.subarray(100, 108), "tar entry mode"); + parseCanonicalTarNumber(header.subarray(108, 116), "tar entry uid", { allowEmpty: true }); + parseCanonicalTarNumber(header.subarray(116, 124), "tar entry gid", { allowEmpty: true }); + parseCanonicalTarNumber(header.subarray(124, 136), "tar entry size"); + parseCanonicalTarNumber(header.subarray(136, 148), "tar entry mtime"); + decodeTarString(header.subarray(157, 257), "tar entry link path"); + decodeTarString(header.subarray(265, 297), "tar entry user name"); + decodeTarString(header.subarray(297, 329), "tar entry group name"); + parseCanonicalTarNumber(header.subarray(329, 337), "tar entry device major"); + parseCanonicalTarNumber(header.subarray(337, 345), "tar entry device minor"); + if (header[475] === 0) { + decodeTarString(header.subarray(345, 475), "tar entry prefix"); + parseCanonicalTarNumber(header.subarray(476, 488), "tar entry access time", { + allowEmpty: true, + }); + parseCanonicalTarNumber(header.subarray(488, 500), "tar entry change time", { + allowEmpty: true, + }); + } else { + decodeTarString(header.subarray(345, 500), "tar entry prefix"); + } +} + +function isZeroTarBlock(bytes) { + return bytes.length === TAR_BLOCK_BYTES && bytes.every((byte) => byte === 0); +} + +function parsePackedJson(bytes, label) { + let value; + try { + value = JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes)); + } catch (error) { + throw new Error( + `${label} is invalid JSON: ${error instanceof Error ? error.message : String(error)}`, + { cause: error }, + ); + } + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error(`${label} must be an object.`); + } + return value; +} + +function firstGzipMemberEnd(bytes, maxOutputLength) { + if ( + bytes.length < 18 || + bytes[0] !== 0x1f || + bytes[1] !== 0x8b || + bytes[2] !== 0x08 || + (bytes[3] & 0xe0) !== 0 + ) { + throw new Error("Invalid gzip header."); + } + + const flags = bytes[3]; + let offset = 10; + if ((flags & 0x04) !== 0) { + if (offset + 2 > bytes.length) { + throw new Error("Truncated gzip extra-field length."); + } + const extraLength = bytes.readUInt16LE(offset); + offset += 2 + extraLength; + if (offset > bytes.length) { + throw new Error("Truncated gzip extra field."); + } + } + for (const flag of [0x08, 0x10]) { + if ((flags & flag) === 0) { + continue; + } + const terminator = bytes.indexOf(0, offset); + if (terminator === -1) { + throw new Error("Unterminated gzip header string."); + } + offset = terminator + 1; + } + if ((flags & 0x02) !== 0) { + offset += 2; + } + if (offset + 8 > bytes.length) { + throw new Error("Truncated gzip member."); + } + + const expanded = inflateRawSync(bytes.subarray(offset), { + info: true, + maxOutputLength, + }); + return offset + expanded.engine.bytesWritten + 8; +} + +export function inspectPackageTarballBytes(inputBytes, options = {}) { + if (!(inputBytes instanceof Uint8Array)) { + throw new Error("Plugin tarball bytes must be a Uint8Array."); + } + const onFile = options.onFile; + if (onFile !== undefined && typeof onFile !== "function") { + throw new Error("Plugin tarball onFile option must be a function."); + } + const tarballBytes = Buffer.from(inputBytes.buffer, inputBytes.byteOffset, inputBytes.byteLength); + const limits = normalizeTarInspectionOptions(options); + if (tarballBytes.length === 0 || tarballBytes.length > limits.maxArchiveBytes) { + throw new Error(`Plugin tarball size is outside the allowed range: ${tarballBytes.length}.`); + } + let memberEnd; + try { + memberEnd = firstGzipMemberEnd(tarballBytes, limits.maxExpandedBytes); + } catch (error) { + throw new Error( + `Plugin tarball is not canonical gzip or expands beyond ${limits.maxExpandedBytes} bytes.`, + { cause: error }, + ); + } + if (memberEnd !== tarballBytes.length) { + throw new Error("Plugin tarball must contain exactly one gzip member."); + } + let tarBytes; + try { + const expanded = gunzipSync(tarballBytes, { + info: true, + maxOutputLength: limits.maxExpandedBytes, + }); + if (expanded.engine.bytesWritten !== tarballBytes.length) { + throw new Error("gzip stream does not consume the full plugin tarball"); + } + tarBytes = expanded.buffer; + } catch (error) { + throw new Error( + `Plugin tarball is not canonical gzip or expands beyond ${limits.maxExpandedBytes} bytes.`, + { cause: error }, + ); + } + + const inventory = []; + const seenPaths = new Set(); + const seenAliases = new Set(); + let packageManifestBytes; + let pluginManifestBytes; + let offset = 0; + let sawEndMarker = false; + let entryCount = 0; + let totalFileBytes = 0; + let totalPathBytes = 0; + + while (offset + TAR_BLOCK_BYTES <= tarBytes.length) { + const header = tarBytes.subarray(offset, offset + TAR_BLOCK_BYTES); + if (isZeroTarBlock(header)) { + const secondEndBlock = tarBytes.subarray( + offset + TAR_BLOCK_BYTES, + offset + TAR_END_MARKER_BYTES, + ); + if ( + !isZeroTarBlock(secondEndBlock) || + !tarBytes.subarray(offset + TAR_END_MARKER_BYTES).every((byte) => byte === 0) + ) { + throw new Error( + "Plugin tarball must end with two zero blocks and contain no trailing entries.", + ); + } + sawEndMarker = true; + break; + } + entryCount += 1; + if (entryCount > limits.maxEntries) { + throw new Error(`Plugin tarball exceeds the ${limits.maxEntries} entry limit.`); + } + verifyTarChecksum(header); + verifyCanonicalTarHeader(header); + const headerName = decodeConsumerTarPathField(header.subarray(0, 100), "tar entry name"); + const headerPrefix = + header[475] === 0 + ? decodeConsumerTarPathField(header.subarray(345, 475), "tar entry prefix") + : decodeConsumerTarPathField(header.subarray(345, 500), "tar entry prefix"); + const headerPath = headerPrefix ? `${headerPrefix}/${headerName}` : headerName; + const headerSize = parseCanonicalTarNumber(header.subarray(124, 136), "tar entry size"); + const typeFlag = String.fromCharCode(header[156] || 0x30); + const linkPath = decodeTarString(header.subarray(157, 257), "tar entry link path"); + if (typeFlag === "x" || typeFlag === "g" || typeFlag === "L" || typeFlag === "K") { + throw new Error("PAX and GNU tar metadata are not supported for plugin artifacts."); + } + if (typeFlag !== "0" && typeFlag !== "5") { + const suffix = linkPath ? ` -> ${linkPath}` : ""; + throw new Error( + `Unsupported plugin tar entry type ${JSON.stringify(typeFlag)}: ${headerPath}${suffix}`, + ); + } + if (typeFlag === "5" && headerSize !== 0) { + throw new Error(`Directory tar entry ${JSON.stringify(headerPath)} must have size zero.`); + } + if (linkPath) { + throw new Error( + `Plugin tar entries must not carry link targets: ${headerPath} -> ${linkPath}`, + ); + } + if (typeFlag !== "5" && headerPath.endsWith("/")) { + throw new Error( + `Non-directory tar entry must not end with a slash: ${JSON.stringify(headerPath)}.`, + ); + } + + const safePath = normalizeConsumerTarPath(headerPath, { directory: typeFlag === "5" }); + if (safePath !== "package" && !safePath.startsWith("package/")) { + throw new Error(`Plugin tar entry must stay under package/: ${safePath}`); + } + const alias = safePath.toLocaleLowerCase("en-US"); + if (seenPaths.has(safePath) || seenAliases.has(alias)) { + throw new Error(`Duplicate or aliased plugin tar entry: ${safePath}`); + } + seenPaths.add(safePath); + seenAliases.add(alias); + totalPathBytes += Buffer.byteLength(safePath, "utf8"); + if (totalPathBytes > limits.maxPathBytes) { + throw new Error(`Plugin tarball paths exceed the ${limits.maxPathBytes} byte limit.`); + } + + if (headerSize > limits.maxEntryBytes) { + throw new Error(`Plugin tar entry ${safePath} exceeds ${limits.maxEntryBytes} bytes.`); + } + const dataOffset = offset + TAR_BLOCK_BYTES; + const dataEnd = dataOffset + headerSize; + const nextOffset = dataOffset + Math.ceil(headerSize / TAR_BLOCK_BYTES) * TAR_BLOCK_BYTES; + if (dataEnd > tarBytes.length || nextOffset > tarBytes.length) { + throw new Error(`Tar entry ${JSON.stringify(safePath)} exceeds the archive boundary.`); + } + const content = tarBytes.subarray(dataOffset, dataEnd); + offset = nextOffset; + + if (typeFlag === "5") { + inventory.push({ path: safePath, sizeBytes: 0, type: "directory" }); + continue; + } + totalFileBytes += content.length; + if (totalFileBytes > limits.maxTotalFileBytes) { + throw new Error(`Plugin tarball file payload exceeds ${limits.maxTotalFileBytes} bytes.`); + } + const entry = { + path: safePath, + sha256: sha256(content), + sizeBytes: content.length, + type: "file", + }; + inventory.push(entry); + onFile?.({ content, path: safePath }); + if (safePath === "package/package.json") { + if (content.length === 0 || content.length > MAX_MANIFEST_BYTES) { + throw new Error( + `Packed package.json size is outside the allowed range: ${content.length}.`, + ); + } + packageManifestBytes = Buffer.from(content); + } else if (safePath === "package/openclaw.plugin.json") { + if (content.length === 0 || content.length > MAX_PLUGIN_MANIFEST_BYTES) { + throw new Error( + `Packed openclaw.plugin.json size is outside the allowed range: ${content.length}.`, + ); + } + pluginManifestBytes = Buffer.from(content); + } + } + + if (!sawEndMarker) { + throw new Error("Plugin tarball is missing its end marker."); + } + if (!packageManifestBytes) { + throw new Error("Plugin tarball must contain exactly one package/package.json."); + } + if (!pluginManifestBytes) { + throw new Error("Plugin tarball must contain exactly one package/openclaw.plugin.json."); + } + inventory.sort((left, right) => compareCodeUnits(left.path, right.path)); + const packageManifest = parsePackedJson(packageManifestBytes, "Packed package.json"); + const pluginManifest = parsePackedJson(pluginManifestBytes, "Packed openclaw.plugin.json"); + return { + inventory, + packageManifest, + packageManifestSha256: sha256(packageManifestBytes), + pluginManifest, + pluginManifestSha256: sha256(pluginManifestBytes), + tarballSizeBytes: tarballBytes.byteLength, + tarballSha256: sha256(tarballBytes), + totalFileBytes, + }; +} + +export function validatePluginPackageManifest(params, packageManifest) { + if (packageManifest.name !== params.packageName || packageManifest.version !== params.version) { + throw new Error( + `Packed plugin identity ${String(packageManifest.name)}@${String(packageManifest.version)} does not match ${params.packageName}@${params.version}.`, + ); + } + if (Object.hasOwn(packageManifest, "tag")) { + throw new Error( + `${params.packageName}: packed package.json must not override the approved publication tag.`, + ); + } + const release = packageManifest.openclaw?.release; + const referencesMetaIdentity = + params.packageName === META_PACKAGE || params.packageDir === META_PACKAGE_DIR; + if ( + referencesMetaIdentity && + (params.packageName !== META_PACKAGE || + params.packageDir !== META_PACKAGE_DIR || + release?.publishToNpm !== true || + release?.publishToClawHub !== true) + ) { + throw new Error( + `${META_PACKAGE}: Meta publication requires ${META_PACKAGE_DIR} with npm and ClawHub enabled.`, + ); + } + if (params.route.startsWith("npm-") && release?.publishToNpm !== true) { + throw new Error(`${params.packageName}: packed plugin is not marked publishToNpm.`); + } + if (params.route.startsWith("clawhub-") && release?.publishToClawHub !== true) { + throw new Error(`${params.packageName}: packed plugin is not marked publishToClawHub.`); + } + if (packageManifest.publishConfig !== undefined) { + throw new Error( + `${params.packageName}: packed package.json must not override publication through publishConfig.`, + ); + } +} + +function normalizePublicationParams(params) { + const targetSha = assertString(params.targetSha, "target SHA"); + if (!SHA_RE.test(targetSha)) { + throw new Error(`Target SHA must be a full lowercase commit SHA: ${targetSha}`); + } + const packageDir = assertSafePackageDir(params.packageDir); + const packageName = assertString(params.packageName, "package name"); + if (!PACKAGE_NAME_RE.test(packageName)) { + throw new Error(`Invalid plugin package name: ${packageName}`); + } + const version = assertString(params.version, "package version"); + if (!VERSION_RE.test(version)) { + throw new Error(`Invalid plugin package version: ${version}`); + } + const route = assertString(params.route, "publication route"); + if (!ROUTES.has(route)) { + throw new Error(`Unsupported plugin publication route: ${route}`); + } + const npmRoutePolicy = NPM_ROUTE_POLICIES.get(route); + let publicationReason = null; + let publisherPolicy = null; + if (npmRoutePolicy) { + if (params.authMode !== undefined && params.authMode !== npmRoutePolicy.authMode) { + throw new Error(`${route} auth mode must be ${npmRoutePolicy.authMode}.`); + } + if (params.capability !== undefined && params.capability !== npmRoutePolicy.capability) { + throw new Error(`${route} capability must be ${npmRoutePolicy.capability}.`); + } + publicationReason = normalizePublicationReason(params.publicationReason); + publisherPolicy = normalizePublisherPolicy(params.publisherPolicy); + } else if ( + params.authMode !== undefined || + params.capability !== undefined || + params.publicationReason !== undefined || + params.publisherPolicy !== undefined + ) { + throw new Error(`${route} must not carry npm publisher-policy controls.`); + } + const publishTag = assertString(params.publishTag, "publish tag"); + const allowedTags = route.startsWith("npm-") ? NPM_TAGS : CLAWHUB_TAGS; + if (!allowedTags.has(publishTag)) { + throw new Error(`Unsupported ${route} publish tag: ${publishTag}`); + } + if (route.startsWith("npm-")) { + const override = publishTag === "extended-stable" ? publishTag : undefined; + const publishPlan = resolveNpmPublishPlan(version, undefined, override); + if (publishPlan.publishTag !== publishTag) { + throw new Error( + `${packageName}@${version}: npm publish tag ${publishTag} does not match release channel ${publishPlan.channel}.`, + ); + } + } else { + const expectedTag = version.includes("-alpha.") + ? "alpha" + : version.includes("-beta.") + ? "beta" + : "latest"; + if (publishTag !== expectedTag) { + throw new Error( + `${packageName}@${version}: ClawHub publish tag ${publishTag} must be ${expectedTag}.`, + ); + } + } + const artifactName = assertSafeArtifactName(params.artifactName); + let bootstrapMode = null; + let requiresManualOverride = false; + const manualOverrideReason = normalizeManualOverrideReason(params.manualOverrideReason); + if (route === "clawhub-token-bootstrap") { + bootstrapMode = assertString(params.bootstrapMode, "bootstrap mode"); + if (bootstrapMode !== "publish" && bootstrapMode !== "configure-only") { + throw new Error(`Unsupported ClawHub bootstrap mode: ${bootstrapMode}`); + } + requiresManualOverride = params.requiresManualOverride === true; + } else if (route === "clawhub-token-release") { + requiresManualOverride = params.requiresManualOverride === true; + } else if ( + params.bootstrapMode !== undefined || + params.requiresManualOverride === true || + manualOverrideReason !== null + ) { + throw new Error(`${route} must not carry ClawHub bootstrap controls.`); + } + if (requiresManualOverride !== (manualOverrideReason !== null)) { + throw new Error( + `${route} must bind a manual override reason exactly when a manual override is required.`, + ); + } + let sourcePackageJsonSha256; + if (params.sourcePackageJsonSha256 !== undefined) { + sourcePackageJsonSha256 = assertString( + params.sourcePackageJsonSha256, + "source package.json SHA-256", + ); + if (!SHA256_RE.test(sourcePackageJsonSha256)) { + throw new Error( + `Source package.json SHA-256 must be 64 lowercase hex characters: ${sourcePackageJsonSha256}`, + ); + } + } + return { + artifactName, + authMode: npmRoutePolicy?.authMode ?? null, + bootstrapMode, + capability: npmRoutePolicy?.capability ?? null, + manualOverrideReason, + packageDir, + packageName, + publicationReason, + publishTag, + publisherPolicy, + requiresManualOverride, + route, + sourcePackageJsonSha256, + targetSha, + version, + }; +} + +function buildManifest(params, tarballName, tarballBytes, inspection) { + validatePluginPackageManifest(params, inspection.packageManifest); + const publication = params.authMode + ? { + route: params.route, + authMode: params.authMode, + capability: params.capability, + reason: params.publicationReason, + tag: params.publishTag, + publisherPolicy: params.publisherPolicy, + } + : { + route: params.route, + tag: params.publishTag, + bootstrapMode: params.bootstrapMode, + manualOverrideReason: params.manualOverrideReason, + requiresManualOverride: params.requiresManualOverride, + }; + return { + schema: MANIFEST_SCHEMA, + schemaVersion: 1, + targetSha: params.targetSha, + package: { + dir: params.packageDir, + name: params.packageName, + version: params.version, + author: inspection.packageManifest.author ?? null, + contributors: inspection.packageManifest.contributors ?? null, + repository: inspection.packageManifest.repository ?? null, + packageJsonSha256: inspection.packageManifestSha256, + pluginManifestSha256: inspection.pluginManifestSha256, + sourcePackageJsonSha256: params.sourcePackageJsonSha256, + }, + publication, + artifact: { + name: params.artifactName, + tarball: tarballName, + npmIntegrity: npmIntegrity(tarballBytes), + npmShasum: npmShasum(tarballBytes), + sha256: inspection.tarballSha256, + sizeBytes: tarballBytes.length, + inventory: inspection.inventory, + }, + }; +} + +function tarInspectionOptionsForRoute(route) { + return route.startsWith("clawhub-") ? CLAWHUB_PUBLICATION_TAR_LIMITS : undefined; +} + +function normalizeExpectedInventory(value) { + if (!Array.isArray(value)) { + throw new Error("Expected plugin tarball inventory must be an array."); + } + const paths = new Set(); + const aliases = new Set(); + const inventory = value.map((entry) => { + if (!entry || typeof entry !== "object" || Array.isArray(entry)) { + throw new Error("Expected plugin tarball inventory entries must be objects."); + } + const expectedKeys = + entry.type === "directory" + ? ["path", "sizeBytes", "type"] + : ["path", "sha256", "sizeBytes", "type"]; + if (JSON.stringify(Object.keys(entry).toSorted()) !== JSON.stringify(expectedKeys)) { + throw new Error("Expected plugin tarball inventory entry shape is invalid."); + } + const path = assertSafeArchivePath(entry.path, "expected tar entry path"); + if (path !== entry.path) { + throw new Error(`Expected plugin tarball path is not canonical: ${entry.path}`); + } + const alias = path.toLocaleLowerCase("en-US"); + if (paths.has(path) || aliases.has(alias)) { + throw new Error(`Expected plugin tarball inventory contains an alias: ${path}`); + } + paths.add(path); + aliases.add(alias); + if ( + !Number.isSafeInteger(entry.sizeBytes) || + entry.sizeBytes < 0 || + (entry.type === "directory" && entry.sizeBytes !== 0) + ) { + throw new Error(`Expected plugin tarball size is invalid for ${path}.`); + } + if (entry.type === "directory") { + return { path, sizeBytes: 0, type: "directory" }; + } + if ( + entry.type !== "file" || + typeof entry.sha256 !== "string" || + !SHA256_RE.test(entry.sha256) + ) { + throw new Error(`Expected plugin tarball file identity is invalid for ${path}.`); + } + return { + path, + sha256: entry.sha256, + sizeBytes: entry.sizeBytes, + type: "file", + }; + }); + return inventory.toSorted((left, right) => compareCodeUnits(left.path, right.path)); +} + +function canonicalManifestText(manifest) { + return `${JSON.stringify(manifest, null, 2)}\n`; +} + +function createFreshOutputDirectory(path, label) { + try { + lstatSync(path); + throw new Error(`${label} must not already exist.`); + } catch (error) { + if (!error || typeof error !== "object" || error.code !== "ENOENT") { + throw error; + } + } + mkdirSync(path, { recursive: true, mode: 0o700 }); + const created = lstatSync(path); + if (!created.isDirectory() || created.isSymbolicLink()) { + throw new Error(`${label} must be a newly created directory.`); + } +} + +export function createPluginPublicationArtifact(params) { + const normalized = normalizePublicationParams(params); + if (!normalized.sourcePackageJsonSha256) { + throw new Error("Plugin publication creation requires source package.json SHA-256."); + } + const artifactDir = resolve(params.artifactDir); + const entries = readdirSync(artifactDir, { withFileTypes: true }); + const tarballs = entries.filter( + (entry) => entry.isFile() && entry.name.endsWith(".tgz") && entry.name !== MANIFEST_FILENAME, + ); + const unexpected = entries.filter((entry) => !(entry.isFile() && entry.name.endsWith(".tgz"))); + if (tarballs.length !== 1 || unexpected.length !== 0) { + throw new Error( + `Plugin publication staging dir must contain exactly one .tgz and no other entries; found ${entries.map((entry) => entry.name).join(", ") || "<empty>"}.`, + ); + } + const tarballName = assertSafeArchivePath(tarballs[0].name, "tarball filename"); + if (basename(tarballName) !== tarballName) { + throw new Error(`Plugin tarball must be at the artifact root: ${tarballName}`); + } + const tarballPath = join(artifactDir, tarballName); + const tarballBytes = readBoundedRegularFile(tarballPath, { + label: "Plugin tarball", + maxBytes: MAX_ARCHIVE_BYTES, + }); + const inspection = inspectPackageTarballBytes( + tarballBytes, + tarInspectionOptionsForRoute(normalized.route), + ); + const manifest = buildManifest(normalized, tarballName, tarballBytes, inspection); + const manifestPath = join(artifactDir, MANIFEST_FILENAME); + writeFileSync(manifestPath, canonicalManifestText(manifest), { mode: 0o600 }); + return { manifest, manifestPath, tarballPath }; +} + +function parseBoundedJsonFile(path, label, maxBytes = MAX_MANIFEST_BYTES) { + let value; + try { + value = JSON.parse( + new TextDecoder("utf-8", { fatal: true }).decode( + readBoundedRegularFile(path, { + label, + maxBytes, + }), + ), + ); + } catch (error) { + throw new Error( + `${label} is invalid JSON: ${error instanceof Error ? error.message : String(error)}`, + { cause: error }, + ); + } + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw new Error(`${label} must be an object.`); + } + return value; +} + +function inspectPluginPublicationArtifactZip(zipBytes) { + return inspectActionsArtifactZipWithPolicy(zipBytes, { + minEntries: 2, + maxEntries: 2, + maxArchiveBytes: MAX_ARCHIVE_BYTES, + maxExpandedBytes: MAX_ARCHIVE_BYTES + MAX_MANIFEST_BYTES, + allowPath: (name) => + name === MANIFEST_FILENAME || (basename(name) === name && name.endsWith(".tgz")), + maxEntryBytes: (name) => (name === MANIFEST_FILENAME ? MAX_MANIFEST_BYTES : MAX_ARCHIVE_BYTES), + }); +} + +export function verifyPluginPublicationArtifact(params) { + const normalized = normalizePublicationParams(params); + let expectedTarballSha256; + if (params.expectedTarballSha256 !== undefined) { + expectedTarballSha256 = assertString(params.expectedTarballSha256, "expected tarball SHA-256"); + if (!SHA256_RE.test(expectedTarballSha256)) { + throw new Error("Expected tarball SHA-256 must be 64 lowercase hex characters."); + } + } + const expectedTarballSizeBytes = + params.expectedTarballSizeBytes === undefined + ? undefined + : assertPositiveInteger(params.expectedTarballSizeBytes, "expected tarball size"); + const expectedInventory = + params.expectedInventory === undefined + ? undefined + : normalizeExpectedInventory(params.expectedInventory); + const artifactId = assertPositiveInteger(params.artifactId, "artifact ID"); + const artifactSizeBytes = assertPositiveInteger(params.artifactSizeBytes, "artifact size"); + const runId = assertPositiveInteger( + params.producerRunId ?? params.runId, + "producer workflow run ID", + ); + const runAttempt = assertPositiveInteger( + params.producerRunAttempt ?? params.runAttempt, + "producer workflow run attempt", + ); + const workflowSha = assertString(params.workflowSha, "workflow SHA"); + if (!SHA_RE.test(workflowSha)) { + throw new Error(`Workflow SHA must be a full lowercase commit SHA: ${workflowSha}`); + } + const expectedArtifactDigest = assertString(params.artifactDigest, "artifact digest"); + if (!ARTIFACT_DIGEST_RE.test(expectedArtifactDigest)) { + throw new Error(`Invalid Actions artifact digest: ${expectedArtifactDigest}`); + } + const metadata = parseBoundedJsonFile(params.artifactMetadataPath, "Actions artifact metadata"); + const workflowRun = parseBoundedJsonFile( + params.workflowRunMetadataPath, + "Actions workflow run metadata", + ); + const expectedBinding = { + artifactDigest: expectedArtifactDigest, + artifactId, + artifactName: normalized.artifactName, + artifactSizeBytes, + consumerRunAttempt: params.consumerRunAttempt, + producerJobName: params.producerJobName, + repository: params.repository, + runStatePolicy: params.runStatePolicy ?? "completed-success", + runAttempt, + runId, + workflowEvent: params.workflowEvent, + workflowHeadBranch: params.workflowHeadBranch, + workflowPath: params.workflowPath, + workflowSha, + }; + validateActionsArtifactBinding({ + artifactMetadata: metadata, + expected: expectedBinding, + workflowRun, + }); + if (expectedBinding.runStatePolicy === "same-run-producer-success") { + const workflowJobs = parseBoundedJsonFile( + params.workflowJobsMetadataPath, + "Actions workflow jobs metadata", + ); + validateActionsArtifactProducerJob({ expected: expectedBinding, workflowJobs }); + } + + const zipBytes = readBoundedRegularFile(params.artifactZipPath, { + label: "Actions artifact ZIP", + maxBytes: MAX_ARCHIVE_BYTES, + }); + if (zipBytes.length !== artifactSizeBytes) { + throw new Error("Actions artifact ZIP size does not match the immutable publish tuple."); + } + const actualArtifactDigest = sha256Digest(zipBytes); + if (actualArtifactDigest !== expectedArtifactDigest) { + throw new Error( + `Actions artifact ZIP digest ${actualArtifactDigest} does not match ${expectedArtifactDigest}.`, + ); + } + const files = inspectPluginPublicationArtifactZip(zipBytes); + const manifestBytes = files.get(MANIFEST_FILENAME); + const tarballNames = [...files.keys()].filter((name) => name.endsWith(".tgz")); + if (!manifestBytes || tarballNames.length !== 1) { + throw new Error( + `Plugin publication artifact must contain ${MANIFEST_FILENAME} and exactly one .tgz.`, + ); + } + if (manifestBytes.length === 0 || manifestBytes.length > MAX_MANIFEST_BYTES) { + throw new Error(`Plugin publication manifest size is outside the allowed range.`); + } + let manifest; + try { + manifest = JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(manifestBytes)); + } catch (error) { + throw new Error( + `Plugin publication manifest is invalid JSON: ${error instanceof Error ? error.message : String(error)}`, + { cause: error }, + ); + } + const manifestSourcePackageJsonSha256 = manifest?.package?.sourcePackageJsonSha256; + if ( + typeof manifestSourcePackageJsonSha256 !== "string" || + !SHA256_RE.test(manifestSourcePackageJsonSha256) + ) { + throw new Error("Plugin publication manifest source package.json SHA-256 is invalid."); + } + if ( + normalized.sourcePackageJsonSha256 !== undefined && + normalized.sourcePackageJsonSha256 !== manifestSourcePackageJsonSha256 + ) { + throw new Error( + "Plugin publication source package.json SHA-256 does not match the approved target source.", + ); + } + const tarballName = tarballNames[0]; + const tarballBytes = files.get(tarballName); + const inspection = inspectPackageTarballBytes( + tarballBytes, + tarInspectionOptionsForRoute(normalized.route), + ); + if ( + expectedTarballSizeBytes !== undefined && + inspection.tarballSizeBytes !== expectedTarballSizeBytes + ) { + throw new Error("Plugin tarball size does not match the approved publication tuple."); + } + if (expectedTarballSha256 !== undefined && inspection.tarballSha256 !== expectedTarballSha256) { + throw new Error("Plugin tarball SHA-256 does not match the approved publication tuple."); + } + if ( + expectedInventory !== undefined && + JSON.stringify(inspection.inventory) !== JSON.stringify(expectedInventory) + ) { + throw new Error("Plugin tarball inventory does not match the approved publication tuple."); + } + const expectedManifest = buildManifest( + { + ...normalized, + sourcePackageJsonSha256: manifestSourcePackageJsonSha256, + }, + tarballName, + tarballBytes, + inspection, + ); + const expectedManifestBytes = Buffer.from(canonicalManifestText(expectedManifest), "utf8"); + if (!manifestBytes.equals(expectedManifestBytes)) { + throw new Error( + "Plugin publication manifest does not canonically bind the approved package and tarball.", + ); + } + + const outputDir = resolve(params.outputDir); + createFreshOutputDirectory(outputDir, "Plugin publication output directory"); + const outputPath = join(outputDir, tarballName); + writeFileSync(outputPath, tarballBytes, { flag: "wx", mode: 0o600 }); + if (!statSync(outputPath).isFile()) { + throw new Error(`Verified plugin tarball was not written: ${outputPath}`); + } + return { + artifactDigest: expectedArtifactDigest, + artifactId, + artifactName: normalized.artifactName, + artifactSizeBytes, + artifactZipSha256: actualArtifactDigest.slice("sha256:".length), + manifest: expectedManifest, + npmIntegrity: expectedManifest.artifact.npmIntegrity, + npmShasum: expectedManifest.artifact.npmShasum, + packageJsonSha256: expectedManifest.package.packageJsonSha256, + pluginManifestSha256: expectedManifest.package.pluginManifestSha256, + producerRunAttempt: runAttempt, + producerRunId: runId, + sourcePackageJsonSha256: expectedManifest.package.sourcePackageJsonSha256, + tarballInventory: inspection.inventory, + tarballName, + tarballPath: outputPath, + tarballSizeBytes: inspection.tarballSizeBytes, + tarballSha256: inspection.tarballSha256, + }; +} + +function parseCliArgs(argv) { + const [command, ...rest] = argv; + if (command !== "create" && command !== "verify") { + throw new Error( + "Usage: plugin-publication-artifact.mjs <create|verify> --artifact-name <name> ...", + ); + } + const values = {}; + for (let index = 0; index < rest.length; index += 2) { + const key = rest[index]; + const value = rest[index + 1]; + if (!key?.startsWith("--") || value === undefined || value.startsWith("--")) { + throw new Error(`Invalid ${command} argument near ${key ?? "<missing>"}.`); + } + const name = key.slice(2).replace(/-([a-z])/gu, (_, letter) => letter.toUpperCase()); + if (values[name] !== undefined) { + throw new Error(`Duplicate ${command} option: ${key}`); + } + values[name] = value; + } + return { command, values }; +} + +function commonCliParams(values) { + return { + artifactName: values.artifactName, + bootstrapMode: values.bootstrapMode, + manualOverrideReason: values.manualOverrideReason, + packageDir: values.packageDir, + packageName: values.packageName, + publishTag: values.publishTag, + requiresManualOverride: + values.requiresManualOverride === undefined + ? false + : parseStrictBooleanArg(values.requiresManualOverride, "requires-manual-override"), + route: values.route, + publicationReason: values.publicationReason, + publisherPolicy: + values.publisherPolicySchema === undefined && + values.publisherPolicyId === undefined && + values.publisherPolicySha256 === undefined + ? undefined + : { + policyId: values.publisherPolicyId, + schema: values.publisherPolicySchema, + sha256: values.publisherPolicySha256, + }, + sourcePackageJsonSha256: values.sourcePackageJsonSha256, + targetSha: values.targetSha, + version: values.packageVersion, + }; +} + +function appendGithubOutput(path, values) { + const lines = Object.entries(values).map(([name, value]) => `${name}=${String(value)}`); + writeFileSync(path, `${lines.join("\n")}\n`, { flag: "a" }); +} + +export function main(argv = process.argv.slice(2)) { + const { command, values } = parseCliArgs(argv); + const common = commonCliParams(values); + if (command === "create") { + const result = createPluginPublicationArtifact({ + ...common, + artifactDir: values.artifactDir, + }); + console.log(`Created canonical plugin publication manifest: ${result.manifestPath}`); + console.log(`Prepared plugin tarball: ${result.tarballPath}`); + return; + } + const result = verifyPluginPublicationArtifact({ + ...common, + artifactDigest: values.artifactDigest, + artifactId: Number(values.artifactId), + artifactMetadataPath: values.artifactMetadata, + artifactSizeBytes: Number(values.artifactSizeBytes), + artifactZipPath: values.artifactZip, + expectedTarballSizeBytes: + values.expectedTarballSizeBytes === undefined + ? undefined + : Number(values.expectedTarballSizeBytes), + expectedTarballSha256: values.expectedTarballSha256, + outputDir: values.outputDir, + consumerRunAttempt: + values.consumerRunAttempt === undefined ? undefined : Number(values.consumerRunAttempt), + producerJobName: values.producerJobName, + producerRunAttempt: Number(values.producerRunAttempt), + producerRunId: Number(values.producerRunId), + repository: values.repository, + workflowEvent: values.workflowEvent, + workflowHeadBranch: values.workflowHeadBranch, + workflowPath: values.workflowPath, + workflowJobsMetadataPath: values.workflowJobsMetadata, + workflowRunMetadataPath: values.workflowRunMetadata, + runStatePolicy: values.runStatePolicy, + workflowSha: values.workflowSha, + }); + if (values.githubOutput) { + appendGithubOutput(values.githubOutput, { + artifact_digest: result.artifactDigest, + artifact_id: result.artifactId, + artifact_name: result.artifactName, + artifact_size_bytes: result.artifactSizeBytes, + artifact_zip_sha256: result.artifactZipSha256, + bootstrap_mode: result.manifest.publication.bootstrapMode ?? "", + manual_override_reason: result.manifest.publication.manualOverrideReason ?? "", + package_name: result.manifest.package.name, + package_json_sha256: result.manifest.package.packageJsonSha256, + package_version: result.manifest.package.version, + publish_route: result.manifest.publication.route, + publish_tag: result.manifest.publication.tag, + producer_run_attempt: result.producerRunAttempt, + producer_run_id: result.producerRunId, + requires_manual_override: result.manifest.publication.requiresManualOverride, + source_package_json_sha256: result.manifest.package.sourcePackageJsonSha256, + tarball_path: result.tarballPath, + tarball_sha256: result.tarballSha256, + }); + } + console.log( + `Verified ${result.manifest.package.name}@${result.manifest.package.version} plugin artifact ${result.artifactId} (${result.artifactDigest}).`, + ); +} + +const entrypoint = process.argv[1] ? pathToFileURL(process.argv[1]).href : undefined; +if (entrypoint === import.meta.url) { + try { + main(); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + } +} diff --git a/scripts/plugin-sdk-api-diff.mts b/scripts/plugin-sdk-api-diff.mts new file mode 100644 index 0000000000000000000000000000000000000000..1d9bb79e257e890a31ab0a4f935e2ddd45450d3c --- /dev/null +++ b/scripts/plugin-sdk-api-diff.mts @@ -0,0 +1,432 @@ +#!/usr/bin/env node +import { spawn, spawnSync } from "node:child_process"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { + createPluginSdkApiDiff, + diffPluginSdkApi, + formatPluginSdkApiDiffReport, + hasPluginSdkApiChanges, + parsePluginSdkApiDiffSurface, + pluginSdkApiAcknowledgement, + renderPluginSdkApiRoot, + type PluginSdkApiDiff, + type PluginSdkApiDiffSurface, +} from "../src/plugin-sdk/api-diff.ts"; +import { runTasksWithConcurrency } from "../src/utils/run-with-concurrency.js"; +import { isConstrainedCiCheckHost } from "./lib/local-check-runtime.mts"; +import { isRecord } from "./lib/record-shared.mjs"; +import { resolveNpmPreflightSdkSelectors } from "./openclaw-npm-extended-stable-release.mjs"; +import { + createPluginSdkApiReleaseEvidence, + createPluginSdkApiReleaseEvidenceSet, +} from "./plugin-sdk-api-release-evidence.mjs"; + +type Args = { + acknowledgement: string | null; + base: string; + bases: { beta: string; latest: string } | null; + evidencePath: string | null; + head: string; + jsonPath: string | null; + requireAcknowledgement: boolean; + summaryPath: string | null; +}; + +const GIT_MAX_BUFFER = 64 * 1024 * 1024; +const SCRIPT_PATH = fileURLToPath(import.meta.url); + +function usage(): never { + console.error( + "Usage: plugin-sdk-api-diff (--base <git-ref> | --bases-json <beta/latest refs>) --head <git-ref> [--evidence <path>] [--json <path>] [--summary <path>] [--require-acknowledgement --acknowledge <8-hex-digest>]", + ); + process.exit(2); +} + +function readValue(argv: string[], index: number, flag: string): string { + const value = argv[index + 1]; + if (!value) { + console.error(`${flag} requires a value.`); + usage(); + } + return value; +} + +function parseArgs(argv: string[]): Args { + let acknowledgement: string | null = null; + let base = ""; + let bases: Args["bases"] = null; + let evidencePath: string | null = null; + let head = ""; + let jsonPath: string | null = null; + let requireAcknowledgement = false; + let summaryPath: string | null = null; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + switch (arg) { + case "--": + break; + case "--acknowledge": + acknowledgement = readValue(argv, index, arg); + index += 1; + break; + case "--base": + base = readValue(argv, index, arg); + index += 1; + break; + case "--bases-json": { + const value: unknown = JSON.parse(readValue(argv, index, arg)); + if ( + !isRecord(value) || + Object.keys(value).length !== 2 || + typeof value.beta !== "string" || + !value.beta || + typeof value.latest !== "string" || + !value.latest + ) { + throw new Error("--bases-json requires exactly beta and latest Git refs"); + } + bases = { beta: value.beta, latest: value.latest }; + index += 1; + break; + } + case "--evidence": + evidencePath = path.resolve(readValue(argv, index, arg)); + index += 1; + break; + case "--head": + head = readValue(argv, index, arg); + index += 1; + break; + case "--json": + jsonPath = path.resolve(readValue(argv, index, arg)); + index += 1; + break; + case "--require-acknowledgement": + requireAcknowledgement = true; + break; + case "--summary": + summaryPath = path.resolve(readValue(argv, index, arg)); + index += 1; + break; + case "-h": + case "--help": + usage(); + default: + console.error(`Unknown argument: ${arg}`); + usage(); + } + } + if ((!base && !bases) || (base && bases) || !head) { + usage(); + } + if (bases && requireAcknowledgement) { + throw new Error( + "Review beta/latest receipts using the selected publication channel's acknowledgement", + ); + } + if (acknowledgement !== null && !/^[a-f0-9]{8}$/u.test(acknowledgement)) { + console.error("--acknowledge must be the 8-character lowercase digest printed by the report."); + usage(); + } + return { + acknowledgement, + base, + bases, + evidencePath, + head, + jsonPath, + requireAcknowledgement, + summaryPath, + }; +} + +function git(repoRoot: string, args: string[]): string { + const result = spawnSync("git", args, { + cwd: repoRoot, + encoding: "utf8", + maxBuffer: GIT_MAX_BUFFER, + }); + if (result.status !== 0) { + throw new Error(result.stderr.trim() || result.stdout.trim() || `git ${args[0]} failed`); + } + return result.stdout.trim(); +} + +async function runAbortableChild(params: { + args: string[]; + command: string; + cwd: string; + failureMessage: string; + signal: AbortSignal; +}): Promise<void> { + let stdout = ""; + let stderr = ""; + let spawnError: Error | undefined; + await new Promise<void>((resolve, reject) => { + const child = spawn(params.command, params.args, { + cwd: params.cwd, + signal: params.signal, + stdio: ["ignore", "pipe", "pipe"], + }); + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk: string) => { + stdout += chunk.slice(0, Math.max(0, GIT_MAX_BUFFER - stdout.length)); + }); + child.stderr.on("data", (chunk: string) => { + stderr += chunk.slice(0, Math.max(0, GIT_MAX_BUFFER - stderr.length)); + }); + child.on("error", (error) => { + spawnError = error; + }); + child.on("close", (code) => { + if (code === 0) { + resolve(); + return; + } + reject(spawnError ?? new Error(stderr.trim() || stdout.trim() || params.failureMessage)); + }); + }); +} + +async function installRevisionDependencies(repoRoot: string, signal: AbortSignal): Promise<void> { + await runAbortableChild({ + command: "pnpm", + args: ["install", "--frozen-lockfile", "--ignore-scripts", "--filter", "openclaw"], + cwd: repoRoot, + failureMessage: "Plugin SDK revision install failed", + signal, + }); +} + +async function writeFile(filePath: string, content: string): Promise<void> { + await fs.mkdir(path.dirname(filePath), { recursive: true }); + await fs.writeFile(filePath, content, "utf8"); +} + +async function renderRevision( + repoRoot: string, + revisionRoot: string, + outputPath: string, + signal: AbortSignal, +): Promise<void> { + await runAbortableChild({ + command: process.execPath, + args: [ + "--max-old-space-size=6144", + "--import", + "tsx", + SCRIPT_PATH, + "--render-root", + revisionRoot, + "--output", + outputPath, + ], + cwd: repoRoot, + failureMessage: "Plugin SDK API render failed", + signal, + }); +} + +async function renderWorker(argv: string[]): Promise<boolean> { + if (argv[0] !== "--render-root") { + return false; + } + const repoRoot = argv[1]; + const outputPath = argv[2] === "--output" ? argv[3] : undefined; + if (!repoRoot || !outputPath || argv.length !== 4) { + throw new Error("Invalid Plugin SDK API renderer invocation"); + } + await writeFile(outputPath, JSON.stringify(await renderPluginSdkApiRoot(repoRoot))); + return true; +} + +async function main(): Promise<void> { + const args = parseArgs(process.argv.slice(2)); + const repoRoot = git(process.cwd(), ["rev-parse", "--show-toplevel"]); + const headCommit = git(repoRoot, ["rev-parse", "--verify", `${args.head}^{commit}`]); + if (args.bases) { + const { version } = JSON.parse(git(repoRoot, ["show", `${headCommit}:package.json`])); + if (resolveNpmPreflightSdkSelectors(version, "beta").length !== 2) { + throw new Error("beta/latest SDK evidence requires a regular final release"); + } + } + const bases = Object.entries(args.bases ?? { base: args.base }).map(([selector, ref]) => ({ + selector, + ref, + commit: git(repoRoot, ["rev-parse", "--verify", `${ref}^{commit}`]), + })); + const temporaryParent = process.env.RUNNER_TEMP ?? os.tmpdir(); + await fs.mkdir(temporaryParent, { recursive: true }); + const temporaryRoot = await fs.mkdtemp( + path.join(temporaryParent, "openclaw-plugin-sdk-api-diff-"), + ); + // A regular release compares two npm predecessors against one frozen head. + // Identical commits need no rendering, even when the caller's checkout is dirty. + const commits = [ + ...new Set(bases.map((base) => base.commit).filter((commit) => commit !== headCommit)), + ]; + if (commits.length > 0) { + commits.push(headCommit); + } + const addedWorktrees: string[] = []; + const abortController = new AbortController(); + let interruptedExitCode: number | undefined; + let cleanupPromise: Promise<void> | undefined; + const cleanup = (): Promise<void> => { + cleanupPromise ??= (async () => { + let cleanupError: Error | undefined; + for (const worktree of addedWorktrees.toReversed()) { + try { + git(repoRoot, ["worktree", "remove", "--force", worktree]); + } catch (error) { + cleanupError ??= + error instanceof Error ? error : new Error("Plugin SDK API worktree cleanup failed"); + } + } + await fs.rm(temporaryRoot, { force: true, recursive: true }); + if (cleanupError) { + throw cleanupError; + } + })(); + return cleanupPromise; + }; + const stop = (exitCode: number): void => { + interruptedExitCode ??= exitCode; + abortController.abort(); + }; + const stopOnInterrupt = (): void => stop(130); + const stopOnTerminate = (): void => stop(143); + process.once("SIGINT", stopOnInterrupt); + process.once("SIGTERM", stopOnTerminate); + + try { + const surfaces = new Map<string, PluginSdkApiDiffSurface>(); + // The shared 8-CPU/24-GiB floor leaves headroom for two 6-GiB render heaps. + const limit = isConstrainedCiCheckHost({ + logicalCpuCount: os.availableParallelism(), + totalMemoryBytes: Math.min(os.totalmem(), process.constrainedMemory() || Infinity), + }) + ? 1 + : 2; + const rendered = await runTasksWithConcurrency({ + limit, + errorMode: "stop", + // Drain aborted siblings before removing their registered worktrees. + throwOnError: false, + onTaskError: () => abortController.abort(), + tasks: commits.map((commit) => async () => { + const worktree = path.join(temporaryRoot, commit); + git(repoRoot, ["worktree", "add", "--detach", "--no-checkout", worktree, commit]); + addedWorktrees.push(worktree); + git(worktree, ["sparse-checkout", "set", "src", "packages", "patches", "scripts"]); + git(worktree, ["checkout", "--detach", commit]); + await installRevisionDependencies(worktree, abortController.signal); + const renderPath = path.join(temporaryRoot, `${commit}.json`); + await renderRevision(repoRoot, worktree, renderPath, abortController.signal); + surfaces.set(commit, parsePluginSdkApiDiffSurface(await fs.readFile(renderPath, "utf8"))); + }), + }); + if (rendered.hasError) { + throw rendered.firstError; + } + const diffs = new Map<string, PluginSdkApiDiff>([ + [ + headCommit, + createPluginSdkApiDiff({ entrypointsAdded: [], entrypointsRemoved: [], exports: [] }), + ], + ]); + const workflowSha = git(repoRoot, ["rev-parse", "HEAD"]); + const comparisons = bases.map((base) => { + let diff = diffs.get(base.commit); + if (!diff) { + const after = surfaces.get(headCommit); + if (!after) { + throw new Error("Plugin SDK API head snapshot is missing"); + } + const before = surfaces.get(base.commit); + if (!before) { + throw new Error("Plugin SDK API predecessor snapshot is missing"); + } + diff = diffPluginSdkApi(before, after); + diffs.set(base.commit, diff); + } + return { + selector: base.selector, + diff, + evidence: createPluginSdkApiReleaseEvidence({ + baseRef: base.ref, + baseSha: base.commit, + diff, + headSha: headCommit, + workflowSha, + }), + report: formatPluginSdkApiDiffReport({ + baseLabel: args.bases + ? `${base.selector}: ${base.ref} (${base.commit.slice(0, 12)})` + : base.commit.slice(0, 12), + diff, + headLabel: headCommit.slice(0, 12), + }), + }; + }); + const primary = comparisons[0]; + if (!primary) { + throw new Error("Plugin SDK API comparison is missing"); + } + const report = comparisons.map((comparison) => comparison.report).join("\n"); + process.stdout.write(report); + if (args.jsonPath) { + const diff = args.bases + ? Object.fromEntries( + comparisons.map((comparison) => [comparison.selector, comparison.diff]), + ) + : primary.diff; + await writeFile(args.jsonPath, `${JSON.stringify(diff, null, 2)}\n`); + } + if (args.evidencePath) { + const evidence = args.bases + ? createPluginSdkApiReleaseEvidenceSet( + Object.fromEntries( + comparisons.map((comparison) => [comparison.selector, comparison.evidence]), + ), + ) + : primary.evidence; + await writeFile(args.evidencePath, `${JSON.stringify(evidence, null, 2)}\n`); + } + if (args.summaryPath) { + await writeFile(args.summaryPath, report); + } + + if (args.requireAcknowledgement && hasPluginSdkApiChanges(primary.diff)) { + const expected = pluginSdkApiAcknowledgement(primary.diff); + if (args.acknowledgement !== expected) { + console.error( + `Plugin SDK API changes require acknowledgement digest ${expected}; rerun with --acknowledge ${expected}.`, + ); + process.exitCode = 1; + } + } + } catch (error) { + if (interruptedExitCode === undefined) { + throw error instanceof Error ? error : new Error("Plugin SDK API diff failed"); + } + } finally { + process.off("SIGINT", stopOnInterrupt); + process.off("SIGTERM", stopOnTerminate); + await cleanup(); + } + if (interruptedExitCode !== undefined) { + process.exitCode = interruptedExitCode; + } +} + +const run = renderWorker(process.argv.slice(2)).then((handled) => (handled ? undefined : main())); +await run.catch((error: unknown) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; +}); diff --git a/scripts/plugin-sdk-surface-report.mts b/scripts/plugin-sdk-surface-report.mts new file mode 100644 index 0000000000000000000000000000000000000000..492c2a0eaeb86e7fb9787bda461565f721df5aec --- /dev/null +++ b/scripts/plugin-sdk-surface-report.mts @@ -0,0 +1,921 @@ +#!/usr/bin/env node + +// Reports plugin SDK export surface metadata. +import fs from "node:fs"; +import { createRequire } from "node:module"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { isRecord } from "@openclaw/normalization-core/record-coerce"; +import type tsTypes from "typescript"; +import { booleanFlag, parseFlagArgs } from "./lib/arg-utils.mts"; +import { + deprecatedBarrelPluginSdkEntrypoints, + deprecatedPublicPluginSdkEntrypoints, + packagedPrivatePluginSdkRuntimeEntrypoints, + pluginSdkEntrypoints, + privateLocalOnlyPluginSdkEntrypoints, + publicPluginSdkEntrypoints, +} from "./lib/plugin-sdk-entries.mts"; +import { resolveRepoRoot } from "./lib/repo-root.mjs"; + +const repoRoot = resolveRepoRoot(import.meta.url); +const require = createRequire(import.meta.url); +let ts: typeof tsTypes; + +type ExportEntryStats = { + callableExports: number; + deprecatedCallableExports: number; + deprecatedExports: number; + exports: number; +}; + +function usage() { + return `Usage: node --import tsx scripts/plugin-sdk-surface-report.mts [--check] + +Reports plugin SDK export surface metadata. + +Options: + --check Fail when SDK surface budgets are exceeded. + -h, --help Show this help. +`; +} + +function parsePluginSdkSurfaceReportArgs(argv: string[]) { + return parseFlagArgs( + argv, + { check: false, help: false }, + [ + booleanFlag("--check", "check", true, { repeatable: true }), + booleanFlag("--help", "help", true, { repeatable: true }), + booleanFlag("-h", "help", true, { repeatable: true }), + ], + { + ignoreDoubleDash: false, + onUnhandledArg(arg: string) { + throw new Error(`Unknown plugin SDK surface report option: ${arg}`); + }, + }, + ); +} +const publicEntrypointSet = new Set(publicPluginSdkEntrypoints); +const localOnlyEntrypointSet = new Set(privateLocalOnlyPluginSdkEntrypoints); +const packagedPrivateRuntimeEntrypointSet = new Set(packagedPrivatePluginSdkRuntimeEntrypoints); +const deprecatedPublicEntrypointSet = new Set(deprecatedPublicPluginSdkEntrypoints); +const deprecatedBarrelEntrypointSet = new Set(deprecatedBarrelPluginSdkEntrypoints); +const forbiddenPublicSubpaths = new Set(["test-utils"]); + +function readPluginSdkSurfaceBudgetEnv( + name: string, + fallback: number, + env: NodeJS.ProcessEnv = process.env, +) { + const raw = env[name]; + if (raw === undefined) { + return fallback; + } + const value = raw.trim(); + if (!/^\d+$/u.test(value)) { + throw new Error(`${name} must be a non-negative integer`); + } + const parsed = Number(value); + if (!Number.isSafeInteger(parsed)) { + throw new Error(`${name} must be a safe non-negative integer`); + } + return parsed; +} + +function readPluginSdkEntrypointBudgetEnv( + name: string, + fallback: Readonly<Record<string, number>>, + env: NodeJS.ProcessEnv = process.env, +) { + const raw = env[name]; + if (raw === undefined) { + return fallback; + } + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + throw new Error(`${name} must be a JSON object of entrypoint integer budgets`); + } + if (!isRecord(parsed)) { + throw new Error(`${name} must be a JSON object of entrypoint integer budgets`); + } + + const overrides: Record<string, number> = {}; + for (const [entrypoint, value] of Object.entries(parsed)) { + if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 0) { + throw new Error(`${name}.${entrypoint} must be a safe non-negative integer`); + } + overrides[entrypoint] = value; + } + return Object.freeze({ ...fallback, ...overrides }); +} + +const defaultPublicDeprecatedExportsByEntrypointBudget = Object.freeze({ + // +1 each: legacy AgentHarness remains projected through the core and plugin-entry + // compatibility barrels while external harnesses migrate to AgentHarnessV2. + core: 3, + "plugin-entry": 1, + routing: 1, + // +4: shipped default/session-agent resolvers remain available through + // compatibility barrels while callers migrate to explicit/sole selection. + health: 1, + "agent-scope-runtime": 4, + // +1: shipped channel setup state-migration declaration during its migration window. + "channel-entry-contract": 1, + "approval-gateway-runtime": 1, + "approval-handler-runtime": 1, + "approval-reply-runtime": 0, + "config-runtime": 115, + "config-contracts": 0, + "inbound-reply-dispatch": 24, + "channel-reply-pipeline": 12, + "interactive-runtime": 11, + // +3: canonical incognito classifier projected through deprecated compatibility barrels. + "infra-runtime": 596, + "ssrf-policy": 1, + "ssrf-runtime": 1, + // +1: deprecated agent media projection re-export during the media migration window. + "media-runtime": 3, + // +3: deprecated media projection type, builder, and local-roots compatibility re-export. + "agent-media-payload": 3, + // +2: deprecated media projection type and builder. + "reply-payload": 2, + "agent-runtime": 4, + "memory-host-core": 2, + // +4: session-write lease no-op compatibility stubs through the 2026.10 train. + // +4: legacy AgentHarness, attempt, embedded-run, and side-question contracts remain + // deprecated while external harnesses migrate to required-capability V2 contracts. + // +1: bounded structured-input compiler/executor for native harness protocol adapters. + "agent-harness": 2, + "agent-harness-runtime": 10, + "command-auth": 78, + discord: 47, + // +4: deprecated media projection type, builder, and turn aliases. + "channel-inbound": 18, + "channel-lifecycle": 23, + // +1: shared ingress error factory projected through the deprecated message barrel. + // +1: shared ingress retention defaults projected through the deprecated message barrel. + // +1: WhatsApp ack-policy bridge counted via channel-message's wildcard re-export. + // Rendering helpers also flow through this shipped wildcard compatibility barrel. + "channel-message": 136, + // +2: Slack progress-draft render bridge (function + mode type). + "channel-outbound": 2, + // +2: WhatsApp ack-policy bridge (function + mode type). + "channel-feedback": 2, + "channel-pairing": 0, + "channel-policy": 7, + "channel-send-result": 1, + "reply-runtime": 1, + "security-runtime": 1, + "session-store-runtime": 4, + // +2: shipped Slack and Discord setup helpers retained through their package migration window. + "setup-runtime": 2, + "reply-history": 6, + "provider-auth": 19, + "telegram-account": 3, +} satisfies Record<string, number>); + +export function readPluginSdkSurfaceBudgets(env: NodeJS.ProcessEnv = process.env) { + const budgets = { + publicEntrypoints: readPluginSdkSurfaceBudgetEnv( + "OPENCLAW_PLUGIN_SDK_MAX_PUBLIC_ENTRYPOINTS", + // +1: session-discussion binds one external discussion provider to sessions. + // +1: focused media-local-roots replacement for the legacy agent-media facade. + // +1: account-aware channel DM policy setup descriptors. + // +1: dependency-light CLI argv parsing for machine-output metadata. + // +1: bounded archive extraction and single-entry reads. + // +1: budgeted root-bounded directory walking. + // +1: pinned secret reads and first-writer-wins creation. + // +2: restore the documented session-catalog and tool-results plugin contracts. + // +1: focused inbound-event delivery correlation for channel plugins. + // +1: dependency-light agent scope helpers for doctor migration enumeration. + // +1: dependency-light channel streaming config readers for doctor closures + // (realtime-voice-activation is private-local and not counted here). + // +1: registry-bound plugin command planning and exact selected execution. + // +1: canonical Computer Use wire contract and node-host provider seam. + // -1: retire the deprecated messaging-targets subpath. + // +2: bounded provider streams and read-only SecretRef resolution. + // +1: diagnostic flag checks without event, trace, or redaction initialization. + // +1: restore the shipped read-only conversation-binding inspection facade. + // +1: canonical node CLI owners for plugin-provided node commands. + // +3: typed feature contracts, backend registration, and native Control UI hosting. + // +1: public provider-owned asynchronous embedding batch contract. + // +1: runtime-neutral WebSocket client/server adapter for bundled and external plugins. + 154, + env, + ), + publicExports: readPluginSdkSurfaceBudgetEnv( + "OPENCLAW_PLUGIN_SDK_MAX_PUBLIC_EXPORTS", + // +5: session discussion state, info, provider, registration, and Control UI path contracts. + // +2: structured media placeholder formatter and its text-fact contract. + // +2: narrow settled-turn finalization result and safe full-attempt projector. + // +1: channel-owned setup contract factory. + // +18: generic schema primitives needed by plugin-owned channel config schemas. + // +2: shared Teams reply-style and TTS schema leaves. + // +2: generic inbound-root and SCP-host schema validators. + // +2: attributed-range renderer and its options contract. + // +1: agent-harness transcript visibility projector. + // +1: outbound formatting capability profile. + // +3: plugin approval reviewer-detail cap/truncator and sanitize-with-status variant. + // +1: canonical incognito session classifier for storage-safe plugin behavior. + // +3: typed channel partial-delivery error, creator, and structural guard. + // +1: closed attempt-terminal merge, normalization, and projection helper. + // +3: harness-native MCP App preview helper and its runtime/catalog contracts. + // +1: canonical unknown-value to Error coercion. + // +6: canonical session delivery normalization, access, and projection helpers. + // +5: focused media-local-roots helpers and typed hook media contracts. + // +1: model-independent agent-harness preflight failure contract. + // +3: channel DM policy factory and its account/patch callback contracts. + // +1: typed owner-required error for session store path resolution. + // +1: native approval messaging target resolver. + // +1: shared plugin SecretRef setup plan helper. + // +2: shared low-cardinality diagnostic dimension normalizers. + // +1: shared plugin SecretRef setup CLI factory. + // +1: shared multi-claim ingress lifecycle fan-in. + // +3: channel prompt-context entry/compat types and channel metadata builder. + // +4: focused CLI root-option constants and parsers. + // +6: model-picker action/capability and authoritative session-apply contracts. + // +1: logger file-transport flush for graceful shutdown drains. + // +1: process-local sessions.changed plugin notification payload. + // +1: loopback-only host classifier for plugin local-machine boundaries. + // +7: bounded archive extraction, entry reads, errors, and policy types. + // +3: root-bounded walk iterator, options, and entry contract. + // +5: pinned secret create/read functions and their options contract. + // +1: canonical Gateway browser-origin acceptance for browser-facing plugin routes. + // +1: watched-sessions prompt block for plugin-owned harness runtimes. + // +11: attributed skill proposal evaluation and committed skill lifecycle contracts. + // +1: inbound media-fact metadata projection for plugin-owned channel ingestion. + // +2: shared ingress error factory through channel-outbound and channel-message. + // +2: shared ingress retention defaults through channel-outbound and channel-message. + // +1: standard raw-event ingress profile replacing two channel-local shells. + // +1: collision-safe MCP server-name assignment for native harness catalogs. + // +45: restore typed session-catalog and tool-results exports promised to plugins. + // +1: forwarding-routed approver-restricted native approval capability factory. + // +1: shared inbound-event delivery correlation factory for channel plugins. + // +1: canonical webhook route identity for plugin-owned target registries. + // +3: canonical ready, blocked, and stopped channel lifecycle patch factories. + // +1: bounded external-content sanitizer for plugin-owned untrusted projections. + // +1: auth-profile preservation decision for native model pickers. + // +2: shared channel question-reaction store and preflight-audio factories. + // +1: shared channel interactive dispatcher with canonical binding authorization. + // +1: simple channel secret contract factory replacing repeated collectors. + // +4: focused agent scope functions for doctor migration enumeration. + // +1: shared transcript credential-safety prompt for plugin-owned agent harnesses. + // +3: channel streaming config reader re-exports and session-agent scope resolver. + // +3: session-catalog terminal-start provider request and Gateway params/result contracts. + // +1: worker desktop endpoint contract for desktop-capable worker leases. + // +1: closed worker desktop app metadata for provider-advertised launchers. + // +1: provider-authored machine option metadata for cloud-session sizing. + // +1: native command spec merger through the native-command-registry facade. + // +8: focused plugin command runtime factory, dispatch symbol, and six readonly contracts. + // -2: remove unused WhatsApp-specific ack policy exports from channel-feedback. + // -7: retire unused and duplicate inbound-dispatch compatibility exports. + // +7: restore still-existing deprecated inbound-dispatch compatibility re-exports. + // +1: channel-account-bound native approval request selection. + // +6: required-capability V2 harness contracts through the focused and runtime barrels, + // including the side-question compatibility split. + // +1: add the account-aware native approval request selector. + // +3: add canonical coercion exports while retaining the shipped asString compatibility name. + // +2: add high-use coercion primitives while retaining shipped object-record exports. + // +2: channel-neutral location and provider-update hook contracts. + // +1: QQBot 2.0.1 operator-approval Gateway client compatibility export. + // +2: narrow channel agent-run terminal reader and outcome contract. + // +5: narrow string, record, and error coercion helpers. + // +1: normalized Gateway public origin resolver for plugin-generated links. + // -2: retire the dead progress-draft render reader; it counted twice via + // channel-outbound and channel-message's wildcard re-export of it. + // +11: Computer Use schemas/types plus parsers, compiler, and provider registration. + // +6: Computer Use v2 action, result, and capability contracts. + // +1: opaque channel participant evidence preservation without mint authority. + // +6: load-only bridges for published pre-split plugin artifacts + // (voice-call/matrix runtime-doctor repair names, WhatsApp ack policy, + // Slack progress-draft render) so installed plugins survive upgrade (#124041 class). + // -1: remove the orphan diagnostic traceparent propagation export. + // +4: registry-owned native compaction registration contracts for Codex harnesses. + // -17: retire the messaging-targets subpath, embedded Pi aliases, and shipped + // channel setup compatibility helpers. + // +1: concrete plugin side-effect owner resolution for agent harness runtimes. + // +1: strict explicit agent-id normalization without default-agent fallback. + // +5: session-catalog paging capability, family/node-host composers, and option contracts. + // +3: two focused primitives and the closed read-only SecretRef result contract. + // -2: remove obsolete transcript display helper exports. + // +2: lightweight agent config resolution and nonthrowing default-agent lookup. + // +1: focused media-store URL/path ingestion (saveMediaSource) off the deprecated barrel. + // +2: structural Gateway transport and request-error guards for plugin CLI routing. + // +1: canonical sensitive-URL redactor so plugin CLI errors never print URL userinfo. + // +1: account-scoped model catalog discovery for native agent harnesses. + // +2: shared delegation policy (mode resolver + section builder) so harness + // runtimes render the same guidance instead of diverging prompt copies. + // +1: shared harness visible-source-reply guidance. + // +1: typed guarded-fetch redirect error for direct-only plugin delivery. + // -1: remove the test-only channel activity reset export. + // +1: named bounded structured-input surface for native harness protocol adapters. + // +1: OpenAI-compatible video execution in the existing media-understanding owner. + // -2: retire the uncalled secret-plan target resolver and its result type. + // +2: restore shipped channel setup helpers until stable packages migrate. + // +1: canonical untrusted audio-transcript formatter for channel plugins. + // +2: embedded foreground prompt context builder and its public context type. + // +1: typed owner-declared approval-scope contract for plugin-authored approvals. + // -5: approval display sanitizers moved to a non-public leaf module + // (exec-approval-text-sanitize) to break the exec-approvals cycle. + // +3: typed ask_user option-index contract and two bounded owner-order resolvers. + // +2: exact-session deletion parameters and synchronous companion mutation contract. + // +2: canonical session-model selection and auxiliary runtime-auth preparation. + // +1: identifier authentication input type for external channel plugins. + // +1: shared channel-account logout config cleanup. + // +1: descriptor-based allowFrom authentication classifier for channel security audits. + // +1: downstream strength mappers need canonical ordering instead of duplicate rank tables. + // +1: focused account media-limit resolver avoids the deprecated barrel on startup. + // +1: shared bounded HTTP rejection transport replaces plugin-local close policies. + // +1: prepared model-provider builder preserves the stable builder's return contract. + // +1: canonical SecretRef default-alias predicate for plugin binding parity. + // +2: strict session-agent resolution aliases preserve shipped Plugin SDK behavior. + // +1: manifest-owned plugin capability secret availability guard. + // +1: canonical diagnostic flag checker through its focused subpath. + // +3: typed system-agent approval request, payload, and resolution contracts for channel plugins. + // +2: focused provider-auth routes for shipped auth ordering and provider-map lookup. + // +2: bounded display-only error diagnostic attachment and rendering. + // +1: shared presentation delivery policy for core and channel plugins. + // +2: shipped conversation-binding inspection function and result type. + // +4: canonical node CLI option, envelope, presentation, and error owners. + // +1: Gateway caller ownership for standalone browser routing. + // +1: canonical temporal context renderer for plugin-owned agent harnesses. + // +1: canonical user-turn operational metadata restoration for native harnesses. + // +2: read-only debug proxy capture reader factory and contract. + // +2: owner-selected channel groups and their authored config path for safe recovery hints. + // +1: canonical conversation-to-session binding read for native channel controls. + // +1: final callable-tool availability projection for native harnesses. + // +44: feature operation/client and native Control UI contribution/host contracts. + // +1: explicit native page history and query preservation options. + // +4: observed session query, result, snapshot, and subscription contracts. + // +2: browser-safe Date timestamp validation and UTF-16 truncation primitives. + // +3: capability catalog descriptors, entry factories, and native host context. + // +2: canonical paragraph grouping and UTF-16 boundaries for channel-owned chunking. + // +1: retained runtime config reader preserves channel owner and scoped config identity. + // +1: shared session-catalog host publication with completion ownership. + // +1: provider-owned local-service reconciliation context. + // +7: card projection plus three rendering helpers on channel-outbound and its shipped barrel. + // +2: shared diff-stat rendering on channel-outbound and its shipped barrel. + // +1: shared static UI guidance, separate from per-turn harness delivery policy. + // +1: shared root/account DM policy refinement for channel schemas. + // -1: add one tool policy object and retire two unused deprecated mode exports. + // -1: one exec policy object replaces two deprecated comparator exports. + // +1: approved bounded TAR inspection through the archive admission owner. + // +8: bounded group-thread coordination, mention/route facts, and participant delivery types. + // +1: canonical runtime-context classifier for native history projection. + // +1: prepared model-specific runtime choices for channel consumers. + // +3: public provider-owned asynchronous embedding batch contract. + // +2: canonical credential-value functions through the narrow secret-input surface. + // +1: shared removed-model choice recovery text for channel consumers. + // +2: shared stored-account key selection and its plugin-owned policy type. + // +1: shared per-connection webhook request ordering for channel listeners. + // +1: approved shared widget CDN policy for core and channel presenters. + // +13: runtime-neutral WebSocket client/server, stream, data, and option contracts. + // +2: approved process-diagnostics predicate and lightweight subsystem logger. + 4481, + env, + ), + publicFunctionExports: readPluginSdkSurfaceBudgetEnv( + "OPENCLAW_PLUGIN_SDK_MAX_PUBLIC_FUNCTION_EXPORTS", + // +2: session discussion provider registration and canonical Control UI path building. + // +1: structured media placeholder formatter for text-only channel carriers. + // +1: settled-turn full-attempt projector. + // +1: channel-owned setup contract factory. + // +4: generic channel schema shape builders. + // +1: plugin-owned sensitive-schema registration. + // +2: generic inbound-root and SCP-host schema validators. + // +1: attributed-range renderer. + // +1: agent-harness transcript visibility projector. + // +2: plugin approval detail truncator and sanitize-with-status variant. + // +1: canonical incognito session classifier for storage-safe plugin behavior. + // +2: channel partial-delivery error creator and structural guard. + // +1: harness-native MCP App preview helper. + // +1: canonical unknown-value to Error coercion. + // +6: canonical session delivery normalization, access, and projection helpers. + // +2: focused media-local-roots helpers. + // +3: channel DM policy factory and its account/patch callbacks. + // +1: native approval messaging target resolver. + // +2: shared low-cardinality diagnostic dimension normalizers. + // +1: shared plugin SecretRef setup CLI factory. + // +1: shared multi-claim ingress lifecycle fan-in. + // +1: channel metadata builder. + // +3: focused CLI root-option parsers. + // +1: authoritative model-picker session-apply operation. + // +1: logger file-transport flush for graceful shutdown drains. + // +1: loopback-only host classifier for plugin local-machine boundaries. + // +2: bounded archive extraction and single-entry reads. + // +1: root-bounded directory walk iterator. + // +4: pinned secret create and synchronous/asynchronous reads. + // +1: canonical Gateway browser-origin acceptance for browser-facing plugin routes. + // +1: watched-sessions prompt block for plugin-owned harness runtimes. + // +1: inbound media-fact metadata projection for plugin-owned channel ingestion. + // +2: shared ingress error factory through channel-outbound and channel-message. + // +1: standard raw-event ingress profile replacing two channel-local shells. + // +1: collision-safe MCP server-name assignment for native harness catalogs. + // +14: restore callable session-catalog and tool-results helpers promised to plugins. + // +1: forwarding-routed approver-restricted native approval capability factory. + // +1: shared inbound-event delivery correlation factory for channel plugins. + // +1: canonical webhook route identity for plugin-owned target registries. + // +3: canonical ready, blocked, and stopped channel lifecycle patch factories. + // +1: bounded external-content sanitizer for plugin-owned untrusted projections. + // +1: auth-profile preservation decision for native model pickers. + // +2: shared channel question-reaction store and preflight-audio factories. + // +1: shared channel interactive dispatcher with canonical binding authorization. + // +1: simple channel secret contract factory replacing repeated collectors. + // +4: focused agent scope functions for doctor migration enumeration. + // +3: channel streaming config reader functions and session-agent scope resolver. + // +1: native command spec merger through the native-command-registry facade. + // +1: focused registry-bound plugin command runtime factory. + // -1: remove the unused WhatsApp-specific ack policy helper. + // -10: collapse inbound-dispatch callable aliases and wrappers. + // +7: restore still-existing deprecated inbound-dispatch callable re-exports. + // -3: keep the generic plugin-command reply carrier opaque and non-callable. + // +1: channel-account-bound native approval request selection. + // +1: add the account-aware native approval request selector. + // +3: add canonical coercion exports while retaining the shipped asString compatibility name. + // +2: add high-use callable coercion primitives while retaining shipped object-record exports. + // +1: QQBot 2.0.1 operator-approval Gateway client compatibility export. + // +1: narrow channel agent-run terminal reader. + // +5: narrow string, record, and error coercion helpers. + // +1: normalized Gateway public origin resolver for plugin-generated links. + // -2: retire the dead progress-draft render reader; it counted twice via + // channel-outbound and channel-message's wildcard re-export of it. + // +4: Computer Use wire parsers, validator compiler, and provider registration. + // +3: load-only bridges for published pre-split plugin artifacts + // (voice-call/matrix runtime-doctor repair names, WhatsApp ack policy, + // Slack progress-draft render) so installed plugins survive upgrade (#124041 class). + // -1: remove the orphan diagnostic traceparent propagation export. + // -12: retire the callable messaging-targets, embedded Pi, and channel setup helpers. + // +1: concrete plugin side-effect owner resolution for agent harness runtimes. + // +1: strict explicit agent-id normalization without default-agent fallback. + // +2: session-catalog family and node-host binding composers. + // +2: bounded provider stream and read-only SecretRef resolver. + // -1: remove the obsolete transcript tool-call predicate. + // +2: lightweight agent config resolution and nonthrowing default-agent lookup. + // +1: focused media-store URL/path ingestion (saveMediaSource) off the deprecated barrel. + // +2: structural Gateway transport and request-error guards for plugin CLI routing. + // +1: canonical sensitive-URL redactor so plugin CLI errors never print URL userinfo. + // +2: shared delegation policy (mode resolver + section builder) so harness + // runtimes render the same guidance instead of diverging prompt copies. + // +1: shared harness visible-source-reply guidance. + // -1: remove the test-only channel activity reset export. + // +1: OpenAI-compatible video execution in the existing media-understanding owner. + // -1: retire the uncalled secret-plan target resolver. + // +2: restore shipped channel setup helpers until stable packages migrate. + // +1: canonical untrusted audio-transcript formatter for channel plugins. + // +1: embedded foreground prompt context builder. + // -4: approval display sanitizers moved to a non-public leaf module + // (exec-approval-text-sanitize) to break the exec-approvals cycle. + // +2: bounded ask_user owner-order map builder and option resolver. + // +2: canonical session-model selection and auxiliary runtime-auth preparation. + // +1: shared channel-account logout config cleanup. + // +1: descriptor-based allowFrom authentication classifier for channel security audits. + // +1: downstream strength mappers need canonical ordering instead of duplicate rank tables. + // +1: focused account media-limit resolver avoids the deprecated barrel on startup. + // +1: shared bounded HTTP rejection transport replaces plugin-local close policies. + // +1: prepared model-provider builder preserves the stable builder's return contract. + // +1: canonical SecretRef default-alias predicate for plugin binding parity. + // +2: strict session-agent resolution aliases preserve shipped Plugin SDK behavior. + // +1: manifest-owned plugin capability secret availability guard. + // +1: canonical diagnostic flag checker through its focused subpath. + // +1: shared approval expiry formatter for native channel prompts. + // +2: focused provider-auth routes for shipped auth ordering and provider-map lookup. + // +2: bounded display-only error diagnostic attachment and rendering. + // +1: shared presentation delivery policy for core and channel plugins. + // +1: shipped read-only conversation-binding inspection function. + // +4: canonical node CLI option, envelope, presentation, and error owners. + // +1: Gateway caller ownership for standalone browser routing. + // +1: canonical temporal context renderer for plugin-owned agent harnesses. + // +1: canonical user-turn operational metadata restoration for native harnesses. + // +1: read-only debug proxy capture reader factory. + // +2: owner-selected channel groups and their authored config path for safe recovery hints. + // +1: canonical conversation-to-session binding read for native channel controls. + // +1: final callable-tool availability projection for native harnesses. + // +4: defineFeatureContract, createFeatureClient, defineFeaturePlugin, defineControlUiPlugin. + // +2: browser-safe Date timestamp validation and UTF-16 truncation primitives. + // +2: canonical paragraph grouping and UTF-16 boundaries for channel-owned chunking. + // +1: retained runtime config reader preserves channel owner and scoped config identity. + // +1: shared session-catalog host publication with completion ownership. + // +7: card projection plus three rendering helpers on channel-outbound and its shipped barrel. + // +2: shared diff-stat rendering on channel-outbound and its shipped barrel. + // +1: shared static UI guidance, separate from per-turn harness delivery policy. + // +1: shared root/account DM policy refinement for channel schemas. + // -2: retire the deprecated mode projection callables. + // -2: exec comparators are members of the shared policy object. + // +1: approved bounded TAR inspection through the archive admission owner. + // +5: group-thread coordinator, config resolution, mention facts, route exclusion, delivery session. + // +1: canonical runtime-context classifier for native history projection. + // +1: prepared model-specific runtime choice reader. + // +2: canonical env-value reader and managed SecretRef marker constructor. + // +1: shared stored-account key selection for channel readers and writers. + // +1: shared per-connection webhook request ordering for channel listeners. + // +4: runtime-neutral WebSocket client/server and stream constructors. + // +2: approved process-diagnostics predicate and lightweight subsystem logger. + 2645, + env, + ), + publicDeprecatedExports: readPluginSdkSurfaceBudgetEnv( + "OPENCLAW_PLUGIN_SDK_MAX_PUBLIC_DEPRECATED_EXPORTS", + // +3: canonical incognito classifier projected through deprecated compatibility barrels. + // +10: named media legacy projection deprecations across public compatibility barrels. + // +2: channel prompt-context type and metadata builder compatibility aliases. + // +1: shared ingress error factory projected through channel-message. + // +1: shared ingress retention defaults projected through channel-message. + // +1: shipped channel setup state-migration declaration during its migration window. + // +4: session-write lease no-op compatibility stubs through the 2026.10 train. + // +7: restore still-existing deprecated inbound-dispatch compatibility re-exports. + // +6: source-compatible harness contracts retained during the V2 migration window. + // +5: shipped default-agent resolver projections retained during explicit-owner migration. + // +5: load-only bridges for published pre-split plugin artifacts + // (voice-call/matrix runtime-doctor repair names, WhatsApp ack policy, + // Slack progress-draft render) so installed plugins survive upgrade (#124041 class). + // -18: retire the expired August compatibility exports and messaging-targets subpath. + // +4: rendering helpers forwarded by the shipped channel-message wildcard. + 1138, + env, + ), + publicWildcardReexports: readPluginSdkSurfaceBudgetEnv( + "OPENCLAW_PLUGIN_SDK_MAX_PUBLIC_WILDCARD_REEXPORTS", + // -1: infra-runtime now names its error exports explicitly. + // -1: infra-runtime excludes the internal system-event receipt API. + // -1: infra-runtime re-exports number coercion directly from its canonical owner. + 50, + env, + ), + }; + const publicDeprecatedExportsByEntrypointBudget = readPluginSdkEntrypointBudgetEnv( + "OPENCLAW_PLUGIN_SDK_MAX_PUBLIC_DEPRECATED_EXPORTS_BY_ENTRYPOINT", + defaultPublicDeprecatedExportsByEntrypointBudget, + env, + ); + return { budgets, publicDeprecatedExportsByEntrypointBudget }; +} + +function entrypointPath(entrypoint: string) { + return path.join(repoRoot, "src", "plugin-sdk", `${entrypoint}.ts`); +} + +function readPackageExportedSubpaths() { + const packageJson = JSON.parse(fs.readFileSync(path.join(repoRoot, "package.json"), "utf8")); + return Object.keys(packageJson.exports ?? {}) + .filter((key) => key.startsWith("./plugin-sdk/")) + .map((key) => key.slice("./plugin-sdk/".length)) + .toSorted(); +} + +function unwrapAlias(checker: tsTypes.TypeChecker, symbol: tsTypes.Symbol) { + return symbol.flags & ts.SymbolFlags.Alias ? checker.getAliasedSymbol(symbol) : symbol; +} + +function hasDeprecatedTag(symbol: tsTypes.Symbol) { + return symbol.getJsDocTags().some((tag) => tag.name === "deprecated"); +} + +function isCallableExport( + checker: tsTypes.TypeChecker, + symbol: tsTypes.Symbol, + sourceFile: tsTypes.SourceFile, +) { + const target = unwrapAlias(checker, symbol); + const declaration = target.valueDeclaration ?? target.declarations?.[0] ?? sourceFile; + const type = checker.getTypeOfSymbolAtLocation(target, declaration); + return checker.getSignaturesOfType(type, ts.SignatureKind.Call).length > 0; +} + +function countWildcardReexports(entrypoints: string[]) { + let count = 0; + const matches: string[] = []; + for (const entrypoint of entrypoints) { + const sourcePath = entrypointPath(entrypoint); + const source = fs.readFileSync(sourcePath, "utf8"); + const lines = source.split(/\r?\n/u); + for (const [index, line] of lines.entries()) { + if (/^\s*export\s+(?:type\s+)?\*\s+from\s+["'][^"']+["']/u.test(line)) { + count += 1; + matches.push(`${path.relative(repoRoot, sourcePath)}:${index + 1}`); + } + } + } + return { count, matches }; +} + +// All three inventories overlap. Lazily reuse one module graph so --help and +// invalid options avoid compiler work without tripling report time and heap. +let exportStatsProgram: tsTypes.Program | undefined; + +function collectExportStats(entrypoints: string[]) { + // CLI validation and help do not need the compiler's startup cost. + const typescript = (ts ??= require("typescript")); + const configPath = path.join(repoRoot, "tsconfig.json"); + const config = typescript.readConfigFile(configPath, (filePath) => + typescript.sys.readFile(filePath), + ); + if (config.error) { + throw new Error(typescript.flattenDiagnosticMessageText(config.error.messageText, "\n")); + } + exportStatsProgram ??= typescript.createProgram(pluginSdkEntrypoints.map(entrypointPath), { + allowJs: false, + baseUrl: repoRoot, + declaration: true, + emitDeclarationOnly: true, + module: typescript.ModuleKind.ESNext, + moduleResolution: typescript.ModuleResolutionKind.Bundler, + noEmit: true, + paths: config.config.compilerOptions?.paths, + skipLibCheck: true, + strict: false, + target: typescript.ScriptTarget.ES2022, + types: [], + }); + const program = exportStatsProgram; + const checker = program.getTypeChecker(); + const byEntrypoint = new Map<string, ExportEntryStats>(); + const uniqueNames = new Set<string>(); + const uniqueCallableNames = new Set<string>(); + + for (const entrypoint of entrypoints) { + const sourceFile = program.getSourceFile(entrypointPath(entrypoint)); + if (!sourceFile) { + byEntrypoint.set(entrypoint, { + exports: 0, + callableExports: 0, + deprecatedExports: 0, + deprecatedCallableExports: 0, + }); + continue; + } + const moduleSymbol = checker.getSymbolAtLocation(sourceFile); + const symbols = moduleSymbol ? checker.getExportsOfModule(moduleSymbol) : []; + let callableExports = 0; + let deprecatedExports = 0; + let deprecatedCallableExports = 0; + const deprecatedEntrypoint = deprecatedPublicEntrypointSet.has(entrypoint); + for (const symbol of symbols) { + const exportName = `${entrypoint}:${symbol.getName()}`; + uniqueNames.add(exportName); + const callable = isCallableExport(checker, symbol, sourceFile); + const deprecated = + deprecatedEntrypoint || + hasDeprecatedTag(symbol) || + hasDeprecatedTag(unwrapAlias(checker, symbol)); + if (callable) { + callableExports += 1; + uniqueCallableNames.add(exportName); + } + if (deprecated) { + deprecatedExports += 1; + if (callable) { + deprecatedCallableExports += 1; + } + } + } + byEntrypoint.set(entrypoint, { + exports: symbols.length, + callableExports, + deprecatedExports, + deprecatedCallableExports, + }); + } + + const totals = { + entrypoints: entrypoints.length, + exports: 0, + callableExports: 0, + deprecatedExports: 0, + deprecatedCallableExports: 0, + uniqueExports: uniqueNames.size, + uniqueCallableExports: uniqueCallableNames.size, + }; + for (const stats of byEntrypoint.values()) { + totals.exports += stats.exports; + totals.callableExports += stats.callableExports; + totals.deprecatedExports += stats.deprecatedExports; + totals.deprecatedCallableExports += stats.deprecatedCallableExports; + } + return { byEntrypoint, totals }; +} + +function selectExportStats( + scannedStats: ReturnType<typeof collectExportStats>, + entrypoints: string[], +) { + const byEntrypoint = new Map<string, ExportEntryStats>(); + const totals = { + entrypoints: entrypoints.length, + exports: 0, + callableExports: 0, + deprecatedExports: 0, + deprecatedCallableExports: 0, + uniqueExports: 0, + uniqueCallableExports: 0, + }; + for (const entrypoint of entrypoints) { + const stats = scannedStats.byEntrypoint.get(entrypoint) ?? { + exports: 0, + callableExports: 0, + deprecatedExports: 0, + deprecatedCallableExports: 0, + }; + byEntrypoint.set(entrypoint, stats); + totals.exports += stats.exports; + totals.callableExports += stats.callableExports; + totals.deprecatedExports += stats.deprecatedExports; + totals.deprecatedCallableExports += stats.deprecatedCallableExports; + } + // Export identities are entrypoint-qualified, so the selected totals are unique. + totals.uniqueExports = totals.exports; + totals.uniqueCallableExports = totals.callableExports; + return { byEntrypoint, totals }; +} + +function formatStats(label: string, stats: ReturnType<typeof collectExportStats>["totals"]) { + return [ + `${label}:`, + ` entrypoints: ${stats.entrypoints}`, + ` exports: ${stats.exports}`, + ` callable exports: ${stats.callableExports}`, + ` deprecated exports: ${stats.deprecatedExports}`, + ` deprecated callable exports: ${stats.deprecatedCallableExports}`, + ` unique entrypoint-qualified exports: ${stats.uniqueExports}`, + ].join("\n"); +} + +function collectDeprecatedEntrypointBudgetFailures( + byEntrypoint: ReturnType<typeof collectExportStats>["byEntrypoint"], + entrypointBudgets: Readonly<Record<string, number>>, +) { + const failures: string[] = []; + for (const [entrypoint, stats] of byEntrypoint) { + const budget = entrypointBudgets[entrypoint] ?? 0; + if (stats.deprecatedExports > budget) { + failures.push( + `public deprecated exports in ${entrypoint} ${stats.deprecatedExports} > ${budget}`, + ); + } + } + return failures; +} + +export function collectPluginSdkSurfaceReport() { + const scannedEntrypoints = [ + ...new Set([ + ...pluginSdkEntrypoints, + ...publicPluginSdkEntrypoints, + ...privateLocalOnlyPluginSdkEntrypoints, + ]), + ]; + const scannedStats = collectExportStats(scannedEntrypoints); + const allStats = selectExportStats(scannedStats, pluginSdkEntrypoints); + const publicStats = selectExportStats(scannedStats, publicPluginSdkEntrypoints); + const localOnlyStats = selectExportStats(scannedStats, privateLocalOnlyPluginSdkEntrypoints); + const publicWildcards = countWildcardReexports(publicPluginSdkEntrypoints); + const leakedForbiddenExports = readPackageExportedSubpaths().filter((subpath) => + forbiddenPublicSubpaths.has(subpath), + ); + const localOnlyStillPublic = privateLocalOnlyPluginSdkEntrypoints.filter( + (entrypoint) => + publicEntrypointSet.has(entrypoint) && !packagedPrivateRuntimeEntrypointSet.has(entrypoint), + ); + const localOnlyMissingFromInventory = [...localOnlyEntrypointSet].filter( + (entrypoint) => !pluginSdkEntrypoints.includes(entrypoint), + ); + const deprecatedMissingFromPublic = [...deprecatedPublicEntrypointSet].filter( + (entrypoint) => !publicEntrypointSet.has(entrypoint), + ); + const deprecatedBarrelMissingFromInventory = [...deprecatedBarrelEntrypointSet].filter( + (entrypoint) => !pluginSdkEntrypoints.includes(entrypoint), + ); + const deprecatedBarrelWithoutWildcard = [...deprecatedBarrelEntrypointSet].filter( + (entrypoint) => { + const source = fs.readFileSync(entrypointPath(entrypoint), "utf8"); + return !/^\s*export\s+(?:type\s+)?\*\s+from\s+["'][^"']+["']/mu.test(source); + }, + ); + return { + allStats, + deprecatedBarrelMissingFromInventory, + deprecatedBarrelWithoutWildcard, + deprecatedMissingFromPublic, + leakedForbiddenExports, + localOnlyMissingFromInventory, + localOnlyStats, + localOnlyStillPublic, + publicStats, + publicWildcards, + }; +} + +export function evaluatePluginSdkSurfaceReport( + report: ReturnType<typeof collectPluginSdkSurfaceReport>, + { + budgets, + publicDeprecatedExportsByEntrypointBudget, + }: ReturnType<typeof readPluginSdkSurfaceBudgets>, +) { + const failures: string[] = []; + if (publicPluginSdkEntrypoints.length > budgets.publicEntrypoints) { + failures.push( + `public entrypoints ${publicPluginSdkEntrypoints.length} > ${budgets.publicEntrypoints}`, + ); + } + if (report.publicStats.totals.exports > budgets.publicExports) { + failures.push(`public exports ${report.publicStats.totals.exports} > ${budgets.publicExports}`); + } + if (report.publicStats.totals.callableExports > budgets.publicFunctionExports) { + failures.push( + `public callable exports ${report.publicStats.totals.callableExports} > ${budgets.publicFunctionExports}`, + ); + } + if (report.publicStats.totals.deprecatedExports > budgets.publicDeprecatedExports) { + failures.push( + `public deprecated exports ${report.publicStats.totals.deprecatedExports} > ${budgets.publicDeprecatedExports}`, + ); + } + failures.push( + ...collectDeprecatedEntrypointBudgetFailures( + report.publicStats.byEntrypoint, + publicDeprecatedExportsByEntrypointBudget, + ), + ); + if (report.publicWildcards.count > budgets.publicWildcardReexports) { + failures.push( + `public wildcard reexports ${report.publicWildcards.count} > ${budgets.publicWildcardReexports}`, + ); + } + if (report.leakedForbiddenExports.length > 0) { + failures.push(`forbidden public subpaths: ${report.leakedForbiddenExports.join(", ")}`); + } + if (report.localOnlyStillPublic.length > 0) { + failures.push(`local-only entrypoints still public: ${report.localOnlyStillPublic.join(", ")}`); + } + if (report.localOnlyMissingFromInventory.length > 0) { + failures.push( + `local-only entrypoints missing from inventory: ${report.localOnlyMissingFromInventory.join(", ")}`, + ); + } + if (report.deprecatedMissingFromPublic.length > 0) { + failures.push( + `deprecated public entrypoints missing from package surface: ${report.deprecatedMissingFromPublic.join(", ")}`, + ); + } + if (report.deprecatedBarrelMissingFromInventory.length > 0) { + failures.push( + `deprecated barrel entrypoints missing from inventory: ${report.deprecatedBarrelMissingFromInventory.join(", ")}`, + ); + } + if (report.deprecatedBarrelWithoutWildcard.length > 0) { + failures.push( + `deprecated barrel entrypoints without wildcard exports: ${report.deprecatedBarrelWithoutWildcard.join(", ")}`, + ); + } + return failures; +} + +function renderPluginSdkSurfaceReport(report: ReturnType<typeof collectPluginSdkSurfaceReport>) { + return [ + formatStats("all SDK entrypoints", report.allStats.totals), + formatStats("public package SDK entrypoints", report.publicStats.totals), + formatStats("local-only SDK entrypoints", report.localOnlyStats.totals), + `deprecated public subpaths: ${deprecatedPublicPluginSdkEntrypoints.length}`, + `deprecated barrel subpaths: ${deprecatedBarrelPluginSdkEntrypoints.length}`, + `public wildcard reexports: ${report.publicWildcards.count}`, + `package-exported forbidden subpaths: ${report.leakedForbiddenExports.length}`, + ].join("\n"); +} + +function main(argv: string[] = process.argv.slice(2), env = process.env) { + const cliArgs = parsePluginSdkSurfaceReportArgs(argv); + if (cliArgs.help) { + process.stdout.write(usage()); + return 0; + } + const budgetConfig = readPluginSdkSurfaceBudgets(env); + const report = collectPluginSdkSurfaceReport(); + process.stdout.write(`${renderPluginSdkSurfaceReport(report)}\n`); + const failures = evaluatePluginSdkSurfaceReport(report, budgetConfig); + if (cliArgs.check && failures.length > 0) { + process.stderr.write(`plugin SDK surface budget failed:\n`); + for (const failure of failures) { + process.stderr.write(`- ${failure}\n`); + } + return 1; + } + return 0; +} + +const isMain = + typeof process.argv[1] === "string" && + process.argv[1].length > 0 && + import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href; + +if (isMain) { + try { + process.exitCode = main(); + } catch (error) { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + process.exitCode = 1; + } +} diff --git a/scripts/pr b/scripts/pr new file mode 100644 index 0000000000000000000000000000000000000000..fe92080cfb7cb434adb9a21a89b0456fbce4ac4b --- /dev/null +++ b/scripts/pr @@ -0,0 +1,626 @@ +#!/usr/bin/env bash +# Bash 5.3+ can deadlock writing heredoc pipes on macOS before the reader starts. +if [[ ${OSTYPE:-} == darwin* && $BASH != /bin/bash ]] && ((BASH_VERSINFO[0] > 5 || (BASH_VERSINFO[0] == 5 && BASH_VERSINFO[1] >= 3))); then + exec /bin/bash "$0" "$@" +fi + +set -euo pipefail + +# This wrapper parses GitHub CLI JSON. Caller shells may force ANSI color globally. +export NO_COLOR=1 +export CLICOLOR=0 +export CLICOLOR_FORCE=0 +export FORCE_COLOR=0 +unset COLORTERM + +# This is the single source of truth for the canonical-wrapper trust boundary. +# Advisory commands may run a mismatched local wrapper only with the explicit +# developer opt-in; landing commands must always use canonical/origin-main code. +# Classification is independent of serialization: ci-dispatch remains locked +# because GitHub exposes neither dispatch deduplication nor a correlation ID. +# PR_SUBCOMMAND_CLASSIFICATIONS_BEGIN +pr_subcommand_classification() { + case "$1" in + ls | ci-dispatch) + printf 'advisory\n' + ;; + gc | lock-recover | review-init | review-checkout-main | review-checkout-pr | review-claim | review-guard | review-artifacts-init | review-validate-artifacts | review-tests | prepare-init | prepare-validate-commit | prepare-gates | prepare-push | prepare-sync-head | prepare-run | merge-verify | merge-run | merge-recover | merge-complete) + printf 'landing\n' + ;; + *) return 1 ;; + esac +} +# PR_SUBCOMMAND_CLASSIFICATIONS_END + +is_locked_pr_command() { + # Advisory trust classification permits local dogfood, but dispatches still + # serialize with landing operations because the remote mutation is not atomic. + if [ "$1" = "ci-dispatch" ]; then + return 0 + fi + local classification + classification=$(pr_subcommand_classification "$1") || return 1 + [ "$classification" = "landing" ] || return 1 + # gc manages per-PR locks itself; lock-recover performs an exact-OID CAS. + [ "$1" != "gc" ] && [ "$1" != "lock-recover" ] +} + +dev_wrapper_opt_in=0 +if [ "${OPENCLAW_PR_DEV_WRAPPER:-}" = "1" ]; then + dev_wrapper_opt_in=1 +fi +if [ "${1-}" = "--dev-wrapper" ]; then + dev_wrapper_opt_in=1 + export OPENCLAW_PR_DEV_WRAPPER=1 + shift +fi +requested_subcommand="${1-}" + +# Select trusted wrapper code independently from the canonical repository root; +# a linked wrapper may be removed by merge-run or gc before supervision ends. +# Physical paths keep helper argv aligned with Node's canonical module URLs. +script_self="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)/$(basename "${BASH_SOURCE[0]}")" +script_parent_dir="$(dirname "$script_self")" +canonical_repo_root="$script_parent_dir/.." +# Bootstrap paths stay fixed so an edited inventory cannot exclude itself from +# the working-tree and anchor checks. The inventory owns the remaining closure. +pr_wrapper_components=(scripts/pr scripts/pr-lib) +# Anchor-exec handoff: this process was exec'd from wrapper bytes materialized +# out of refs/remotes/origin/main by the selection logic below in a previous +# wrapper. The env var conveys repository addressing only — code trust came +# from the parent's per-blob verification of the materialized copy, the same +# verify-then-exec contract as the canonical-checkout substitution. Skip +# re-selection: the running bytes ARE the anchor, and the temp copy has no git +# context of its own to select against. +if [ -n "${OPENCLAW_PR_ANCHOR_REPO_ROOT:-}" ]; then + if ! git -C "$OPENCLAW_PR_ANCHOR_REPO_ROOT" rev-parse --is-inside-work-tree >/dev/null 2>&1; then + echo "OPENCLAW_PR_ANCHOR_REPO_ROOT is not a git work tree: $OPENCLAW_PR_ANCHOR_REPO_ROOT" >&2 + exit 1 + fi + canonical_repo_root="$OPENCLAW_PR_ANCHOR_REPO_ROOT" +elif common_git_dir=$(git -C "$script_parent_dir" rev-parse --path-format=absolute --git-common-dir 2>/dev/null); then + canonical_repo_root="$(dirname "$common_git_dir")" + canonical_self="$canonical_repo_root/scripts/$(basename "${BASH_SOURCE[0]}")" + if [ "$script_self" != "$canonical_self" ] && [ -x "$canonical_self" ]; then + linked_inventory=$(git -C "$script_parent_dir" show HEAD:scripts/pr-lib/wrapper-components.txt) + while IFS= read -r wrapper_component; do + pr_wrapper_components+=("$wrapper_component") + done <<< "$linked_inventory" + if ! git -C "$script_parent_dir" diff --quiet HEAD -- "${pr_wrapper_components[@]/#/:(top)}"; then + echo "scripts/pr wrapper files have uncommitted changes in this worktree." >&2 + echo "Refusing to run unreviewed wrapper code from: $script_parent_dir" >&2 + exit 1 + fi + linked_wrapper_revision=$( + git -C "$script_parent_dir" rev-parse "${pr_wrapper_components[@]/#/HEAD:}" 2>/dev/null || true + ) + canonical_wrapper_revision=$( + git -C "$canonical_repo_root" rev-parse "${pr_wrapper_components[@]/#/HEAD:}" 2>/dev/null || true + ) + if [ -n "$linked_wrapper_revision" ] && + [ "$linked_wrapper_revision" = "$canonical_wrapper_revision" ] && + git -C "$canonical_repo_root" diff --quiet HEAD -- "${pr_wrapper_components[@]/#/:(top)}"; then + exec "$canonical_self" "$@" + fi + # The canonical checkout can be parked on another branch or carry local + # edits (release trains move it); maintainer-controlled origin/main is the + # trust anchor then. Matching code can run in place when its command + # does not need the materialized dependency context. + # refs/remotes/... explicitly: bare "origin/main" is a DWIM name that a + # local branch or tag named origin/main could shadow, spoofing the anchor. + anchor_commit=$(git -C "$script_parent_dir" rev-parse --verify refs/remotes/origin/main^{commit} 2>/dev/null || true) + anchor_wrapper_revision="" + if [ -n "$anchor_commit" ] && + anchor_inventory=$(git -C "$script_parent_dir" show "$anchor_commit:scripts/pr-lib/wrapper-components.txt" 2>/dev/null); then + # A newer anchor can add helpers outside the caller's inventory. Its own + # closure governs canonical substitution as well as extraction below. + anchor_components=(scripts/pr scripts/pr-lib) + while IFS= read -r anchor_component; do + anchor_components+=("$anchor_component") + done <<< "$anchor_inventory" + anchor_wrapper_revision=$( + git -C "$script_parent_dir" rev-parse "${anchor_components[@]/#/$anchor_commit:}" 2>/dev/null || true + ) + canonical_wrapper_revision=$( + git -C "$canonical_repo_root" rev-parse "${anchor_components[@]/#/HEAD:}" 2>/dev/null || true + ) + fi + linked_matches_anchor=0 + if [ -n "$linked_wrapper_revision" ] && + [ "$linked_wrapper_revision" = "$anchor_wrapper_revision" ]; then + linked_matches_anchor=1 + fi + # Node resolves helper imports from their source directory, not the Git owner. + # Reuse the verified materializer for code-only linked PR operations. + if [ "$linked_matches_anchor" != "1" ] || + { is_locked_pr_command "$requested_subcommand" && [ ! -d "$script_parent_dir/../node_modules" ]; }; then + wrapper_route_reason="differs from origin/main" + if [ "$linked_matches_anchor" = "1" ]; then + wrapper_route_reason="matches origin/main but has no node_modules directory" + fi + requested_classification=$(pr_subcommand_classification "$requested_subcommand" 2>/dev/null || true) + if [ "$linked_matches_anchor" != "1" ] && [ "$dev_wrapper_opt_in" = "1" ] && [ "$requested_classification" = "advisory" ]; then + if [ "${OPENCLAW_PR_DEV_WRAPPER_BANNER_SHOWN:-}" != "1" ]; then + local_head_revision=$(git -C "$script_parent_dir" rev-parse HEAD 2>/dev/null || printf 'unknown') + echo "WARNING: running local scripts/pr revision $local_head_revision via dev-wrapper opt-in." >&2 + echo "subcommand '$requested_subcommand' is classified advisory." >&2 + echo "The local wrapper differs from the canonical checkout and origin/main; landing subcommands remain refused." >&2 + export OPENCLAW_PR_DEV_WRAPPER_BANNER_SHOWN=1 + fi + else + if [ "$linked_matches_anchor" != "1" ] && [ "$dev_wrapper_opt_in" = "1" ] && [ -n "$requested_classification" ]; then + echo "subcommand '$requested_subcommand' is classified $requested_classification; dev-wrapper opt-in is unavailable." >&2 + fi + # Worktrees routinely sit on a base that predates (or carries) wrapper + # changes relative to main. When the canonical checkout is byte-identical + # to the fetched origin/main anchor, exec-ing it runs exactly the trusted + # anchor code; announce the substitution so it is never silent. + if [ -n "$anchor_wrapper_revision" ] && + [ "$canonical_wrapper_revision" = "$anchor_wrapper_revision" ] && + git -C "$canonical_repo_root" diff --quiet HEAD -- "${anchor_components[@]/#/:(top)}"; then + echo "scripts/pr wrapper in this worktree $wrapper_route_reason; running the canonical checkout's wrapper (matches the origin/main trust anchor): $canonical_repo_root" >&2 + exec "$canonical_self" "$@" + fi + # The canonical checkout cannot supply trusted code and dependency context. + # Materialize the anchor wrapper bytes directly + # from refs/remotes/origin/main and exec that copy: trust flows from + # the fetched ref itself, so a queued PR stops paying a rebase + CI lap + # for unrelated wrapper drift on main. Requires the anchor entrypoint + # to understand the handoff; older anchors fall through to the refusal. + # Archive emits paths relative to its cwd, so run it (and the + # verification listing) from the worktree toplevel to keep component + # paths intact in the materialized tree. + anchor_git_root=$(git -C "$script_parent_dir" rev-parse --show-toplevel 2>/dev/null || true) + if [ -n "$anchor_wrapper_revision" ] && [ -n "$anchor_git_root" ] && + anchor_exec_dir=$(mktemp -d "${TMPDIR:-/tmp}/openclaw-pr-anchor.XXXXXX" 2>/dev/null); then + anchor_extraction_valid=0 + # BSD tar can stop at the end marker before Git finishes writing padding, + # causing SIGPIPE under pipefail. Require producer completion before reading. + if git -C "$anchor_git_root" archive "$anchor_commit" -- \ + "${anchor_components[@]}" > "$anchor_exec_dir/anchor.tar" 2>/dev/null && + tar -xf "$anchor_exec_dir/anchor.tar" -C "$anchor_exec_dir" 2>/dev/null; then + rm "$anchor_exec_dir/anchor.tar" + anchor_extraction_valid=1 + anchor_verified_entries=0 + # Verify every extracted file byte-matches its anchor blob so an + # archive/extract fault can never smuggle different code past the + # trust boundary. The temp dir is 0700 and exec'd immediately — + # the same TOCTOU class as the canonical-substitution exec above. + while IFS=$'\t' read -r anchor_meta anchor_path; do + anchor_blob="${anchor_meta##* }" + extracted_blob=$(git -C "$anchor_git_root" hash-object -- \ + "$anchor_exec_dir/$anchor_path" 2>/dev/null || true) + if [ -z "$extracted_blob" ] || [ "$extracted_blob" != "$anchor_blob" ]; then + anchor_extraction_valid=0 + break + fi + anchor_verified_entries=$((anchor_verified_entries + 1)) + done < <(git -C "$anchor_git_root" ls-tree -r --full-tree "$anchor_commit" -- "${anchor_components[@]}") + # An empty listing means the anchor read failed; never exec unverified bytes. + if [ "$anchor_verified_entries" = "0" ]; then + anchor_extraction_valid=0 + fi + fi + if [ "$anchor_extraction_valid" = "1" ] && + grep -q "OPENCLAW_PR_ANCHOR_REPO_ROOT" "$anchor_exec_dir/scripts/pr" 2>/dev/null; then + # Only third-party tooling comes from the installed canonical tree. + # Pin package directories: a later install may replace top-level aliases. + # A whole node_modules link could load unanchored workspace source. + if ! node --input-type=module - "$canonical_repo_root/node_modules" "$anchor_exec_dir/node_modules" <<'EOF_NODE' +import { mkdirSync, realpathSync, statSync, symlinkSync } from "node:fs"; +import { join } from "node:path"; +const dependencies = ["tsx", "zod", "minimatch", "yaml"].map((dependency) => { + const installedPath = join(process.argv[2], dependency); + try { + const target = realpathSync(installedPath); + if (!statSync(target).isDirectory()) throw new Error("not a package directory"); + return { dependency, target }; + } catch { + console.error(`Cannot resolve installed scripts/pr dependency '${dependency}' at ${installedPath}.`); + console.error("Restore frozen dependencies in a clean trusted-main checkout before retrying; no dependencies were installed."); + process.exit(1); + } +}); +mkdirSync(process.argv[3]); +for (const { dependency, target } of dependencies) { + // Native junctions avoid Git Bash copying package directories on Windows. + symlinkSync(target, join(process.argv[3], dependency), + process.platform === "win32" ? "junction" : "dir"); +} +EOF_NODE + then + rm -rf "$anchor_exec_dir" + exit 1 + fi + # Not cleaned by trap: exec replaces this shell, and bash re-reads + # the script file during execution, so the copy must outlive the + # whole supervised run. OS tmp reaping owns this directory. + echo "scripts/pr wrapper in this worktree $wrapper_route_reason; running wrapper code materialized from the refs/remotes/origin/main trust anchor at revision $(git -C "$script_parent_dir" rev-parse --short "$anchor_commit")." >&2 + OPENCLAW_PR_ANCHOR_REPO_ROOT="$canonical_repo_root" exec "$anchor_exec_dir/scripts/pr" "$@" + fi + rm -rf "$anchor_exec_dir" + fi + # HEAD blobs are authoritative here: the uncommitted-wrapper guard above + # already exited for any staged or unstaged edit to these paths, so + # the working tree matches HEAD and this list matches what was rejected. + differing_wrapper_components=() + for wrapper_component in "${pr_wrapper_components[@]}"; do + linked_component_revision=$(git -C "$script_parent_dir" rev-parse "HEAD:$wrapper_component" 2>/dev/null || true) + anchor_component_revision=$(git -C "$script_parent_dir" rev-parse "refs/remotes/origin/main:$wrapper_component" 2>/dev/null || true) + if [ -z "$linked_component_revision" ] || + [ -z "$anchor_component_revision" ] || + [ "$linked_component_revision" != "$anchor_component_revision" ]; then + differing_wrapper_components+=("$wrapper_component") + fi + done + if [ "$linked_matches_anchor" = "1" ]; then + echo "scripts/pr wrapper matches origin/main, but its linked checkout has no node_modules directory." >&2 + else + echo "scripts/pr implementation differs between this worktree and the canonical checkout, and does not match origin/main." >&2 + fi + echo "differing wrapper components vs origin/main: ${differing_wrapper_components[*]}" >&2 + echo "Refusing to silently substitute canonical wrapper code from: $canonical_repo_root" >&2 + echo "Run scripts/pr from a checkout with installed dependencies whose wrapper matches the canonical checkout or a fetched origin/main." >&2 + exit 1 + fi + fi + fi +fi + +# Keep the verified owner available after cleanup removes a linked wrapper. +canonical_repo_root="$(cd "$canonical_repo_root" && pwd -P)" +readonly canonical_repo_root + +is_main_only_pr_command() { + case "$1" in + prepare-init | prepare-validate-commit | prepare-gates | prepare-push | prepare-sync-head | prepare-run | merge-verify | merge-run | merge-recover) return 0 ;; + *) return 1 ;; + esac +} + +is_supervised_pr_process() { + [ "${OPENCLAW_PR_DEDICATED_PROCESS_GROUP:-}" = "1" ] && + [ "${OPENCLAW_PR_LOCK_NOTIFY_FD:-}" = "3" ] && + [ "${OPENCLAW_PR_LOCK_SUPERVISOR_PID:-}" = "$PPID" ] +} + +# The supervisor changes cwd to the canonical repo. Resolve an explicit body +# relative to the operator's caller before crossing that boundary; do not read it. +if [ "${1-}" = merge-run ] || [ "${1-}" = merge-recover ]; then + merge_cli_args=() merge_cli_body_next=false + for merge_cli_arg in "$@"; do + if [ "$merge_cli_body_next" = true ] && [ -n "$merge_cli_arg" ]; then + merge_cli_arg=$(node -e 'process.stdout.write(require("node:path").resolve(process.argv[1]))' -- "$merge_cli_arg") || exit 1 + fi + merge_cli_args+=("$merge_cli_arg") + merge_cli_body_next=false + if [ "$merge_cli_arg" = --body-file ] || [ "$merge_cli_arg" = --legacy-refusal ]; then merge_cli_body_next=true; fi + done + set -- "${merge_cli_args[@]}" +fi + +if [ "${1-}" = "gc" ] || is_locked_pr_command "${1-}"; then + if is_supervised_pr_process; then + # operation-lock.sh consumes the one-shot marker when it installs the + # leader-only completion trap, before PR command tools can inherit it. + : + else + unset OPENCLAW_PR_DEDICATED_PROCESS_GROUP + unset OPENCLAW_PR_LOCK_NOTIFY_FD + unset OPENCLAW_PR_LOCK_SUPERVISOR_PID + command -v node >/dev/null 2>&1 || { echo "Missing required command: node" >&2; exit 1; } + exec node "$script_parent_dir/pr-lib/process-group-runner.mjs" "$canonical_repo_root" "$script_self" "$@" + fi +fi + +# shellcheck disable=SC1091 +source "$script_parent_dir/lib/plain-gh.sh" + +usage() { + cat <<USAGE +Usage: + scripts/pr [--dev-wrapper] <subcommand> ... + scripts/pr ls + scripts/pr gc [--dry-run] + scripts/pr lock-recover <PR> <OWNER_OID> --confirmed-no-running-tools + scripts/pr review-init <PR> + scripts/pr review-checkout-main <PR> + scripts/pr review-checkout-pr <PR> + scripts/pr review-claim <PR> + scripts/pr review-guard <PR> + scripts/pr review-artifacts-init <PR> + scripts/pr review-validate-artifacts <PR> + scripts/pr review-tests <PR> <test-file> [<test-file> ...] + scripts/pr prepare-init <PR> + scripts/pr prepare-validate-commit <PR> + scripts/pr prepare-gates <PR> + scripts/pr prepare-push <PR> + scripts/pr prepare-sync-head <PR> + scripts/pr prepare-run <PR> + scripts/pr ci-dispatch <PR> [--backend crabbox --run-id <id> --lease-id <id> --bootstrap-sha256 <hash>] + scripts/pr merge-verify <PR> + scripts/pr merge-run <PR> [--auto-merge] [--body-file <path>] + OPENCLAW_PR_MERGE_METHOD=merge|rebase preserves the PR commit series. + --auto-merge selects pinned immediate squash for MERGEABLE/CLEAN, auto for MERGEABLE/BEHIND. + OPENCLAW_PR_AUTO_MERGE=1 is equivalent. + --body-file snapshots a regular UTF-8 file relative to the caller, replacing squash prose. + Empty files are valid. Explicit/source co-authors and preview credit backed by tree-changing PR commits or PR authorship are retained; known machine credit is excluded. Queue merges reject it. + Repeated merge-run reconciles retained outcomes; it never retries an uncertain dispatch. + scripts/pr merge-recover <PR> <OUTCOME_OID> --confirmed-operator-recovery [--replacement-head <SHA>] [--body-file <path>] [--legacy-refusal <evidence-directory>] + Explicitly authorize one new attempt after inspecting the retained outcome and remote history. + Defaults to the retained head; replacement requires an explicitly approved full lowercase 40-character SHA, + freshly authored exact-head review/preparation and completed CI proof. Never rebases automatically. + Keeps prior evidence and reruns admission; requires the same method and no auto/queue/admin route. + scripts/pr merge-complete <PR> <OUTCOME_OID> --confirmed-operator-completion + Finish a verified merge receipt after cleanup; never dispatches a merge or deletes resources. + Posts a first completion comment only from merged; uncertain comment attempts are lookup-only. + + --dev-wrapper permits a mismatched local wrapper only for subcommands + classified advisory. OPENCLAW_PR_DEV_WRAPPER=1 is equivalent. + +Required commands: git, gh, jq, rg (ripgrep), pnpm, node. +USAGE +} + +require_cmds() { + local missing=() + local cmd + for cmd in git gh jq rg pnpm node; do + if ! command -v "$cmd" >/dev/null 2>&1; then + missing+=("$cmd") + fi + done + if ! resolve_plain_gh_bin >/dev/null; then + missing+=("gh") + fi + + if [ "${#missing[@]}" -gt 0 ]; then + echo "Missing required command(s): ${missing[*]}" >&2 + if [[ " ${missing[*]} " = *" rg "* ]]; then + echo "Install ripgrep and retry: https://github.com/BurntSushi/ripgrep#installation" >&2 + fi + exit 1 + fi +} + +require_main_target_pr() { + local pr="$1" + local base base_json + base_json=$(read_pr_view_json "$pr" "baseRefName") || exit 1 + base=$(pr_view_string_field "$base_json" "baseRefName" "$pr" "Retry the scripts/pr command.") || exit 1 + if [ "$base" != "main" ]; then + echo "scripts/pr prepare and merge commands only support PRs targeting main; PR #$pr targets $base." >&2 + echo "Use the reviewed release-branch landing flow for non-main PRs." >&2 + exit 1 + fi +} + +# shellcheck disable=SC1091 +source "$script_parent_dir/pr-lib/worktree.sh" +# shellcheck disable=SC1091 +source "$script_parent_dir/pr-lib/operation-lock.sh" +# shellcheck disable=SC1091 +source "$script_parent_dir/pr-lib/common.sh" +# shellcheck disable=SC1091 +source "$script_parent_dir/pr-lib/changelog.sh" +# shellcheck disable=SC1091 +source "$script_parent_dir/pr-lib/gates.sh" +# shellcheck disable=SC1091 +source "$script_parent_dir/pr-lib/push.sh" +# shellcheck disable=SC1091 +source "$script_parent_dir/pr-lib/review.sh" +# shellcheck disable=SC1091 +source "$script_parent_dir/pr-lib/prepare-core.sh" +# shellcheck disable=SC1091 +source "$script_parent_dir/pr-lib/merge.sh" + +main() { + if [ "$#" -lt 1 ]; then + usage + exit 2 + fi + + local cmd="${1-}" + shift || true + + if [ "$cmd" = "lock-recover" ]; then + local pr="${1-}" + local owner_oid="${2-}" + local confirmation="${3-}" + [ -n "$pr" ] && [ -n "$owner_oid" ] && [ "$#" -eq 3 ] || { usage; exit 2; } + recover_pr_operation_lock "$pr" "$owner_oid" "$confirmation" + return + fi + + case "$cmd" in + ls) ;; + gc) + [ "$#" -eq 0 ] || { [ "$#" -eq 1 ] && [ "$1" = "--dry-run" ]; } || { + usage + exit 2 + } + ;; + review-tests) + [ "$#" -ge 2 ] || { usage; exit 2; } + ;; + merge-complete) + [ "$#" -eq 3 ] && is_canonical_pr_number "$1" && + [[ "$2" =~ ^[0-9a-f]{40}$ ]] && [ "$3" = --confirmed-operator-completion ] || { usage; exit 2; } + ;; + merge-run | merge-recover) + local merge_pr="${1-}" auto_merge=false recovery_oid="" replacement_head="" body_path="" legacy_directory="" + [ -n "$merge_pr" ] || { usage; exit 2; } + shift + if [ "$cmd" = merge-recover ]; then + [ "$#" -ge 2 ] && is_canonical_pr_number "$merge_pr" && + [[ "$1" =~ ^[0-9a-f]{40}$ ]] && [ "$2" = --confirmed-operator-recovery ] || { usage; exit 2; } + recovery_oid="$1" + shift 2 + fi + while [ "$#" -gt 0 ]; do + case "$1" in + --auto-merge) + [ "$cmd" = merge-run ] && [ "$auto_merge" = false ] || { usage; exit 2; } + auto_merge=true + shift + ;; + --body-file) + [ "$#" -ge 2 ] && [ -n "$2" ] && [ -z "$body_path" ] || { usage; exit 2; } + body_path="$2" + shift 2 + ;; + --legacy-refusal) + [ "$cmd" = merge-recover ] && [ "$#" -ge 2 ] && [ -n "$2" ] && [ -z "$legacy_directory" ] || { usage; exit 2; } + legacy_directory="$2" + shift 2 + ;; + --replacement-head) + [ "$cmd" = merge-recover ] && [ "$#" -ge 2 ] && + [[ "$2" =~ ^[0-9a-f]{40}$ ]] && [ -z "$replacement_head" ] || { usage; exit 2; } + replacement_head="$2" + shift 2 + ;; + *) usage; exit 2 ;; + esac + done + if [ "$cmd" = merge-run ] && [ "${OPENCLAW_PR_AUTO_MERGE:-}" = 1 ]; then + auto_merge=true + fi + [ -z "$legacy_directory" ] || [ -n "$replacement_head" ] || { usage; exit 2; } + set -- "$merge_pr" + ;; + review-init | review-checkout-main | review-checkout-pr | review-claim | review-guard | review-artifacts-init | review-validate-artifacts | prepare-init | prepare-validate-commit | prepare-gates | prepare-push | prepare-sync-head | prepare-run | ci-dispatch | merge-verify) + [ "$#" -ge 1 ] || { usage; exit 2; } + ;; + *) + usage + exit 2 + ;; + esac + + require_cmds + + # merge-run reconciles retained outcomes before reading disposable artifacts or base guards. + if [ "$cmd" != merge-run ] && is_main_only_pr_command "$cmd"; then + require_main_target_pr "${1-}" + fi + + if is_locked_pr_command "$cmd"; then + local locked_pr="${1-}" + acquire_pr_operation_lock "$locked_pr" + begin_pr_operation_validation_phase + # Temp-backed shell redirections must fail while the supervisor can auto-release. + validate_pr_temp_storage + trap 'exit 129' HUP + trap 'exit 130' INT + trap 'exit 131' QUIT + trap 'exit 143' TERM + fi + + case "$cmd" in + ls) + list_pr_worktrees + ;; + gc) + local dry_run=false + if [ "$#" -eq 1 ]; then + dry_run=true + fi + gc_pr_worktrees "$dry_run" + ;; + review-init) + local pr="${1-}" + [ -n "$pr" ] || { usage; exit 2; } + review_init "$pr" + ;; + review-checkout-main) + local pr="${1-}" + [ -n "$pr" ] || { usage; exit 2; } + review_checkout_main "$pr" + ;; + review-checkout-pr) + local pr="${1-}" + [ -n "$pr" ] || { usage; exit 2; } + review_checkout_pr "$pr" + ;; + review-claim) + local pr="${1-}" + [ -n "$pr" ] || { usage; exit 2; } + review_claim "$pr" + ;; + review-guard) + local pr="${1-}" + [ -n "$pr" ] || { usage; exit 2; } + review_guard "$pr" + ;; + review-artifacts-init) + local pr="${1-}" + [ -n "$pr" ] || { usage; exit 2; } + review_artifacts_init "$pr" + ;; + review-validate-artifacts) + local pr="${1-}" + [ -n "$pr" ] || { usage; exit 2; } + review_validate_artifacts "$pr" + ;; + review-tests) + local pr="${1-}" + [ -n "$pr" ] || { usage; exit 2; } + shift || true + review_tests "$pr" "$@" + ;; + prepare-init) + local pr="${1-}" + [ -n "$pr" ] || { usage; exit 2; } + prepare_init "$pr" + ;; + prepare-validate-commit) + local pr="${1-}" + [ -n "$pr" ] || { usage; exit 2; } + prepare_validate_commit "$pr" + ;; + prepare-gates) + local pr="${1-}" + [ -n "$pr" ] || { usage; exit 2; } + prepare_gates "$pr" + ;; + prepare-push) + local pr="${1-}" + [ -n "$pr" ] || { usage; exit 2; } + prepare_push "$pr" + ;; + prepare-sync-head) + local pr="${1-}" + [ -n "$pr" ] || { usage; exit 2; } + prepare_sync_head "$pr" + ;; + prepare-run) + local pr="${1-}" + [ -n "$pr" ] || { usage; exit 2; } + prepare_run "$pr" + ;; + ci-dispatch) + local pr="${1-}" + [ -n "$pr" ] || { usage; exit 2; } + shift + ci_dispatch "$pr" "$@" + ;; + merge-verify) + local pr="${1-}" + [ -n "$pr" ] || { usage; exit 2; } + merge_verify "$pr" + ;; + merge-run | merge-recover) + merge_run "$merge_pr" "$auto_merge" "$recovery_oid" "$replacement_head" "$body_path" "$legacy_directory" + ;; + merge-complete) + merge_complete "$1" "$2" + ;; + *) + usage + exit 2 + ;; + esac +} + +main "$@" diff --git a/scripts/pr-merge b/scripts/pr-merge new file mode 100644 index 0000000000000000000000000000000000000000..ee6fdb3ea6cb74e5580a16e633c83979ffd45c93 --- /dev/null +++ b/scripts/pr-merge @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$(cd "$(dirname "$0")" && pwd)" +base="$script_dir/pr" + +usage() { + cat <<USAGE +Usage: + scripts/pr-merge <PR> # verify only (backward compatible) + scripts/pr-merge verify <PR> # verify only + scripts/pr-merge run <PR> # verify + merge + post-merge checks + cleanup +USAGE +} + +if [ "$#" -eq 1 ]; then + exec "$base" merge-verify "$1" +fi + +if [ "$#" -eq 2 ]; then + mode="$1" + pr="$2" + case "$mode" in + verify) + exec "$base" merge-verify "$pr" + ;; + run) + exec "$base" merge-run "$pr" + ;; + *) + usage + exit 2 + ;; + esac +fi + +usage +exit 2 diff --git a/scripts/pr-prepare b/scripts/pr-prepare new file mode 100644 index 0000000000000000000000000000000000000000..56b9e1000414c1d9b9233211c905429dbf29020c --- /dev/null +++ b/scripts/pr-prepare @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [ "$#" -ne 2 ]; then + echo "Usage: scripts/pr-prepare <init|validate-commit|gates|push|run> <PR>" + exit 2 +fi + +mode="$1" +pr="$2" +script_dir="$(cd "$(dirname "$0")" && pwd)" +base="$script_dir/pr" + +case "$mode" in + init) + exec "$base" prepare-init "$pr" + ;; + validate-commit) + exec "$base" prepare-validate-commit "$pr" + ;; + gates) + exec "$base" prepare-gates "$pr" + ;; + push) + exec "$base" prepare-push "$pr" + ;; + run) + exec "$base" prepare-run "$pr" + ;; + *) + echo "Usage: scripts/pr-prepare <init|validate-commit|gates|push|run> <PR>" + exit 2 + ;; +esac diff --git a/scripts/preflight-frozen-target-contracts.mjs b/scripts/preflight-frozen-target-contracts.mjs new file mode 100644 index 0000000000000000000000000000000000000000..820fd703f92160241a0f7fed9216cc2add620016 --- /dev/null +++ b/scripts/preflight-frozen-target-contracts.mjs @@ -0,0 +1,1348 @@ +#!/usr/bin/env node + +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { lstatSync, readFileSync, realpathSync, statSync } from "node:fs"; +import { dirname, join, relative, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const ownRoot = realpathSync(resolve(dirname(fileURLToPath(import.meta.url)), "..")); +const entryPath = "scripts/preflight-frozen-target-contracts.mjs"; +const readerPath = "scripts/lib/frozen-target-source.mjs"; +const toolingClosure = [ + entryPath, + readerPath, + "scripts/lib/docker-e2e-plan.mts", + "scripts/lib/docker-e2e-scenarios.mts", + "scripts/lib/official-external-channel-catalog.json", + "scripts/lib/upgrade-survivor-policy.mjs", + "scripts/lib/release-version.mjs", + "scripts/lib/frozen-target-compat.sh", + "scripts/resolve-frozen-codex-live-suite.mjs", + "scripts/resolve-fs-safe-native-contract.mjs", + "scripts/e2e/lib/upgrade-survivor/config-recipe.mts", + "scripts/windows-cmd-helpers.mjs", + "package.json", + "pnpm-lock.yaml", +]; +const workflowToolingClosure = [ + "scripts/plan-release-workflow-matrix.mjs", + "scripts/lib/direct-run.mjs", + "scripts/lib/plugin-prerelease-test-plan.mts", + "scripts/plan-targeted-docker-lane-groups.mjs", + "scripts/lib/numeric-options.mjs", +]; +const maxRecordBytes = 256 * 1024; +const prefix = "OPENCLAW_FROZEN_TARGET_"; +const shellOwners = { + onboard: ["onboard_contract", [`${prefix}ONBOARD_CASES`]], + "release-typed-onboarding": [ + "typed_onboarding_contract", + [ + `${prefix}ONBOARD_SESSION_MEMORY_HOOK_MODE`, + `${prefix}TYPED_ONBOARDING_SCENARIO_PATH`, + `${prefix}TYPED_ONBOARDING_ASSERTIONS_PATH`, + `${prefix}TYPED_ONBOARDING_ASSERTION_FILES_PATH`, + `${prefix}TYPED_ONBOARDING_MOCK_CONFIG_PATH`, + ], + ], + "session-runtime-context": [ + ["runtime_context_contract", "session_cold_storage_contract"], + [ + `${prefix}RUNTIME_CONTEXT_INPUT_MODE`, + `${prefix}SESSION_REPAIR_MODE`, + `${prefix}SESSION_COLD_STORAGE_MODE`, + ], + ], + "openai-chat-tools": ["session_cold_storage_contract", [`${prefix}SESSION_COLD_STORAGE_MODE`]], + "mcp-code-mode-gateway": [ + "mcp_code_mode_contract", + [`${prefix}MCP_MEMORY_CONFIG_MODE`, `${prefix}MCP_CODE_MODE_CATALOG_MODE`], + ], + "agent-bundle-mcp-tools": [ + "agent_bundle_mcp_contract", + [`${prefix}AGENT_BUNDLE_MCP_MODE`, `${prefix}AGENT_BUNDLE_MCP_CLIENT_PATH`], + ], + "gateway-network": ["gateway_network_layout", [`${prefix}GATEWAY_NETWORK_LEGACY_LIB`]], + plugins: [ + "plugin_harness_capabilities", + [`${prefix}PLUGIN_UNINSTALL_MODE`, "OPENCLAW_FROZEN_PLUGIN_PRERELEASE_FIXTURE_DIALECT"], + ], + "live-cli-backend": ["live_cli_backend_package_mode", [`${prefix}LIVE_CLI_BACKEND_PACKAGE_MODE`]], + "update-channel-switch": [ + "update_channel_dry_run_mode", + [ + "OPENCLAW_UPDATE_CHANNEL_DRY_RUN_PACKAGE_COMPAT", + "OPENCLAW_UPDATE_CHANNEL_DIRTY_BLOCK_EXIT_ZERO_COMPAT", + ], + ], + "upgrade-survivor": [ + "upgrade_survivor_capabilities", + ["OPENCLAW_FROZEN_UPGRADE_SURVIVOR_CLAWHUB_MODE"], + ], +}; + +// These are the existing wrappers' generic resolver arguments, not new dialect rules. +const targetFiles = { + "npm-onboard-channel-agent": [ + ["scripts/e2e/lib/npm-onboard-channel-agent/assertions.mjs"], + ["scripts/e2e/lib/fixtures/mock-openai-config.mjs"], + ], + "codex-on-demand": [ + ["scripts/e2e/lib/codex-on-demand/assertions.mjs"], + ["scripts/e2e/lib/codex-on-demand/doctor-checks.mjs", ""], + ], + "update-corrupt-plugin": [["scripts/e2e/lib/plugin-update/corrupt-update-scenario.sh"]], + "kitchen-sink-plugin": [["scripts/e2e/lib/kitchen-sink-plugin/assertions.mjs"]], +}; +const supportFiles = { + "npm-onboard-channel-agent": [ + "agent-turn-output.mjs", + "auth-profile-store-assertions.mjs", + "env-limits.mjs", + "text-file-utils.mjs", + "openclaw-state-paths.mjs", + "fixtures/common.mjs", + ], + "codex-on-demand": [ + "auth-profile-store-assertions.mjs", + "codex-install-utils.mjs", + "codex-release-package-assertions.mjs", + "openclaw-state-paths.mjs", + "plugin-index-sqlite.mjs", + "fixtures/common.mjs", + "env-limits.mjs", + "text-file-utils.mjs", + ], + "kitchen-sink-plugin": [ + "env-limits.mjs", + "openclaw-state-paths.mjs", + "plugin-index-sqlite.mjs", + "plugin-uninstall-assertions.mjs", + "text-file-utils.mjs", + ], + "update-corrupt-plugin": [ + "plugins/fixtures.sh", + "plugin-update/probe.mjs", + "plugin-index-sqlite.mjs", + "update-first-hop-package-fixtures.mjs", + "release-scenarios/assertions.mjs", + "agent-turn-output.mjs", + "auth-profile-store-assertions.mjs", + "fixtures/mock-openai-config.mjs", + "env-limits.mjs", + "text-file-utils.mjs", + "openclaw-state-paths.mjs", + "plugin-uninstall-assertions.mjs", + "release-assertion-files.mjs", + "package-compat.mjs", + "plugin-update/consent-scenario.mjs", + "plugin-update/process-observer.mjs", + ], +}; + +// Acquisition inputs only. Dialect decisions remain in the shared resolver owners. +const selectedMetadata = { + onboard: ["src/config/zod-schema.ts"], + "release-typed-onboarding": [ + "src/commands/onboard-hooks.ts", + "scripts/e2e/lib/release-typed-onboarding/scenario.sh", + "scripts/e2e/lib/release-scenarios/assertions.mjs", + "scripts/e2e/lib/release-assertion-files.mjs", + "scripts/e2e/lib/fixtures/mock-openai-config.mjs", + ], + "session-runtime-context": [ + "src/state/openclaw-agent-db-session-migrations.ts", + "src/commands/doctor-session-transcripts.ts", + "src/agents/embedded-agent-runner/run/runtime-context-prompt.ts", + "src/config/zod-schema.session.ts", + "src/config/zod-schema.session-config.ts", + ], + "openai-chat-tools": [ + "src/config/zod-schema.session.ts", + "src/config/zod-schema.session-config.ts", + ], + "mcp-code-mode-gateway": ["src/agents/memory-search.ts", "src/agents/code-mode-namespaces.ts"], + "agent-bundle-mcp-tools": [ + "package.json", + "scripts/e2e/agent-bundle-mcp-tools-docker-client.ts", + "test/e2e/qa-lab/runtime/agent-bundle-mcp-tools-docker-client.ts", + "scripts/e2e/lib/temp-state-dir.ts", + "src/agents/agent-bundle-mcp-manager-api.ts", + "src/agents/agent-bundle-mcp-runtime.ts", + ], + "gateway-network": [ + "scripts/e2e/lib/gateway-network/client.mjs", + "scripts/e2e/lib/gateway-network/client.mts", + ], + plugins: [ + "scripts/e2e/lib/plugins/assertions.mjs", + "src/config/types.messages.ts", + "src/config/types.plugins.ts", + "src/plugin-sdk/session-store-runtime.ts", + "src/plugins/uninstall-package-plan.ts", + ], + "live-cli-backend": ["scripts/print-cli-backend-live-metadata.ts"], + "update-channel-switch": ["src/cli/update-cli/update-command.ts"], + "upgrade-survivor": [ + "package.json", + "src/infra/clawhub-install-trust.ts", + "src/plugins/clawhub.ts", + "scripts/e2e/lib/upgrade-survivor", + "scripts/lib/npm-publish-plan.mjs", + "scripts/windows-cmd-helpers.mjs", + "scripts/e2e/lib/plugin-index-sqlite.mjs", + "scripts/e2e/lib/env-limits.mjs", + "scripts/e2e/lib/text-file-utils.mjs", + ], +}; + +function workflowRequest(env) { + const serialized = env.ADMISSION_INPUTS ?? "{}"; + if (typeof serialized !== "string" || Buffer.byteLength(serialized, "utf8") > 48 * 1024) { + throw new Error("invalid workflow inputs"); + } + const raw = JSON.parse(serialized); + if ( + !raw || + Array.isArray(raw) || + typeof raw !== "object" || + Object.values(raw).some((value) => !["string", "boolean", "number"].includes(typeof value)) + ) { + throw new Error("invalid workflow inputs"); + } + const get = (name, fallback = "") => raw[name] ?? fallback; + const flag = (value) => value === true || value === "true"; + const profile = + env.ADMISSION_RELEASE_PROFILE || get("release_test_profile", get("release_profile", "stable")); + const options = { + releaseProfile: profile === "minimum" ? "beta" : profile, + phase: get("phase", "all"), + rerunGroup: get("rerun_group", "all"), + runReleaseSoak: flag(get("run_release_soak")) || profile === "stable" || profile === "full", + qaFilterSeen: flag(env.ADMISSION_QA_FILTER_SEEN), + liveSuiteFilter: env.ADMISSION_REPO_LIVE_SUITE_FILTER ?? get("live_suite_filter"), + liveModelsOnly: flag(get("live_models_only")), + liveModelProviders: get("live_model_providers"), + includeLiveSuites: flag(get("include_live_suites", true)), + includeReleasePathSuites: flag(get("include_release_path_suites", true)), + includeOpenWebUI: flag(get("include_openwebui")), + includeRepoE2e: flag(get("include_repo_e2e", true)), + prepareOnly: flag(get("prepare_only")), + dockerLanes: get("docker_lanes"), + targetedDockerLaneGroupSize: String(get("targeted_docker_lane_group_size", 1)), + suiteProfile: get("suite_profile", "package"), + telegramMode: get("telegram_mode", "none"), + telegramScenarios: get("telegram_scenarios"), + upgradeSurvivorBaseline: + env.ADMISSION_BASELINE ?? get("published_upgrade_survivor_baseline", "openclaw@latest"), + upgradeSurvivorBaselines: + env.ADMISSION_BASELINES ?? get("published_upgrade_survivor_baselines"), + upgradeSurvivorBaselineScope: + env.ADMISSION_BASELINE_SCOPE ?? + get("published_upgrade_survivor_baseline_scope", "all-scenarios"), + upgradeSurvivorScenarios: get("published_upgrade_survivor_scenarios"), + baselinesResolved: env.ADMISSION_BASELINES_RESOLVED === "true", + packageOverride: Boolean(String(get("release_package_spec")).trim()), + acceptanceOverride: Boolean(String(get("package_acceptance_package_spec")).trim()), + }; + const workflow = env.ADMISSION_WORKFLOW; + if (workflow === "parent") { + options.upgradeSurvivorScenarios = options.runReleaseSoak ? "reported-issues" : ""; + } + return { + version: 2, + repository: text(env.GITHUB_REPOSITORY, "repository"), + selected: { + root: text(env.ADMISSION_SELECTED_ROOT, "selected root"), + sha: text(env.ADMISSION_SELECTED_SHA, "selected SHA"), + }, + tooling: { + root: text(env.ADMISSION_TOOLING_ROOT, "tooling root"), + sha: text(env.ADMISSION_TOOLING_SHA, "tooling SHA"), + }, + allowFrozenTargetScenarioOmissions: + flag(get("allow_frozen_target_scenario_omissions")) || + (workflow === "parent" && Boolean(get("target_context_ref"))), + workflow, + options, + requestedBaselines: { + baseline: get("published_upgrade_survivor_baseline", "openclaw@latest"), + baselines: get("published_upgrade_survivor_baselines"), + scope: get("published_upgrade_survivor_baseline_scope", "all-scenarios"), + scenarios: options.upgradeSurvivorScenarios, + }, + binding: { + workflowRef: text(env.ADMISSION_WORKFLOW_REF, "workflow ref"), + inputsDigest: createHash("sha256") + .update( + JSON.stringify( + Object.fromEntries( + Object.keys(raw) + .toSorted() + .map((key) => [key, raw[key]]), + ), + ), + ) + .digest("hex"), + coveragePolicy: text(env.ADMISSION_COVERAGE_POLICY ?? "", "coverage policy"), + candidateRequestDigest: text( + env.ADMISSION_CANDIDATE_REQUEST_DIGEST ?? "", + "candidate request digest", + ), + packageSourceSha: text(env.ADMISSION_PACKAGE_SOURCE_SHA ?? "", "package source SHA"), + packageSha256: text(env.ADMISSION_PACKAGE_SHA256 ?? "", "package digest"), + packageVersion: text(env.ADMISSION_PACKAGE_VERSION ?? "", "package version"), + stage: text(env.ADMISSION_STAGE ?? "source", "admission stage"), + }, + provenance: { + runId: text(env.GITHUB_RUN_ID, "run id"), + runAttempt: text(env.GITHUB_RUN_ATTEMPT, "run attempt"), + }, + }; +} + +async function planWorkflowAdmission(input) { + object( + input, + [ + "version", + "repository", + "selected", + "tooling", + "allowFrozenTargetScenarioOmissions", + "workflow", + "options", + "requestedBaselines", + "binding", + "provenance", + ], + "workflow admission", + ); + if ( + input.version !== 2 || + input.repository !== "openclaw/openclaw" || + !["parent", "release-checks", "reusable", "package"].includes(input.workflow) + ) { + throw new Error("invalid workflow admission identity"); + } + for (const key of ["selected", "tooling"]) { + object(input[key], ["root", "sha"], `${key} identity`); + text(input[key].root, `${key} root`); + if (!/^[a-f0-9]{40}$/u.test(input[key].sha)) { + throw new Error(`missing exact ${key} source SHA`); + } + } + const allow = boolean(input.allowFrozenTargetScenarioOmissions); + const options = object( + input.options, + [ + "releaseProfile", + "phase", + "rerunGroup", + "runReleaseSoak", + "qaFilterSeen", + "liveSuiteFilter", + "liveModelsOnly", + "liveModelProviders", + "includeLiveSuites", + "includeReleasePathSuites", + "includeOpenWebUI", + "includeRepoE2e", + "prepareOnly", + "dockerLanes", + "targetedDockerLaneGroupSize", + "suiteProfile", + "telegramMode", + "telegramScenarios", + "upgradeSurvivorBaseline", + "upgradeSurvivorBaselines", + "upgradeSurvivorBaselineScope", + "upgradeSurvivorScenarios", + "baselinesResolved", + "packageOverride", + "acceptanceOverride", + ], + "workflow selection", + ); + for (const [key, value] of Object.entries(options)) { + if (typeof value === "string") { + if ( + [ + "dockerLanes", + "liveModelProviders", + "telegramScenarios", + "upgradeSurvivorBaselines", + "upgradeSurvivorScenarios", + ].includes(key) + ) { + tokenListText(value, "selection value"); + } else { + text(value, "selection value"); + } + } else { + boolean(value); + } + } + object( + input.binding, + [ + "workflowRef", + "inputsDigest", + "coveragePolicy", + "candidateRequestDigest", + "packageSourceSha", + "packageSha256", + "packageVersion", + "stage", + ], + "workflow binding", + ); + for (const value of Object.values(input.binding)) { + text(value, "binding"); + } + if ( + !/^[a-f0-9]{64}$/u.test(input.binding.inputsDigest) || + !input.binding.workflowRef?.startsWith(`openclaw/openclaw/.github/workflows/`) + ) { + throw new Error("missing workflow input binding"); + } + if ( + input.workflow === "package" && + input.binding.stage !== "known-source" && + (!/^[a-f0-9]{40}$/u.test(input.binding.packageSourceSha) || + !/^[a-f0-9]{64}$/u.test(input.binding.packageSha256) || + !input.binding.packageVersion?.trim()) + ) { + throw new Error("complete resolved package identity is required"); + } + object(input.provenance, ["runId", "runAttempt"], "attempt provenance"); + if ( + !/^[1-9][0-9]*$/u.test(input.provenance.runId) || + !/^[1-9][0-9]*$/u.test(input.provenance.runAttempt) + ) { + throw new Error("invalid run attempt provenance"); + } + if (input.binding.packageSourceSha && input.binding.packageSourceSha !== input.selected.sha) { + throw new Error("package source differs from selected source"); + } + const obligations = []; + const requestedBaselines = object( + input.requestedBaselines, + ["baseline", "baselines", "scope", "scenarios"], + "requested baselines", + ); + for (const [key, value] of Object.entries(requestedBaselines)) { + if (key === "baselines" || key === "scenarios") { + tokenListText(value, "requested baseline"); + } else { + text(value, "requested baseline"); + } + } + // Planning precedes selected-object acquisition; only tooling is read here. + await loadVerifiedTooling(input.tooling, true); + const { + createPackageAcceptanceSelection, + createReleaseCheckSelection, + createReleaseSourceSelection, + RELEASE_PACKAGE_ACCEPTANCE_LANES, + } = await import("./plan-release-workflow-matrix.mjs"); + const { releasePathChunkLanes } = await import("./lib/docker-e2e-scenarios.mts"); + const { createPluginPrereleaseTestPlan } = await import("./lib/plugin-prerelease-test-plan.mts"); + const { parseUpgradeSurvivorScenarios } = await import("./lib/upgrade-survivor-policy.mjs"); + const baselineOptions = options.baselinesResolved + ? options + : { + ...options, + upgradeSurvivorBaseline: "", + upgradeSurvivorBaselines: "", + upgradeSurvivorBaselineScope: "all-scenarios", + }; + const selections = []; + const add = (overrides) => + selections.push(createReleaseSourceSelection({ ...baselineOptions, ...overrides })); + if (input.workflow === "reusable") { + add({}); + } else if (input.workflow === "package") { + const profile = createPackageAcceptanceSelection(options); + add({ + dockerLanes: profile.docker_lanes, + includeOpenWebUI: profile.include_openwebui, + includeReleasePathSuites: profile.include_release_path_suites, + includeLiveSuites: false, + }); + } else { + const releaseGroups = [ + "all", + "install-smoke", + "cross-os", + "live-e2e", + "package", + "qa", + "qa-parity", + "qa-live", + ]; + if (releaseGroups.includes(options.rerunGroup)) { + const groups = createReleaseCheckSelection({ + ...options, + repoLiveSuiteFilter: options.liveSuiteFilter, + phase: input.workflow === "parent" ? "all" : options.phase, + }); + if (groups.live_e2e_scheduled) { + add({ + includeLiveSuites: true, + includeReleasePathSuites: false, + includeOpenWebUI: false, + dockerLanes: "", + }); + } + if (groups.docker_required && !options.packageOverride) { + add({ + includeLiveSuites: false, + includeReleasePathSuites: true, + includeOpenWebUI: options.releaseProfile !== "beta", + dockerLanes: "", + }); + } + if ( + groups.package_acceptance_scheduled && + !options.packageOverride && + !options.acceptanceOverride + ) { + add({ + includeLiveSuites: false, + includeReleasePathSuites: false, + includeOpenWebUI: false, + dockerLanes: RELEASE_PACKAGE_ACCEPTANCE_LANES, + }); + } + if (groups.package_required && (options.packageOverride || options.acceptanceOverride)) { + obligations.push({ kind: "package-source", status: "UNRESOLVED" }); + } + } + if (input.workflow === "parent" && ["all", "plugin-prerelease"].includes(options.rerunGroup)) { + add({ + includeLiveSuites: false, + includeReleasePathSuites: false, + includeOpenWebUI: false, + dockerLanes: createPluginPrereleaseTestPlan().dockerLanes.join(" "), + }); + } + } + const docker = selections.flatMap((selection) => selection.docker); + const explicitConsumers = [...new Set(selections.flatMap((selection) => selection.consumers))]; + const consumers = new Set(explicitConsumers); + const codexSuites = [...new Set(selections.flatMap((selection) => selection.codexSuites))]; + const possibleLanes = []; + let mobilePairingSelected = false; + for (const selection of docker) { + const lanes = + selection.lanes ?? + releasePathChunkLanes(selection.chunk, { + releaseProfile: selection.releaseProfile, + includeOpenWebUI: selection.includeOpenWebUI, + }).map(({ name }) => name); + possibleLanes.push(...lanes); + const survivorLanes = lanes.filter((lane) => + /^(published-upgrade-survivor|update-migration)(-|$)/u.test(lane), + ); + if ( + allow && + survivorLanes.length && + (survivorLanes.some((lane) => lane.includes("mobile-pairing-reconnect")) || + parseUpgradeSurvivorScenarios(selection.scenarios ?? "").includes( + "mobile-pairing-reconnect", + )) + ) { + mobilePairingSelected = true; + } + } + for (const name of possibleLanes) { + const consumer = consumerForLane(name); + if (consumer) { + consumers.add(consumer); + } + } + if ( + possibleLanes.some( + (lane) => + lane === "root-managed-vps-upgrade" || + lane === "update-restart-auth" || + /^(published-upgrade-survivor|update-migration)(-|$)/u.test(lane), + ) + ) { + if (!options.baselinesResolved) { + obligations.push({ + kind: "upgrade-baselines", + status: "UNRESOLVED", + requested: requestedBaselines, + }); + } else { + const { normalizeUpgradeSurvivorBaselineSpec, parseUpgradeSurvivorBaselineSpecs } = + await import("./lib/upgrade-survivor-policy.mjs"); + const specs = [ + normalizeUpgradeSurvivorBaselineSpec(options.upgradeSurvivorBaseline), + ...parseUpgradeSurvivorBaselineSpecs(options.upgradeSurvivorBaselines), + ]; + if (!specs[0] || specs.some((spec) => !/^openclaw@[0-9]/u.test(spec))) { + throw new Error("unresolved upgrade baselines at the execution boundary"); + } + } + } + const sourcePaths = new Set(); + if (allow) { + for (const consumer of consumers) { + for (const path of selectedMetadata[ + consumer === "kitchen-sink-plugin" ? "plugins" : consumer + ] ?? []) { + sourcePaths.add(path); + } + for (const [path] of targetFiles[consumer] ?? []) { + sourcePaths.add(path); + } + } + if (codexSuites.length) { + sourcePaths.add("extensions/codex/provider-catalog.ts"); + sourcePaths.add("scripts/test-live-codex-harness-docker.sh"); + } + } + if (consumers.has("codex-on-demand")) { + sourcePaths.add("extensions/codex/package.json"); + } + const fsSafeNative = selections.some((selection) => selection.fsSafeNative); + if (fsSafeNative && allow) { + sourcePaths.add("package.json"); + sourcePaths.add("src/infra/fs-safe-defaults.ts"); + } + for (const [lane, path] of [ + ["update-first-hop-compat", "scripts/runtime-postbuild.mts"], + ["update-corrupt-plugin", "src/cli/update-cli/update-command-plugin-preflight.ts"], + ]) { + if (possibleLanes.includes(lane)) { + sourcePaths.add(path); + } + } + if (mobilePairingSelected) { + sourcePaths.add("src/gateway/node-command-policy.ts"); + } + if ( + possibleLanes.some((lane) => /^(published-upgrade-survivor|update-migration)(-|$)/u.test(lane)) + ) { + sourcePaths.add("scripts/e2e/lib/upgrade-survivor/assertions.mjs"); + } + if (docker.length > 256) { + throw new Error("too many selected Docker groups"); + } + return { + docker, + consumers: [...consumers].toSorted((left, right) => (left < right ? -1 : left > right ? 1 : 0)), + explicitConsumers, + codexSuites, + fsSafeNative, + preparationLanes: [...new Set(selections.flatMap((selection) => selection.preparationLanes))], + sourcePaths: [...sourcePaths].toSorted((left, right) => + left < right ? -1 : left > right ? 1 : 0, + ), + sourceHistory: fsSafeNative && allow, + parserRequired: allow && consumers.has("agent-bundle-mcp-tools"), + obligations, + requestedBaselines, + }; +} + +async function preflightWorkflow(input) { + const plan = await planWorkflowAdmission(input); + if ( + (input.workflow === "reusable" || input.binding.stage === "resolved-package") && + plan.obligations.some((obligation) => obligation.kind === "upgrade-baselines") + ) { + throw new Error("unresolved upgrade baselines at the execution boundary"); + } + const common = { + version: 1, + repository: input.repository, + selected: input.selected, + tooling: input.tooling, + allowFrozenTargetScenarioOmissions: input.allowFrozenTargetScenarioOmissions, + }; + const verified = await loadVerifiedTooling(input.tooling, true); + // Share verified evidence, not the reader's deadline or cumulative read set. + const verifiedTooling = { + sha: input.tooling.sha, + createFrozenTargetSource: verified.createFrozenTargetSource, + identities: Object.freeze(verified.source.blobIdentities().map(Object.freeze)), + }; + // Evaluate each baseline/scenario group without collapsing the execution owner's pairing. + const evaluations = []; + for (const docker of plan.docker) { + evaluations.push( + await preflightFrozenTargetContracts( + { ...common, selection: { docker } }, + true, + verifiedTooling, + ), + ); + } + evaluations.push( + await preflightFrozenTargetContracts( + { + ...common, + selection: { + consumers: plan.explicitConsumers, + codexSuites: plan.codexSuites, + fsSafeNative: plan.fsSafeNative, + }, + }, + true, + verifiedTooling, + ), + ); + const executionStates = evaluations.flatMap((evaluation) => [ + ...(evaluation.docker ? [evaluation.docker.status] : []), + ...evaluation.contracts + .filter((contract) => contract.consumer !== "fs-safe-native") + .map((contract) => contract.status), + ]); + const sources = { selected: [], tooling: [] }; + const identityIndexes = { selected: new Map(), tooling: new Map() }; + const compactEvaluations = evaluations.map((evaluation) => { + const sourceRefs = {}; + for (const role of ["selected", "tooling"]) { + sourceRefs[role] = evaluation.sources[role].map((identity) => { + const key = JSON.stringify(identity); + if (!identityIndexes[role].has(key)) { + identityIndexes[role].set(key, sources[role].length); + sources[role].push(identity); + } + return identityIndexes[role].get(key); + }); + } + return Object.assign( + { + selection: evaluation.selection, + contracts: evaluation.contracts, + }, + evaluation.docker ? { docker: evaluation.docker } : {}, + { sourceRefs, digest: evaluation.digest }, + ); + }); + const content = { + version: 2, + repository: input.repository, + selectedSha: input.selected.sha, + toolingSha: input.tooling.sha, + workflow: input.workflow, + binding: input.binding, + options: input.options, + requestedBaselines: plan.requestedBaselines, + obligations: plan.obligations, + preparationLanes: plan.preparationLanes, + evaluationIdentity: { + version: 1, + repository: input.repository, + selectedSha: input.selected.sha, + toolingSha: input.tooling.sha, + }, + sources, + evaluations: compactEvaluations, + status: plan.obligations.length + ? "UNRESOLVED" + : executionStates.includes("ADMITTED") + ? "ADMITTED" + : executionStates.length + ? "NOT RUN" + : plan.preparationLanes.length + ? "PREPARATION ONLY" + : "UNSELECTED", + }; + const bytes = JSON.stringify(content); + const record = { + ...content, + digest: createHash("sha256").update(bytes).digest("hex"), + provenance: input.provenance, + }; + if (Buffer.byteLength(`${JSON.stringify(record)}\n`) > maxRecordBytes) { + throw new Error("workflow admission record exceeds limit"); + } + return record; +} + +function object(value, keys, label) { + if ( + !value || + typeof value !== "object" || + Array.isArray(value) || + Object.keys(value).some((key) => !keys.includes(key)) + ) { + throw new Error(`invalid ${label}`); + } + return value; +} + +function text(value, label, limit = 4096) { + if (typeof value !== "string" || value.length > limit) { + throw new Error(`invalid ${label}`); + } + for (let index = 0; index < value.length; index += 1) { + if (value.charCodeAt(index) < 32) { + throw new Error(`invalid ${label}`); + } + } + return value; +} + +function tokenListText(value, label) { + if (typeof value !== "string") { + throw new Error(`invalid ${label}`); + } + // Preserve list whitespace for the owning parser without admitting other controls. + text(value.replace(/[\t\r\n]/gu, " "), label); + return value; +} + +function strings(value, label) { + if (!Array.isArray(value) || value.length > 256) { + throw new Error(`invalid ${label}`); + } + return [...new Set(value.map((entry) => text(entry, label, 256)))].toSorted(); +} + +function boolean(value, fallback = false) { + if (value === undefined) { + return fallback; + } + if (typeof value !== "boolean") { + throw new Error("expected boolean selection"); + } + return value; +} + +function required(source, path) { + const content = source.readText(path); + if (content === null) { + throw new Error(`missing required contract file: ${path}`); + } + return content; +} + +function verifyToolingFile(path, committed) { + const file = join(ownRoot, path); + let info; + try { + info = lstatSync(file); + if (!info.isFile() || realpathSync.native(file) !== file) { + throw new Error("not an owned regular file"); + } + } catch { + throw new Error(`tooling closure requires an owned regular file: ${path}`); + } + if (info.size !== committed.length || !readFileSync(file).equals(committed)) { + throw new Error(`tooling closure does not match committed source: ${path}`); + } +} + +function verifyReaderBootstrap(sha) { + if (!/^[0-9a-f]{40}$/.test(sha)) { + throw new Error("tooling source requires a full lowercase commit SHA"); + } + const env = Object.fromEntries( + Object.entries(process.env).filter(([key]) => !key.startsWith("GIT_")), + ); + Object.assign(env, { + GIT_NO_LAZY_FETCH: "1", + GIT_NO_REPLACE_OBJECTS: "1", + GIT_OPTIONAL_LOCKS: "0", + GIT_CONFIG_NOSYSTEM: "1", + GIT_CONFIG_GLOBAL: "/dev/null", + }); + const deadline = Date.now() + 30_000; + const git = (...args) => { + try { + const timeout = deadline - Date.now(); + if (timeout <= 0) { + throw new Error("bootstrap read limit exceeded"); + } + // Unsupported no-lazy-fetch flags fail closed before the reader can load. + return execFileSync( + "git", + ["--no-lazy-fetch", "--no-replace-objects", "-C", ownRoot, ...args], + { + env, + stdio: ["ignore", "pipe", "pipe"], + timeout, + maxBuffer: 16 * 1024 * 1024, + }, + ); + } catch { + throw new Error("unable to read committed tooling bootstrap"); + } + }; + // The launched bootstrap and checkout are trusted; this binds their working + // bytes, not hostile bootstrap code or concurrent writers. The verified reader + // still owns Git version, HEAD, commit/tree hashes, and all other source reads. + for (const path of [entryPath, readerPath]) { + const entry = /^(100644|100755) blob ([0-9a-f]{40})\t([^\0]+)\0$/.exec( + git("ls-tree", "-z", sha, "--", path).toString("utf8"), + ); + if (!entry || entry[3] !== path) { + throw new Error(`tooling bootstrap requires a regular committed file: ${path}`); + } + const content = git("cat-file", "blob", entry[2]); + const oid = createHash("sha1").update(`blob ${content.length}\0`).update(content).digest("hex"); + if (oid !== entry[2]) { + throw new Error("unable to read committed tooling bootstrap (object hash mismatch)"); + } + verifyToolingFile(path, content); + } +} + +async function loadVerifiedTooling(identity, workflow = false) { + object(identity, ["root", "sha"], "tooling identity"); + if (realpathSync(text(identity.root, "tooling root")) !== ownRoot) { + throw new Error("tooling identity does not own this evaluator"); + } + verifyReaderBootstrap(identity.sha); + const { createFrozenTargetSource } = await import("./lib/frozen-target-source.mjs"); + const source = createFrozenTargetSource(ownRoot, identity.sha); + const recipes = source.readDirectory("scripts/e2e/lib/upgrade-survivor/config-recipe"); + if (recipes === null) { + throw new Error("missing required tooling recipe directory"); + } + for (const path of [...toolingClosure, ...(workflow ? workflowToolingClosure : []), ...recipes]) { + verifyToolingFile(path, Buffer.from(required(source, path), "utf8")); + } + return { source, createFrozenTargetSource }; +} + +function consumerForLane(name) { + if ( + name === "root-managed-vps-upgrade" || + name === "update-restart-auth" || + /^(published-upgrade-survivor|update-migration)(-|$)/u.test(name) + ) { + return "upgrade-survivor"; + } + if (name.startsWith("npm-onboard-")) { + return "npm-onboard-channel-agent"; + } + if (name === "live-mcp-code-mode-gateway") { + return "mcp-code-mode-gateway"; + } + if (name === "live-gateway") { + return "live-cli-backend"; + } + if ( + name === "plugins-offline" || + name === "mcp-channels" || + name === "kitchen-sink-rpc" || + /^bundled-plugin-install-uninstall(-|$)/u.test(name) + ) { + return "plugins"; + } + return Object.hasOwn(shellOwners, name) || Object.hasOwn(targetFiles, name) ? name : null; +} + +async function preflightFrozenTargetContracts(input, workflow = false, verifiedTooling = null) { + object( + input, + [ + "version", + "repository", + "selected", + "tooling", + "allowFrozenTargetScenarioOmissions", + "selection", + ], + "admission request", + ); + if (input.version !== 1 || input.repository !== "openclaw/openclaw") { + throw new Error("invalid admission identity"); + } + const allow = boolean(input.allowFrozenTargetScenarioOmissions); + object(input.selected, ["root", "sha"], "selected identity"); + const roots = { + selected: realpathSync(text(input.selected.root, "selected root")), + tooling: ownRoot, + }; + if (verifiedTooling) { + object(input.tooling, ["root", "sha"], "tooling identity"); + if ( + realpathSync(text(input.tooling.root, "tooling root")) !== ownRoot || + input.tooling.sha !== verifiedTooling.sha + ) { + throw new Error("verified tooling identity differs from evaluation"); + } + } + const { source, createFrozenTargetSource } = verifiedTooling + ? { + source: verifiedTooling.createFrozenTargetSource(ownRoot, input.tooling.sha), + createFrozenTargetSource: verifiedTooling.createFrozenTargetSource, + } + : await loadVerifiedTooling(input.tooling, workflow); + const sources = { + tooling: source, + selected: createFrozenTargetSource(roots.selected, input.selected.sha), + }; + const { + DEFAULT_LIVE_RETRIES, + parseLaneSelection, + parseLiveMode, + parseProfile, + resolveDockerE2ePlan, + } = await import("./lib/docker-e2e-plan.mts"); + const { classifyReleaseTrain, parseReleaseVersion } = await import("./lib/release-version.mjs"); + const { resolveFrozenCodexCompatibility } = await import("./resolve-frozen-codex-live-suite.mjs"); + const { resolveFsSafeNativeContract } = await import("./resolve-fs-safe-native-contract.mjs"); + if (allow && input.selected.sha === input.tooling.sha) { + throw new Error("frozen omissions require distinct identities"); + } + const selection = object( + input.selection, + ["docker", "consumers", "codexSuites", "fsSafeNative"], + "selection", + ); + const consumers = new Set(strings(selection.consumers ?? [], "consumers")); + for (const consumer of consumers) { + if (!Object.hasOwn(shellOwners, consumer) && !Object.hasOwn(targetFiles, consumer)) { + throw new Error(`unknown selected contract: ${consumer}`); + } + } + const codexSuites = strings(selection.codexSuites ?? [], "Codex suites"); + if ( + codexSuites.some( + (suite) => !/^live-codex-harness(?:-gpt56-(?:sol|luna|terra))?-docker$/u.test(suite), + ) + ) { + throw new Error("unknown selected Codex suite"); + } + const fsSafeNative = boolean(selection.fsSafeNative); + let docker; + let normalizedDocker; + if (selection.docker !== undefined) { + const value = object( + selection.docker, + [ + "lanes", + "profile", + "releaseProfile", + "chunk", + "planReleaseAll", + "liveMode", + "includeOpenWebUI", + "baselines", + "scenarios", + ], + "Docker selection", + ); + normalizedDocker = { + lanes: parseLaneSelection(strings(value.lanes ?? [], "Docker lanes").join(",")), + profile: parseProfile(text(value.profile ?? "all", "Docker profile")), + releaseProfile: text(value.releaseProfile ?? "full", "release profile"), + chunk: text(value.chunk ?? "core", "release chunk"), + planReleaseAll: boolean(value.planReleaseAll), + liveMode: parseLiveMode(text(value.liveMode ?? "all", "live mode")), + includeOpenWebUI: boolean(value.includeOpenWebUI), + baselines: tokenListText(value.baselines ?? "", "baselines"), + scenarios: tokenListText(value.scenarios ?? "", "scenarios"), + }; + const result = resolveDockerE2ePlan({ + allowFrozenTargetScenarioOmissions: allow, + frozenTarget: { mode: "inert", source: sources.selected }, + includeOpenWebUI: normalizedDocker.includeOpenWebUI, + liveMode: normalizedDocker.liveMode, + liveRetries: DEFAULT_LIVE_RETRIES, + orderLanes: (lanes) => lanes, + planReleaseAll: normalizedDocker.planReleaseAll, + profile: normalizedDocker.profile, + releaseChunk: normalizedDocker.chunk, + releaseProfile: normalizedDocker.releaseProfile, + selectedLaneNames: normalizedDocker.lanes, + upgradeSurvivorBaselines: normalizedDocker.baselines, + upgradeSurvivorScenarios: normalizedDocker.scenarios, + }); + docker = { + lanes: result.scheduledLanes.map((lane) => lane.name), + omitted: result.omittedUnsupportedLaneNames, + status: result.scheduledLanes.length ? "ADMITTED" : "NOT RUN", + }; + for (const lane of docker.lanes) { + const consumer = consumerForLane(lane); + if (consumer) { + consumers.add(consumer); + } + } + } + const env = { + PATH: process.env.PATH, + LANG: "C.UTF-8", + OPENCLAW_ALLOW_FROZEN_TARGET_SCENARIO_OMISSIONS: allow ? "1" : "0", + OPENCLAW_SELECTED_SHA: input.selected.sha, + OPENCLAW_TOOLING_SHA: input.tooling.sha, + }; + const deadline = Date.now() + 60_000; + const shell = (command, args) => + execFileSync( + "bash", + [ + "--noprofile", + "--norc", + "-c", + `set -euo pipefail; source "$1"; shift; ${command}`, + "admission", + join(roots.tooling, "scripts/lib/frozen-target-compat.sh"), + ...args, + ], + { + cwd: roots.tooling, + env, + encoding: "utf8", + timeout: Math.max(1, deadline - Date.now()), + maxBuffer: maxRecordBytes, + stdio: ["ignore", "pipe", "pipe"], + }, + ); + const rootOrder = ["tooling", "selected"].toSorted((a, b) => roots[b].length - roots[a].length); + const bindPath = (absolute) => { + if (!absolute) { + return null; + } + for (const key of rootOrder) { + const path = relative(roots[key], absolute); + if (!path.startsWith("../") && path !== ".." && !path.startsWith("/")) { + required(sources[key], path); + return { source: key, path }; + } + } + throw new Error("resolved contract path escaped its source"); + }; + const contracts = []; + for (const consumer of [...consumers].toSorted((a, b) => (a < b ? -1 : a > b ? 1 : 0))) { + const contract = { consumer, status: "ADMITTED", modes: {}, files: [] }; + const owner = shellOwners[consumer === "kitchen-sink-plugin" ? "plugins" : consumer]; + if (owner) { + const [fn, names] = owner; + const output = shell( + `${[fn] + .flat() + .map((name) => `openclaw_resolve_frozen_${name} "$1" "$2";`) + .join(" ")} shift 2; for key in "$@"; do printf "%s\\0" "\${!key}"; done`, + [roots.selected, roots.tooling, ...names], + ).split("\0"); + if (output.pop() !== "" || output.length !== names.length) { + throw new Error("invalid contract result"); + } + for (const [index, name] of names.entries()) { + const value = output[index]; + if (name.endsWith("_PATH")) { + const absolute = name.endsWith("_CLIENT_PATH") + ? join(allow ? roots.selected : roots.tooling, value) + : value; + contract.files.push(bindPath(absolute)); + } else if (name.endsWith("_LIB")) { + if (value) { + const path = relative(roots.selected, value); + required(sources.selected, `${path}/gateway-network/client.mjs`); + contract.modes[name] = path; + } + } else { + contract.modes[name] = text(value, "contract mode", 1024); + } + } + } + for (const [path, missing] of targetFiles[consumer] ?? []) { + const args = [roots.selected, path, join(roots.tooling, path)]; + if (missing !== undefined) { + args.push(missing); + } + const value = shell('openclaw_resolve_frozen_target_file "$@"', args).trimEnd(); + contract.files.push(bindPath(value)); + } + if (consumer === "codex-on-demand") { + const path = "extensions/codex/package.json"; + required(sources.selected, path); + contract.files.push({ source: "selected", path }); + } + for (const path of supportFiles[consumer] ?? []) { + required(sources.tooling, `scripts/e2e/lib/${path}`); + } + if ( + ["npm-onboard-channel-agent", "codex-on-demand", "update-corrupt-plugin"].includes(consumer) + ) { + required(sources.tooling, "scripts/lib/record-shared.mjs"); + } + if (consumer === "update-corrupt-plugin") { + required(sources.tooling, "scripts/lib/update-compat-contract.mjs"); + required(sources.tooling, "scripts/lib/openclaw-e2e-instance.sh"); + required(sources.tooling, "scripts/lib/direct-run.mjs"); + } + if (consumer === "upgrade-survivor" && allow) { + const version = JSON.parse(required(sources.selected, "package.json")).version; + const parsed = typeof version === "string" ? parseReleaseVersion(version) : null; + if (!parsed) { + throw new Error("selected upgrade target has an invalid release version"); + } + const train = classifyReleaseTrain(parsed); + if (train === "unsupported-extended-stable-correction") { + throw new Error("unsupported extended-stable correction"); + } + contract.modes.releaseTrain = train; + if (train === "extended-stable") { + const dir = "scripts/e2e/lib/upgrade-survivor"; + const selected = shell('openclaw_resolve_frozen_target_file "$@"', [ + roots.selected, + dir, + ]).trimEnd(); + if (!selected) { + throw new Error("selected extended-stable target lacks its scenario"); + } + for (const file of ["run.sh", "assertions.mjs", "probe-gateway.mjs"]) { + required(sources.selected, `${dir}/${file}`); + } + const files = sources.selected.readDirectory(dir); + const recipes = ["config-recipe.mjs", "config-recipe.mts"].filter((file) => + files.includes(`${dir}/${file}`), + ); + if (recipes.length !== 1) { + throw new Error("missing or ambiguous shipped survivor recipe"); + } + const recipeFiles = sources.selected.readDirectory(`${dir}/config-recipe`); + if (!recipeFiles?.length) { + throw new Error("missing shipped survivor recipe data"); + } + for (const section of [ + "agents", + "channels-discord", + "channels-feishu", + "channels-matrix", + "channels-telegram", + "channels-whatsapp", + "gateway", + "models-openai", + "plugins-configured-installs", + "plugins-feishu", + "plugins", + "skills", + ]) { + required(sources.selected, `${dir}/config-recipe/${section}.json`); + } + for (const path of [ + "scripts/lib/npm-publish-plan.mjs", + "scripts/windows-cmd-helpers.mjs", + "scripts/e2e/lib/plugin-index-sqlite.mjs", + "scripts/e2e/lib/env-limits.mjs", + "scripts/e2e/lib/text-file-utils.mjs", + ]) { + const selectedFile = shell('openclaw_resolve_frozen_target_file "$@"', [ + roots.selected, + path, + ]).trimEnd(); + if (!selectedFile) { + throw new Error(`missing shipped survivor support: ${path}`); + } + contract.files.push(bindPath(selectedFile)); + } + } + } + contracts.push(contract); + } + for (const suiteId of codexSuites) { + const result = allow + ? resolveFrozenCodexCompatibility({ + suiteId, + readSource: (path) => sources.selected.readText(path), + }) + : { runLane: true }; + contracts.push({ + consumer: suiteId, + status: result.runLane ? "ADMITTED" : "NOT RUN", + ...(result.model ? { model: result.model } : {}), + }); + } + if (fsSafeNative) { + contracts.push({ + consumer: "fs-safe-native", + mode: resolveFsSafeNativeContract({ + selectedSha: input.selected.sha, + workflowSha: input.tooling.sha, + allowFrozenSource: allow, + containingBranches: () => + sources.selected.containingBranches("refs/remotes/origin/extended-stable"), + readSource: (path) => sources.selected.readText(path), + }), + }); + } + const record = { + version: 1, + repository: input.repository, + selectedSha: input.selected.sha, + toolingSha: input.tooling.sha, + selection: { + consumers: [...consumers].toSorted((a, b) => (a < b ? -1 : a > b ? 1 : 0)), + codexSuites, + fsSafeNative, + allowFrozenTargetScenarioOmissions: allow, + ...(normalizedDocker ? { docker: normalizedDocker } : {}), + }, + contracts, + ...(docker ? { docker } : {}), + sources: { + selected: sources.selected.blobIdentities(), + tooling: verifiedTooling + ? [ + ...new Map( + [...verifiedTooling.identities, ...sources.tooling.blobIdentities()].map( + (identity) => [identity.path, identity], + ), + ).values(), + ].toSorted((a, b) => a.path.localeCompare(b.path)) + : sources.tooling.blobIdentities(), + }, + }; + const serialized = JSON.stringify(record); + if (Buffer.byteLength(serialized) > maxRecordBytes) { + throw new Error("admission record exceeds limit"); + } + return { ...record, digest: createHash("sha256").update(serialized).digest("hex") }; +} + +let invokedAsMain = false; +if (process.argv[1]) { + try { + invokedAsMain = + realpathSync.native(fileURLToPath(import.meta.url)) === realpathSync.native(process.argv[1]); + } catch { + // Inline and stdin importers need not have a filesystem entrypoint. + } +} + +if (invokedAsMain) { + try { + const args = process.argv.slice(2); + if (args.length === 1 && args[0] === "--workflow-request") { + process.stdout.write(`${JSON.stringify(workflowRequest(process.env))}\n`); + process.exit(0); + } + if (args[0] === "--verify-tooling") { + if (args.length !== 3) { + throw new Error("expected exact tooling root and SHA"); + } + await loadVerifiedTooling({ root: args[1], sha: args[2] }, true); + process.exit(0); + } + const planOnly = args[0] === "--plan"; + const [file, ...extra] = planOnly ? args.slice(1) : args; + if (!file || extra.length || !statSync(file).isFile() || statSync(file).size > 64 * 1024) { + throw new Error("expected one bounded admission request file"); + } + const input = JSON.parse(readFileSync(file, "utf8")); + const result = planOnly + ? await planWorkflowAdmission(input) + : input.version === 2 + ? await preflightWorkflow(input) + : await preflightFrozenTargetContracts(input); + process.stdout.write(`${JSON.stringify(result)}\n`); + } catch (error) { + console.error(`frozen admission: ${error.message}`); + process.exitCode = 1; + } +} diff --git a/scripts/prepare-codex-ci-auth.ts b/scripts/prepare-codex-ci-auth.ts new file mode 100644 index 0000000000000000000000000000000000000000..5d3f9b1fd0732361effef6fb8d48ef9e78d42020 --- /dev/null +++ b/scripts/prepare-codex-ci-auth.ts @@ -0,0 +1,110 @@ +#!/usr/bin/env -S node --import tsx +// Prepare Codex Ci Auth script supports OpenClaw repository automation. +import fs from "node:fs/promises"; +import path from "node:path"; + +type CodexAuthJson = { + tokens?: { + account_id?: unknown; + id_token?: unknown; + }; +}; + +type JwtParts = { + header: string; + payload: Record<string, unknown>; + signature: string; +}; + +function decodeBase64UrlJson(value: string): Record<string, unknown> { + const decoded = Buffer.from(value, "base64url").toString("utf-8"); + const parsed: unknown = JSON.parse(decoded); + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + throw new Error("JWT payload is not a JSON object."); + } + return parsed as Record<string, unknown>; +} + +function encodeBase64UrlJson(value: Record<string, unknown>): string { + return Buffer.from(JSON.stringify(value), "utf-8").toString("base64url"); +} + +function parseJwt(value: string): JwtParts { + const parts = value.split("."); + if (parts.length !== 3 || !parts[0] || !parts[1]) { + throw new Error("id_token is not a JWT."); + } + return { + header: parts[0], + payload: decodeBase64UrlJson(parts[1]), + signature: parts[2] ?? "", + }; +} + +function stringifyJwt(parts: JwtParts): string { + return [parts.header, encodeBase64UrlJson(parts.payload), parts.signature].join("."); +} + +export function patchCodexAuthForCi(auth: CodexAuthJson): { + auth: CodexAuthJson; + changed: boolean; +} { + const tokens = auth.tokens; + if (!tokens) { + return { auth, changed: false }; + } + const accountId = typeof tokens.account_id === "string" ? tokens.account_id.trim() : ""; + const idToken = typeof tokens.id_token === "string" ? tokens.id_token.trim() : ""; + if (!accountId || !idToken) { + return { auth, changed: false }; + } + + const jwt = parseJwt(idToken); + if (typeof jwt.payload.chatgpt_account_id === "string" && jwt.payload.chatgpt_account_id) { + return { auth, changed: false }; + } + + return { + auth: { + ...auth, + tokens: { + ...tokens, + // Newer Codex app-server builds read ChatGPT account metadata from + // id_token claims. Older local auth files can have the same value only + // at tokens.account_id, so patch the staged Docker copy for CI. + id_token: stringifyJwt({ + ...jwt, + payload: { + ...jwt.payload, + chatgpt_account_id: accountId, + }, + }), + }, + }, + changed: true, + }; +} + +export async function prepareCodexCiAuth(authPath: string): Promise<boolean> { + const raw = await fs.readFile(authPath, "utf-8"); + const parsed = JSON.parse(raw) as CodexAuthJson; + const { auth, changed } = patchCodexAuthForCi(parsed); + if (!changed) { + return false; + } + const stat = await fs.stat(authPath); + await fs.writeFile(authPath, `${JSON.stringify(auth, null, 2)}\n`, "utf-8"); + await fs.chmod(authPath, stat.mode); + return true; +} + +if (path.basename(process.argv[1] ?? "") === "prepare-codex-ci-auth.ts") { + const authPath = process.argv[2]; + if (!authPath) { + throw new Error("Usage: node --import tsx scripts/prepare-codex-ci-auth.ts <auth-json-path>"); + } + const changed = await prepareCodexCiAuth(authPath); + if (changed) { + console.error("Prepared staged Codex auth metadata for CI."); + } +} diff --git a/scripts/prepare-codex-ci-config.ts b/scripts/prepare-codex-ci-config.ts new file mode 100644 index 0000000000000000000000000000000000000000..c2412da4c73134dd9f937c2fae86d0523a669c73 --- /dev/null +++ b/scripts/prepare-codex-ci-config.ts @@ -0,0 +1,57 @@ +// Prepare Codex Ci Config script supports OpenClaw repository automation. +import fs from "node:fs/promises"; +import path from "node:path"; +import { expectDefined } from "../packages/normalization-core/src/expect.js"; + +function tomlString(value: string): string { + return JSON.stringify(value); +} + +export function buildCiSafeCodexConfig(params: { + projectPath: string; + approvalPolicy?: string; + modelReasoningEffort?: string; + sandboxMode?: string; +}): string { + if (!params.projectPath || typeof params.projectPath !== "string") { + throw new Error("projectPath is required."); + } + const resolvedProjectPath = path.resolve(params.projectPath); + const approvalPolicy = params.approvalPolicy ?? "never"; + const modelReasoningEffort = params.modelReasoningEffort ?? "low"; + const sandboxMode = params.sandboxMode ?? "workspace-write"; + return [ + "# Generated for Codex CI runs.", + "# Keep the checked-out repo trusted while avoiding maintainer-local", + "# provider/profile overrides that do not exist on CI runners.", + `approval_policy = ${tomlString(approvalPolicy)}`, + `sandbox_mode = ${tomlString(sandboxMode)}`, + `model_reasoning_effort = ${tomlString(modelReasoningEffort)}`, + "", + `[projects.${tomlString(resolvedProjectPath)}]`, + 'trust_level = "trusted"', + "", + ].join("\n"); +} + +export async function writeCiSafeCodexConfig(params: { + outputPath: string; + projectPath: string; + approvalPolicy?: string; + modelReasoningEffort?: string; + sandboxMode?: string; +}): Promise<string> { + if (!params.outputPath || typeof params.outputPath !== "string") { + throw new Error("outputPath is required."); + } + const rendered = buildCiSafeCodexConfig(params); + await fs.mkdir(path.dirname(params.outputPath), { recursive: true }); + await fs.writeFile(params.outputPath, rendered, "utf-8"); + return rendered; +} + +if (path.basename(process.argv[1] ?? "") === "prepare-codex-ci-config.ts") { + const outputPath = expectDefined(process.argv[2], "Codex CI config output path"); + const projectPath = process.argv[3] ?? process.cwd(); + await writeCiSafeCodexConfig({ outputPath, projectPath }); +} diff --git a/scripts/prepare-git-hooks.mjs b/scripts/prepare-git-hooks.mjs new file mode 100644 index 0000000000000000000000000000000000000000..29ac558176116439e19d1ca5dd1122671d4c4cf2 --- /dev/null +++ b/scripts/prepare-git-hooks.mjs @@ -0,0 +1,58 @@ +// Initializes this checkout's Git hooks path without replacing an existing selection. +import { spawnSync } from "node:child_process"; +import { existsSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +const scriptDir = dirname(fileURLToPath(import.meta.url)); +const DEFAULT_PACKAGE_ROOT = join(scriptDir, ".."); + +/** + * Initializes an unset hooks path and returns a structured reason if skipped. + */ +export function configurePrepareGitHooks(params = {}) { + const cwd = params.cwd ?? DEFAULT_PACKAGE_ROOT; + const exists = params.existsSync ?? existsSync; + const gitBin = params.gitBin ?? "git"; + const spawn = params.spawnSync ?? spawnSync; + const warn = params.warn ?? console.warn; + const runGit = (args) => + spawn(gitBin, args, { cwd, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] }); + + if (!exists(join(cwd, "git-hooks"))) { + return { configured: false, reason: "missing-hooks-dir" }; + } + + const worktree = runGit(["rev-parse", "--is-inside-work-tree"]); + if (worktree.error?.code === "ENOENT") { + return { configured: false, reason: "missing-git" }; + } + if (worktree.status !== 0 || String(worktree.stdout ?? "").trim() !== "true") { + return { configured: false, reason: "not-worktree" }; + } + + // Inherited and explicitly empty selections already have an owner. + const existing = runGit(["config", "--get", "core.hooksPath"]); + if (existing.status === 0) { + return { configured: false, reason: "already-configured" }; + } + // Git refuses a shared write when multiple checkouts lack private worktree config. + const configured = + existing.status === 1 + ? runGit(["config", "--worktree", "core.hooksPath", "git-hooks"]) + : existing; + if (configured.error?.code === "ENOENT") { + return { configured: false, reason: "missing-git" }; + } + if (configured.status !== 0) { + const stderr = String(configured.stderr ?? "").trim(); + warn(`[prepare] could not configure git hooks${stderr ? `: ${stderr}` : ""}`); + return { configured: false, reason: "config-failed" }; + } + + return { configured: true, reason: "configured" }; +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { + configurePrepareGitHooks(); +} diff --git a/scripts/prepush-ci.sh b/scripts/prepush-ci.sh new file mode 100644 index 0000000000000000000000000000000000000000..6af3e9059dcc8fa5297e58a14d81035ed81d49db --- /dev/null +++ b/scripts/prepush-ci.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash + +set -euo pipefail + +ROOT_DIR="$(git rev-parse --show-toplevel 2>/dev/null || pwd)" +cd "$ROOT_DIR" + +log_step() { + printf '\n==> %s\n' "$*" +} + +run_step() { + log_step "$*" + "$@" +} + +changed_paths_need_apple_build() { + node --input-type=module -e ' + import { readFileSync } from "node:fs"; + import { detectChangedScope } from "./scripts/ci-changed-scope.mjs"; + const scope = detectChangedScope(readFileSync(0, "utf8").split("\n")); + process.exit(scope.runMacos || scope.runIosBuild ? 0 : 1); + ' +} + +has_native_swift_changes() { + if git rev-parse --verify --quiet origin/main >/dev/null; then + if git diff --name-only --relative origin/main...HEAD | changed_paths_need_apple_build; then + return 0 + fi + fi + + if git rev-parse --verify --quiet HEAD^ >/dev/null; then + git diff --name-only --relative HEAD^..HEAD | changed_paths_need_apple_build + return $? + fi + + git show --name-only --relative --pretty='' HEAD | changed_paths_need_apple_build +} + +run_linux_ci_mirror() { + run_step pnpm check + run_step pnpm build:strict-smoke + run_step pnpm lint:ui:no-raw-window-open + run_step pnpm protocol:gen + run_step pnpm protocol:check:swift + run_step pnpm plugins:assets:build + run_step node scripts/run-vitest.mjs run --config test/vitest/vitest.extensions.config.ts --maxWorkers=1 + run_step env CI=true node scripts/run-vitest.mjs run --config test/vitest/vitest.unit.config.ts --maxWorkers=1 + + log_step "OPENCLAW_VITEST_MAX_WORKERS=${OPENCLAW_VITEST_MAX_WORKERS:-1} NODE_OPTIONS=${NODE_OPTIONS:---max-old-space-size=6144} pnpm test" + OPENCLAW_VITEST_MAX_WORKERS="${OPENCLAW_VITEST_MAX_WORKERS:-1}" \ + NODE_OPTIONS="${NODE_OPTIONS:---max-old-space-size=6144}" \ + pnpm test +} + +run_macos_ci_mirror() { + if [[ "${OPENCLAW_PREPUSH_SKIP_MACOS:-0}" == "1" ]]; then + log_step "Skipping macOS mirror because OPENCLAW_PREPUSH_SKIP_MACOS=1" + return 0 + fi + + if [[ "$(uname -s)" != "Darwin" ]]; then + log_step "Skipping macOS mirror on non-Darwin host" + return 0 + fi + + if ! has_native_swift_changes; then + log_step "Skipping macOS mirror because no native Swift paths changed" + return 0 + fi + + run_step pnpm lint:swift + run_step pnpm format:swift + run_step node scripts/prepare-apple-mermaid.mjs + run_step swift build --package-path apps/macos --configuration release + echo "Native tests were NOT run: use the disposable macos-swift GitHub CI job for this exact commit." >&2 + echo "Local lint/build passed; prepush cannot certify the native suite on an operator desktop." >&2 + return 1 +} + +main() { + run_linux_ci_mirror + run_macos_ci_mirror +} + +main "$@" diff --git a/scripts/print-live-docker-plugin-selection.mjs b/scripts/print-live-docker-plugin-selection.mjs new file mode 100644 index 0000000000000000000000000000000000000000..181c8dd96e4673b7153cdb902321e00e8d0484ab --- /dev/null +++ b/scripts/print-live-docker-plugin-selection.mjs @@ -0,0 +1,24 @@ +// Keeps shared and standalone live images aligned with the release provider roster. +import path from "node:path"; +import { resolveDockerPluginSelection } from "./lib/docker-plugin-selection.mjs"; +import { createReleaseWorkflowMatrixPlan } from "./plan-release-workflow-matrix.mjs"; + +// Standalone shards reuse one image, so a focused invocation must retain the full roster. +const plan = createReleaseWorkflowMatrixPlan({ releaseProfile: "full", includeLiveSuites: true }); +const providers = [ + "deepseek", // Gateway fixture provider outside the direct-model roster. + ...plan.liveModels.matrix.include.map((entry) => entry.providers), + process.env.OPENCLAW_LIVE_PROVIDERS ?? "", + process.env.OPENCLAW_LIVE_GATEWAY_PROVIDERS ?? "", + process.env.OPENCLAW_LIVE_MODELS ?? "", + process.env.OPENCLAW_LIVE_GATEWAY_MODELS ?? "", +].flatMap((value) => value.split(/[\s,]+/u).map((ref) => ref.split("/")[0])); + +// Gateway/CLI fixtures also require these plugin artifacts. +const selection = `matrix,acpx,anthropic,${process.argv[3] ?? ""}`; +const selected = resolveDockerPluginSelection({ + extensionsRoot: path.join(process.argv[2] ?? process.cwd(), "extensions"), + selection, + providers, +}); +console.log(selected.join(",")); diff --git a/scripts/private-codex-latest-probe.ts b/scripts/private-codex-latest-probe.ts new file mode 100644 index 0000000000000000000000000000000000000000..42bd75db762c49b84968981c17a946de41f30369 --- /dev/null +++ b/scripts/private-codex-latest-probe.ts @@ -0,0 +1,642 @@ +// Parent-operated acceptance client: no config loaders, local servers, or credential files. +import { randomUUID } from "node:crypto"; +import { writeFile } from "node:fs/promises"; +import { pathToFileURL } from "node:url"; +import WebSocket, { type ClientOptions } from "ws"; +import { z } from "zod"; +import { rawDataToString } from "../packages/gateway-client/src/websocket-data.ts"; +import { PROTOCOL_VERSION } from "../packages/gateway-protocol/src/version.ts"; + +const ALIAS = "codex-latest"; +const SENTINEL = "synthetic-private-probe-selector"; +const MAX_BYTES = 1_048_576; +const headersSchema = z.record(z.string().max(128), z.string().max(8192)); +const inputSchema = z.strictObject({ + isolatedCellReady: z.literal(true), + verifiedExternalProxyReady: z.literal(true), + allowInference: z.literal(true), + gatewayUrl: z.url(), + facadeBase: z.url(), + ownerHeaders: headersSchema, + nonownerHeaders: headersSchema, + workloadToken: z.string().min(1).max(8192), + openclawAgent: z.string().regex(/^[a-z0-9-]{1,64}$/), + codexAgent: z + .string() + .regex(/^[a-z0-9-]{1,64}$/) + .optional(), + nativeSafetyContractApproved: z.boolean().default(false), + privateTarget: z.string().min(8).max(512).optional(), +}); +type ProbeInput = z.infer<typeof inputSchema>; +const frameSchema = z.object({ + type: z.string(), + id: z.string().optional(), + event: z.string().optional(), + ok: z.boolean().optional(), + payload: z.unknown().optional(), + error: z.object({ details: z.object({ code: z.string().optional() }).optional() }).optional(), +}); +const recordSchema = z.record(z.string(), z.unknown()); +const reportTemplate = { + schema_version: 1, + preflight_pass: 0, + owner_admitted: 0, + nonowner_denied: 0, + facade_catalog_pass: 0, + raw_model_rejected: 0, + malformed_request_rejected: 0, + alias_sse_pass: 0, + openclaw_picker_pass: 0, + openclaw_runtime_pass: 0, + openclaw_tool_pass: 0, + openclaw_second_turn_pass: 0, + codex_picker_pass: 0, + codex_runtime_pass: 0, + codex_tool_pass: 0, + codex_second_turn_pass: 0, + native_blocked: 1, + native_restart_resume_tested: 0, + persisted_state_scanned: 0, + report_scanned: 0, + observed_bytes: 0, + leak_hits: 0, + explicit_model_failures: 0, + errors: 0, + protocol_pass: 0, +}; +type ProbeReport = typeof reportTemplate; + +function observe(text: string, needles: string[], report: ProbeReport) { + report.observed_bytes += Buffer.byteLength(text); + if (report.observed_bytes > 16 * MAX_BYTES) { + report.errors++; + throw new Error("budget"); + } + // Scan both raw wire text and decoded JSON strings; never include matches in an error. + const inspect = (value: unknown): void => { + const remaining = [value]; + while (remaining.length) { + const item = remaining.pop(); + if (typeof item === "string") { + if ( + needles.some( + (needle) => item.includes(needle) || item.includes(encodeURIComponent(needle)), + ) + ) { + report.leak_hits++; + } + } else if (item && typeof item === "object") { + for (const [key, child] of Object.entries(item)) { + if ( + ["model", "model_id", "modelid", "upstream", "openai-model", "retry_model"].includes( + key.toLowerCase(), + ) && + typeof child === "string" && + ![ALIAS, `openai/${ALIAS}`, `clawrouter/${ALIAS}`].includes(child) + ) { + report.explicit_model_failures++; + } + remaining.push(key, child); + } + } + } + }; + inspect(text); + try { + inspect(JSON.parse(text)); + } catch { + /* Malformed bodies still get the raw scan. */ + } +} + +async function readResponse(response: Response, scan: (value: string) => void) { + scan(JSON.stringify(Object.fromEntries(response.headers))); + const reader = response.body?.getReader(); + if (!reader) { + return ""; + } + let body = ""; + let bytes = 0; + const decoder = new TextDecoder(); + try { + while (true) { + const chunk = await reader.read(); + if (chunk.done) { + break; + } + bytes += chunk.value.byteLength; + if (bytes > MAX_BYTES) { + throw new Error("budget"); + } + body += decoder.decode(chunk.value, { stream: true }); + } + body += decoder.decode(); + scan(body); + for (const line of body.split(/\r?\n/u)) { + if (line.startsWith("data: ")) { + scan(line.slice(6)); + } + } + return body; + } finally { + await reader.cancel().catch(() => {}); + } +} + +type GatewayWebSocketFactory = (url: string, options: ClientOptions) => WebSocket; + +const createGatewayWebSocket: GatewayWebSocketFactory = (url, options) => + new WebSocket(url, options); + +function gatewayClient( + url: string, + headers: Record<string, string>, + scan: (text: string) => void, + createWebSocket: GatewayWebSocketFactory, +) { + const ws = createWebSocket(url, { + headers, + handshakeTimeout: 12_000, + maxPayload: MAX_BYTES, + followRedirects: false, + }); + const pending = new Map< + string, + { + resolve: (value: z.infer<typeof frameSchema>) => void; + reject: () => void; + timer: ReturnType<typeof setTimeout>; + } + >(); + let denied = false; + let challengeResolve: () => void; + let challengeReject: () => void; + const challenge = new Promise<void>((resolve, reject) => { + challengeResolve = resolve; + challengeReject = reject; + }); + const timer = setTimeout(() => { + challengeReject(); + ws.terminate(); + }, 12_000); + const fail = () => { + challengeReject(); + for (const waiter of pending.values()) { + clearTimeout(waiter.timer); + waiter.reject(); + } + pending.clear(); + }; + const scanFrame = (text: string) => { + try { + scan(text); + } catch { + fail(); + ws.terminate(); + } + }; + ws.on("unexpected-response", (_request, response) => { + scanFrame(JSON.stringify(response.headers)); + denied = response.statusCode === 401 || response.statusCode === 403; + let body = ""; + let bytes = 0; + const decoder = new TextDecoder(); + response.on("data", (chunk: Buffer) => { + bytes += chunk.byteLength; + if (bytes > MAX_BYTES) { + denied = false; + response.destroy(); + fail(); + return; + } + body += decoder.decode(chunk, { stream: true }); + }); + response.on("end", () => { + body += decoder.decode(); + scanFrame(body); + fail(); + }); + response.on("error", fail); + }); + ws.on("upgrade", (response) => { + scanFrame(JSON.stringify(response.headers)); + }); + ws.on("message", (data) => { + try { + const text = rawDataToString(data); + scan(text); + const frame = frameSchema.parse(JSON.parse(text)); + if (frame.type === "event" && frame.event === "connect.challenge") { + challengeResolve(); + } + if (frame.type === "res" && frame.id) { + const waiter = pending.get(frame.id); + if (waiter) { + clearTimeout(waiter.timer); + pending.delete(frame.id); + waiter.resolve(frame); + } + } + } catch { + fail(); + ws.terminate(); + } + }); + ws.on("error", fail); + ws.on("close", (_code, reason) => { + scanFrame(reason.toString("utf8")); + fail(); + }); + async function request(method: string, params: unknown, timeoutMs = 12_000) { + const id = randomUUID(); + return await new Promise<z.infer<typeof frameSchema>>((resolve, reject) => { + const requestTimer = setTimeout(() => { + pending.delete(id); + reject(new Error("timeout")); + }, timeoutMs); + pending.set(id, { + resolve, + reject: () => reject(new Error("transport")), + timer: requestTimer, + }); + ws.send(JSON.stringify({ type: "req", id, method, params }), (error) => { + if (error) { + fail(); + } + }); + }); + } + return { + async connect() { + try { + await challenge; + const response = await request("connect", { + minProtocol: PROTOCOL_VERSION, + maxProtocol: PROTOCOL_VERSION, + client: { id: "cli", version: "private-alias-probe", platform: "node", mode: "cli" }, + role: "operator", + scopes: ["operator.read", "operator.write"], + caps: [], + }); + // Only an edge HTTP denial proves the owner fence ran before Gateway admission. + return response.ok === true ? "admitted" : "failed"; + } catch { + return denied ? "denied" : "failed"; + } finally { + clearTimeout(timer); + } + }, + async rpc(method: string, params: unknown, timeoutMs?: number) { + const response = await request(method, params, timeoutMs); + if (response.ok !== true) { + throw new Error("rpc"); + } + return recordSchema.parse(response.payload); + }, + close() { + clearTimeout(timer); + fail(); + ws.terminate(); + }, + }; +} + +const messageSchema = z.object({ + role: z.string(), + content: z.unknown().optional(), + toolCallId: z.string().optional(), + isError: z.boolean().optional(), +}); +const contentSchema = z.union([ + z.string(), + z.array(z.unknown()), + z.record(z.string(), z.unknown()), +]); +const successfulToolResultSchema = messageSchema.extend({ + role: z.literal("toolResult"), + isError: z.literal(false), + toolCallId: z.string(), + content: contentSchema, +}); +function toolProof(messages: z.infer<typeof messageSchema>[], nonce: string) { + const calls = new Set<string>(); + for (const message of messages) { + if (message.role !== "assistant" || !Array.isArray(message.content)) { + continue; + } + for (const part of message.content) { + const parsed = z.object({ type: z.literal("toolCall"), id: z.string() }).safeParse(part); + if (parsed.success) { + calls.add(parsed.data.id); + } + } + } + return messages.some((message) => { + const result = successfulToolResultSchema.safeParse(message); + return ( + result.success && + calls.has(result.data.toolCallId) && + JSON.stringify(result.data.content).includes(nonce) + ); + }); +} + +async function probeHarness( + input: ProbeInput, + native: boolean, + scan: (text: string) => void, + report: ProbeReport, + createWebSocket: GatewayWebSocketFactory, +) { + const prefix = native ? "codex" : "openclaw"; + const agentId = native ? input.codexAgent : input.openclawAgent; + if (!agentId) { + throw new Error("agent"); + } + let client = gatewayClient(input.gatewayUrl, input.ownerHeaders, scan, createWebSocket); + try { + if ((await client.connect()) !== "admitted") { + throw new Error("admission"); + } + const catalog = await client.rpc("models.list", { + agentId, + preparedOnly: true, + view: "configured", + }); + const rows = z.array(z.object({ id: z.string(), name: z.string() })).parse(catalog.models); + report[`${prefix}_picker_pass`] = Number( + rows.some((row) => row.id === ALIAS && row.name === "Codex (Latest)"), + ); + if (!report[`${prefix}_picker_pass`]) { + return; + } + const created = await client.rpc("sessions.create", { + agentId, + idempotencyKey: randomUUID(), + label: "Private alias acceptance probe", + model: `${native ? "openai" : "clawrouter"}/${ALIAS}`, + }); + const key = z.string().min(1).parse(created.key); + const nonce = `probe-tool-${randomUUID()}`; + const first = await client.rpc("sessions.send", { + key, + agentId, + idempotencyKey: randomUUID(), + message: `Run one read-only shell tool command: printf '%s' '${nonce}'. Do not read files or use the network. Then reply with its output.`, + }); + const firstRun = z.string().parse(first.runId); + const done = await client.rpc("agent.wait", { runId: firstRun, timeoutMs: 60_000 }, 65_000); + if (done.status !== "ok") { + throw new Error("turn"); + } + const history = await client.rpc("chat.history", { sessionKey: key, agentId, limit: 100 }); + const runtimeSchema = z.object({ + model: z.literal(ALIAS), + modelProvider: z.literal(native ? "openai" : "clawrouter"), + agentRuntime: z.object({ id: z.literal(native ? "codex" : "openclaw") }), + }); + report[`${prefix}_runtime_pass`] = Number(runtimeSchema.safeParse(history.sessionInfo).success); + if (!report[`${prefix}_runtime_pass`]) { + return; + } + const messages = z.array(messageSchema).parse(history.messages); + report[`${prefix}_tool_pass`] = Number(toolProof(messages, nonce)); + if (!report[`${prefix}_tool_pass`]) { + return; + } + client.close(); + client = gatewayClient(input.gatewayUrl, input.ownerHeaders, scan, createWebSocket); + if ((await client.connect()) !== "admitted") { + throw new Error("admission"); + } + const second = await client.rpc("sessions.send", { + key, + agentId, + idempotencyKey: randomUUID(), + message: + "Reply with the exact nonce produced by the tool in the previous turn. Do not call tools.", + }); + const secondDone = await client.rpc( + "agent.wait", + { runId: z.string().parse(second.runId), timeoutMs: 60_000 }, + 65_000, + ); + if (secondDone.status !== "ok") { + throw new Error("turn"); + } + const continued = await client.rpc("chat.history", { sessionKey: key, agentId, limit: 100 }); + if (!runtimeSchema.safeParse(continued.sessionInfo).success) { + report[`${prefix}_runtime_pass`] = 0; + return; + } + const next = z.array(messageSchema).parse(continued.messages); + const previousAssistants = messages.filter((message) => message.role === "assistant").length; + const assistants = next.filter((message) => message.role === "assistant"); + const last = z.object({ content: contentSchema }).safeParse(assistants.at(-1)); + report[`${prefix}_second_turn_pass`] = Number( + assistants.length > previousAssistants && + last.success && + JSON.stringify(last.data.content).includes(nonce), + ); + } finally { + client.close(); + } +} + +export async function runPrivateCodexProbe( + raw: unknown, + options: { createWebSocket?: GatewayWebSocketFactory } = {}, +): Promise<ProbeReport> { + const report = { ...reportTemplate }; + let needles = [SENTINEL]; + const scan = (text: string) => observe(text, needles, report); + try { + const input = inputSchema.parse(raw); + const gateway = new URL(input.gatewayUrl); + const facade = new URL(input.facadeBase); + // No redirects or plain-text remote endpoints may receive parent-supplied credentials. + for (const url of [gateway, facade]) { + const loopback = ["127.0.0.1", "[::1]", "localhost"].includes(url.hostname); + if ( + (!loopback && !["wss:", "https:"].includes(url.protocol)) || + url.username || + url.password || + url.search || + url.hash + ) { + throw new Error("endpoint"); + } + } + if ( + !["ws:", "wss:"].includes(gateway.protocol) || + !["http:", "https:"].includes(facade.protocol) || + facade.pathname.replace(/\/$/u, "") !== "/private" + ) { + throw new Error("endpoint"); + } + if (JSON.stringify(input.ownerHeaders) === JSON.stringify(input.nonownerHeaders)) { + throw new Error("identity"); + } + needles = [SENTINEL, ...(input.privateTarget ? [input.privateTarget] : [])]; + report.preflight_pass = 1; + const webSocketFactory = options.createWebSocket ?? createGatewayWebSocket; + const owner = gatewayClient(input.gatewayUrl, input.ownerHeaders, scan, webSocketFactory); + try { + report.owner_admitted = Number((await owner.connect()) === "admitted"); + } finally { + owner.close(); + } + const nonowner = gatewayClient(input.gatewayUrl, input.nonownerHeaders, scan, webSocketFactory); + try { + report.nonowner_denied = Number((await nonowner.connect()) === "denied"); + } finally { + nonowner.close(); + } + if ( + !report.owner_admitted || + !report.nonowner_denied || + report.leak_hits || + report.explicit_model_failures + ) { + throw new Error("admission"); + } + const http = async (path: string, body?: string) => { + const response = await fetch(`${input.facadeBase.replace(/\/$/u, "")}/v1/${path}`, { + method: body === undefined ? "GET" : "POST", + redirect: "manual", + signal: AbortSignal.timeout(20_000), + headers: { + authorization: `Bearer ${input.workloadToken}`, + "content-type": "application/json", + }, + body, + }); + return { status: response.status, body: await readResponse(response, scan) }; + }; + const catalog = await http("catalog"); + const parsed = z + .object({ + providers: z.array( + z.object({ + models: z.array( + z.object({ id: z.string(), displayName: z.string(), upstream: z.string() }), + ), + }), + ), + }) + .parse(JSON.parse(catalog.body)); + const rows = parsed.providers.flatMap((provider) => provider.models); + report.facade_catalog_pass = Number( + catalog.status === 200 && + rows.length === 1 && + rows.every( + (row) => + row.id === ALIAS && row.upstream === ALIAS && row.displayName === "Codex (Latest)", + ), + ); + const responseRequest = { + model: ALIAS, + instructions: "Reply briefly.", + input: [{ role: "user", content: [{ type: "input_text", text: "Reply OK" }] }], + stream: true, + store: false, + }; + const rejected = await http( + "responses", + JSON.stringify({ ...responseRequest, model: SENTINEL }), + ); + report.raw_model_rejected = Number([400, 403, 404].includes(rejected.status)); + const malformed = await http("responses", "{"); + report.malformed_request_rejected = Number([400, 422].includes(malformed.status)); + if ( + !report.facade_catalog_pass || + !report.raw_model_rejected || + !report.malformed_request_rejected || + report.leak_hits || + report.explicit_model_failures + ) { + throw new Error("facade"); + } + const stream = await http("responses", JSON.stringify(responseRequest)); + report.alias_sse_pass = Number( + stream.status === 200 && + stream.body.split(/\r?\n/u).some((line) => { + if (!line.startsWith("data: ")) { + return false; + } + try { + return z + .object({ type: z.literal("response.completed") }) + .safeParse(JSON.parse(line.slice(6))).success; + } catch { + return false; + } + }), + ); + if (!report.alias_sse_pass || report.leak_hits || report.explicit_model_failures) { + throw new Error("stream"); + } + await probeHarness(input, false, scan, report, webSocketFactory); + if ( + input.nativeSafetyContractApproved && + input.codexAgent && + !report.leak_hits && + !report.explicit_model_failures + ) { + report.native_blocked = 0; + await probeHarness(input, true, scan, report, webSocketFactory); + } + } catch { + report.errors++; + } + report.protocol_pass = Number( + report.errors === 0 && + report.leak_hits === 0 && + report.explicit_model_failures === 0 && + report.native_blocked === 0 && + report.openclaw_tool_pass === 1 && + report.openclaw_second_turn_pass === 1 && + report.codex_tool_pass === 1 && + report.codex_second_turn_pass === 1, + ); + try { + scan(JSON.stringify(report)); + report.report_scanned = 1; + if (report.leak_hits || report.explicit_model_failures) { + report.protocol_pass = 0; + } + } catch { + report.protocol_pass = 0; + } + return report; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + let report = { ...reportTemplate }; + try { + if (process.argv.length !== 4 || process.argv[2] !== "-o") { + throw new Error("arguments"); + } + const outputPath = z.string().min(1).parse(process.argv[3]); + let input = ""; + const inputTimer = setTimeout(() => process.stdin.destroy(new Error("input")), 12_000); + try { + for await (const chunk of process.stdin) { + input += String(chunk); + if (Buffer.byteLength(input) > 65_536) { + throw new Error("input"); + } + } + } finally { + clearTimeout(inputTimer); + } + report = await runPrivateCodexProbe(JSON.parse(input)); + await writeFile(outputPath, `${JSON.stringify(report)}\n`, { flag: "wx", mode: 0o600 }); + } catch { + report.errors++; + report.protocol_pass = 0; + } + process.stdout.write(`${JSON.stringify(report)}\n`); + process.exitCode = report.protocol_pass ? 0 : 1; +} diff --git a/scripts/process-warning-filter.mts b/scripts/process-warning-filter.mts new file mode 100644 index 0000000000000000000000000000000000000000..04dd851e68bd085ed48778a75b06fc34d0f8e493 --- /dev/null +++ b/scripts/process-warning-filter.mts @@ -0,0 +1,45 @@ +// Installs a process-wide warning filter for dependency warnings that are known +// noise in current toolchains. +const warningFilterKey = Symbol.for("openclaw.warning-filter"); +type EmitWarningArgs = + | [warning: string | Error, ctor?: Function] + | [warning: string | Error, type?: string, ctor?: Function] + | [warning: string | Error, options?: NodeJS.EmitWarningOptions] + | [warning: string | Error, type?: string, code?: string, ctor?: Function]; + +/** + * Suppresses punycode deprecation warnings while preserving all other warnings. + */ +export function installProcessWarningFilter(): void { + const state: unknown = Reflect.get(globalThis, warningFilterKey); + if (state && typeof state === "object" && "installed" in state && state.installed === true) { + return; + } + + const originalEmitWarning = process.emitWarning.bind(process); + const filteredEmitWarning = ((...args: EmitWarningArgs): void => { + const [warningArg, secondArg, thirdArg] = args; + const options = typeof secondArg === "object" ? secondArg : undefined; + const warning = + warningArg instanceof Error + ? { + name: warningArg.name, + message: warningArg.message, + code: "code" in warningArg ? warningArg.code : undefined, + } + : { + name: typeof secondArg === "string" ? secondArg : options?.type, + message: typeof warningArg === "string" ? warningArg : undefined, + code: typeof thirdArg === "string" ? thirdArg : options?.code, + }; + + if (warning.code === "DEP0040" && warning.message?.includes("punycode")) { + return; + } + + Reflect.apply(originalEmitWarning, process, args); + }) satisfies typeof process.emitWarning; + process.emitWarning = filteredEmitWarning; + + Reflect.set(globalThis, warningFilterKey, { installed: true }); +} diff --git a/scripts/profile-tsgo.mts b/scripts/profile-tsgo.mts new file mode 100644 index 0000000000000000000000000000000000000000..26ad8912d0f712205f5662cc2eec2e8bd47d7661 --- /dev/null +++ b/scripts/profile-tsgo.mts @@ -0,0 +1,497 @@ +#!/usr/bin/env node + +// Profiles selected tsgo graphs and writes diagnostics/trace artifacts for +// TypeScript graph size and performance investigations. +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { applyLocalTsgoPolicy, resolveRepoToolBinPath } from "./lib/local-check-runtime.mts"; +import { createManagedCommandInvocation } from "./lib/managed-child-process.mts"; +import { resolveRepoRoot } from "./lib/repo-root.mjs"; +import { TSGO_CORE_TEST_SHARDS, type TsgoCoreTestShard } from "./lib/tsgo-core-test-shards.mts"; +const repoRoot = resolveRepoRoot(import.meta.url); +const artifactRoot = path.resolve(repoRoot, ".artifacts/tsgo-profile"); +const tsgoPath = resolveRepoToolBinPath("tsgo", { cwd: repoRoot }); + +type GraphDefinition = { config: string; description: string }; +type CoreTestGraphName = `core-test-${TsgoCoreTestShard["name"]}`; +const CORE_TEST_GRAPH_DEFINITIONS = Object.fromEntries( + TSGO_CORE_TEST_SHARDS.map((shard) => [ + `core-test-${shard.name}`, + { + config: shard.config, + description: `bounded core test shard: ${shard.name}`, + }, + ]), +) as Record<CoreTestGraphName, GraphDefinition>; + +const GRAPH_DEFINITIONS = { + core: { + config: "tsconfig.core.json", + description: "core production graph", + }, + ui: { + config: "tsconfig.ui.json", + description: "UI production graph", + }, + ...CORE_TEST_GRAPH_DEFINITIONS, + extensions: { + config: "tsconfig.extensions.json", + description: "bundled extension production graph", + }, + "extensions-test": { + config: "test/tsconfig/tsconfig.extensions.test.json", + description: "bundled extension colocated test graph", + }, +} as const; + +type GraphName = keyof typeof GRAPH_DEFINITIONS; +const DEFAULT_GRAPHS = [ + ...TSGO_CORE_TEST_SHARDS.map((shard) => `core-test-${shard.name}` as CoreTestGraphName), + "extensions-test", +] satisfies GraphName[]; +type ProfileOptions = { + all: boolean; + deep: boolean; + explain: boolean; + json: boolean; + reuse: boolean; + outDir: string; +}; +type Diagnostics = Record<string, number>; +type ProfileGraphResult = ReturnType<typeof profileGraph>; +type ProfileReport = { + generatedAt: string; + options: { graphs: GraphName[]; deep: boolean; explain: boolean; reuse: boolean }; + graphs: ProfileGraphResult[]; + paths: { json?: string; text?: string }; +}; + +function usage(): string { + return [ + "Usage: pnpm tsgo:profile [graph...] [options]", + "", + "Graphs:", + ...Object.entries(GRAPH_DEFINITIONS).map( + ([name, graph]) => ` ${name.padEnd(26)} ${graph.description}`, + ), + "", + "Options:", + " --all Profile all graphs", + " --reuse Reuse profile tsbuildinfo files instead of forcing fresh checks", + " --deep Also write --generateTrace and --pprofDir artifacts", + " --explain Also write list-only --explainFiles artifacts", + " --out=<dir> Output directory (default: .artifacts/tsgo-profile)", + " --json Print JSON report to stdout", + " --help Show this help", + "", + "Default graphs: all bounded core-test shards and extensions-test", + ].join("\n"); +} + +function parseArgs(argv: string[]): { options: ProfileOptions; selectedGraphs: GraphName[] } { + const graphNames: GraphName[] = []; + const options: ProfileOptions = { + all: false, + deep: false, + explain: false, + json: false, + reuse: false, + outDir: artifactRoot, + }; + + for (const arg of argv) { + if (arg === "--help" || arg === "-h") { + throw new Error(usage()); + } + if (arg === "--all") { + options.all = true; + continue; + } + if (arg === "--deep") { + options.deep = true; + continue; + } + if (arg === "--explain") { + options.explain = true; + continue; + } + if (arg === "--json") { + options.json = true; + continue; + } + if (arg === "--reuse") { + options.reuse = true; + continue; + } + if (arg.startsWith("--out=")) { + options.outDir = path.resolve(repoRoot, arg.slice("--out=".length)); + continue; + } + if (!(arg in GRAPH_DEFINITIONS)) { + throw new Error(`Unknown graph: ${arg}\n\n${usage()}`); + } + graphNames.push(arg as GraphName); + } + + const selectedGraphs = options.all + ? (Object.keys(GRAPH_DEFINITIONS) as GraphName[]) + : graphNames.length > 0 + ? graphNames + : DEFAULT_GRAPHS; + + return { options, selectedGraphs }; +} + +function ensureDirs(outDir: string): void { + fs.mkdirSync(outDir, { recursive: true }); + fs.mkdirSync(path.join(outDir, "cache"), { recursive: true }); +} + +function removeIfFreshMode(filePath: string, reuse: boolean): void { + if (!reuse) { + fs.rmSync(filePath, { force: true }); + } +} + +function runTsgo( + label: string, + args: string[], + params: { maxBuffer?: number } = {}, +): { elapsedMs: number; stdout: string; stderr: string } { + const { args: finalArgs, env } = applyLocalTsgoPolicy(args, process.env, { + logicalCpuCount: + typeof os.availableParallelism === "function" ? os.availableParallelism() : os.cpus().length, + totalMemoryBytes: os.totalmem(), + }); + const startedAt = Date.now(); + const tsgo = createManagedCommandInvocation({ + args: finalArgs, + bin: tsgoPath, + env, + }); + const result = spawnSync(tsgo.command, tsgo.args, { + cwd: repoRoot, + env, + encoding: "utf8", + maxBuffer: params.maxBuffer ?? 128 * 1024 * 1024, + shell: tsgo.shell, + windowsVerbatimArguments: tsgo.windowsVerbatimArguments, + }); + const elapsedMs = Date.now() - startedAt; + const stdout = result.stdout ?? ""; + const stderr = result.stderr ?? ""; + if (result.error) { + throw result.error; + } + if ((result.status ?? 1) !== 0) { + const output = [stdout, stderr].filter(Boolean).join("\n"); + throw new Error(`${label} failed with exit code ${result.status ?? 1}\n${output}`); + } + return { elapsedMs, stdout, stderr }; +} + +function parseDiagnostics(output: string): Diagnostics { + const diagnostics: Diagnostics = {}; + for (const line of output.split(/\r?\n/u)) { + const match = /^(.+?):\s+([0-9.]+)(K|s)?\s*$/u.exec(line.trim()); + if (!match) { + continue; + } + const [, rawKey, rawValue, unit] = match; + const key = rawKey!.trim().replaceAll(/\s+/gu, " "); + const value = Number(rawValue); + diagnostics[key] = unit === "K" ? value * 1024 : value; + } + return diagnostics; +} + +function normalizeFilePath(filePath: string): string { + const normalized = filePath.trim().replaceAll("\\", "/"); + const normalizedRoot = repoRoot.replaceAll("\\", "/"); + if (normalized.startsWith(`${normalizedRoot}/`)) { + return normalized.slice(normalizedRoot.length + 1); + } + return normalized; +} + +function packageNameFromNodeModule(parts: string[], startIndex: number): string { + const first = parts[startIndex + 1]; + if (!first) { + return "node_modules"; + } + if (first.startsWith("@")) { + return `${first}/${parts[startIndex + 2] ?? ""}`.replace(/\/$/u, ""); + } + return first; +} + +function classifyFile(relativePath: string): string { + const parts = relativePath.split("/"); + const first = parts[0]; + if (relativePath.includes("/node_modules/") || first === "node_modules") { + const nodeModulesIndex = parts.indexOf("node_modules"); + return `node_modules/${packageNameFromNodeModule(parts, nodeModulesIndex)}`; + } + if (first === "extensions") { + return `extensions/${parts[1] ?? "(root)"}`; + } + if (first === "packages") { + return `packages/${parts[1] ?? "(root)"}`; + } + if (first === "src") { + return `src/${parts[1] ?? "(root)"}`; + } + if (first === "ui") { + return `ui/${parts[1] ?? "(root)"}`; + } + if (first === "test") { + return `test/${parts[1] ?? "(root)"}`; + } + if (first?.startsWith("/") || (first !== undefined && /^[A-Za-z]:/u.test(first))) { + return "(external)"; + } + return first || "(unknown)"; +} + +function countBy<T>(values: T[], keyFn: (value: T) => string) { + const counts = new Map<string, number>(); + for (const value of values) { + const key = keyFn(value); + counts.set(key, (counts.get(key) ?? 0) + 1); + } + return [...counts.entries()] + .map(([key, count]) => ({ key, count })) + .toSorted((left, right) => right.count - left.count || left.key.localeCompare(right.key)); +} + +function summarizeFiles(stdout: string) { + const files = stdout + .split(/\r?\n/u) + .map(normalizeFilePath) + .filter(Boolean) + .filter((line) => !line.startsWith("Files:")); + + const projectRelativeFiles = files.filter( + (file) => !path.isAbsolute(file) && !/^[A-Za-z]:/u.test(file), + ); + const testFiles = projectRelativeFiles.filter((file) => /\.test\.[cm]?[tj]sx?$/u.test(file)); + return { + totalFiles: files.length, + projectRelativeFiles: projectRelativeFiles.length, + testFiles: testFiles.length, + groups: countBy(projectRelativeFiles, classifyFile).slice(0, 40), + }; +} + +function diffDiagnostics(check: Diagnostics, noCheck: Diagnostics) { + const totalDelta = (check["Total time"] ?? 0) - (noCheck["Total time"] ?? 0); + const checkTime = check["Check time"] ?? 0; + return { + checkTimeSeconds: checkTime, + totalDeltaSeconds: totalDelta, + typeShareOfTotal: + check["Total time"] && checkTime ? Number((checkTime / check["Total time"]).toFixed(3)) : 0, + }; +} + +function formatSeconds(value: number): string { + return `${value.toFixed(2)}s`; +} + +function renderTextReport(report: ProfileReport): string { + const lines = [ + "# tsgo profile", + "", + `Generated: ${report.generatedAt}`, + `Fresh profile caches: ${report.options.reuse ? "no" : "yes"}`, + "", + ]; + + for (const graph of report.graphs) { + const check = graph.check.diagnostics; + const noCheck = graph.noCheck.diagnostics; + lines.push(`## ${graph.name}`); + lines.push(`Config: ${graph.config}`); + lines.push( + `Check: wall ${formatSeconds(graph.check.elapsedMs / 1000)}, compiler total ${formatSeconds( + check["Total time"] ?? 0, + )}, check ${formatSeconds(check["Check time"] ?? 0)}, memory ${Math.round( + (check["Memory used"] ?? 0) / 1024 / 1024, + )} MiB`, + ); + lines.push( + `NoCheck: wall ${formatSeconds( + graph.noCheck.elapsedMs / 1000, + )}, compiler total ${formatSeconds(noCheck["Total time"] ?? 0)}`, + ); + lines.push( + `Files: compiler ${check.Files ?? "?"}, listed ${graph.files.totalFiles}, project-relative ${graph.files.projectRelativeFiles}, tests ${graph.files.testFiles}`, + ); + lines.push(`File list: ${graph.files.artifact}`); + lines.push( + `Type cost: check ${formatSeconds(graph.typeCost.checkTimeSeconds)}, total delta ${formatSeconds( + graph.typeCost.totalDeltaSeconds, + )}, share ${(graph.typeCost.typeShareOfTotal * 100).toFixed(1)}%`, + ); + lines.push("Top file groups:"); + for (const group of graph.files.groups.slice(0, 15)) { + lines.push(`- ${group.key}: ${group.count}`); + } + if (graph.deep) { + lines.push(`Deep artifacts: ${graph.deep.traceDir}, ${graph.deep.profileDir}`); + } + if (graph.explain) { + lines.push(`Explain: ${graph.explain.artifact}`); + } + lines.push(""); + } + + lines.push(`JSON: ${report.paths.json ?? ""}`); + lines.push(""); + return `${lines.join("\n")}\n`; +} + +function profileGraph(name: GraphName, options: ProfileOptions) { + const graph = GRAPH_DEFINITIONS[name]; + const outDir = options.outDir; + const graphCacheRoot = path.join(outDir, "cache"); + const checkBuildInfo = path.join(graphCacheRoot, `${name}-check.tsbuildinfo`); + const noCheckBuildInfo = path.join(graphCacheRoot, `${name}-nocheck.tsbuildinfo`); + const configPath = graph.config; + + removeIfFreshMode(checkBuildInfo, options.reuse); + removeIfFreshMode(noCheckBuildInfo, options.reuse); + + const baseArgs = ["-p", configPath, "--pretty", "false"]; + const listFiles = runTsgo(`${name}:listFilesOnly`, [...baseArgs, "--listFilesOnly"], { + maxBuffer: 256 * 1024 * 1024, + }); + const filesArtifact = path.join(outDir, `${name}.files.txt`); + fs.writeFileSync(filesArtifact, listFiles.stdout); + const noCheck = runTsgo(`${name}:noCheck`, [ + ...baseArgs, + "--noCheck", + "--incremental", + "--tsBuildInfoFile", + noCheckBuildInfo, + "--extendedDiagnostics", + ]); + + const checkArgs = [ + ...baseArgs, + "--incremental", + "--tsBuildInfoFile", + checkBuildInfo, + "--extendedDiagnostics", + ]; + let deep: { profileDir: string; traceDir: string } | undefined; + if (options.deep) { + const traceDir = path.join(outDir, `${name}-trace`); + const profileDir = path.join(outDir, `${name}-pprof`); + fs.rmSync(traceDir, { force: true, recursive: true }); + fs.rmSync(profileDir, { force: true, recursive: true }); + fs.mkdirSync(profileDir, { recursive: true }); + checkArgs.push("--generateTrace", traceDir, "--pprofDir", profileDir); + deep = { + traceDir: path.relative(repoRoot, traceDir), + profileDir: path.relative(repoRoot, profileDir), + }; + } + const check = runTsgo(`${name}:check`, checkArgs); + let explain: { artifact: string; elapsedMs: number } | undefined; + if (options.explain) { + const explainArtifact = path.join(outDir, `${name}.explain.txt`); + const explainResult = runTsgo( + `${name}:explainFiles`, + [...baseArgs, "--listFilesOnly", "--explainFiles"], + { + maxBuffer: 256 * 1024 * 1024, + }, + ); + fs.writeFileSync(explainArtifact, `${explainResult.stdout}${explainResult.stderr}`); + explain = { + artifact: path.relative(repoRoot, explainArtifact), + elapsedMs: explainResult.elapsedMs, + }; + } + + const checkDiagnostics = parseDiagnostics(`${check.stdout}\n${check.stderr}`); + const noCheckDiagnostics = parseDiagnostics(`${noCheck.stdout}\n${noCheck.stderr}`); + return { + name, + config: configPath, + description: graph.description, + files: { + ...summarizeFiles(listFiles.stdout), + artifact: path.relative(repoRoot, filesArtifact), + }, + noCheck: { + elapsedMs: noCheck.elapsedMs, + diagnostics: noCheckDiagnostics, + }, + check: { + elapsedMs: check.elapsedMs, + diagnostics: checkDiagnostics, + }, + typeCost: diffDiagnostics(checkDiagnostics, noCheckDiagnostics), + ...(deep ? { deep } : {}), + ...(explain ? { explain } : {}), + }; +} + +async function main(argv: string[]): Promise<void> { + const { options, selectedGraphs } = parseArgs(argv); + ensureDirs(options.outDir); + const report: ProfileReport = { + generatedAt: new Date().toISOString(), + options: { + graphs: selectedGraphs, + deep: options.deep, + explain: options.explain, + reuse: options.reuse, + }, + graphs: [], + paths: {}, + }; + + for (const graphName of selectedGraphs) { + process.stderr.write(`[tsgo-profile] profiling ${graphName}\n`); + report.graphs.push(profileGraph(graphName, options)); + } + + const timestamp = new Date() + .toISOString() + .replaceAll(":", "") + .replaceAll(".", "") + .replace("T", "-") + .replace("Z", ""); + const jsonPath = path.join(options.outDir, `tsgo-profile-${timestamp}.json`); + const textPath = path.join(options.outDir, `tsgo-profile-${timestamp}.md`); + report.paths = { + json: path.relative(repoRoot, jsonPath), + text: path.relative(repoRoot, textPath), + }; + + fs.writeFileSync(jsonPath, `${JSON.stringify(report, null, 2)}\n`); + fs.writeFileSync(textPath, renderTextReport(report)); + fs.writeFileSync( + path.join(options.outDir, "latest.json"), + `${JSON.stringify(report, null, 2)}\n`, + ); + fs.writeFileSync(path.join(options.outDir, "latest.md"), renderTextReport(report)); + + if (options.json) { + process.stdout.write(`${JSON.stringify(report, null, 2)}\n`); + } else { + process.stdout.write(renderTextReport(report)); + } +} + +try { + await main(process.argv.slice(2)); +} catch (error) { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + process.exit(1); +} diff --git a/scripts/protocol-gen.ts b/scripts/protocol-gen.ts new file mode 100644 index 0000000000000000000000000000000000000000..8bcd34aa80c1399a06dc325dc0e2d35aa87d7aeb --- /dev/null +++ b/scripts/protocol-gen.ts @@ -0,0 +1,66 @@ +// Protocol Gen script supports OpenClaw repository automation. +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { ProtocolSchemas } from "../packages/gateway-protocol/src/schema/protocol-schemas.js"; +import { listCoreGatewayMethodMetadata } from "../src/gateway/methods/core-descriptors.js"; +import { writeGeneratedOutput } from "./lib/generated-output-utils.mts"; +import { + assertProtocolSchemaDocument, + buildProtocolSchemaDocument, +} from "./lib/protocol-schema-document.mts"; + +const scriptDir = path.dirname(fileURLToPath(import.meta.url)); +const repoRoot = path.resolve(scriptDir, ".."); +const defaultOutputPath = path.join(repoRoot, "dist", "protocol.schema.json"); + +function resolveOutputPath(args: string[]): string { + let outputPath = defaultOutputPath; + let hasOutputPath = false; + for (let index = 0; index < args.length; index += 1) { + const arg = args[index]; + if (arg !== "--out") { + throw new Error(`Unknown argument: ${arg}`); + } + if (hasOutputPath) { + throw new Error("--out may only be specified once."); + } + const value = args[index + 1]?.trim(); + if (!value || value === "--out") { + throw new Error("--out requires a path."); + } + outputPath = path.resolve(value); + hasOutputPath = true; + index += 1; + } + return outputPath; +} + +function main() { + const document = buildProtocolSchemaDocument({ + methods: listCoreGatewayMethodMetadata(), + schemas: ProtocolSchemas, + }); + // The artifact is a build output with no committed baseline, so this contract + // check is the only guard between a degraded registry and the published + // schema; a regenerate-then-diff guard on it can never fail. + assertProtocolSchemaDocument(document); + const result = writeGeneratedOutput({ + check: false, + next: JSON.stringify(document, null, 2), + outputPath: resolveOutputPath(process.argv.slice(2)), + repoRoot, + }); + const displayPath = path.relative(repoRoot, result.outputPath); + console.log( + result.wrote + ? `[protocol-gen] wrote ${displayPath}` + : `[protocol-gen] unchanged ${displayPath}`, + ); +} + +try { + main(); +} catch (err: unknown) { + console.error(err); + process.exit(1); +} diff --git a/scripts/prune-docker-plugin-dist.mjs b/scripts/prune-docker-plugin-dist.mjs new file mode 100644 index 0000000000000000000000000000000000000000..323711c5b9d61dd4af60cdf4fcf87c3ec9ac4e9f --- /dev/null +++ b/scripts/prune-docker-plugin-dist.mjs @@ -0,0 +1,286 @@ +// Prunes omitted bundled plugin files and their unshared runtime dependencies +// from Docker-oriented production package output, then links the retained +// externally distributed plugins' own dependencies under their packaged roots. +import fs from "node:fs"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { collectRootPackageExcludedExtensionDirs } from "./lib/bundled-plugin-build-entries.mjs"; +import { linkSourcePluginDependencies } from "./lib/bundled-plugin-dependency-links.mjs"; +import { assertRealOutputRoot } from "./lib/output-root-guard.mjs"; +import { removePathIfExists } from "./runtime-postbuild-shared.mjs"; + +const RUNTIME_DEPENDENCY_FIELDS = ["dependencies", "optionalDependencies"]; + +function parsePluginList(value) { + if (typeof value !== "string") { + return new Set(); + } + return new Set( + value + .split(/[\s,]+/u) + .map((entry) => entry.trim()) + .filter(Boolean), + ); +} + +/** + * Parses OPENCLAW_EXTENSIONS into the bundled plugin ids that Docker should keep. + */ +export function parseDockerPluginKeepList(value) { + return parsePluginList(value); +} + +function readPackageJson(filePath) { + if (!fs.existsSync(filePath)) { + return null; + } + return JSON.parse(fs.readFileSync(filePath, "utf8")); +} + +function collectRuntimeDependencyNames(packageJson, options = {}) { + const dependencies = new Set(); + for (const field of RUNTIME_DEPENDENCY_FIELDS) { + for (const dependencyName of Object.keys(packageJson?.[field] ?? {})) { + dependencies.add(dependencyName); + } + } + for (const dependencyName of Object.keys(packageJson?.peerDependencies ?? {})) { + const optional = packageJson?.peerDependenciesMeta?.[dependencyName]?.optional === true; + if (options.includeOptionalPeers === true || !optional) { + dependencies.add(dependencyName); + } + } + return dependencies; +} + +function nodeModulePath(repoRoot, packageName) { + return path.join(repoRoot, "node_modules", ...packageName.split("/")); +} + +// Follow Node's importer-relative lookup: hoisted installs can contain several versions, +// and root-only traversal can misclassify a kept dependency as exclusive to an omitted plugin. +function resolveNodeModulePackageDir(importerDir, packageName) { + let currentDir = fs.realpathSync(importerDir); + + while (true) { + const packageDir = path.join(currentDir, "node_modules", ...packageName.split("/")); + if (fs.existsSync(path.join(packageDir, "package.json"))) { + return fs.realpathSync(packageDir); + } + const parentDir = path.dirname(currentDir); + if (parentDir === currentDir) { + return null; + } + currentDir = parentDir; + } +} + +function removeEmptyScopeDir(repoRoot, packageName) { + if (!packageName.startsWith("@")) { + return; + } + const [scope] = packageName.split("/"); + const scopeDir = path.join(repoRoot, "node_modules", scope); + try { + fs.rmdirSync(scopeDir); + } catch { + // Scope still has other packages or does not exist. + } +} + +function collectPackageRuntimeClosure(repoRoot, seeds, options = {}) { + const packageDirs = new Set(); + const rootPackageNames = new Set(); + const stack = [...seeds]; + + while (stack.length > 0) { + const entry = stack.pop(); + const packageDir = resolveNodeModulePackageDir(entry.importerDir, entry.packageName); + if (!packageDir) { + continue; + } + + const rootPackageDir = nodeModulePath(repoRoot, entry.packageName); + if ( + fs.existsSync(path.join(rootPackageDir, "package.json")) && + fs.realpathSync(rootPackageDir) === packageDir + ) { + rootPackageNames.add(entry.packageName); + } + if (packageDirs.has(packageDir)) { + continue; + } + packageDirs.add(packageDir); + + const packageJson = readPackageJson(path.join(packageDir, "package.json")); + for (const dependencyName of collectRuntimeDependencyNames(packageJson, options)) { + stack.push({ importerDir: packageDir, packageName: dependencyName }); + } + } + + return { packageDirs, rootPackageNames }; +} + +function collectWorkspacePackageRuntimeSeeds(repoRoot, workspaceDir, excludedPluginIds) { + const seeds = []; + const workspaceRoot = path.join(repoRoot, workspaceDir); + if (!fs.existsSync(workspaceRoot)) { + return seeds; + } + + for (const entry of fs.readdirSync(workspaceRoot, { withFileTypes: true })) { + if (!entry.isDirectory() || excludedPluginIds.has(entry.name)) { + continue; + } + const importerDir = path.join(workspaceRoot, entry.name); + const packageJson = readPackageJson(path.join(importerDir, "package.json")); + if (typeof packageJson?.name === "string") { + seeds.push({ importerDir, packageName: packageJson.name }); + } + for (const packageName of collectRuntimeDependencyNames(packageJson)) { + seeds.push({ importerDir, packageName }); + } + } + return seeds; +} + +function pruneNodeModulesForOmittedPlugins(repoRoot, bundledPluginDir, omittedPluginIds) { + const rootPackageJson = readPackageJson(path.join(repoRoot, "package.json")); + const omittedPackageNames = new Set(); + const omittedSeeds = []; + + for (const pluginId of omittedPluginIds) { + const importerDir = path.join(repoRoot, bundledPluginDir, pluginId); + const packageJson = readPackageJson(path.join(importerDir, "package.json")); + if (typeof packageJson?.name === "string") { + omittedPackageNames.add(packageJson.name); + } + for (const packageName of collectRuntimeDependencyNames(packageJson)) { + omittedSeeds.push({ importerDir, packageName }); + } + } + + const keptSeeds = [...collectRuntimeDependencyNames(rootPackageJson)].map((packageName) => ({ + importerDir: repoRoot, + packageName, + })); + keptSeeds.push(...collectWorkspacePackageRuntimeSeeds(repoRoot, "packages", new Set())); + keptSeeds.push( + ...collectWorkspacePackageRuntimeSeeds(repoRoot, bundledPluginDir, omittedPluginIds), + ); + + const keptClosure = collectPackageRuntimeClosure(repoRoot, keptSeeds); + // Hoisted workspace dev dependencies can satisfy optional peers of omitted + // plugins. Treat those installed peer-only branches as removal candidates; + // the kept runtime closure below remains authoritative. + const omittedClosure = collectPackageRuntimeClosure(repoRoot, omittedSeeds, { + includeOptionalPeers: true, + }); + const removed = []; + const removalCandidates = new Set([...omittedPackageNames, ...omittedClosure.rootPackageNames]); + + for (const packageName of [...removalCandidates].toSorted((left, right) => + left.localeCompare(right), + )) { + const packageDir = nodeModulePath(repoRoot, packageName); + if (!fs.existsSync(packageDir)) { + continue; + } + if (keptClosure.packageDirs.has(fs.realpathSync(packageDir))) { + continue; + } + removePathIfExists(packageDir); + removeEmptyScopeDir(repoRoot, packageName); + removed.push(path.relative(repoRoot, packageDir).replaceAll("\\", "/")); + } + + return removed; +} + +// Docker compiles selected externally distributed plugins into the unified dist +// graph, but their dependencies stay plugin-local under the isolated pnpm install +// instead of the root node_modules that dist/extensions/<id> can reach. Link them +// under the packaged root, as isolated source checkouts do, and fail closed when a +// declared dependency still does not resolve from there: the plugin would otherwise +// ship loadable-looking but be rejected by dependency diagnostics at runtime. +function linkRetainedPluginDependencies(repoRoot, bundledPluginDir, retainedPluginIds) { + const unreachable = []; + for (const pluginId of [...retainedPluginIds].toSorted((left, right) => + left.localeCompare(right), + )) { + const distPluginDir = path.join(repoRoot, "dist", "extensions", pluginId); + if (!fs.existsSync(distPluginDir)) { + continue; + } + const pluginDir = path.join(repoRoot, bundledPluginDir, pluginId); + const distNodeModules = path.join(distPluginDir, "node_modules"); + fs.rmSync(distNodeModules, { recursive: true, force: true }); + linkSourcePluginDependencies(pluginDir, distNodeModules); + const packageJson = readPackageJson(path.join(pluginDir, "package.json")); + for (const packageName of Object.keys(packageJson?.dependencies ?? {})) { + if ( + !(packageName in (packageJson.optionalDependencies ?? {})) && + !resolveNodeModulePackageDir(distPluginDir, packageName) + ) { + unreachable.push(`${pluginId}: ${packageName}`); + } + } + } + if (unreachable.length > 0) { + throw new Error( + `plugin dependencies are not reachable from their packaged dist roots:\n${unreachable.join("\n")}`, + ); + } +} + +/** + * Removes omitted plugin dist trees plus node_modules packages not needed by kept runtime code, + * then links retained externally distributed plugin dependencies under their packaged roots. + */ +export function pruneDockerPluginDist(params = {}) { + const repoRoot = params.cwd ?? params.repoRoot ?? process.cwd(); + const env = params.env ?? process.env; + const bundledPluginDir = env.OPENCLAW_BUNDLED_PLUGIN_DIR ?? "extensions"; + const keepPluginIds = parseDockerPluginKeepList(env.OPENCLAW_EXTENSIONS); + const excludedPluginIds = collectRootPackageExcludedExtensionDirs({ cwd: repoRoot }); + const omittedPluginIds = new Set( + [...excludedPluginIds].filter((pluginId) => !keepPluginIds.has(pluginId)), + ); + const removed = []; + + // The removals below recurse into dist/ and dist-runtime/ plugin trees; + // refuse to follow a symlinked output root into its target. + assertRealOutputRoot(path.join(repoRoot, "dist")); + assertRealOutputRoot(path.join(repoRoot, "dist-runtime")); + + removed.push(...pruneNodeModulesForOmittedPlugins(repoRoot, bundledPluginDir, omittedPluginIds)); + + for (const pluginId of [...omittedPluginIds].toSorted((left, right) => + left.localeCompare(right), + )) { + for (const pluginPath of [ + path.join(bundledPluginDir, pluginId), + path.join("dist", "extensions", pluginId), + path.join("dist-runtime", "extensions", pluginId), + ]) { + const absolutePluginPath = path.join(repoRoot, pluginPath); + if (!fs.existsSync(absolutePluginPath)) { + continue; + } + removePathIfExists(absolutePluginPath); + removed.push(path.relative(repoRoot, absolutePluginPath).replaceAll("\\", "/")); + } + } + + linkRetainedPluginDependencies( + repoRoot, + bundledPluginDir, + [...excludedPluginIds].filter((pluginId) => keepPluginIds.has(pluginId)), + ); + + return removed; +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { + pruneDockerPluginDist(); +} diff --git a/scripts/release-beta-smoke.ts b/scripts/release-beta-smoke.ts new file mode 100644 index 0000000000000000000000000000000000000000..d83e22ff64772a2d34ddb06cafbe4519fe91282a --- /dev/null +++ b/scripts/release-beta-smoke.ts @@ -0,0 +1,461 @@ +#!/usr/bin/env -S pnpm tsx +// Release Beta Smoke script supports OpenClaw repository automation. +import { spawnSync } from "node:child_process"; +import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from "node:fs"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { + booleanFlag, + parseFlagArgs, + stringFlag, + stripLeadingPackageManagerSeparator, +} from "./lib/arg-utils.mts"; + +type Options = { + beta: string; + model: string; + providerMode: string; + ref: string; + repo: string; + skipParallels: boolean; + skipTelegram: boolean; +}; + +export type RunOptions = { + capture?: boolean; + timeoutMs?: number; +}; + +type WorkflowRunInfo = { + conclusion: string | null; + html_url: string; + status: string; + updated_at: string; +}; + +type PollRunOptions = { + pollIntervalMs?: number; + readRun?: (repo: string, runId: string) => WorkflowRunInfo; + sleep?: (ms: number) => Promise<void>; + timeoutMs?: number; + now?: () => number; +}; + +function usage(): string { + return `Usage: pnpm release:beta-smoke -- --beta beta4 [options] + +Options: + --beta <beta|betaN|version> Beta target. Default: beta + --model <provider/model> Parallels agent-turn model. Default: openai/gpt-5.4 + --provider-mode <mode> Telegram workflow provider mode. Default: mock-openai + --ref <ref> GitHub workflow dispatch ref. Default: main + --repo <owner/repo> GitHub repo. Default: openclaw/openclaw + --skip-parallels Only run Telegram workflow + --skip-telegram Only run Parallels beta validation + -h, --help Show help +`; +} + +export function parseArgs(argv: string[]): Options { + const args = stripLeadingPackageManagerSeparator(argv); + const terminatorIndex = args.indexOf("--"); + const cliArgs = terminatorIndex === -1 ? args : args.slice(0, terminatorIndex); + const options: Options = { + beta: "beta", + model: "openai/gpt-5.4", + providerMode: "mock-openai", + ref: "main", + repo: "openclaw/openclaw", + skipParallels: false, + skipTelegram: false, + }; + const helpIndex = cliArgs.findIndex((arg: string) => arg === "-h" || arg === "--help"); + parseFlagArgs( + helpIndex === -1 ? cliArgs : cliArgs.slice(0, helpIndex), + options, + [ + ...( + [ + ["--beta", "beta"], + ["--model", "model"], + ["--provider-mode", "providerMode"], + ["--ref", "ref"], + ["--repo", "repo"], + ] as const + ).map(([flag, key]) => + stringFlag(flag, key, { + allowInline: false, + rejectShortOptions: true, + repeatable: true, + }), + ), + booleanFlag("--skip-parallels", "skipParallels", true, { repeatable: true }), + booleanFlag("--skip-telegram", "skipTelegram", true, { repeatable: true }), + ], + { + onUnhandledArg(arg: string) { + throw new Error(`unknown option: ${arg}`); + }, + }, + ); + if (helpIndex !== -1) { + process.stdout.write(usage()); + process.exit(0); + } + if (options.skipParallels && options.skipTelegram) { + throw new Error("--skip-parallels and --skip-telegram cannot be used together"); + } + return options; +} + +const CAPTURE_MAX_BUFFER_BYTES = 32 * 1024 * 1024; +const DEFAULT_COMMAND_TIMEOUT_MS = readPositiveInt( + process.env.OPENCLAW_RELEASE_BETA_SMOKE_COMMAND_MS, + 10 * 60_000, + "OPENCLAW_RELEASE_BETA_SMOKE_COMMAND_MS", +); +const TELEGRAM_POLL_INTERVAL_MS = readPositiveInt( + process.env.OPENCLAW_RELEASE_BETA_SMOKE_POLL_INTERVAL_MS, + 30_000, + "OPENCLAW_RELEASE_BETA_SMOKE_POLL_INTERVAL_MS", +); +const TELEGRAM_POLL_TIMEOUT_MS = readPositiveInt( + process.env.OPENCLAW_RELEASE_BETA_SMOKE_POLL_TIMEOUT_MS, + 4 * 60 * 60_000, + "OPENCLAW_RELEASE_BETA_SMOKE_POLL_TIMEOUT_MS", +); + +export function readPositiveInt( + raw: string | undefined, + fallback: number, + label = "value", +): number { + const text = (raw ?? "").trim(); + if (!text) { + return fallback; + } + if (!/^\d+$/u.test(text)) { + throw new Error(`${label} must be a positive integer. Got: ${JSON.stringify(raw)}`); + } + const parsed = Number(text); + if (!Number.isSafeInteger(parsed) || parsed <= 0) { + throw new Error(`${label} must be a positive integer. Got: ${JSON.stringify(raw)}`); + } + return parsed; +} + +export function run(command: string, args: string[], input?: RunOptions): string { + const timeoutMs = input?.timeoutMs ?? DEFAULT_COMMAND_TIMEOUT_MS; + const result = spawnSync(command, args, { + encoding: "utf8", + killSignal: "SIGKILL", + maxBuffer: CAPTURE_MAX_BUFFER_BYTES, + stdio: input?.capture ? ["ignore", "pipe", "pipe"] : "inherit", + timeout: timeoutMs, + }); + if (result.error || result.status !== 0) { + const errorCode = (result.error as NodeJS.ErrnoException | undefined)?.code; + const reason = + errorCode === "ETIMEDOUT" + ? `timed out after ${timeoutMs}ms` + : (result.status ?? result.signal ?? result.error?.message ?? "unknown"); + const stderr = result.stderr ? `\n${result.stderr}` : ""; + throw new Error(`${command} ${args.join(" ")} failed with ${reason}${stderr}`); + } + return result.stdout ?? ""; +} + +function shellQuote(value: string): string { + return `'${value.replace(/'/g, "'\\''")}'`; +} + +const TELEGRAM_BETA_WORKFLOW_FILE = "npm-telegram-beta-e2e.yml"; + +function resolveBetaVersion(beta: string): string { + const value = beta.trim().replace(/^openclaw@/, ""); + if (/^\d{4}\.\d+\.\d+-beta\.\d+$/u.test(value)) { + return value; + } + if (value === "beta") { + return run("npm", ["view", "openclaw@beta", "version"], { capture: true }).trim(); + } + const betaMatch = /^(?:beta)?(\d+)$/u.exec(value); + if (!betaMatch) { + return run("npm", ["view", `openclaw@${value}`, "version"], { capture: true }).trim(); + } + const suffix = `-beta.${betaMatch[1]}`; + const versions = JSON.parse( + run("npm", ["view", "openclaw", "versions", "--json"], { capture: true }), + ) as string[]; + const match = versions + .filter((version) => version.endsWith(suffix)) + .toSorted((a, b) => a.localeCompare(b, undefined, { numeric: true })) + .at(-1); + if (!match) { + throw new Error(`no openclaw registry version found for ${beta}`); + } + return match; +} + +function timeoutCommand(): string { + return run("bash", ["-lc", "command -v gtimeout || command -v timeout"], { + capture: true, + }).trim(); +} + +function runParallels(beta: string, model: string): void { + const timeoutBin = timeoutCommand(); + const forwarded = [ + "pnpm", + "test:parallels:npm-update", + "--", + "--beta-validation", + beta, + "--model", + model, + "--json", + ]; + const command = [ + 'set -a; source "$HOME/.profile" >/dev/null 2>&1 || true; set +a;', + "exec", + shellQuote(timeoutBin), + "--foreground", + "150m", + ...forwarded.map(shellQuote), + ].join(" "); + run("bash", ["-lc", command], { timeoutMs: 155 * 60_000 }); +} + +function ghJson(repo: string, pathSuffix: string): unknown { + const url = `https://api.github.com/repos/${repo}/${pathSuffix}`; + const result = spawnSync( + "bash", + [ + "-lc", + [ + "set -euo pipefail", + 'token="$(gh auth token)"', + 'curl -fsS -H "Authorization: Bearer ${token}" -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "${OPENCLAW_GITHUB_REST_URL}"', + ].join("\n"), + ], + { + encoding: "utf8", + env: { ...process.env, OPENCLAW_GITHUB_REST_URL: url }, + killSignal: "SIGKILL", + maxBuffer: CAPTURE_MAX_BUFFER_BYTES, + stdio: ["ignore", "pipe", "pipe"], + timeout: DEFAULT_COMMAND_TIMEOUT_MS, + }, + ); + if (result.error || result.status !== 0) { + const reason = result.status ?? result.signal ?? result.error?.message ?? "unknown"; + const stderr = result.stderr ? `\n${result.stderr}` : ""; + throw new Error(`GitHub REST request failed for ${pathSuffix} with ${reason}${stderr}`); + } + return JSON.parse(result.stdout ?? ""); +} + +export function parseWorkflowRunIdFromOutput(output: string): string | undefined { + return /\/actions\/runs\/(\d+)/u.exec(output)?.[1]; +} + +export function requireWorkflowRunIdFromOutput(output: string, workflow: string): string { + const runId = parseWorkflowRunIdFromOutput(output); + if (!runId) { + throw new Error( + `gh workflow run ${workflow} did not return an Actions run URL; refusing to guess from recent workflow_dispatch runs`, + ); + } + return runId; +} + +async function dispatchTelegram(options: Options, packageSpec: string): Promise<string> { + const output = run( + "gh", + [ + "workflow", + "run", + TELEGRAM_BETA_WORKFLOW_FILE, + "--repo", + options.repo, + "--ref", + options.ref, + "-f", + `package_spec=${packageSpec}`, + "-f", + `package_label=${packageSpec}`, + "-f", + `provider_mode=${options.providerMode}`, + ], + { capture: true }, + ); + return requireWorkflowRunIdFromOutput(output, TELEGRAM_BETA_WORKFLOW_FILE); +} + +export async function pollRun( + repo: string, + runId: string, + options: PollRunOptions = {}, +): Promise<void> { + const started = (options.now ?? Date.now)(); + const timeoutMs = Math.max(1, options.timeoutMs ?? TELEGRAM_POLL_TIMEOUT_MS); + const pollIntervalMs = Math.max(1, options.pollIntervalMs ?? TELEGRAM_POLL_INTERVAL_MS); + const sleep = + options.sleep ?? + ((ms: number) => + new Promise<void>((resolve) => { + setTimeout(resolve, ms); + })); + const readRun = + options.readRun ?? + ((currentRepo: string, currentRunId: string) => + ghJson(currentRepo, `actions/runs/${currentRunId}`) as WorkflowRunInfo); + for (;;) { + const info = readRun(repo, runId); + console.log( + `Telegram workflow ${runId}: ${info.status}${info.conclusion ? `/${info.conclusion}` : ""} updated=${info.updated_at}`, + ); + if (info.status === "completed") { + if (info.conclusion !== "success") { + throw new Error( + `Telegram workflow failed: ${info.conclusion ?? "unknown"} ${info.html_url}`, + ); + } + console.log(info.html_url); + return; + } + const elapsedMs = (options.now ?? Date.now)() - started; + if (elapsedMs >= timeoutMs) { + throw new Error(`Telegram workflow ${runId} did not complete within ${timeoutMs}ms`); + } + await sleep(Math.min(pollIntervalMs, timeoutMs - elapsedMs)); + } +} + +function downloadTelegramArtifact(repo: string, runId: string): string { + const artifacts = ( + ghJson(repo, `actions/runs/${runId}/artifacts`) as { + artifacts: Array<{ expired: boolean; name: string }>; + } + ).artifacts; + const artifact = artifacts.find( + (entry) => !entry.expired && entry.name.startsWith(`npm-telegram-beta-e2e-${runId}-`), + ); + if (!artifact) { + throw new Error(`no npm Telegram artifact found for run ${runId}`); + } + const outputDir = path.join(".artifacts", "qa-e2e", artifact.name); + mkdirSync(outputDir, { recursive: true }); + run("gh", [ + "run", + "download", + runId, + "--repo", + repo, + "--name", + artifact.name, + "--dir", + outputDir, + ]); + return outputDir; +} + +function findFile(root: string, basename: string): string { + for (const entry of readdirSync(root, { withFileTypes: true })) { + const filePath = path.join(root, entry.name); + if (entry.isFile() && entry.name === basename) { + return filePath; + } + if (entry.isDirectory()) { + const nested = findFile(filePath, basename); + if (nested) { + return nested; + } + } + } + return ""; +} + +export function mergeTelegramProofIntoReleaseBody(body: string, telegramLine: string): string { + if (body.includes(telegramLine)) { + return body; + } + + const marker = "### Release verification"; + const telegramProofPattern = /^- npm Telegram beta E2E: .*$/mu; + if (telegramProofPattern.test(body)) { + return body.replace(telegramProofPattern, telegramLine); + } + if (!body.includes(marker)) { + return `${body.trimEnd()}\n\n${marker}\n\n${telegramLine}\n`; + } + + const markerIndex = body.indexOf(marker); + const afterMarkerIndex = markerIndex + marker.length; + const nextHeading = /\n#{1,6} /u.exec(body.slice(afterMarkerIndex)); + const insertionIndex = nextHeading === null ? -1 : afterMarkerIndex + nextHeading.index; + if (insertionIndex === -1) { + return `${body.trimEnd()}\n${telegramLine}\n`; + } + return `${body.slice(0, insertionIndex).trimEnd()}\n${telegramLine}\n${body.slice(insertionIndex)}`; +} + +function appendTelegramProofToRelease(repo: string, version: string, runId: string): void { + const tag = `v${version}`; + const release = ghJson(repo, `releases/tags/${encodeURIComponent(tag)}`) as { + body?: string; + html_url?: string; + }; + const body = release.body ?? ""; + const telegramLine = `- npm Telegram beta E2E: https://github.com/${repo}/actions/runs/${runId}`; + const notesFile = path.join( + "/tmp", + `openclaw-${version.replace(/[^a-zA-Z0-9.-]/g, "-")}-release-notes-${process.pid}.md`, + ); + const nextBody = mergeTelegramProofIntoReleaseBody(body, telegramLine); + if (nextBody === body) { + return; + } + writeFileSync(notesFile, nextBody); + run("gh", ["release", "edit", tag, "--repo", repo, "--notes-file", notesFile]); + console.log( + `Updated release proof: ${release.html_url ?? `https://github.com/${repo}/releases/tag/${tag}`}`, + ); +} + +async function main(): Promise<void> { + const options = parseArgs(process.argv.slice(2)); + const version = resolveBetaVersion(options.beta); + const packageSpec = `openclaw@${version}`; + console.log(`Resolved beta target: ${packageSpec}`); + + let telegramRunId: string | undefined; + if (!options.skipTelegram) { + telegramRunId = await dispatchTelegram(options, packageSpec); + console.log( + `Dispatched Telegram workflow: https://github.com/${options.repo}/actions/runs/${telegramRunId}`, + ); + } + + if (!options.skipParallels) { + runParallels(version, options.model); + } + + if (telegramRunId) { + await pollRun(options.repo, telegramRunId); + const artifactDir = downloadTelegramArtifact(options.repo, telegramRunId); + const report = findFile(artifactDir, "qa-suite-report.md"); + if (report && existsSync(report)) { + console.log(`\nTelegram report: ${report}\n`); + console.log(readFileSync(report, "utf8")); + } + appendTelegramProofToRelease(options.repo, version, telegramRunId); + } +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { + await main().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); + }); +} diff --git a/scripts/release-candidate-checklist.mts b/scripts/release-candidate-checklist.mts new file mode 100644 index 0000000000000000000000000000000000000000..5c38cc52abbe065be1d6be569c03aba03850b129 --- /dev/null +++ b/scripts/release-candidate-checklist.mts @@ -0,0 +1,2505 @@ +#!/usr/bin/env node +// Coordinates release-candidate validation runs and emits the publish command +// only after required local, CI, npm, plugin, and E2E evidence is green. +import { spawnSync } from "node:child_process"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + realpathSync, + renameSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { createRequire } from "node:module"; +import { tmpdir } from "node:os"; +import { basename, dirname, join, resolve as resolvePath } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { isDeepStrictEqual } from "node:util"; +import { isRecord } from "@openclaw/normalization-core/record-coerce"; +import { parse as parseYaml } from "yaml"; +import { + normalizePublicationIntent, + publicationAdmissionContract, + publicationDispatchEnvelope, + publicationSourceContract, + type PublicationSelection, +} from "./full-release-publication-contract.mjs"; +import { + booleanFlag, + parseFlagArgs, + stringFlag, + stringListFlag, + stripLeadingPackageManagerSeparator, +} from "./lib/arg-utils.mts"; +import { readBoundedResponseText } from "./lib/bounded-response.mjs"; +import { parsePluginReleaseSelection } from "./lib/plugin-npm-release.ts"; +import { loadChangelogCollection, loadReleaseChangelog } from "./lib/release-changelog.mjs"; +import { releaseBranchForTag } from "./lib/release-context.mjs"; +import { classifyReleaseTrain, parseReleaseVersion } from "./lib/release-version.mjs"; +import { + downloadFullReleaseNpmPreflight, + verifyNpmPreflightProducer, + validateReleasePreflightTagIdentity, +} from "./npm-preflight-tooling-identity.mjs"; +import { validateNpmPreflightDistTag } from "./openclaw-npm-extended-stable-release.mjs"; +import { validatePluginSdkApiReleaseEvidence } from "./plugin-sdk-api-release-evidence.mjs"; +import { verifyReleaseToolingIdentity } from "./release-tooling-identity.mjs"; +import { + dedicatedSectionVersionForTag, + extractChangelogReleaseSections, + extractChangelogSection, + formatShippedBaselineExclusions, + loadReleaseNotesForTag, + parseContributionRecordProvenance, + parseShippedBaselineExclusions, + releaseNotesSectionForTag, + releaseNotesVersionForTag, + renderGithubReleaseNotes, +} from "./render-github-release-notes.mts"; +import { + isShaPinnedReleaseValidationBranch, + authenticateFullReleaseValidationEvidence, +} from "./validate-full-release-validation-evidence.mjs"; + +type JsonRecord = Record<string, unknown>; +type StringFields<K extends string> = Record<K, string>; +type CandidateState = ReturnType<typeof buildReleaseCandidateState>; +type WorkflowArtifact = Awaited<ReturnType<typeof runArtifacts>>[number]; +type GithubFetch = (url: string, init?: RequestInit) => Promise<Response>; +type GithubApiOptions = Partial<{ + fetchImpl: GithubFetch; + maxBodyBytes: number; + timeoutMs: number; + token: string; +}>; +type TarballDescriptor = Record< + "packageName" | "packageVersion" | "tarballName" | "tarballSha256", + string +>; +type TelegramArtifact = Partial<WorkflowArtifact>; +type TelegramArtifactParams = { + artifact: TelegramArtifact; + manifest: JsonRecord; + runAttempt: number; + runId: string; + sourceSha: string; +}; +type TelegramResult = { status: string } & Partial< + StringFields<"artifactName" | "providerMode" | "runId" | "url"> +>; +type LocalCheckResult = + | { status: "passed"; command: string; reason?: never } + | { status: "skipped"; reason: string; command?: never }; +type PreflightTarballs = Partial<Record<"corePackageTarballs" | "dependencyTarballs", unknown>>; +const DEFAULT_REPO = "openclaw/openclaw"; +const DEFAULT_PROVIDER = "openai"; +const DEFAULT_MODE = "both"; +const DEFAULT_NPM_DIST_TAG = "beta"; +const DEFAULT_PLUGIN_SCOPE = "all-publishable"; +const DEFAULT_TELEGRAM_PROVIDER_MODE = "mock-openai"; +const DEFAULT_GITHUB_API_TIMEOUT_MS = 30_000; +const DEFAULT_GITHUB_API_RESPONSE_BODY_MAX_BYTES = 16 * 1024 * 1024; +const COMMAND_CAPTURE_MAX_BUFFER_BYTES = 16 * 1024 * 1024; +const TOOLING_ROOT = fileURLToPath(new URL("../", import.meta.url)); +const TIDECLAW_ALPHA_WORKFLOW_REF_PATTERN = + /^tideclaw\/alpha\/[0-9]{4}-[0-9]{2}-[0-9]{2}-[0-9]{4}Z$/u; +const WINDOWS_NODE_TAG_PATTERN = /^v[0-9]+\.[0-9]+\.[0-9]+([-.][0-9A-Za-z]+([.-][0-9A-Za-z]+)*)?$/u; +const WINDOWS_NODE_REPO = "openclaw/openclaw-windows-node"; +const WINDOWS_NODE_REQUIRED_ASSETS = [ + "OpenClawCompanion-Setup-x64.exe", + "OpenClawCompanion-Setup-arm64.exe", +]; +const SHA256_DIGEST_PATTERN = /^sha256:[a-f0-9]{64}$/u; +const SHA256_HEX_PATTERN = /^[a-f0-9]{64}$/u; +const RELEASE_CANDIDATE_STATE_VERSION = 2; +const RELEASE_CANDIDATE_STATE_FILE = "release-candidate-state.json"; +const TRUSTED_TOOLING_SHA_ENV = "OPENCLAW_RELEASE_CANDIDATE_TRUSTED_TOOLING_SHA"; +const RELEASE_CANDIDATE_STATE_KEYS = [ + "repo", + "tag", + "targetSha", + "toolingSha", + "workflowRef", + "publishWorkflowRef", + "provider", + "mode", + "releaseProfile", + "npmDistTag", + "pluginPublishScope", + "plugins", + "parallelsRegistryPackageArtifacts", + "windowsNodeTag", + "skipParallels", + "skipTelegram", + "telegramProviderMode", +] as const satisfies readonly (keyof CandidateState)[]; + +function formatJsonValue(value: unknown): string { + if (typeof value === "string") { + return value; + } + if (typeof value === "number" || typeof value === "boolean" || typeof value === "bigint") { + return String(value); + } + if (value === null) { + return "null"; + } + return JSON.stringify(value) ?? "<undefined>"; +} + +function usage() { + return `Usage: pnpm release:candidate -- --tag vYYYY.M.PATCH-beta.N [options] + +Dispatches or consumes release validation runs, validates the prepared npm tarball, +builds plugin publish plans, writes a green evidence bundle, then prints the exact +selected publication command. --publication-route prepared selects the +prepare-once release button for complete regular beta/stable releases. + +Options: + --tag <tag> Planned release tag. The tag must not exist yet. + --target-sha <sha> Frozen release SHA. Defaults to the current HEAD. + --workflow-ref <ref> Trusted workflow ref. Default: main; matching Tideclaw branch required for alpha. + --publish-workflow-ref <tag> Protected publication tooling tag matching the trusted helper checkout. + --publication-route <normal|prepared> + Intended publication route. Default: normal; not inferred from a protected ref. + --repo <owner/repo> GitHub repo. Default: ${DEFAULT_REPO} + --full-release-run <id> Reuse successful Full Release Validation run. + --npm-preflight-run <id> Reuse successful OpenClaw NPM Release preflight run. + --plugin-sdk-api-acknowledgement <digest> + 8-character digest from the Plugin SDK API diff report. + --windows-node-tag <tag> Optional exact Windows Node tag for postpublish asset promotion. + --skip-dispatch Require Full Release Validation run; separate npm run only for historical recovery. + --skip-local-generated-check Do not run local generated release baseline checks before dispatch. + --run-parallels Force candidate Parallels smoke; beta defaults to postpublish release:beta-smoke. + --skip-parallels Force-skip candidate Parallels smoke; stable/full run by default. + --parallels-registry-package-artifact <dir> + Add a verified plugin npm preflight artifact directory. Repeatable. + --skip-telegram Do not run NPM Telegram E2E against the prepared tarball. + --telegram-provider-mode <mode> mock-openai|live-frontier. Default: ${DEFAULT_TELEGRAM_PROVIDER_MODE} + --provider <provider> Full validation provider. Default: ${DEFAULT_PROVIDER} + --mode <fresh|upgrade|both> Full validation cross-OS mode. Default: ${DEFAULT_MODE} + --release-profile <beta|stable|full> Default: beta for prereleases; stable otherwise. + --npm-dist-tag <alpha|beta|latest> Default: ${DEFAULT_NPM_DIST_TAG} + --plugin-publish-scope <scope> selected|all-publishable. Default: ${DEFAULT_PLUGIN_SCOPE} + --plugins <names> Required when plugin scope is selected. + --output-dir <dir> Evidence output dir. Default: .artifacts/release-candidate/<tag> +`; +} + +/** + * Parses release-candidate validation options and enforces publish-scope policy. + */ +export function parseArgs(argv: string[]) { + const args = stripLeadingPackageManagerSeparator(argv); + const terminatorIndex = args.indexOf("--"); + const cliArgs = terminatorIndex === -1 ? args : args.slice(0, terminatorIndex); + const options = { + repo: DEFAULT_REPO, + provider: DEFAULT_PROVIDER, + mode: DEFAULT_MODE, + releaseProfile: "", + npmDistTag: DEFAULT_NPM_DIST_TAG, + pluginPublishScope: DEFAULT_PLUGIN_SCOPE, + plugins: "", + parallelsRegistryPackageArtifactDirs: new Array<string>(), + parallelsRegistryPackageArtifacts: new Array< + ReturnType<typeof validateParallelsRegistryPackageArtifact> + >(), + skipDispatch: false, + skipLocalGeneratedCheck: false, + runParallels: false, + skipParallels: false, + parallelsMode: "auto" as "auto" | "run" | "skip", + parallelsSkipReason: "", + skipTelegram: false, + telegramProviderMode: DEFAULT_TELEGRAM_PROVIDER_MODE, + tag: "", + targetSha: "", + workflowRef: "", + publishWorkflowRef: "", + publicationRoute: "normal", + fullReleaseRunId: "", + npmPreflightRunId: "", + pluginSdkApiAcknowledgement: "", + windowsNodeTag: "", + windowsNodeInstallerDigests: "", + outputDir: "", + }; + const helpIndex = cliArgs.findIndex((arg) => arg === "-h" || arg === "--help"); + parseFlagArgs( + helpIndex === -1 ? cliArgs : cliArgs.slice(0, helpIndex), + options, + [ + ...( + [ + ["--tag", "tag"], + ["--target-sha", "targetSha"], + ["--workflow-ref", "workflowRef"], + ["--publish-workflow-ref", "publishWorkflowRef"], + ["--publication-route", "publicationRoute"], + ["--repo", "repo"], + ["--full-release-run", "fullReleaseRunId"], + ["--npm-preflight-run", "npmPreflightRunId"], + ["--plugin-sdk-api-acknowledgement", "pluginSdkApiAcknowledgement"], + ["--windows-node-tag", "windowsNodeTag"], + ["--telegram-provider-mode", "telegramProviderMode"], + ["--provider", "provider"], + ["--mode", "mode"], + ["--release-profile", "releaseProfile"], + ["--npm-dist-tag", "npmDistTag"], + ["--plugin-publish-scope", "pluginPublishScope"], + ["--plugins", "plugins"], + ["--output-dir", "outputDir"], + ] satisfies Array<[string, string]> + ).map(([flag, key]) => + stringFlag(flag, key, { allowInline: false, rejectShortOptions: true }), + ), + stringListFlag( + "--parallels-registry-package-artifact", + "parallelsRegistryPackageArtifactDirs", + { allowInline: false, rejectShortOptions: true }, + ), + booleanFlag("--skip-dispatch", "skipDispatch"), + booleanFlag("--skip-local-generated-check", "skipLocalGeneratedCheck"), + booleanFlag("--run-parallels", "runParallels"), + booleanFlag("--skip-parallels", "skipParallels"), + booleanFlag("--skip-telegram", "skipTelegram"), + ], + { + onUnhandledArg(arg: string) { + throw new Error(`unknown option: ${arg}`); + }, + }, + ); + if (helpIndex !== -1) { + process.stdout.write(usage()); + process.exit(0); + } + if (!options.tag) { + throw new Error("--tag is required"); + } + if (!["normal", "prepared"].includes(options.publicationRoute)) { + throw new Error("--publication-route must be normal or prepared"); + } + if ( + options.publicationRoute === "prepared" && + (options.tag.includes("-alpha.") || + options.npmDistTag === "extended-stable" || + options.pluginPublishScope !== "all-publishable" || + options.plugins.trim() || + !/^release-publish\/[a-f0-9]{12}-[1-9][0-9]*$/u.test(options.publishWorkflowRef)) + ) { + throw new Error( + "Prepared publication requires a protected tooling tag and a complete regular-release roster.", + ); + } + if (options.targetSha && !/^[a-f0-9]{40}$/u.test(options.targetSha)) { + throw new Error("--target-sha must be a full lowercase commit SHA"); + } + if ( + options.pluginSdkApiAcknowledgement && + !/^[a-f0-9]{8}$/u.test(options.pluginSdkApiAcknowledgement) + ) { + throw new Error("--plugin-sdk-api-acknowledgement must be an 8-character lowercase digest"); + } + if (options.tag.includes("-alpha.")) { + if (!TIDECLAW_ALPHA_WORKFLOW_REF_PATTERN.test(options.workflowRef)) { + throw new Error( + "--workflow-ref must be the matching tideclaw/alpha/YYYY-MM-DD-HHMMZ branch for alpha release candidates", + ); + } + } else { + options.workflowRef ||= "main"; + } + if (!options.tag.includes("-alpha.") && options.workflowRef !== "main") { + throw new Error("--workflow-ref must be main for regular beta and stable release candidates"); + } + if ( + options.publishWorkflowRef && + (options.tag.includes("-alpha.") || + !/^release-publish\/[a-f0-9]{12}-[1-9][0-9]*$/u.test(options.publishWorkflowRef)) + ) { + throw new Error( + "--publish-workflow-ref must name a protected release-publish tag for a regular release", + ); + } + options.releaseProfile ||= + options.tag.includes("-alpha.") || options.tag.includes("-beta.") ? "beta" : "stable"; + if (!["beta", "stable", "full"].includes(options.releaseProfile)) { + throw new Error("--release-profile must be beta, stable, or full"); + } + if (options.runParallels && options.skipParallels) { + throw new Error("--run-parallels and --skip-parallels cannot be combined"); + } + options.parallelsMode = options.runParallels ? "run" : options.skipParallels ? "skip" : "auto"; + options.skipParallels = + options.parallelsMode === "skip" || + (options.parallelsMode === "auto" && options.releaseProfile === "beta"); + options.parallelsSkipReason = options.skipParallels + ? options.parallelsMode === "auto" + ? "deferred to postpublish release:beta-smoke" + : "operator skipped --skip-parallels" + : ""; + if (options.skipDispatch && !options.fullReleaseRunId) { + throw new Error("--skip-dispatch requires --full-release-run"); + } + if (options.pluginPublishScope === "selected" && !options.plugins.trim()) { + throw new Error("--plugin-publish-scope selected requires --plugins"); + } + if (options.pluginPublishScope === "selected") { + throw new Error( + "--plugin-publish-scope selected is only for plugin-only repair publishes; release candidates publish OpenClaw with --plugin-publish-scope all-publishable", + ); + } + if (options.pluginPublishScope === "all-publishable" && options.plugins.trim()) { + throw new Error("--plugins is only valid with --plugin-publish-scope selected"); + } + // Apps can attach after npm and GitHub publication; only an explicitly selected + // Windows promotion needs a source tag and its immutable installer digests. + if (options.windowsNodeTag && !WINDOWS_NODE_TAG_PATTERN.test(options.windowsNodeTag)) { + throw new Error("--windows-node-tag must be an explicit version tag, not latest"); + } + if (!["mock-openai", "live-frontier"].includes(options.telegramProviderMode)) { + throw new Error("--telegram-provider-mode must be mock-openai or live-frontier"); + } + return options; +} + +export function run( + command: string, + args: string[], + options: { capture?: boolean; cwd?: string; env?: NodeJS.ProcessEnv } = {}, +) { + const result = spawnSync(command, args, { + cwd: options.cwd, + encoding: "utf8", + env: options.env ? { ...process.env, ...options.env } : process.env, + maxBuffer: COMMAND_CAPTURE_MAX_BUFFER_BYTES, + stdio: options.capture ? ["ignore", "pipe", "pipe"] : "inherit", + }); + if (result.status !== 0) { + throw new Error( + `${command} ${args.join(" ")} failed with ${result.status ?? result.signal}\n${result.stderr ?? ""}`, + ); + } + return result.stdout ?? ""; +} + +function readJson(path: string, label: string) { + try { + const value: unknown = JSON.parse(readFileSync(path, "utf8")); + if (!isRecord(value)) { + throw new Error("expected an object"); + } + return value; + } catch (error) { + throw new Error( + `${label} is invalid JSON: ${error instanceof Error ? error.message : String(error)}`, + { cause: error }, + ); + } +} + +export function validateParallelsRegistryPackageArtifact( + artifactDir: string, + params: { targetSha: string; targetVersion: string }, +) { + const resolvedDir = resolvePath(artifactDir); + const manifestPath = join(resolvedDir, "plugin-publication-manifest.json"); + const manifest = readJson(manifestPath, "plugin npm preflight manifest"); + const artifact = isRecord(manifest.artifact) ? manifest.artifact : undefined; + const packageInfo = isRecord(manifest.package) ? manifest.package : undefined; + const artifactName = artifact?.name; + const tarballName = artifact?.tarball; + const tarballSha256 = artifact?.sha256; + const packageName = packageInfo?.name; + const packageVersion = packageInfo?.version; + if ( + manifest.schema !== "openclaw.plugin-publication-artifact/v1" || + manifest.schemaVersion !== 1 || + manifest.targetSha !== params.targetSha || + typeof artifactName !== "string" || + typeof packageName !== "string" || + packageVersion !== params.targetVersion || + typeof tarballName !== "string" || + tarballName !== basename(tarballName) || + typeof tarballSha256 !== "string" || + !SHA256_HEX_PATTERN.test(tarballSha256) + ) { + throw new Error(`plugin npm preflight artifact identity is invalid: ${resolvedDir}`); + } + const tarballPath = join(resolvedDir, tarballName); + const compareFileNames = (left: string, right: string) => left.localeCompare(right); + const files = readdirSync(resolvedDir).toSorted(compareFileNames); + const expectedFiles = [basename(manifestPath), tarballName].toSorted(compareFileNames); + if (!isDeepStrictEqual(files, expectedFiles) || !existsSync(tarballPath)) { + throw new Error(`plugin npm preflight artifact inventory is invalid: ${resolvedDir}`); + } + const actualSha256 = sha256(tarballPath); + if (actualSha256 !== tarballSha256) { + throw new Error( + `plugin npm preflight tarball digest mismatch for ${packageName}: expected ${tarballSha256}, got ${actualSha256}`, + ); + } + let packedPackage: unknown; + try { + packedPackage = JSON.parse( + run("tar", ["-xOf", tarballPath, "package/package.json"], { capture: true }), + ); + } catch (error) { + throw new Error(`plugin npm preflight tarball package metadata is invalid: ${tarballPath}`, { + cause: error, + }); + } + if ( + !isRecord(packedPackage) || + packedPackage.name !== packageName || + packedPackage.version !== packageVersion + ) { + throw new Error( + `plugin npm preflight tarball identity mismatch: manifest=${packageName}@${packageVersion} packed=${formatJsonValue(isRecord(packedPackage) ? (packedPackage.name ?? "<missing>") : "<missing>")}@${formatJsonValue(isRecord(packedPackage) ? (packedPackage.version ?? "<missing>") : "<missing>")}`, + ); + } + return { + artifactDir: resolvedDir, + artifactName, + manifestPath, + packageName, + packageVersion, + tarballPath, + tarballSha256, + }; +} + +export function buildReleaseCandidateState( + options: ReturnType<typeof parseArgs>, + { targetSha, toolingSha }: { targetSha: string; toolingSha: string }, +) { + return { + version: RELEASE_CANDIDATE_STATE_VERSION, + phase: "validated", + repo: options.repo, + tag: options.tag, + targetSha, + toolingSha, + workflowRef: options.workflowRef, + publishWorkflowRef: options.publishWorkflowRef, + publicationRoute: options.publicationRoute, + provider: options.provider, + mode: options.mode, + releaseProfile: options.releaseProfile, + npmDistTag: options.npmDistTag, + pluginPublishScope: options.pluginPublishScope, + plugins: options.plugins, + parallelsRegistryPackageArtifacts: options.parallelsRegistryPackageArtifacts, + windowsNodeTag: options.windowsNodeTag, + skipParallels: options.skipParallels, + skipTelegram: options.skipTelegram, + telegramProviderMode: options.telegramProviderMode, + fullReleaseRunId: options.fullReleaseRunId, + npmPreflightRunId: options.npmPreflightRunId, + }; +} + +export function reconcileReleaseCandidateState(saved: unknown, expected: CandidateState) { + if (!saved) { + return expected; + } + if (!isRecord(saved) || saved.version !== RELEASE_CANDIDATE_STATE_VERSION) { + throw new Error("release candidate state has an unsupported schema"); + } + // Historical state emitted the normal command by default. It did not record + // source admission, and cannot silently become a newly selected prepared route. + if ((saved.publicationRoute ?? "normal") !== expected.publicationRoute) { + throw new Error("release candidate state mismatch for publicationRoute"); + } + for (const key of RELEASE_CANDIDATE_STATE_KEYS) { + if (!isDeepStrictEqual(saved[key], expected[key])) { + throw new Error( + `release candidate state mismatch for ${key}: saved=${JSON.stringify(saved[key])} current=${JSON.stringify(expected[key])}`, + ); + } + } + for (const key of ["fullReleaseRunId", "npmPreflightRunId"] as const) { + if (saved[key] && expected[key] && saved[key] !== expected[key]) { + throw new Error(`release candidate state mismatch for ${key}`); + } + } + return { + ...expected, + phase: typeof saved.phase === "string" ? saved.phase : expected.phase, + fullReleaseRunId: + expected.fullReleaseRunId || + (typeof saved.fullReleaseRunId === "string" ? saved.fullReleaseRunId : ""), + npmPreflightRunId: + expected.npmPreflightRunId || + (typeof saved.npmPreflightRunId === "string" ? saved.npmPreflightRunId : ""), + }; +} + +function writeReleaseCandidateState(path: string, state: unknown) { + mkdirSync(join(path, ".."), { recursive: true }); + const temporaryPath = `${path}.tmp-${process.pid}`; + writeFileSync(temporaryPath, `${JSON.stringify(state, null, 2)}\n`); + renameSync(temporaryPath, path); +} + +function updateReleaseCandidateState( + path: string, + state: CandidateState, + phase: string, + runIds: Partial<Pick<CandidateState, "fullReleaseRunId" | "npmPreflightRunId">> = {}, +) { + const next = { ...state, ...runIds, phase }; + writeReleaseCandidateState(path, next); + return next; +} + +function githubApiTimeoutMs() { + const raw = process.env.OPENCLAW_RELEASE_CANDIDATE_GITHUB_API_TIMEOUT_MS; + if (!raw) { + return DEFAULT_GITHUB_API_TIMEOUT_MS; + } + if (!/^[1-9]\d*$/u.test(raw)) { + throw new Error("OPENCLAW_RELEASE_CANDIDATE_GITHUB_API_TIMEOUT_MS must be a positive integer"); + } + const value = Number(raw); + if (!Number.isSafeInteger(value)) { + throw new Error("OPENCLAW_RELEASE_CANDIDATE_GITHUB_API_TIMEOUT_MS must be a positive integer"); + } + return value; +} + +function githubApiTimedOut(error: unknown) { + return ( + error instanceof DOMException && (error.name === "AbortError" || error.name === "TimeoutError") + ); +} + +/** + * Calls the GitHub REST API with the gh-auth token and a bounded timeout. + */ +export async function githubApi(path: string, options: GithubApiOptions = {}): Promise<unknown> { + const token = options.token ?? run("gh", ["auth", "token"], { capture: true }).trim(); + const timeoutMs = options.timeoutMs ?? githubApiTimeoutMs(); + const maxBodyBytes = options.maxBodyBytes ?? DEFAULT_GITHUB_API_RESPONSE_BODY_MAX_BYTES; + const controller = new AbortController(); + const fetchImpl = options.fetchImpl ?? fetch; + const request = (requestToken: string) => + fetchImpl(`https://api.github.com/${path}`, { + signal: controller.signal, + headers: { + Accept: "application/vnd.github+json", + ...(requestToken ? { Authorization: `Bearer ${requestToken}` } : {}), + "X-GitHub-Api-Version": "2022-11-28", + }, + }); + let timeout: ReturnType<typeof setTimeout> | undefined; + const timeoutPromise = new Promise<never>((_, reject) => { + timeout = setTimeout(() => { + controller.abort(new DOMException("request timed out", "TimeoutError")); + reject(new DOMException("request timed out", "TimeoutError")); + }, timeoutMs); + timeout.unref?.(); + }); + try { + let response = await Promise.race([request(token), timeoutPromise]); + let text = await readBoundedResponseText(response, `GitHub API ${path}`, maxBodyBytes, { + signal: controller.signal, + timeoutPromise, + }); + const primaryRateLimitExhausted = + (response.status === 403 || response.status === 429) && + (/API rate limit exceeded/iu.test(text) || + response.headers.get("x-ratelimit-remaining") === "0"); + if (primaryRateLimitExhausted) { + // Public release evidence remains readable without auth when one maintainer + // token is exhausted. Mutating gh commands keep their authenticated path. + response = await Promise.race([request(""), timeoutPromise]); + text = await readBoundedResponseText(response, `GitHub API ${path}`, maxBodyBytes, { + signal: controller.signal, + timeoutPromise, + }); + } + if (!response.ok) { + throw new Error(`GitHub API ${path} failed with ${response.status}: ${text}`); + } + return JSON.parse(text); + } catch (error) { + if (githubApiTimedOut(error)) { + throw new Error(`GitHub API ${path} timed out after ${timeoutMs}ms`, { cause: error }); + } + throw error; + } finally { + clearTimeout(timeout); + } +} + +/** + * Validates the immutable Windows source release contract for a stable candidate. + */ +export async function validateWindowsSourceRelease(tag: string, options: GithubApiOptions = {}) { + const release = await githubApi( + `repos/${WINDOWS_NODE_REPO}/releases/tags/${encodeURIComponent(tag)}`, + options, + ); + if (!isRecord(release)) { + throw new Error(`Windows source release tag mismatch: expected ${tag}, got undefined`); + } + if (release.tag_name !== tag) { + throw new Error( + `Windows source release tag mismatch: expected ${tag}, got ${formatJsonValue(release.tag_name)}`, + ); + } + if (release.draft) { + throw new Error(`Windows source release ${tag} must be published`); + } + if (release.prerelease) { + throw new Error(`Windows source release ${tag} must not be a prerelease`); + } + + const releaseAssets = Array.isArray(release.assets) ? release.assets.filter(isRecord) : []; + const assets = WINDOWS_NODE_REQUIRED_ASSETS.map((name) => { + const matches = releaseAssets.filter((entry) => entry.name === name); + if (matches.length !== 1) { + throw new Error( + `Windows source release ${tag} must contain exactly one required asset ${name}; found ${matches.length}`, + ); + } + const [asset] = matches; + if (!asset) { + throw new Error(`Windows source release ${tag} is missing required asset ${name}`); + } + if (typeof asset.digest !== "string" || !SHA256_DIGEST_PATTERN.test(asset.digest)) { + throw new Error(`Windows source release ${tag} asset ${name} is missing its SHA-256 digest`); + } + return { name, digest: asset.digest }; + }); + return { + tag, + url: release.html_url, + assets, + }; +} + +function gitRevParse(ref: string, cwd: string = process.cwd()) { + return run("git", ["rev-parse", ref], { capture: true, cwd }).trim(); +} + +function gitTopLevel(cwd: string) { + return run("git", ["rev-parse", "--show-toplevel"], { capture: true, cwd }).trim(); +} + +function gitTrackedStatus(cwd: string) { + return run("git", ["status", "--porcelain=v1", "--untracked-files=no"], { + capture: true, + cwd, + }); +} + +function fetchTrustedWorkflowSha(workflowRef: string, toolingRoot: string) { + const remoteRef = `refs/remotes/origin/${workflowRef}`; + run("git", ["fetch", "--no-tags", "origin", `+refs/heads/${workflowRef}:${remoteRef}`], { + cwd: toolingRoot, + }); + return gitRevParse(`${remoteRef}^{commit}`, toolingRoot); +} + +function runFromTrustedTooling( + argv: string[], + { targetRoot, workflowRef }: { targetRoot: string; workflowRef: string }, +) { + const trustedToolingSha = fetchTrustedWorkflowSha(workflowRef, targetRoot); + const tempRoot = mkdtempSync(join(tmpdir(), "openclaw-release-tooling-")); + const toolingRoot = join(tempRoot, "checkout"); + let worktreeAdded = false; + try { + run("git", ["worktree", "add", "--detach", toolingRoot, trustedToolingSha], { + cwd: targetRoot, + }); + worktreeAdded = true; + // The tooling worktree installs its own frozen graph: borrowing the target's + // node_modules would resolve workspace package links back into the target + // checkout and let candidate code run inside the trusted helper. + run("pnpm", ["install", "--frozen-lockfile", "--ignore-scripts", "--prefer-offline"], { + cwd: toolingRoot, + }); + const tsxLoader = pathToFileURL( + createRequire(join(toolingRoot, "package.json")).resolve("tsx"), + ).href; + const result = spawnSync( + process.execPath, + [ + "--import", + tsxLoader, + join(toolingRoot, "scripts/release-candidate-checklist.mts"), + ...argv, + ], + { + cwd: targetRoot, + env: { ...process.env, [TRUSTED_TOOLING_SHA_ENV]: trustedToolingSha }, + stdio: "inherit", + }, + ); + if (result.status !== 0) { + throw new Error( + `trusted release candidate tooling failed with ${result.status ?? result.signal}`, + ); + } + } finally { + if (worktreeAdded) { + const cleanup = spawnSync("git", ["worktree", "remove", "--force", toolingRoot], { + cwd: targetRoot, + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }); + if (cleanup.status !== 0) { + console.warn( + `could not remove temporary trusted tooling worktree: ${cleanup.stderr?.trim() || cleanup.signal || cleanup.status}`, + ); + } + } + rmSync(tempRoot, { force: true, recursive: true }); + } +} + +export function isDirectReleaseCandidateExecution( + directPath: string | undefined, + modulePath: string, + resolveRealPath: (path: string) => string = realpathSync, +) { + if (!directPath) { + return false; + } + try { + return resolveRealPath(directPath) === resolveRealPath(modulePath); + } catch { + return false; + } +} + +export function validateCandidateCheckout({ + targetSha, + targetHeadSha, + targetTrackedStatus, + toolingSha, + trustedToolingSha, + toolingTrackedStatus, + workflowRef, +}: StringFields< + | "targetSha" + | "targetHeadSha" + | "targetTrackedStatus" + | "toolingSha" + | "trustedToolingSha" + | "toolingTrackedStatus" + | "workflowRef" +>) { + if (targetHeadSha !== targetSha) { + throw new Error( + `release candidate target is ${targetSha}, but target worktree HEAD is ${targetHeadSha}`, + ); + } + if (targetTrackedStatus.trim()) { + throw new Error( + "release candidate validation requires a clean tracked target worktree at the frozen release SHA", + ); + } + if (toolingSha !== trustedToolingSha) { + throw new Error( + `release candidate tooling HEAD ${toolingSha} does not match trusted ${workflowRef} ${trustedToolingSha}`, + ); + } + if (toolingTrackedStatus.trim()) { + throw new Error( + "release candidate validation requires a clean tracked tooling checkout at the trusted workflow ref", + ); + } + return { status: "passed", targetSha, toolingSha, workflowRef }; +} + +/** + * Keeps release validation pre-publication: the final immutable tag is created + * only after this helper has recorded green evidence for the frozen SHA. + */ +export function assertPlannedReleaseTagIsAbsent( + tag: string, + checkRemoteTagExists: (tag: string) => boolean, +) { + if (checkRemoteTagExists(tag)) { + throw new Error( + `release candidate tag ${tag} already exists; validate a new patch instead of reusing a published tag`, + ); + } +} + +function remoteTagExists(tag: string, cwd: string) { + const result = spawnSync( + "git", + ["ls-remote", "--exit-code", "--tags", "origin", `refs/tags/${tag}`], + { + cwd, + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }, + ); + if (result.status === 0) { + return true; + } + if (result.status === 2) { + return false; + } + throw new Error( + `could not determine whether planned release tag ${tag} already exists: ${result.stderr.trim() || result.stdout.trim() || `git exited ${result.status ?? "without a status"}`}`, + ); +} + +function gitIsAncestor(ancestor: string, target: string) { + const result = spawnSync( + "git", + ["merge-base", "--is-ancestor", `${ancestor}^{commit}`, `${target}^{commit}`], + { + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }, + ); + if (result.status === 0) { + return true; + } + if (result.status === 1) { + return false; + } + throw new Error( + `could not validate changelog provenance ${ancestor}..${target}: ${ + result.stderr?.trim() || result.signal || result.status + }`, + ); +} + +export function validateTrustedToolingPin({ + toolingSha, + pinnedToolingSha, + latestTrustedToolingSha, + isAncestor = gitIsAncestor, +}: { + toolingSha: string; + pinnedToolingSha: string; + latestTrustedToolingSha: string; + isAncestor?: (ancestor: string, target: string) => boolean; +}) { + if (!/^[a-f0-9]{40}$/u.test(pinnedToolingSha)) { + throw new Error("release candidate trusted tooling pin is missing or invalid"); + } + if (toolingSha !== pinnedToolingSha) { + throw new Error( + `release candidate tooling HEAD ${toolingSha} does not match pinned tooling ${pinnedToolingSha}`, + ); + } + if ( + pinnedToolingSha !== latestTrustedToolingSha && + !isAncestor(pinnedToolingSha, latestTrustedToolingSha) + ) { + throw new Error( + `pinned release candidate tooling ${pinnedToolingSha} is not reachable from trusted workflow tip ${latestTrustedToolingSha}`, + ); + } + return pinnedToolingSha; +} + +export async function validateNpmPreflightRunSource( + { + repository, + runId, + workflowRun, + workflowRef, + isTrustedWorkflowAncestor = gitIsAncestor, + }: { + repository: string; + runId: string; + workflowRun: Omit<RunInfo, "jobs" | "url">; + workflowRef: string; + isTrustedWorkflowAncestor?: (ancestor: string, target: string) => boolean; + }, + apiOptions: GithubApiOptions = {}, +) { + const [workflowPath, fullRef] = String(workflowRun.workflowPath).split("@", 2); + if ( + String(workflowRun.databaseId) !== runId || + !Number.isSafeInteger(workflowRun.runAttempt) || + workflowRun.runAttempt < 1 || + workflowRun.repository !== repository || + workflowRun.workflowName !== "OpenClaw NPM Release" || + workflowPath !== ".github/workflows/openclaw-npm-release.yml" || + workflowRun.event !== "workflow_dispatch" || + workflowRun.status !== "completed" || + workflowRun.conclusion !== "success" || + !/^[a-f0-9]{40}$/u.test(workflowRun.headSha) + ) { + throw new Error(`npm preflight run ${runId} has invalid workflow identity`); + } + const ref = workflowRun.headBranch ?? ""; + const protectedTag = workflowRef === "main" && ref.startsWith("release-publish/"); + const expectedFullRef = `refs/${protectedTag ? "tags" : "heads"}/${ref}`; + if ((!protectedTag && ref !== workflowRef) || (fullRef && fullRef !== expectedFullRef)) { + throw new Error(`npm preflight run ${runId} workflow ref mismatch`); + } + const trustedRef = `refs/remotes/origin/${workflowRef}`; + if (!isTrustedWorkflowAncestor(workflowRun.headSha, trustedRef)) { + throw new Error( + `npm preflight workflow SHA ${workflowRun.headSha} is not reachable from trusted ${workflowRef}`, + ); + } + if (protectedTag) { + const [tagRef, branches] = await Promise.all([ + githubApi(`repos/${repository}/git/ref/tags/${ref}`, apiOptions), + githubApi(`repos/${repository}/git/matching-refs/heads/${ref}`, apiOptions), + ]); + validateReleasePreflightTagIdentity({ + branches, + workflowRef: ref, + workflowFullRef: expectedFullRef, + workflowSha: workflowRun.headSha, + tagRef, + }); + } + return { status: "passed", headSha: workflowRun.headSha, workflowRef: ref }; +} + +function candidateContributionRecordPullRequests( + section: string, + label: string, + { requireExactProvenance = true }: { requireExactProvenance?: boolean } = {}, +) { + const recordStart = section.search(/\n### Complete contribution record\r?$/m); + if (recordStart < 0) { + throw new Error(`${label} is missing ### Complete contribution record`); + } + const record = section.slice(recordStart); + const rowNumbers = [...record.matchAll(/^- \*\*PR #(?<number>[0-9]+)\*\*/gmu)].map((match) => + Number(match.groups?.number), + ); + const rows = new Set(rowNumbers); + if (rows.size !== rowNumbers.length) { + const duplicate = rowNumbers.find((number, index) => rowNumbers.indexOf(number) !== index); + throw new Error(`${label} contains duplicate contribution record PR #${duplicate}`); + } + if (!requireExactProvenance) { + return rows; + } + const provenance = parseContributionRecordProvenance(record); + if (!provenance || !/^[0-9a-f]{40}$/u.test(provenance.target)) { + throw new Error(`${label} is missing exact complete contribution record provenance`); + } + return rows; +} + +export function candidateCumulativeShippedPullRequests(changelog: string, label: string) { + const pullRequests = new Set<number>(); + for (const section of extractChangelogReleaseSections(changelog)) { + if ( + section.version === "Unreleased" || + !section.source.includes("\n### Complete contribution record") + ) { + continue; + } + for (const number of candidateContributionRecordPullRequests( + section.source, + `${label} section ${section.version}`, + { requireExactProvenance: false }, + )) { + pullRequests.add(number); + } + } + return pullRequests; +} + +function requireString(value: unknown, label: string) { + if (typeof value !== "string") { + throw new TypeError(`${label} must be a string`); + } + return value; +} + +export function loadCandidateShippedBaseline(ref: string, rootDir = process.cwd()) { + const tagRef = `refs/tags/${ref}`; + gitRevParse(`${tagRef}^{commit}`, rootDir); + const changelog = loadChangelogCollection({ rootDir, ref: tagRef, recordsOnly: true }); + const version = requireString(releaseNotesVersionForTag(ref), "release notes version"); + const source = loadReleaseChangelog({ rootDir, ref: tagRef, version }); + candidateContributionRecordPullRequests( + source.record ?? source.section, + `shipped baseline ${ref}`, + ); + const pullRequests = candidateCumulativeShippedPullRequests(changelog, `shipped baseline ${ref}`); + return { ref, pullRequests }; +} + +export function validateCandidateReleaseNotes({ + changelog, + repository, + tag, + contributionRecordPath, +}: StringFields<"changelog" | "repository" | "tag"> & { contributionRecordPath?: string }) { + const rendered = renderGithubReleaseNotes({ + changelog, + version: releaseNotesVersionForTag(tag), + tag, + repository, + contributionRecordPath, + }); + return { + status: "passed", + mode: rendered.mode, + characters: rendered.size.characters, + bytes: rendered.size.bytes, + }; +} + +export function validateCandidateChangelogProvenance({ + changelog, + version, + tag, + targetSha, + isAncestor = gitIsAncestor, + loadShippedBaseline = loadCandidateShippedBaseline, +}: { + changelog: string; + version: string; + tag: string; + targetSha: string; + isAncestor?: (ancestor: string, target: string) => boolean; + loadShippedBaseline?: (ref: string) => { pullRequests: Set<number> }; +}) { + // Validate the same section the renderer publishes: alpha and correction + // tags may carry their own heading, and alpha tags may fall back to + // Unreleased. + let section: string | undefined; + let sectionVersion = version; + let usesAlphaUnreleasedFallback = false; + const dedicatedVersion = dedicatedSectionVersionForTag(tag); + if (typeof dedicatedVersion === "string" && dedicatedVersion !== version) { + try { + section = requireString( + extractChangelogSection(changelog, dedicatedVersion), + "changelog section", + ); + sectionVersion = dedicatedVersion; + } catch { + // No dedicated section; validate the base section. + } + } + if (section === undefined) { + try { + section = requireString(extractChangelogSection(changelog, version), "changelog section"); + } catch (error) { + if (!/-alpha\.[1-9][0-9]*$/u.test(tag)) { + throw error; + } + section = requireString( + releaseNotesSectionForTag(changelog, version, tag), + "release notes section", + ); + usesAlphaUnreleasedFallback = true; + } + } + if (section === undefined) { + throw new Error(`CHANGELOG.md ## ${sectionVersion} could not be resolved`); + } + const recordStart = section.search(/\n### Complete contribution record\r?$/m); + if (recordStart < 0) { + if (usesAlphaUnreleasedFallback) { + return { + status: "skipped", + reason: "alpha release uses the explicit Unreleased fallback", + shippedBaselines: [], + base: undefined, + target: undefined, + }; + } + throw new Error( + `CHANGELOG.md ## ${sectionVersion} is missing ### Complete contribution record`, + ); + } + const record = section.slice(recordStart); + const recordedPullRequests = candidateContributionRecordPullRequests( + section, + `CHANGELOG.md ## ${sectionVersion}`, + ); + const provenance = parseContributionRecordProvenance(record); + const base = provenance?.base; + const recordedTarget = provenance?.target; + if (!base || !recordedTarget || !/^[0-9a-f]{40}$/u.test(recordedTarget)) { + throw new Error( + `CHANGELOG.md ## ${sectionVersion} is missing exact complete contribution record provenance`, + ); + } + const shippedBaselines = parseShippedBaselineExclusions(record); + const sectionShippedBaselines = parseShippedBaselineExclusions(section); + if ( + formatShippedBaselineExclusions(sectionShippedBaselines) !== + formatShippedBaselineExclusions(shippedBaselines) + ) { + throw new Error( + "shipped baseline exclusions must appear inside the complete contribution record", + ); + } + if (!isAncestor(base, recordedTarget)) { + throw new Error( + `CHANGELOG.md contribution record base ${base} is not an ancestor of recorded target ${recordedTarget}`, + ); + } + // The record is generated before its own changelog/finalization commit. Require + // reachability so the tag can contain that bounded release-only follow-up. + if (!isAncestor(recordedTarget, targetSha)) { + throw new Error( + `CHANGELOG.md contribution record target ${recordedTarget} is not reachable from release tag ${targetSha}`, + ); + } + // The verifier persists associated and text-linked PR exclusions together. + // Revalidate that exact inventory here instead of rediscovering a narrower set from git text. + const excludedPullRequests = new Set<number>(); + for (const baseline of shippedBaselines) { + const loaded = loadShippedBaseline(baseline.ref); + if (!(loaded.pullRequests instanceof Set)) { + throw new Error(`shipped baseline ${baseline.ref} did not provide a PR inventory`); + } + const duplicateExclusions = baseline.pullRequests.filter((number) => + excludedPullRequests.has(number), + ); + if (duplicateExclusions.length > 0) { + throw new Error( + `release contribution record repeats shipped PR exclusions across baselines: ${duplicateExclusions.map((number) => `#${number}`).join(", ")}`, + ); + } + const absent = baseline.pullRequests.filter((number) => !loaded.pullRequests.has(number)); + if (absent.length > 0) { + throw new Error( + `release contribution record lists PRs absent from shipped baseline ${baseline.ref}: ${absent.map((number) => `#${number}`).join(", ")}`, + ); + } + const retained = [...recordedPullRequests].filter((number) => loaded.pullRequests.has(number)); + if (retained.length > 0) { + throw new Error( + `release contribution record still contains shipped PRs from ${baseline.ref}: ${retained.map((number) => `#${number}`).join(", ")}`, + ); + } + for (const number of baseline.pullRequests) { + excludedPullRequests.add(number); + } + } + return { + status: "passed", + base, + target: recordedTarget, + shippedBaselines, + reason: undefined, + }; +} + +async function runArtifacts(repo: string, runId: string) { + const data = await githubApi(`repos/${repo}/actions/runs/${runId}/artifacts?per_page=100`); + const artifacts = isRecord(data) && Array.isArray(data.artifacts) ? data.artifacts : []; + return artifacts.flatMap((artifact) => { + if (!isRecord(artifact) || typeof artifact.name !== "string") { + return []; + } + const workflowRun = isRecord(artifact.workflow_run) ? artifact.workflow_run : undefined; + return [ + { + digest: typeof artifact.digest === "string" ? artifact.digest : undefined, + expired: typeof artifact.expired === "boolean" ? artifact.expired : undefined, + id: typeof artifact.id === "number" ? artifact.id : undefined, + name: artifact.name, + workflowRunId: typeof workflowRun?.id === "number" ? workflowRun.id : undefined, + }, + ]; + }); +} + +/** + * Chooses the expected artifact name, allowing one same-prefix fallback per run. + */ +export function resolveArtifactName( + artifacts: Array<{ expired?: boolean; name: string }>, + preferredName: string, + prefix: string, +) { + const available = artifacts + .filter((artifact) => artifact.expired !== true) + .map((artifact) => artifact.name); + if (available.includes(preferredName)) { + return preferredName; + } + const candidates = available.filter((name) => name.startsWith(prefix)); + if (candidates.length === 1) { + console.warn(`artifact ${preferredName} not found; using ${candidates[0]} from the same run`); + return candidates[0]; + } + const candidateList = + available.length > 0 ? available.map((name) => `- ${name}`).join("\n") : "- <none>"; + throw new Error( + `artifact ${preferredName} not found in run. Expected ${preferredName} or exactly one ${prefix}* fallback.\nAvailable artifacts:\n${candidateList}`, + ); +} + +async function resolveRunArtifact( + repo: string, + runId: string, + preferredName: string, + prefix: string, +) { + const artifacts = await runArtifacts(repo, runId); + const name = resolveArtifactName(artifacts, preferredName, prefix); + const artifact = artifacts.find((candidate) => candidate.name === name); + if (!artifact) { + throw new Error(`resolved artifact ${name} disappeared from run ${runId}`); + } + return artifact; +} + +function runAndEcho(command: string, args: string[]) { + const result = spawnSync(command, args, { + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }); + if (result.stdout) { + process.stdout.write(result.stdout); + } + if (result.stderr) { + process.stderr.write(result.stderr); + } + if (result.status !== 0) { + throw new Error( + `${command} ${args.join(" ")} failed with ${result.status ?? result.signal}\n${ + result.stderr ?? "" + }`, + ); + } + return `${result.stdout ?? ""}${result.stderr ?? ""}`; +} + +function runLocalGeneratedCheckIfNeeded(options: ReturnType<typeof parseArgs>): LocalCheckResult { + if (options.skipLocalGeneratedCheck) { + return { status: "skipped", reason: "operator skipped --skip-local-generated-check" }; + } + run("pnpm", ["release:generated:check"]); + return { status: "passed", command: "pnpm release:generated:check" }; +} + +/** + * Extracts a GitHub Actions run id from gh workflow dispatch output. + */ +export function parseRunIdFromDispatchOutput(output: string) { + return output.match(/actions\/runs\/([0-9]+)/u)?.[1] ?? ""; +} + +export function requireRunIdFromDispatchOutput(output: string, workflowFile: string) { + const runId = parseRunIdFromDispatchOutput(output); + if (!runId) { + throw new Error( + `gh workflow run ${workflowFile} did not return an Actions run URL; refusing to guess from recent workflow_dispatch runs`, + ); + } + return runId; +} + +export function fullReleaseTrustedWorkflowFields({ + workflowRef, + workflowSha, + workflowSource, + publicationIntent, +}: { + workflowRef: string; + workflowSha: string; + workflowSource: string; + publicationIntent?: import("./full-release-publication-contract.mjs").PublicationIntent; +}) { + const workflow: unknown = parseYaml(workflowSource); + const env = isRecord(workflow) && isRecord(workflow.env) ? workflow.env : undefined; + const contract = formatJsonValue(env?.RELEASE_ISOLATION_TOOLING_CONTRACT ?? ""); + if (contract === "1") { + return {}; + } + if (contract !== "2") { + throw new Error( + "Full Release Validation does not declare a supported release tooling contract", + ); + } + const workflowDispatch = + isRecord(workflow) && isRecord(workflow.on) && isRecord(workflow.on.workflow_dispatch) + ? workflow.on.workflow_dispatch + : undefined; + const inputs = + workflowDispatch && isRecord(workflowDispatch.inputs) ? workflowDispatch.inputs : undefined; + if (!inputs || !Object.hasOwn(inputs, "trusted_workflow_json")) { + throw new Error(`Full Release Validation contract ${contract} requires trusted_workflow_json`); + } + if (!/^[a-f0-9]{40}$/u.test(workflowSha)) { + throw new Error("Full Release Validation trusted workflow SHA must be a full lowercase SHA"); + } + const identity = { ref: workflowRef, fullRef: `refs/heads/${workflowRef}`, sha: workflowSha }; + if (publicationSourceContract(workflowSource) === "1") { + if (!publicationIntent) { + throw new Error("Fresh FRV dispatch requires explicit source intent"); + } + return { trusted_workflow_json: publicationDispatchEnvelope(identity, publicationIntent) }; + } + return { trusted_workflow_json: JSON.stringify(identity) }; +} + +async function wait(ms: number) { + await new Promise<void>((resolve) => { + setTimeout(resolve, ms); + }); +} + +function dispatchWorkflow( + repo: string, + workflowFile: string, + workflowRef: string, + fields: Record<string, unknown>, +) { + const args = ["workflow", "run", workflowFile, "--repo", repo, "--ref", workflowRef]; + for (const [key, value] of Object.entries(fields)) { + args.push("-f", `${key}=${String(value)}`); + } + return requireRunIdFromDispatchOutput(runAndEcho("gh", args), workflowFile); +} + +async function runInfo(repo: string, runId: string) { + const [runData, jobsData] = await Promise.all([ + githubApi(`repos/${repo}/actions/runs/${runId}`), + githubApi(`repos/${repo}/actions/runs/${runId}/jobs?per_page=100`), + ]); + if (!isRecord(runData) || !isRecord(jobsData)) { + throw new Error(`run ${runId} returned invalid GitHub API data`); + } + const repository = isRecord(runData.repository) ? runData.repository : undefined; + const jobs = Array.isArray(jobsData.jobs) ? jobsData.jobs.filter(isRecord) : []; + return { + databaseId: typeof runData.id === "number" ? runData.id : undefined, + runAttempt: typeof runData.run_attempt === "number" ? runData.run_attempt : 0, + workflowName: typeof runData.name === "string" ? runData.name : undefined, + workflowPath: runData.path, + repository: repository?.full_name, + headBranch: typeof runData.head_branch === "string" ? runData.head_branch : undefined, + headSha: typeof runData.head_sha === "string" ? runData.head_sha : "", + event: runData.event, + status: typeof runData.status === "string" ? runData.status : undefined, + conclusion: typeof runData.conclusion === "string" ? runData.conclusion : undefined, + url: typeof runData.html_url === "string" ? runData.html_url : undefined, + jobs: jobs.map((job) => ({ + name: job.name, + status: job.status, + conclusion: job.conclusion, + url: job.html_url, + })), + }; +} +type RunInfo = Awaited<ReturnType<typeof runInfo>>; + +async function pendingDeployments(repo: string, runId: string) { + try { + return await githubApi(`repos/${repo}/actions/runs/${runId}/pending_deployments`); + } catch { + return []; + } +} + +function summarizePendingDeployments(repo: string, runId: string, deployments: unknown) { + if (!Array.isArray(deployments) || deployments.length === 0) { + return ""; + } + return deployments + .map((deploymentValue) => { + const deployment = isRecord(deploymentValue) ? deploymentValue : {}; + const environment = isRecord(deployment.environment) ? deployment.environment : {}; + return [ + `- pending approval: env=${formatJsonValue(environment.name ?? "<unknown>")} canApprove=${formatJsonValue(deployment.current_user_can_approve ?? "<unknown>")}`, + ` approve: gh api -X POST repos/${repo}/actions/runs/${runId}/pending_deployments -F 'environment_ids[]=${formatJsonValue(environment.id ?? "<id>")}' -f state=approved -f comment='Approve release gate'`, + ].join("\n"); + }) + .join("\n"); +} + +function summarizeFailedRun(info: RunInfo) { + const failedJobs = (info.jobs ?? []).filter( + (job) => job.conclusion && job.conclusion !== "success" && job.conclusion !== "skipped", + ); + return [ + `${info.workflowName} ${info.databaseId} ended ${info.status}/${info.conclusion}: ${info.url}`, + ...failedJobs.map( + (job) => + `- ${formatJsonValue(job.name)}: ${formatJsonValue(job.conclusion)} ${formatJsonValue(job.url ?? "")}`, + ), + ].join("\n"); +} + +async function waitForSuccessfulRun( + repo: string, + runId: string, + expected: { + allowShaPinnedWorkflowRef?: boolean; + workflowName: string; + workflowRef: string; + validateSource?: (info: RunInfo) => ReturnType<typeof validateNpmPreflightRunSource>; + }, +) { + let lastState = ""; + for (;;) { + const info = await runInfo(repo, runId); + const state = `${info.status}:${info.conclusion ?? ""}`; + if (state !== lastState) { + console.log( + `${info.workflowName} ${runId}: ${info.status}${info.conclusion ? `/${info.conclusion}` : ""} ${info.url}`, + ); + const pending = summarizePendingDeployments( + repo, + runId, + await pendingDeployments(repo, runId), + ); + if (pending) { + console.log(pending); + } + lastState = state; + } + if (info.status === "completed") { + if (info.conclusion !== "success") { + throw new Error(summarizeFailedRun(info)); + } + if (info.workflowName !== expected.workflowName) { + throw new Error( + `run ${runId} workflow mismatch: expected ${expected.workflowName}, got ${info.workflowName}`, + ); + } + if (expected.validateSource) { + return { run: info, source: await expected.validateSource(info) }; + } + const acceptsPinnedWorkflow = + expected.allowShaPinnedWorkflowRef && isShaPinnedReleaseValidationBranch(info.headBranch); + if (info.headBranch !== expected.workflowRef && !acceptsPinnedWorkflow) { + throw new Error( + `run ${runId} branch mismatch: expected ${expected.workflowRef}, got ${info.headBranch}`, + ); + } + return { run: info, source: undefined }; + } + await wait(30_000); + } +} + +function downloadArtifact(repo: string, runId: string, name: string, dir: string) { + rmSync(dir, { force: true, recursive: true }); + mkdirSync(dir, { recursive: true }); + run("gh", ["run", "download", runId, "--repo", repo, "--name", name, "--dir", dir]); +} + +async function downloadResolvedArtifact( + repo: string, + runId: string, + preferredName: string, + prefix: string, + dir: string, +) { + const artifact = await resolveRunArtifact(repo, runId, preferredName, prefix); + downloadArtifact(repo, runId, artifact.name, dir); + return artifact; +} + +function sha256(path: string) { + return run("shasum", ["-a", "256", path], { capture: true }).trim().split(/\s+/u)[0] ?? ""; +} + +function pluginPlanArgs(options: ReturnType<typeof parseArgs>) { + const args = ["--selection-mode", options.pluginPublishScope]; + if (options.pluginPublishScope === "selected") { + args.push("--plugins", options.plugins); + } + return args; +} + +function collectPluginPlan(script: string, options: ReturnType<typeof parseArgs>): unknown { + const plan: unknown = JSON.parse( + run("node", ["--import", "tsx", join(TOOLING_ROOT, script), ...pluginPlanArgs(options)], { + capture: true, + }), + ); + console.log(formatPluginPlanSummary(script, plan).join("\n")); + return plan; +} + +async function collectPluginPlanWithRetry(script: string, options: ReturnType<typeof parseArgs>) { + let lastError: unknown; + for (let attempt = 1; attempt <= 3; attempt += 1) { + try { + return collectPluginPlan(script, options); + } catch (error) { + lastError = error; + if (attempt === 3) { + break; + } + console.warn( + `${script} failed on attempt ${attempt}; retrying: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + await wait(5_000 * attempt); + } + } + throw lastError; +} + +function formatPluginPlanSummary(label: string, plan: unknown): string[] { + const count = isRecord(plan) && Array.isArray(plan.all) ? plan.all.length : 0; + const warnings = isRecord(plan) && Array.isArray(plan.warnings) ? plan.warnings : []; + return [ + `- ${label}: ${count} packages`, + ...warnings + .filter((warning): warning is string => typeof warning === "string") + .map((warning) => `- Warning: ${warning}`), + ]; +} + +function shellQuote(value: unknown) { + return `'${String(value).replace(/'/gu, "'\\''")}'`; +} + +function publicationSelectionForChecklist( + options: ReturnType<typeof parseArgs>, +): PublicationSelection { + const selection = { + route: + options.npmDistTag === "extended-stable" + ? "extended-stable" + : options.tag.includes("-alpha.") + ? "alpha" + : options.publicationRoute, + npmDistTag: options.npmDistTag, + publishOpenclawNpm: true, + pluginPublishScope: options.pluginPublishScope, + plugins: parsePluginReleaseSelection(options.plugins), + ...(options.windowsNodeTag + ? { + windowsNodeTag: options.windowsNodeTag, + windowsNodeInstallerDigests: JSON.parse(options.windowsNodeInstallerDigests), + } + : {}), + }; + return normalizePublicationIntent("publish", JSON.stringify(selection)).publicationSelection!; +} + +/** + * Builds the final release publish workflow command once validation evidence is ready. + */ +export function buildPublishCommand( + options: ReturnType<typeof parseArgs> & { + fullReleaseRunAttempt?: number; + npmTelegramRunId?: string; + }, + npmPreflightSource?: Awaited<ReturnType<typeof validateNpmPreflightRunSource>>, + mode: "publish" | "prepare" = "publish", +) { + const workflowRef = + options.publishWorkflowRef || npmPreflightSource?.workflowRef || options.workflowRef; + const publishRefPattern = options.tag.includes("-alpha.") + ? TIDECLAW_ALPHA_WORKFLOW_REF_PATTERN + : /^release-publish\/[a-f0-9]{12}-[1-9][0-9]*$/u; + if (!publishRefPattern.test(workflowRef)) { + throw new Error( + options.tag.includes("-alpha.") + ? "alpha release publish requires a matching tideclaw/alpha/YYYY-MM-DD-HHMMZ workflow ref" + : "regular release publish requires protected tooling; supply --publish-workflow-ref release-publish/<sha12>-<epoch> after creating and pushing the tag at the trusted tooling SHA", + ); + } + const fields: Array<[string, string | number | undefined]> = [ + ["tag", options.tag], + ["preflight_run_id", options.npmPreflightRunId], + ["plugin_sdk_api_acknowledgement", options.pluginSdkApiAcknowledgement], + ["full_release_validation_run_id", options.fullReleaseRunId], + ["full_release_validation_run_attempt", options.fullReleaseRunAttempt], + ["npm_dist_tag", options.npmDistTag], + ["plugin_publish_scope", options.pluginPublishScope], + ["publish_openclaw_npm", "true"], + ["release_profile", "from-validation"], + ["wait_for_clawhub", "false"], + ]; + if (options.npmTelegramRunId) { + fields.push(["npm_telegram_run_id", options.npmTelegramRunId]); + } + if (options.windowsNodeTag) { + fields.push(["windows_node_tag", options.windowsNodeTag]); + } + if (options.windowsNodeInstallerDigests) { + fields.push(["windows_node_installer_digests", options.windowsNodeInstallerDigests]); + } + if (options.plugins.trim()) { + fields.push(["plugins", options.plugins]); + } + if ( + mode === "prepare" && + (!/^release-publish\/[a-f0-9]{12}-[1-9][0-9]*$/u.test(workflowRef) || + options.pluginPublishScope !== "all-publishable" || + options.tag.includes("-alpha.") || + options.npmDistTag === "extended-stable") + ) { + throw new Error( + "Prepared publication requires a protected tooling tag and the complete regular-release plugin roster.", + ); + } + if (mode === "prepare") { + const version = parseReleaseVersion(options.tag.slice(1)); + if (!version || !["beta", "stable"].includes(classifyReleaseTrain(version))) { + throw new Error("Prepared publication requires a regular beta or stable release train."); + } + } + return [ + "gh", + "workflow", + "run", + mode === "prepare" ? "openclaw-release-prepare.yml" : "openclaw-release-publish.yml", + "--repo", + options.repo, + "--ref", + workflowRef, + ...(mode === "prepare" + ? [ + "-f", + `publish_inputs=${JSON.stringify(Object.fromEntries(fields.filter(([, value]) => value !== undefined).map(([key, value]) => [key, String(value)])))}`, + ] + : fields.flatMap(([key, value]) => ["-f", `${key}=${String(value)}`])), + ] + .map(shellQuote) + .join(" "); +} + +export function validatePreflightManifest(manifest: JsonRecord, params: JsonRecord) { + if (manifest.releaseTag !== params.tag) { + throw new Error( + `npm preflight tag mismatch: expected ${formatJsonValue(params.tag)}, got ${formatJsonValue(manifest.releaseTag)}`, + ); + } + if (manifest.releaseSha !== params.targetSha) { + throw new Error( + `npm preflight SHA mismatch: expected ${formatJsonValue(params.targetSha)}, got ${formatJsonValue(manifest.releaseSha)}`, + ); + } + validateNpmPreflightDistTag({ manifest, npmDistTag: params.npmDistTag }); + if (!manifest.tarballName || !manifest.tarballSha256) { + throw new Error("npm preflight manifest missing tarball metadata"); + } + const corePackageTarballs = preflightCorePackageTarballs(manifest); + const dependencyTarballs = preflightDependencyTarballs(manifest); + for (const dependency of [...corePackageTarballs, ...dependencyTarballs]) { + if ( + !dependency?.packageName || + !dependency.packageVersion || + !dependency.tarballName || + !dependency.tarballSha256 || + dependency.tarballName !== basename(dependency.tarballName) + ) { + throw new Error("npm preflight manifest contains invalid dependency tarball metadata"); + } + } + const corePackageDescriptors = new Set(corePackageTarballs.map(preflightTarballDescriptorKey)); + for (const dependency of dependencyTarballs) { + if (!corePackageDescriptors.has(preflightTarballDescriptorKey(dependency))) { + throw new Error( + `npm preflight dependency tarball metadata does not match the core package manifest: ${dependency.packageName}`, + ); + } + } +} + +function preflightTarballDescriptorKey(tarball: TarballDescriptor) { + return JSON.stringify([ + tarball.packageName, + tarball.packageVersion, + tarball.tarballName, + tarball.tarballSha256, + ]); +} + +function isTarballDescriptor(value: unknown): value is TarballDescriptor { + return ( + isRecord(value) && + typeof value.packageName === "string" && + typeof value.packageVersion === "string" && + typeof value.tarballName === "string" && + typeof value.tarballSha256 === "string" + ); +} + +function preflightTarballs( + manifest: PreflightTarballs, + preferred: "corePackageTarballs" | "dependencyTarballs", + fallback: "corePackageTarballs" | "dependencyTarballs", +) { + const tarballs = Object.hasOwn(manifest, preferred) ? manifest[preferred] : manifest[fallback]; + if (!Array.isArray(tarballs) || !tarballs.every(isTarballDescriptor)) { + throw new Error("npm preflight manifest missing dependency tarball metadata"); + } + return tarballs; +} + +export function preflightCorePackageTarballs(manifest: PreflightTarballs) { + return preflightTarballs(manifest, "corePackageTarballs", "dependencyTarballs"); +} + +export function preflightDependencyTarballs(manifest: PreflightTarballs) { + return preflightTarballs(manifest, "dependencyTarballs", "corePackageTarballs"); +} + +export function validateFullManifest(manifest: JsonRecord, params: JsonRecord) { + if (manifest.workflowName !== "Full Release Validation") { + throw new Error(`full validation workflow mismatch: ${formatJsonValue(manifest.workflowName)}`); + } + if (manifest.targetSha !== params.targetSha) { + throw new Error( + `full validation SHA mismatch: expected ${formatJsonValue(params.targetSha)}, got ${formatJsonValue(manifest.targetSha)}`, + ); + } + if (manifest.releaseProfile !== params.releaseProfile) { + throw new Error( + `full validation profile mismatch: expected ${formatJsonValue(params.releaseProfile)}, got ${formatJsonValue(manifest.releaseProfile)}`, + ); + } + if (manifest.rerunGroup !== "all") { + throw new Error( + `full validation must use rerun_group=all, got ${formatJsonValue(manifest.rerunGroup)}`, + ); + } + if ( + (params.releaseProfile === "stable" || params.releaseProfile === "full") && + manifest.runReleaseSoak !== "true" + ) { + throw new Error( + `full validation must record runReleaseSoak=true for ${formatJsonValue(params.releaseProfile)} release candidates`, + ); + } + const controls = isRecord(manifest.controls) ? manifest.controls : undefined; + if (params.releaseProfile !== "beta" && controls?.performanceBlocking !== true) { + throw new Error("full validation manifest must record blocking product performance evidence"); + } +} + +export function candidateParallelsArgs( + tarballPath: string, + dependencyTarballPaths: string[] = [], + toolingRoot = TOOLING_ROOT, + registryPackageTarballPaths: string[] = [], + macosSnapshotHint = "", +) { + return [ + "exec", + "tsx", + join(toolingRoot, "scripts/e2e/parallels/npm-update-smoke.ts"), + "--target-tarball", + tarballPath, + ...dependencyTarballPaths.flatMap((dependency) => ["--dependency-tarball", dependency]), + ...registryPackageTarballPaths.flatMap((registryPackage) => [ + "--registry-package-tarball", + registryPackage, + ]), + ...(macosSnapshotHint ? ["--macos-snapshot-hint", macosSnapshotHint] : []), + "--json", + ]; +} + +export function candidateParallelsShellCommand( + tarballPath: string, + timeoutBin: string, + dependencyTarballPaths: string[] = [], + registryPackageTarballPaths: string[] = [], + macosSnapshotHint = "", +) { + // Login shells can replace the candidate's supported Node with ambient host Node. + // Keep the invoking Node first so pnpm and npm use the validated runtime. + const nodeBinDir = dirname(process.execPath); + return [ + 'set -a; source "$HOME/.profile" >/dev/null 2>&1 || true; set +a;', + `export PATH=${shellQuote(nodeBinDir)}:"$PATH";`, + "exec", + shellQuote(timeoutBin), + "--foreground", + "150m", + "pnpm", + ...candidateParallelsArgs( + tarballPath, + dependencyTarballPaths, + TOOLING_ROOT, + registryPackageTarballPaths, + macosSnapshotHint, + ).map(shellQuote), + ].join(" "); +} + +async function runParallelsIfNeeded( + options: ReturnType<typeof parseArgs>, + tarballPath: string, + dependencyTarballPaths: string[], + registryPackageTarballPaths: string[], +): Promise<LocalCheckResult> { + if (options.skipParallels) { + return { status: "skipped", reason: options.parallelsSkipReason }; + } + const timeoutBin = run("bash", ["-lc", "command -v gtimeout || command -v timeout"], { + capture: true, + }).trim(); + const command = candidateParallelsShellCommand( + tarballPath, + timeoutBin, + dependencyTarballPaths, + registryPackageTarballPaths, + process.env.OPENCLAW_PARALLELS_MACOS_SNAPSHOT_HINT?.trim() ?? "", + ); + run("bash", ["-lc", command], { + env: { + OPENCLAW_PARALLELS_ARTIFACT_ROOT: join(process.cwd(), ".artifacts", "parallels"), + }, + }); + return { + status: "passed", + command, + }; +} + +export function buildTelegramArtifactInputs({ + artifact, + manifest, + runAttempt, + runId, + sourceSha, +}: TelegramArtifactParams) { + const artifactDigest = artifact.digest?.match(/^sha256:([0-9a-f]{64})$/u)?.[1]; + if ( + typeof artifact.id !== "number" || + !Number.isInteger(artifact.id) || + artifact.id < 1 || + !artifactDigest + ) { + throw new Error(`npm preflight artifact ${artifact.name} is missing immutable identity`); + } + if (!/^[1-9][0-9]*$/u.test(runId) || String(artifact.workflowRunId) !== runId) { + throw new Error( + `npm preflight artifact ${artifact.name} belongs to run ${artifact.workflowRunId}, not ${runId}`, + ); + } + if (!Number.isInteger(runAttempt) || runAttempt < 1) { + throw new Error(`npm preflight run ${runId} has invalid attempt`); + } + if ( + !artifact.name || + typeof manifest.tarballName !== "string" || + typeof manifest.tarballSha256 !== "string" || + typeof manifest.packageVersion !== "string" + ) { + throw new Error("npm preflight artifact manifest is incomplete"); + } + return { + package_artifact_digest: artifactDigest, + package_artifact_id: artifact.id, + package_artifact_name: artifact.name, + package_artifact_run_attempt: runAttempt, + package_artifact_run_id: runId, + package_file_name: manifest.tarballName, + package_sha256: manifest.tarballSha256, + package_source_sha: sourceSha, + package_version: manifest.packageVersion, + }; +} + +async function runTelegramIfNeeded( + options: ReturnType<typeof parseArgs>, + artifact: TelegramArtifact, + manifest: JsonRecord, + runAttempt: number, + sourceSha: string, + coveragePolicy: string | undefined, +): Promise<TelegramResult> { + if (options.skipTelegram) { + return { status: "skipped" }; + } + // Only the admitted evidence policy defers this wait; legacy beta evidence + // still requires the separate Telegram qualification. + if (coveragePolicy === "npm-beta-v1") { + return { status: "deferred-postpublish" }; + } + const workflowFile = "npm-telegram-beta-e2e.yml"; + const artifactInputs = buildTelegramArtifactInputs({ + artifact, + manifest, + runAttempt, + runId: String(artifact.workflowRunId), + sourceSha, + }); + const runId = dispatchWorkflow(options.repo, workflowFile, options.workflowRef, { + package_spec: `openclaw@${options.tag.replace(/^v/u, "")}`, + package_label: options.tag, + ...artifactInputs, + harness_ref: options.workflowRef, + provider_mode: options.telegramProviderMode, + }); + const { run: runLocal } = await waitForSuccessfulRun(options.repo, runId, { + workflowName: "NPM Telegram Beta E2E", + workflowRef: options.workflowRef, + }); + return { + status: "passed", + runId, + url: runLocal.url, + artifactName: artifact.name, + providerMode: options.telegramProviderMode, + }; +} + +function checkCandidateAndroidVersion(targetSha: string, tag: string) { + const release = parseReleaseVersion(tag.replace(/^v/u, "")); + if (release?.channel !== "stable") { + return undefined; + } + const manifest: unknown = JSON.parse( + run("git", ["show", `${targetSha}:apps/android/version.json`], { capture: true }), + ); + const androidVersion = requireString( + isRecord(manifest) ? manifest.version : undefined, + "Android version", + ); + const targetVersion = release.baseVersion; + const matches = androidVersion === targetVersion; + return { + status: matches ? "passed" : "warning", + androidVersion, + targetVersion, + message: matches + ? `PASS: Android version ${androidVersion} matches release train ${targetVersion}.` + : `WARNING: Android version ${androidVersion} does not match release train ${targetVersion}; run node --import tsx scripts/mobile-release-version.ts --prepare --version ${targetVersion} --write before tagging, or accept that Android will not ship for this release.`, + }; +} + +async function main() { + const options = parseArgs(process.argv.slice(2)); + const targetRoot = gitTopLevel(process.cwd()); + const toolingRoot = gitTopLevel(TOOLING_ROOT); + if (targetRoot === toolingRoot) { + runFromTrustedTooling(process.argv.slice(2), { + targetRoot, + workflowRef: options.workflowRef, + }); + return; + } + options.outputDir ||= join(".artifacts", "release-candidate", options.tag); + const targetSha = gitRevParse(options.targetSha || "HEAD", targetRoot); + assertPlannedReleaseTagIsAbsent(options.tag, (tag) => remoteTagExists(tag, targetRoot)); + const toolingSha = gitRevParse("HEAD", TOOLING_ROOT); + const latestTrustedToolingSha = fetchTrustedWorkflowSha(options.workflowRef, TOOLING_ROOT); + // The outer process pins a clean main commit before creating this tooling checkout. + // A newer main tip must not invalidate that immutable, still-trusted ancestor mid-run. + const trustedToolingSha = validateTrustedToolingPin({ + toolingSha, + pinnedToolingSha: process.env[TRUSTED_TOOLING_SHA_ENV] ?? latestTrustedToolingSha, + latestTrustedToolingSha, + }); + validateCandidateCheckout({ + targetSha, + targetHeadSha: gitRevParse("HEAD", targetRoot), + targetTrackedStatus: gitTrackedStatus(targetRoot), + toolingSha, + trustedToolingSha, + toolingTrackedStatus: gitTrackedStatus(TOOLING_ROOT), + workflowRef: options.workflowRef, + }); + // Publication may use repaired tooling while the prepared tarball retains its original producer. + const publishWorkflowIdentity = options.publishWorkflowRef + ? verifyReleaseToolingIdentity({ + repository: options.repo, + workflowRef: options.publishWorkflowRef, + workflowFullRef: `refs/tags/${options.publishWorkflowRef}`, + workflowSha: toolingSha, + }) + : undefined; + options.parallelsRegistryPackageArtifacts = options.parallelsRegistryPackageArtifactDirs.map( + (artifactDir) => + validateParallelsRegistryPackageArtifact(artifactDir, { + targetSha, + targetVersion: options.tag.replace(/^v/u, ""), + }), + ); + const registryPackageNames = new Set( + options.parallelsRegistryPackageArtifacts.map((artifact) => artifact.packageName), + ); + if (registryPackageNames.size !== options.parallelsRegistryPackageArtifacts.length) { + throw new Error("Parallels registry package artifacts must have unique package names"); + } + const statePath = join(options.outputDir, RELEASE_CANDIDATE_STATE_FILE); + const expectedState = buildReleaseCandidateState(options, { targetSha, toolingSha }); + let candidateState = reconcileReleaseCandidateState( + existsSync(statePath) ? readJson(statePath, "release candidate state") : undefined, + expectedState, + ); + options.fullReleaseRunId = candidateState.fullReleaseRunId; + options.npmPreflightRunId = candidateState.npmPreflightRunId; + if (!options.fullReleaseRunId && !options.skipDispatch) { + const workflowSource = readFileSync( + join(TOOLING_ROOT, ".github/workflows/full-release-validation.yml"), + "utf8", + ); + if ( + publicationSourceContract(workflowSource) !== "1" || + publicationAdmissionContract(workflowSource) !== "1" + ) { + throw new Error( + "Fresh checklist dispatch requires source and registry admission in frozen tooling; existing run recovery is unchanged.", + ); + } + const version = parseReleaseVersion(options.tag.replace(/^v/u, "")); + const train = version && classifyReleaseTrain(version); + if (train === "unsupported-extended-stable-correction") { + throw new Error( + `Extended-stable correction suffixes are invalid (${options.tag}); use a new monthly maintenance patch. See the monthly Gateway extended-stable procedure in docs/reference/RELEASING.md.`, + ); + } + if (options.npmDistTag === "extended-stable" || train === "extended-stable") { + throw new Error( + "Fresh extended-stable checklist launches are not supported. Use the monthly Gateway extended-stable procedure in docs/reference/RELEASING.md: Full Release Validation, then the separate plugin npm and core npm publication owners.", + ); + } + } + writeReleaseCandidateState(statePath, candidateState); + const androidVersionCheck = checkCandidateAndroidVersion(targetSha, options.tag); + if (androidVersionCheck) { + console.log(androidVersionCheck.message); + } + const releaseNotesVersion = releaseNotesVersionForTag(options.tag); + const releaseChangelog = loadReleaseNotesForTag({ + rootDir: targetRoot, + ref: targetSha, + tag: options.tag, + version: releaseNotesVersion, + }); + const releaseNotesCheck = validateCandidateReleaseNotes({ + changelog: releaseChangelog.section, + repository: options.repo, + tag: options.tag, + contributionRecordPath: releaseChangelog.recordPath ?? undefined, + }); + const releaseNotesProvenance = validateCandidateChangelogProvenance({ + changelog: releaseChangelog.record ?? releaseChangelog.section, + version: releaseNotesVersion, + tag: options.tag, + targetSha, + }); + const windowsNodeSourceRelease = options.windowsNodeTag + ? await validateWindowsSourceRelease(options.windowsNodeTag) + : undefined; + options.windowsNodeInstallerDigests = windowsNodeSourceRelease + ? JSON.stringify( + Object.fromEntries( + windowsNodeSourceRelease.assets.map((asset) => [asset.name, asset.digest]), + ), + ) + : ""; + const localGeneratedCheck = runLocalGeneratedCheckIfNeeded(options); + + if (!options.fullReleaseRunId && !options.skipDispatch) { + const workflowFile = "full-release-validation.yml"; + const targetContextRef = releaseBranchForTag(options.tag); + const trustedWorkflowFields = fullReleaseTrustedWorkflowFields({ + workflowRef: options.workflowRef, + workflowSha: toolingSha, + workflowSource: readFileSync( + join(TOOLING_ROOT, ".github", "workflows", workflowFile), + "utf8", + ), + publicationIntent: { + validationPurpose: "publish", + publicationSelection: publicationSelectionForChecklist(options), + }, + }); + options.fullReleaseRunId = dispatchWorkflow(options.repo, workflowFile, options.workflowRef, { + ref: targetSha, + ...(targetContextRef ? { target_context_ref: targetContextRef } : {}), + ...trustedWorkflowFields, + provider: options.provider, + mode: options.mode, + release_profile: options.releaseProfile, + run_release_soak: + options.releaseProfile === "stable" || options.releaseProfile === "full" ? "true" : "false", + rerun_group: "all", + }); + candidateState = updateReleaseCandidateState(statePath, candidateState, "dispatching", { + fullReleaseRunId: options.fullReleaseRunId, + }); + } + + // Full validation qualifies its package producer. Explicit separate run IDs + // remain the recovery contract for releases prepared by older tooling. + options.npmPreflightRunId ||= options.fullReleaseRunId; + candidateState = updateReleaseCandidateState(statePath, candidateState, "waiting", { + fullReleaseRunId: options.fullReleaseRunId, + npmPreflightRunId: options.npmPreflightRunId, + }); + + const { run: fullRun } = await waitForSuccessfulRun(options.repo, options.fullReleaseRunId, { + workflowName: "Full Release Validation", + workflowRef: options.workflowRef, + allowShaPinnedWorkflowRef: true, + }); + const npmUsesFullRun = options.npmPreflightRunId === options.fullReleaseRunId; + const { run: selectedNpmRun, source: npmPreflightSource } = npmUsesFullRun + ? { + run: fullRun, + source: { status: "passed", headSha: fullRun.headSha, workflowRef: options.workflowRef }, + } + : await waitForSuccessfulRun(options.repo, options.npmPreflightRunId, { + workflowName: "OpenClaw NPM Release", + workflowRef: options.workflowRef, + validateSource: (workflowRun) => + validateNpmPreflightRunSource({ + repository: options.repo, + runId: options.npmPreflightRunId, + workflowRun, + workflowRef: options.workflowRef, + }), + }); + + const npmDir = join(options.outputDir, "npm-preflight"); + const pluginSdkApiDir = join(options.outputDir, "plugin-sdk-api-evidence"); + const fullDir = join(options.outputDir, "full-release-validation"); + if (!Number.isInteger(fullRun.runAttempt) || fullRun.runAttempt < 1) { + throw new Error(`Full Release Validation run ${options.fullReleaseRunId} has invalid attempt.`); + } + const fullArtifactName = `full-release-validation-${options.fullReleaseRunId}-${fullRun.runAttempt}`; + downloadArtifact(options.repo, options.fullReleaseRunId, fullArtifactName, fullDir); + const fullManifest = readJson( + join(fullDir, "full-release-validation-manifest.json"), + "full validation manifest", + ); + run("git", ["fetch", "--no-tags", "origin", "+refs/heads/main:refs/remotes/origin/main"], { + capture: true, + }); + const fullValidationEvidence = await authenticateFullReleaseValidationEvidence({ + run: fullRun, + manifest: fullManifest, + expectedRepository: options.repo, + expectedRunId: options.fullReleaseRunId, + expectedRunAttempt: fullRun.runAttempt, + expectedTargetSha: targetSha, + expectedReleaseTag: options.tag, + expectedWorkflowBranch: options.workflowRef, + expectedPublicationSelection: () => publicationSelectionForChecklist(options), + isTrustedMainAncestor: (sha: string) => gitIsAncestor(sha, "refs/remotes/origin/main"), + manifestPath: join(fullDir, "full-release-validation-manifest.json"), + verifierSourceSha: toolingSha, + verifierSourceContent: readFileSync(join(TOOLING_ROOT, "scripts/release-ci-summary.mjs")), + }); + if (fullValidationEvidence.source === "direct" && fullRun.headSha !== targetSha) { + throw new Error(`run SHA mismatch: tag=${targetSha} full=${fullRun.headSha}`); + } + // Only exact historical producers retain local, non-authoritative planning. + // B recovery consumes its original hosted observations without another sweep. + const publicationAdmission = fullValidationEvidence.publicationAdmission; + const pluginNpmPlan = publicationAdmission + ? publicationAdmission.observations.plans.npm + : await collectPluginPlanWithRetry("scripts/plugin-npm-release-plan.ts", options); + const pluginClawHubPlan = publicationAdmission + ? publicationAdmission.observations.plans.clawhub + : await collectPluginPlanWithRetry("scripts/plugin-clawhub-release-plan.ts", options); + if (npmUsesFullRun) { + rmSync(npmDir, { recursive: true, force: true }); + } + const qualifiedPreflight = npmUsesFullRun + ? await downloadFullReleaseNpmPreflight({ + manifest: fullManifest, + repository: options.repo, + runId: options.fullReleaseRunId, + runAttempt: fullRun.runAttempt, + sourceSha: targetSha, + toolingSha: fullRun.headSha, + outputDir: npmDir, + token: run("gh", ["auth", "token"], { capture: true }).trim(), + }) + : undefined; + const npmProducerRunId = qualifiedPreflight?.producer.runId ?? options.npmPreflightRunId; + const npmRun = qualifiedPreflight + ? { + ...selectedNpmRun, + databaseId: Number(npmProducerRunId), + runAttempt: Number(qualifiedPreflight.producer.runAttempt), + workflowName: qualifiedPreflight.run.name, + workflowPath: qualifiedPreflight.run.path, + headBranch: qualifiedPreflight.run.head_branch, + headSha: qualifiedPreflight.run.head_sha, + url: qualifiedPreflight.run.html_url, + } + : selectedNpmRun; + const npmArtifact = qualifiedPreflight + ? { ...qualifiedPreflight.artifact, workflowRunId: Number(npmProducerRunId) } + : await downloadResolvedArtifact( + options.repo, + npmProducerRunId, + `openclaw-npm-preflight-${options.tag}`, + "openclaw-npm-preflight-", + npmDir, + ); + const npmArtifactName = npmArtifact.name; + if (!Number.isInteger(npmRun.runAttempt) || npmRun.runAttempt < 1) { + throw new Error(`OpenClaw npm preflight run ${npmProducerRunId} has invalid attempt.`); + } + downloadArtifact( + options.repo, + npmProducerRunId, + `plugin-sdk-api-release-diff-${npmProducerRunId}-${npmRun.runAttempt}`, + pluginSdkApiDir, + ); + const npmManifest = readJson(join(npmDir, "preflight-manifest.json"), "npm preflight manifest"); + const npmPreflightProducer = verifyNpmPreflightProducer({ + manifest: npmManifest, + repository: options.repo, + workflowFullRef: `refs/${npmRun.headBranch?.startsWith("release-publish/") ? "tags" : "heads"}/${npmRun.headBranch}`, + workflowSha: npmRun.headSha, + runId: npmProducerRunId, + runAttempt: npmRun.runAttempt, + workflowPath: String(npmRun.workflowPath).split("@", 1)[0], + fullReleaseManifest: fullManifest, + fullReleaseRunId: options.fullReleaseRunId, + fullReleaseRunAttempt: fullRun.runAttempt, + manifestSha256: sha256(join(npmDir, "preflight-manifest.json")), + }); + const immutablePluginSdkApiEvidence = readJson( + join(pluginSdkApiDir, "plugin-sdk-api-release-evidence.json"), + "immutable Plugin SDK API evidence", + ); + if (!isDeepStrictEqual(npmManifest.pluginSdkApi, immutablePluginSdkApiEvidence)) { + throw new Error( + "npm preflight manifest Plugin SDK API evidence does not match its immutable artifact", + ); + } + validatePreflightManifest(npmManifest, { + tag: options.tag, + targetSha, + npmDistTag: options.npmDistTag, + }); + const pluginSdkApiValidation = validatePluginSdkApiReleaseEvidence({ + acknowledgement: options.pluginSdkApiAcknowledgement, + evidence: npmManifest.pluginSdkApi, + expectedHeadSha: targetSha, + expectedWorkflowSha: npmRun.headSha, + npmDistTag: options.npmDistTag, + }); + validateFullManifest(fullManifest, { + targetSha, + releaseProfile: options.releaseProfile, + }); + const tarballPath = join( + npmDir, + requireString(npmManifest.tarballName, "npm preflight tarball name"), + ); + if (!existsSync(tarballPath)) { + throw new Error(`prepared tarball missing: ${tarballPath}`); + } + const actualTarballSha = sha256(tarballPath); + if (actualTarballSha !== npmManifest.tarballSha256) { + throw new Error( + `prepared tarball digest mismatch: expected ${formatJsonValue(npmManifest.tarballSha256)}, got ${actualTarballSha}`, + ); + } + const corePackageTarballPaths = new Map( + preflightCorePackageTarballs(npmManifest).map((dependency) => { + const dependencyPath = join(npmDir, dependency.tarballName); + if (!existsSync(dependencyPath)) { + throw new Error(`prepared dependency tarball missing: ${dependencyPath}`); + } + const actualDependencySha = sha256(dependencyPath); + if (actualDependencySha !== dependency.tarballSha256) { + throw new Error( + `prepared dependency tarball digest mismatch for ${dependency.packageName}: expected ${dependency.tarballSha256}, got ${actualDependencySha}`, + ); + } + return [preflightTarballDescriptorKey(dependency), dependencyPath]; + }), + ); + const dependencyTarballPaths = preflightDependencyTarballs(npmManifest).map((dependency) => { + const dependencyPath = corePackageTarballPaths.get(preflightTarballDescriptorKey(dependency)); + if (!dependencyPath) { + throw new Error( + `prepared dependency tarball is missing from the core package manifest: ${dependency.tarballName}`, + ); + } + return dependencyPath; + }); + + const revalidatedRegistryArtifacts = options.parallelsRegistryPackageArtifactDirs.map( + (artifactDir) => + validateParallelsRegistryPackageArtifact(artifactDir, { + targetSha, + targetVersion: options.tag.replace(/^v/u, ""), + }), + ); + if (!isDeepStrictEqual(revalidatedRegistryArtifacts, options.parallelsRegistryPackageArtifacts)) { + throw new Error("Parallels registry package artifacts changed during candidate validation"); + } + const parallels = await runParallelsIfNeeded( + options, + tarballPath, + dependencyTarballPaths, + revalidatedRegistryArtifacts.map((artifact) => artifact.tarballPath), + ); + const npmTelegram = await runTelegramIfNeeded( + options, + npmArtifact, + npmManifest, + npmRun.runAttempt, + targetSha, + fullValidationEvidence.coveragePolicy, + ); + const publicationOptions = { + ...options, + fullReleaseRunAttempt: fullRun.runAttempt, + npmTelegramRunId: npmTelegram.runId, + }; + const publishCommand = + options.publicationRoute === "normal" + ? buildPublishCommand(publicationOptions, npmPreflightSource) + : undefined; + const prepareCommand = + options.publicationRoute === "prepared" + ? buildPublishCommand(publicationOptions, npmPreflightSource, "prepare") + : undefined; + const evidence = { + version: 1, + tag: options.tag, + targetSha, + workflowRef: options.workflowRef, + publishWorkflowIdentity, + publicationRoute: options.publicationRoute, + sourceAdmission: fullManifest.sourceAdmission ?? null, + publicationAdmission, + npmDistTag: options.npmDistTag, + fullReleaseValidationRunId: options.fullReleaseRunId, + fullReleaseValidationRunAttempt: fullRun.runAttempt, + npmPreflightRunId: npmProducerRunId, + npmPreflightAuthorizationRunId: options.npmPreflightRunId, + windowsNodeTag: options.windowsNodeTag || undefined, + windowsNodeSourceRelease, + fullReleaseValidationUrl: fullRun.url, + fullReleaseValidationControls: fullManifest.controls, + npmPreflightUrl: npmRun.url, + npmPreflightSource, + pluginSdkApi: npmManifest.pluginSdkApi, + npmPreflightProducer, + pluginSdkApiValidation, + artifacts: { + npmPreflight: npmArtifactName, + fullReleaseValidation: fullArtifactName, + }, + releaseNotesCheck, + releaseNotesProvenance, + androidVersionCheck, + localGeneratedCheck, + tarball: { + name: basename(tarballPath), + sha256: actualTarballSha, + path: tarballPath, + }, + parallels, + parallelsRegistryPackageArtifacts: revalidatedRegistryArtifacts, + npmTelegram, + pluginNpmPlan, + pluginClawHubPlan, + publishCommand, + prepareCommand, + }; + mkdirSync(options.outputDir, { recursive: true }); + const evidencePath = join(options.outputDir, "release-candidate-evidence.json"); + const evidenceMarkdownPath = join(options.outputDir, "release-candidate-evidence.md"); + writeFileSync(evidencePath, `${JSON.stringify(evidence, null, 2)}\n`); + writeFileSync( + evidenceMarkdownPath, + [ + `# ${options.tag} release candidate evidence`, + "", + `- target SHA: ${targetSha}`, + ...(androidVersionCheck ? [`- **${androidVersionCheck.message}**`] : []), + `- full release validation: ${options.fullReleaseRunId} ${fullRun.url}`, + `- npm preflight: ${npmProducerRunId} ${npmRun.url}`, + ...(windowsNodeSourceRelease + ? [ + `- Windows Node source release: ${windowsNodeSourceRelease.tag} ${formatJsonValue(windowsNodeSourceRelease.url)}`, + ...windowsNodeSourceRelease.assets.map( + (asset) => `- Windows Node source asset: ${asset.name} ${asset.digest}`, + ), + ] + : []), + `- npm preflight artifact: ${npmArtifactName}`, + `- Plugin SDK API evidence: ${pluginSdkApiValidation.status}${ + pluginSdkApiValidation.digest ? ` (${pluginSdkApiValidation.digest})` : "" + }`, + `- full release artifact: ${fullArtifactName}`, + `- GitHub release notes: ${releaseNotesCheck.status} (${releaseNotesCheck.mode}, ${releaseNotesCheck.characters} characters, ${releaseNotesCheck.bytes} bytes)`, + releaseNotesProvenance.status === "passed" + ? `- changelog provenance: passed (${releaseNotesProvenance.base}..${releaseNotesProvenance.target})` + : `- changelog provenance: skipped (${releaseNotesProvenance.reason})`, + `- ${ + formatShippedBaselineExclusions(releaseNotesProvenance.shippedBaselines) || + "Shipped baseline exclusions: none" + }`, + `- local generated release checks: ${localGeneratedCheck.status}${ + localGeneratedCheck.reason ? ` (${localGeneratedCheck.reason})` : "" + }`, + `- tarball: ${basename(tarballPath)}`, + `- tarball sha256: ${actualTarballSha}`, + `- npm dist-tag: ${options.npmDistTag}`, + `- intended publication route: ${options.publicationRoute}`, + publicationAdmission + ? "- FRV registry observations are authenticated retained admission evidence, not publication authority. Pending owner actions and final publisher checks remain required." + : "- Historical FRV source admission and local registry plans are not registry eligibility or publication authority.", + ...formatPluginPlanSummary("plugin npm plan", pluginNpmPlan), + ...formatPluginPlanSummary("ClawHub plan", pluginClawHubPlan), + `- Parallels: ${parallels.status}${parallels.reason ? ` (${parallels.reason})` : ""}`, + `- NPM Telegram E2E: ${npmTelegram.status}${ + npmTelegram.runId ? ` ${npmTelegram.runId} ${npmTelegram.url}` : "" + }`, + "", + ...(prepareCommand + ? [ + "Prepare once for the release button (after creating the frozen release tag):", + "", + "```bash", + prepareCommand, + "```", + "", + "When preparation succeeds, its summary supplies the single input for OpenClaw Release Button.", + "", + ] + : []), + ...(publishCommand + ? ["Direct publication / recovery command:", "", "```bash", publishCommand, "```", ""] + : []), + ].join("\n"), + ); + updateReleaseCandidateState(statePath, candidateState, "completed"); + + console.log(`release candidate evidence: ${evidencePath}`); + console.log(`release candidate summary: ${evidenceMarkdownPath}`); + if (androidVersionCheck) { + console.log(androidVersionCheck.message); + } + if (prepareCommand) { + console.log("prepare once for the release button (after creating the frozen release tag):"); + console.log(prepareCommand); + } + if (publishCommand) { + console.log("direct publication / recovery command:"); + console.log(publishCommand); + } +} + +if (isDirectReleaseCandidateExecution(process.argv[1], fileURLToPath(import.meta.url))) { + await main().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); + }); +} diff --git a/scripts/release-changelog.mjs b/scripts/release-changelog.mjs new file mode 100644 index 0000000000000000000000000000000000000000..72bffa35ad56188e5a935d3d86213009cca36be8 --- /dev/null +++ b/scripts/release-changelog.mjs @@ -0,0 +1,77 @@ +#!/usr/bin/env node +import fs from "node:fs"; +import { parseArgs } from "node:util"; +import { + checkChangelogLayout, + loadChangelogCollection, + loadReleaseChangelog, + splitChangelog, + writeReleaseChangelog, +} from "./lib/release-changelog.mjs"; + +try { + const { positionals, values } = parseArgs({ + allowPositionals: true, + options: { + root: { type: "string" }, + ref: { type: "string" }, + version: { type: "string" }, + file: { type: "string" }, + record: { type: "boolean" }, + json: { type: "boolean" }, + "records-only": { type: "boolean" }, + check: { type: "boolean" }, + }, + }); + const [command] = positionals; + if (positionals.length !== 1) { + throw new Error("Expected one command: read, collection, write, split, check"); + } + const params = { + rootDir: values.root ?? process.cwd(), + ref: values.ref, + version: values.version, + }; + if (command === "read") { + const result = loadReleaseChangelog(params); + if (values.record && result.record === null) { + throw new Error(`No contribution record for ${result.version}`); + } + process.stdout.write( + values.json ? `${JSON.stringify(result)}\n` : values.record ? result.record : result.section, + ); + } else if (command === "collection") { + process.stdout.write( + loadChangelogCollection({ ...params, recordsOnly: values["records-only"] }), + ); + } else if (command === "check") { + console.log(JSON.stringify(checkChangelogLayout(params))); + } else if (command === "split" || command === "write") { + if (values.ref) { + throw new Error("Writes cannot target a Git ref"); + } + if (command === "split") { + console.log(JSON.stringify(splitChangelog({ rootDir: params.rootDir, check: values.check }))); + } else { + if (!values.file) { + throw new Error("write requires --file (use - for stdin)"); + } + const result = writeReleaseChangelog({ + ...params, + section: fs.readFileSync(values.file === "-" ? 0 : values.file, "utf8"), + }); + console.log( + JSON.stringify({ + version: result.version, + sourcePath: result.sourcePath, + recordPath: result.recordPath, + }), + ); + } + } else { + throw new Error("Expected command: read, collection, write, split, check"); + } +} catch (error) { + console.error(error.message); + process.exitCode = 1; +} diff --git a/scripts/release-ci-summary.mjs b/scripts/release-ci-summary.mjs new file mode 100644 index 0000000000000000000000000000000000000000..c4a9eb739ad24956bbdc617bfe30adf13116f1db --- /dev/null +++ b/scripts/release-ci-summary.mjs @@ -0,0 +1,3504 @@ +#!/usr/bin/env node +/** + * Release CI summary helper that prints parent and child workflow status for a + * full release run. + */ +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { closeSync, mkdtempSync, openSync, readFileSync, rmSync, statSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import process from "node:process"; +import { fileURLToPath } from "node:url"; +import { validateFullReleaseCandidateBinding } from "./full-release-candidate-contract.mjs"; +import { + publicationAdmissionContract, + publicationObservationJson, + publicationSourceContract, + publicationSourceJson, + publicationSourceReuseIdentity, + validatePublicationAdmissionBinding, + validatePublicationSourceBinding, +} from "./full-release-publication-contract.mjs"; +import { + classifyReleaseGhTransportError, + compareReleaseJobsByName, + composeReleaseChildAttemptEvidence, + formatReleaseStateOutcome, + isReleaseCheckJobAdvisory, + isReleaseGhArtifactMissingError, + isSplitChangelogEvidenceDelta, + classifyReleaseChangelogEvidenceComparison, + SPLIT_CHANGELOG_EVIDENCE_REUSE_POLICY, + MAX_RELEASE_ARTIFACT_BYTES, + normalizeReleaseCoveragePolicy, + normalizeReleaseTelegramWaiver, + releaseCompositeJobsSha256, + terminalPolicyPass, + validateReleaseChildDispatchBinding, + validateReleaseCoveragePolicyBinding, + validateReleaseExecutionPlanArtifact, + validateReleaseChildRunProvenance, + validateReleaseStateArtifact, + validateReleaseTelegramWaiverBinding, +} from "./full-release-validation-policy.mjs"; +import { inspectActionsArtifactZip } from "./lib/actions-artifact-archive.mjs"; +import { sortJsonValueKeys } from "./lib/canonical-json.mjs"; +import { + execGhRead, + execGhReadAsync, + plainGhAuthenticatedEnv, + resolvePlainGhBin, +} from "./lib/plain-gh.mjs"; +import { resolveReleaseContextIdentity } from "./lib/release-context.mjs"; + +const sortReleaseJsonValueKeys = /** @type {<T>(value: T) => T} */ (sortJsonValueKeys); // Validated release JSON preserves its structural type. +const DEFAULT_REPO = process.env.OPENCLAW_RELEASE_REPO || "openclaw/openclaw"; +const RELEASE_EVIDENCE_SCHEMA = "openclaw.release-validation-evidence/v3"; +const PHASED_RELEASE_EVIDENCE_SCHEMA = "openclaw.release-validation-evidence/v4"; +const SHA_PINNED_BRANCH_PATTERN = /^release-ci\/[a-f0-9]{12}-[1-9][0-9]*$/u; +const TRUSTED_RELEASE_PUBLISH_TAG_PATTERN = + /^refs\/tags\/release-publish\/([a-f0-9]{12})-[1-9][0-9]*$/u; +const RELEASE_EVIDENCE_SCRIPT = "scripts/release-ci-summary.mjs"; +const RELEASE_EVIDENCE_FILE = fileURLToPath(import.meta.url); +const RELEASE_EVIDENCE_REPO_ROOT = resolve(dirname(RELEASE_EVIDENCE_FILE), ".."); +const MANIFEST_ARTIFACT_ENTRY = "full-release-validation-manifest.json"; +const MAX_MANIFEST_ENTRY_LIST_BYTES = 8 * 1024; +const MAX_MANIFEST_ARTIFACT_ZIP_BYTES = MAX_RELEASE_ARTIFACT_BYTES + MAX_MANIFEST_ENTRY_LIST_BYTES; +// Release evidence lookups run during full release validation, so keep enough +// headroom for GitHub latency while preventing one stalled read from consuming +// the workflow budget. +const GH_COMMAND_TIMEOUT_MS = 60_000; +const ARTIFACT_DOWNLOAD_MIN_BYTES_PER_SECOND = 256 * 1024; +const ARTIFACT_DOWNLOAD_OVERHEAD_MS = 60_000; +const ARTIFACT_DOWNLOAD_MAX_TIMEOUT_MS = 30 * 60_000; +const ARTIFACT_DOWNLOAD_ATTEMPTS = 2; +const SUCCESSFUL_PARENT_JOB_CONCLUSIONS = new Set(["neutral", "skipped", "success"]); + +const LEGACY_CHILD_DISPATCHES = [ + { + manifestKey: "normalCi", + name: "CI", + parentJobName: "Run normal full CI", + suffix: "-ci", + trustedRef: "parent", + workflow: "ci.yml", + }, + { + manifestKey: "releaseChecks", + name: "OpenClaw Release Checks", + parentJobName: "Run release/live/Docker/QA validation", + suffix: "-release-checks", + trustedRef: "parent", + workflow: "openclaw-release-checks.yml", + }, + { + manifestKey: "pluginPrerelease", + name: "Plugin Prerelease", + parentJobName: "Run plugin prerelease validation", + suffix: "-plugin-prerelease", + trustedRef: "parent", + workflow: "plugin-prerelease.yml", + }, + { + manifestKey: "npmTelegram", + name: "NPM Telegram Beta E2E", + parentJobName: "Run package Telegram E2E", + suffix: "-npm-telegram", + trustedRef: "parent", + workflow: "npm-telegram-beta-e2e.yml", + }, + { + manifestKey: "productPerformance", + name: "OpenClaw Performance", + parentJobName: "Run product performance evidence", + suffix: "", + trustedRef: "parent", + workflow: "openclaw-performance.yml", + }, +]; + +const PHASED_CHILD_DISPATCHES = [ + LEGACY_CHILD_DISPATCHES.find((child) => child.manifestKey === "normalCi"), + { + manifestKey: "pluginPrereleaseIndependent", + name: "Plugin Prerelease", + parentJobName: "Run plugin prerelease independent validation", + suffix: "-plugin-prerelease-independent", + trustedRef: "parent", + workflow: "plugin-prerelease.yml", + }, + { + manifestKey: "pluginPrereleaseCandidate", + name: "Plugin Prerelease", + parentJobName: "Run plugin prerelease candidate validation", + suffix: "-plugin-prerelease-candidate", + trustedRef: "parent", + workflow: "plugin-prerelease.yml", + }, + { + manifestKey: "releaseChecksIndependent", + name: "OpenClaw Release Checks", + parentJobName: "Run release checks independent validation", + suffix: "-release-checks-independent", + trustedRef: "parent", + workflow: "openclaw-release-checks.yml", + }, + { + manifestKey: "releaseChecksCandidate", + name: "OpenClaw Release Checks", + parentJobName: "Run release checks candidate validation", + suffix: "-release-checks-candidate", + trustedRef: "parent", + workflow: "openclaw-release-checks.yml", + }, + LEGACY_CHILD_DISPATCHES.find((child) => child.manifestKey === "npmTelegram"), + LEGACY_CHILD_DISPATCHES.find((child) => child.manifestKey === "productPerformance"), +]; +// One phased child set plus current and reused parents. +const MAX_EXPECTED_RUN_ATTEMPTS = PHASED_CHILD_DISPATCHES.length + 2; +const MAX_EXPECTED_RUN_ATTEMPTS_JSON_BYTES = 4 * 1024; + +class ReleaseEvidenceRefreshRequiredError extends Error { + constructor(message) { + super(message); + this.name = "ReleaseEvidenceRefreshRequiredError"; + this.refreshable = true; + } +} + +const EXACT_TARGET_EVIDENCE_REUSE_POLICY = "exact-target-full-validation-v1"; +const CHANGELOG_ONLY_EVIDENCE_REUSE_POLICY = "changelog-only-release-v1"; +const EVIDENCE_REUSE_POLICIES = new Set([ + EXACT_TARGET_EVIDENCE_REUSE_POLICY, + CHANGELOG_ONLY_EVIDENCE_REUSE_POLICY, + SPLIT_CHANGELOG_EVIDENCE_REUSE_POLICY, +]); + +const RERUN_GROUP_CHILD_KEYS = new Map([ + ["all", ["normalCi", "releaseChecks", "pluginPrerelease", "productPerformance"]], + ["ci", ["normalCi"]], + ["plugin-prerelease", ["pluginPrerelease"]], + ["install-smoke", ["releaseChecks"]], + ["cross-os", ["releaseChecks"]], + ["live-e2e", ["releaseChecks"]], + ["package", ["releaseChecks"]], + ["qa-parity", ["releaseChecks"]], + ["qa-live", ["releaseChecks"]], + ["npm-telegram", ["npmTelegram"]], + ["performance", ["productPerformance"]], +]); + +const PHASED_RERUN_GROUP_CHILD_KEYS = new Map([ + [ + "all", + [ + "normalCi", + "pluginPrereleaseIndependent", + "pluginPrereleaseCandidate", + "releaseChecksIndependent", + "releaseChecksCandidate", + "productPerformance", + ], + ], + ["ci", ["normalCi"]], + ["plugin-prerelease", ["pluginPrereleaseIndependent", "pluginPrereleaseCandidate"]], + ["install-smoke", ["releaseChecksIndependent"]], + ["cross-os", ["releaseChecksCandidate"]], + ["live-e2e", ["releaseChecksIndependent", "releaseChecksCandidate"]], + ["package", ["releaseChecksCandidate"]], + ["qa-parity", ["releaseChecksIndependent"]], + ["qa-live", ["releaseChecksIndependent"]], + ["npm-telegram", ["npmTelegram"]], + ["performance", ["productPerformance"]], +]); + +const HISTORICAL_MANIFEST_RERUN_GROUP_CHILD_KEYS = new Map([ + ["release-checks", ["releaseChecks"]], + ["qa", ["releaseChecks"]], +]); + +export function runReleaseCiGh(args, params = {}) { + const execFileSyncImpl = params.execFileSyncImpl ?? execFileSync; + const timeoutMs = params.timeoutMs ?? GH_COMMAND_TIMEOUT_MS; + const stdio = params.stdio ?? ["ignore", "pipe", "pipe"]; + return execGhRead( + args, + { + encoding: "utf8", + killSignal: "SIGKILL", + maxBuffer: 64 * 1024 * 1024, + stdio, + timeout: timeoutMs, + }, + { execFileSyncImpl }, + ); +} + +function gh(args) { + return runReleaseCiGh(args); +} + +function ghAsync(args) { + return execGhReadAsync(args, { + killSignal: "SIGKILL", + maxBuffer: 64 * 1024 * 1024, + timeout: GH_COMMAND_TIMEOUT_MS, + }); +} + +function jsonGh(args) { + return JSON.parse(gh(args)); +} + +export function githubRestArgs(pathSuffix, repository = DEFAULT_REPO) { + return ["api", `repos/${repository}/${pathSuffix}`]; +} + +function githubRestJson(pathSuffix, repository = DEFAULT_REPO) { + return jsonGh(githubRestArgs(pathSuffix, repository)); +} + +async function githubRestJsonAsync(pathSuffix, repository = DEFAULT_REPO) { + return JSON.parse(await ghAsync(githubRestArgs(pathSuffix, repository))); +} + +export function artifactDownloadArgs(artifactId, repository = DEFAULT_REPO) { + return ["api", `repos/${repository}/actions/artifacts/${artifactId}/zip`]; +} + +export function artifactDownloadTimeoutMs(sizeInBytes) { + const size = Number(sizeInBytes); + if (!Number.isSafeInteger(size) || size < 1) { + throw new Error("artifact download size is invalid"); + } + return Math.min( + ARTIFACT_DOWNLOAD_MAX_TIMEOUT_MS, + Math.max( + GH_COMMAND_TIMEOUT_MS, + Math.ceil((size / ARTIFACT_DOWNLOAD_MIN_BYTES_PER_SECOND) * 1000) + + ARTIFACT_DOWNLOAD_OVERHEAD_MS, + ), + ); +} + +function downloadArtifactZip(artifactId, destination, sizeInBytes, repository = DEFAULT_REPO) { + const timeout = sizeInBytes ? artifactDownloadTimeoutMs(sizeInBytes) : GH_COMMAND_TIMEOUT_MS; + for (let attempt = 1; attempt <= ARTIFACT_DOWNLOAD_ATTEMPTS; attempt += 1) { + const output = openSync(destination, "w"); + try { + execFileSync(resolvePlainGhBin(), artifactDownloadArgs(artifactId, repository), { + env: plainGhAuthenticatedEnv(), + killSignal: "SIGKILL", + maxBuffer: 64 * 1024 * 1024, + stdio: ["ignore", output, "pipe"], + timeout, + }); + return; + } catch (error) { + if ( + attempt === ARTIFACT_DOWNLOAD_ATTEMPTS || + classifyReleaseGhTransportError(error) !== "transient" + ) { + throw error; + } + } finally { + closeSync(output); + } + } +} + +function tryDownloadExecutionPlan(runId, repository = DEFAULT_REPO) { + const artifactName = `full-release-execution-plan-${runId}`; + const downloadDir = mkdtempSync(join(tmpdir(), "openclaw-release-execution-plan-")); + try { + try { + runReleaseCiGh( + [ + "run", + "download", + String(runId), + "--repo", + repository, + "--name", + artifactName, + "--dir", + downloadDir, + ], + { stdio: ["ignore", "ignore", "pipe"] }, + ); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + if (isReleaseGhArtifactMissingError(error)) { + return undefined; + } + throw new Error(`release execution plan artifact read failed: ${message}`, { + cause: error, + }); + } + const path = join(downloadDir, "full-release-execution-plan.json"); + if (!statSync(path, { throwIfNoEntry: false })) { + throw new Error(`release execution plan artifact ${artifactName} omitted its manifest`); + } + if (statSync(path).size > MAX_RELEASE_ARTIFACT_BYTES) { + throw new Error(`release execution plan artifact ${artifactName} exceeds the size limit`); + } + return JSON.parse(readFileSync(path, "utf8")); + } finally { + rmSync(downloadDir, { force: true, recursive: true }); + } +} + +function readExecutionPlanEvidence(runId, repository) { + const name = `full-release-execution-plan-${runId}`; + const artifacts = []; + let total; + for (let page = 1; page <= 10; page += 1) { + const response = githubRestJson( + `actions/runs/${runId}/artifacts?per_page=100&page=${page}`, + repository, + ); + if ( + !Number.isSafeInteger(response?.total_count) || + response.total_count < 0 || + !Array.isArray(response.artifacts) || + response.artifacts.length > 100 || + (total !== undefined && total !== response.total_count) + ) { + throw new Error("publication execution plan artifact enumeration is invalid"); + } + total = response.total_count; + artifacts.push(...response.artifacts); + if (artifacts.length === total) { + break; + } + if (artifacts.length > total || response.artifacts.length < 100) { + throw new Error("publication execution plan artifact enumeration is incomplete"); + } + } + if (artifacts.length !== total || new Set(artifacts.map((entry) => entry.id)).size !== total) { + throw new Error("publication execution plan artifact enumeration is incomplete or duplicated"); + } + const matches = artifacts.filter((artifact) => artifact.name === name); + if (matches.length !== 1) { + throw new Error( + "publication original execution plan is missing or ambiguous; use a fresh parent", + ); + } + const listed = matches[0]; + const artifact = githubRestJson(`actions/artifacts/${listed.id}`, repository); + if ( + String(artifact.id) !== String(listed.id) || + artifact.name !== name || + artifact.digest !== listed.digest || + artifact.size_in_bytes !== listed.size_in_bytes || + artifact.expired !== false || + String(artifact.workflow_run?.id) !== String(runId) || + !/^sha256:[a-f0-9]{64}$/u.test(artifact.digest) || + !Number.isSafeInteger(artifact.size_in_bytes) || + artifact.size_in_bytes < 1 || + artifact.size_in_bytes > MAX_MANIFEST_ARTIFACT_ZIP_BYTES + ) { + throw new Error("publication execution plan artifact identity mismatch"); + } + const directory = mkdtempSync(join(tmpdir(), "openclaw-publication-plan-")); + try { + const path = join(directory, "plan.zip"); + downloadArtifactZip(String(artifact.id), path, artifact.size_in_bytes, repository); + const archive = readFileSync(path); + if ( + archive.length !== artifact.size_in_bytes || + `sha256:${createHash("sha256").update(archive).digest("hex")}` !== artifact.digest + ) { + throw new Error("publication execution plan artifact bytes differ"); + } + const entry = "full-release-execution-plan.json"; + const files = inspectActionsArtifactZip(archive, [entry], { + maxArchiveBytes: MAX_MANIFEST_ARTIFACT_ZIP_BYTES, + maxCompressedEntryBytes: MAX_MANIFEST_ARTIFACT_ZIP_BYTES, + maxEntryBytes: MAX_RELEASE_ARTIFACT_BYTES, + maxExpandedBytes: MAX_RELEASE_ARTIFACT_BYTES, + }); + return { + artifact, + plan: JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(files.get(entry))), + }; + } finally { + rmSync(directory, { force: true, recursive: true }); + } +} + +export function validatePublicationObservationArtifactIdentity(artifact, source, uploaded) { + if ( + source.repository !== DEFAULT_REPO || + String(artifact?.id) !== String(uploaded.id) || + artifact.digest !== uploaded.digest || + artifact.name !== + `full-release-publication-observations-${source.runId}-${source.runAttempt}` || + artifact.expired !== false || + String(artifact.workflow_run?.id) !== source.runId || + artifact.workflow_run?.head_sha !== source.workflow.sha || + artifact.workflow_run?.head_branch !== + source.workflow.ref.replace(/^refs\/(?:heads|tags)\//u, "") || + !Number.isSafeInteger(artifact.size_in_bytes) || + artifact.size_in_bytes < 1 || + artifact.size_in_bytes > MAX_MANIFEST_ARTIFACT_ZIP_BYTES + ) { + throw new Error("publication observation upload identity mismatch"); + } + return { + id: String(artifact.id), + name: artifact.name, + digest: artifact.digest, + sizeInBytes: artifact.size_in_bytes, + }; +} + +export async function restoreOriginalPublicationAdmission({ request, client }) { + const evidenceClient = client ?? createReleaseEvidenceClient(request.repository); + const original = await evidenceClient.getRunAttempt(request.runId, 1); + const branch = request.workflow.ref.replace(/^refs\/(?:heads|tags)\//u, ""); + if ( + request.repository !== "openclaw/openclaw" || + original.repository?.full_name !== request.repository || + original.head_repository?.full_name !== request.repository || + String(original.id) !== request.runId || + original.run_attempt !== 1 || + original.event !== "workflow_dispatch" || + original.path !== ".github/workflows/full-release-validation.yml" || + original.head_sha !== request.workflow.sha || + original.head_branch !== branch + ) { + throw new Error("publication original parent identity mismatch"); + } + const workflow = evidenceClient.getWorkflowSource(request.workflow.sha); + if ( + publicationAdmissionContract(workflow) !== "1" || + publicationSourceContract(workflow) !== "1" + ) { + throw new Error("publication original parent lacks the required source-qualified contract"); + } + const jobs = await evidenceClient.getRunAttemptJobs(request.runId, 1, { requireComplete: true }); + const resolutions = jobs.filter((job) => job.name === "Resolve target ref"); + if ( + resolutions.length !== 1 || + resolutions[0].run_attempt !== 1 || + resolutions[0].status !== "completed" || + resolutions[0].conclusion !== "success" || + resolutions[0].steps?.filter( + (step) => step.name === "Finalize publication admission" && step.conclusion === "success", + ).length !== 1 + ) { + throw new Error("publication original admission did not succeed; use a fresh parent"); + } + const sealers = jobs.filter((job) => job.name === "Seal release execution plan"); + const sealer = sealers[0]; + const seals = + sealer?.steps?.filter((step) => step.name === "Seal immutable release execution plan") ?? []; + const uploads = + sealer?.steps?.filter((step) => step.name === "Upload immutable release execution plan") ?? []; + const seal = seals[0]; + const upload = uploads[0]; + const sealStart = Date.parse(seal?.started_at); + const sealEnd = Date.parse(seal?.completed_at); + const uploadStart = Date.parse(upload?.started_at); + const uploadEnd = Date.parse(upload?.completed_at); + // A failed/cancelled sealer may have written its complete interruption + // checkpoint. The guarded successful upload, not overall job success, seals it. + if ( + sealers.length !== 1 || + sealer.run_attempt !== 1 || + sealer.status !== "completed" || + seals.length !== 1 || + seal.status !== "completed" || + !["success", "failure", "cancelled"].includes(seal.conclusion) || + uploads.length !== 1 || + upload.status !== "completed" || + upload.conclusion !== "success" || + !Number.isSafeInteger(seal.number) || + !Number.isSafeInteger(upload.number) || + upload.number <= seal.number || + ![sealStart, sealEnd, uploadStart, uploadEnd].every(Number.isFinite) || + sealStart > sealEnd || + sealEnd > uploadStart || + uploadStart > uploadEnd + ) { + throw new Error("publication original execution plan sealer/upload did not succeed"); + } + const retained = evidenceClient.loadExecutionPlanEvidence(request.runId); + const created = Date.parse(retained?.artifact.created_at); + if ( + !retained || + retained.artifact.workflow_run?.head_sha !== original.head_sha || + retained.artifact.workflow_run?.head_branch !== original.head_branch || + !Number.isFinite(created) || + created < uploadStart || + created > uploadEnd + ) { + throw new Error("publication original plan artifact producer mismatch"); + } + const plan = validateReleaseExecutionPlanArtifact(retained.plan, { + publicationAdmissionContract: "1", + sourceAdmissionContract: "1", + repository: request.repository, + parentRunId: request.runId, + sourceParentRunAttempt: 1, + targetSha: request.candidateSha, + targetContextRef: request.targetContextRef, + trustedWorkflowFullRef: request.tooling.ref, + trustedWorkflowSha: request.tooling.sha, + workflowRef: branch, + workflowSha: request.workflow.sha, + releaseProfile: request.coverage.release_profile, + rerunGroup: request.coverage.rerun_group, + }); + const source = validatePublicationSourceBinding(plan); + if ( + publicationSourceJson(source.coverage) !== publicationSourceJson(request.coverage) || + publicationSourceJson(source.publicationSelection) !== + publicationSourceJson(request.publicationSelection) || + source.validationPurpose !== request.validationPurpose + ) { + throw new Error("publication original admission operands differ from this attempt"); + } + const admission = validatePublicationAdmissionBinding(plan, { + publicationAdmissionContract: "1", + }); + return { source, admission, plan }; +} + +function rate() { + try { + return jsonGh(["api", "rate_limit"]).resources.core; + } catch { + return undefined; + } +} + +export function validateParentRunBinding(parentView, parentRest, expectedRunId) { + const boundWorkflowPath = String(parentRest.path ?? "").split("@", 1)[0]; + if ( + String(parentRest.id) !== String(expectedRunId) || + parentRest.event !== "workflow_dispatch" || + boundWorkflowPath !== ".github/workflows/full-release-validation.yml" || + Number(parentRest.run_attempt) !== Number(parentView.attempt) || + parentRest.head_branch !== parentView.headBranch || + parentRest.head_sha !== parentView.headSha + ) { + throw new Error(`full release parent run binding mismatch: ${expectedRunId}`); + } + return parentRest; +} + +function childDispatchesForPhaseVersion(childPhaseVersion) { + return childPhaseVersion === 3 ? PHASED_CHILD_DISPATCHES : LEGACY_CHILD_DISPATCHES; +} + +export function expectedChildDispatches( + parentRunId, + parentRunAttempt, + parentWorkflowRef, + childPhaseVersion = 2, +) { + if (!/^[1-9][0-9]*$/u.test(String(parentRunId))) { + throw new Error("parent run ID must be a positive decimal"); + } + if (!Number.isSafeInteger(parentRunAttempt) || parentRunAttempt < 1) { + throw new Error("parent run attempt must be a positive integer"); + } + if (typeof parentWorkflowRef !== "string" || parentWorkflowRef.length === 0) { + throw new Error("parent workflow ref is required"); + } + const dispatchPrefix = `full-release-validation-${parentRunId}-${parentRunAttempt}`; + return childDispatchesForPhaseVersion(childPhaseVersion).map((child) => + Object.assign({}, child, { + displayTitle: `${child.name} ${dispatchPrefix}${child.suffix}`, + headBranch: child.trustedRef === "main" ? "main" : parentWorkflowRef, + }), + ); +} + +export function requiredChildKeysForRerunGroup( + rerunGroup, + validationInputs = {}, + childPhaseVersion = 2, +) { + const childKeys = ( + childPhaseVersion === 3 ? PHASED_RERUN_GROUP_CHILD_KEYS : RERUN_GROUP_CHILD_KEYS + ).get(rerunGroup); + if (!childKeys) { + throw new Error(`release validation manifest rerun group is invalid: ${rerunGroup}`); + } + const selectedKeys = new Set(childKeys); + if ( + childPhaseVersion === 3 && + rerunGroup === "live-e2e" && + typeof validationInputs.liveSuiteFilter === "string" && + validationInputs.liveSuiteFilter.trim().length > 0 + ) { + selectedKeys.delete("releaseChecksCandidate"); + } + if ( + rerunGroup === "all" && + !validationInputs.telegramWaiver && + ((typeof validationInputs.npmTelegramPackageSpec === "string" && + validationInputs.npmTelegramPackageSpec.length > 0) || + (typeof validationInputs.releasePackageSpec === "string" && + validationInputs.releasePackageSpec.length > 0)) + ) { + selectedKeys.add("npmTelegram"); + } + return selectedKeys; +} + +function requiredChildKeysForManifest(manifest) { + if ( + [2, 3].includes(manifest.version) && + HISTORICAL_MANIFEST_RERUN_GROUP_CHILD_KEYS.has(manifest.rerunGroup) + ) { + return new Set(HISTORICAL_MANIFEST_RERUN_GROUP_CHILD_KEYS.get(manifest.rerunGroup)); + } + const selectedKeys = requiredChildKeysForRerunGroup( + manifest.rerunGroup, + manifest.validationInputs, + manifest.version === 4 ? 3 : 2, + ); + // validateParentManifest authenticates the explicit policy before selection; + // an older beta receipt without this marker still requires its full child set. + if (manifest.validationInputs?.coveragePolicy === "npm-beta-v1") { + selectedKeys.delete("productPerformance"); + selectedKeys.delete("npmTelegram"); + } + return selectedKeys; +} + +export function expectedSelectedChildDispatches( + parentRunId, + parentRunAttempt, + parentWorkflowRef, + selectedKeys, + childPhaseVersion = 2, +) { + return expectedChildDispatches( + parentRunId, + parentRunAttempt, + parentWorkflowRef, + childPhaseVersion, + ).filter((child) => selectedKeys.has(child.manifestKey)); +} + +export function selectExactChildRun(runs, expectedDisplayTitle, expectedHeadBranch) { + const matches = runs.filter( + (run) => + run.event === "workflow_dispatch" && + run.display_title === expectedDisplayTitle && + run.head_branch === expectedHeadBranch, + ); + if (matches.length > 1) { + throw new Error( + `multiple child runs have exact dispatch title and branch: ${expectedDisplayTitle} (${expectedHeadBranch})`, + ); + } + return matches[0]; +} + +export function selectExactChildRunFromPages(runPages, expectedDisplayTitle, expectedHeadBranch) { + let exactMatch; + for (const runs of runPages) { + const match = selectExactChildRun(runs, expectedDisplayTitle, expectedHeadBranch); + if (match) { + if (exactMatch) { + throw new Error( + `multiple child runs have exact dispatch title and branch: ${expectedDisplayTitle} (${expectedHeadBranch})`, + ); + } + exactMatch = match; + } + if (runs.length < 100) { + break; + } + } + return exactMatch; +} + +function findExactChildRun(child, repository = DEFAULT_REPO) { + const runPages = []; + for (let page = 1; page <= 10; page += 1) { + const query = new URLSearchParams({ + event: "workflow_dispatch", + branch: child.headBranch, + page: String(page), + per_page: "100", + }); + const runs = + githubRestJson(`actions/workflows/${child.workflow}/runs?${query.toString()}`, repository) + .workflow_runs ?? []; + runPages.push(runs); + if (runs.length < 100) { + break; + } + } + return selectExactChildRunFromPages(runPages, child.displayTitle, child.headBranch); +} + +async function findParentJobsAll(parentRunId, repository = DEFAULT_REPO) { + const jobs = []; + for (let page = 1; page <= 10; page += 1) { + const query = new URLSearchParams({ + filter: "all", + page: String(page), + per_page: "100", + }); + const pageJobs = + ( + await githubRestJsonAsync( + `actions/runs/${parentRunId}/jobs?${query.toString()}`, + repository, + ) + ).jobs ?? []; + jobs.push(...pageJobs); + if (pageJobs.length < 100) { + break; + } + } + return jobs; +} + +async function findRunAttemptJobsAll( + runId, + runAttempt, + repository = DEFAULT_REPO, + requireComplete = false, +) { + const jobs = []; + let total; + for (let page = 1; page <= 10; page += 1) { + const query = new URLSearchParams({ + page: String(page), + per_page: "100", + }); + const response = await githubRestJsonAsync( + `actions/runs/${runId}/attempts/${runAttempt}/jobs?${query.toString()}`, + repository, + ); + const pageJobs = response.jobs ?? []; + if ( + requireComplete && + (!Array.isArray(response.jobs) || + !Number.isSafeInteger(response.total_count) || + response.total_count < 0 || + (total !== undefined && total !== response.total_count)) + ) { + throw new Error("publication original jobs enumeration is invalid"); + } + total = response.total_count; + jobs.push(...pageJobs); + if (pageJobs.length < 100) { + break; + } + } + if ( + requireComplete && + (jobs.length !== total || new Set(jobs.map((job) => job.id)).size !== total) + ) { + throw new Error("publication original jobs enumeration is incomplete or duplicated"); + } + return jobs; +} + +function parentJobLogArgs(jobId, repository = DEFAULT_REPO, allowEscapeSequences = true) { + const args = ["api", `repos/${repository}/actions/jobs/${jobId}/logs`]; + if (allowEscapeSequences) { + args.push("--allow-escape-sequences"); + } + return args; +} + +function isUnknownAllowEscapeSequencesFlag(error) { + if (typeof error !== "object" || error === null || !("stderr" in error)) { + return false; + } + const stderr = error.stderr; + return ( + typeof stderr === "string" && + stderr.replace(/\r\n?/gu, "\n").split("\n").includes("unknown flag: --allow-escape-sequences") + ); +} + +async function parentJobLog(jobId, repository = DEFAULT_REPO) { + try { + return await ghAsync(parentJobLogArgs(jobId, repository)); + } catch (error) { + if (!isUnknownAllowEscapeSequencesFlag(error)) { + throw error; + } + return ghAsync(parentJobLogArgs(jobId, repository, false)); + } +} + +function normalizeOptionalRunId(value, label) { + if (value === "") { + return ""; + } + if (!/^[1-9][0-9]*$/u.test(String(value))) { + throw new Error(`${label} must be empty or a positive decimal run ID`); + } + return String(value); +} + +function normalizeRequiredRunId(value, label) { + const runId = normalizeOptionalRunId(value, label); + if (!runId) { + throw new Error(`${label} is required`); + } + return runId; +} + +function normalizeRepository(value) { + const repository = String(value ?? ""); + if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(repository)) { + throw new Error("repository must use the owner/name form"); + } + return repository; +} + +function normalizeWorkflowRef(value, label) { + const workflowRef = String(value ?? ""); + const hasForbiddenCharacter = Array.from(workflowRef).some((character) => { + const codePoint = character.codePointAt(0) ?? 0; + return ( + codePoint <= 0x1f || + codePoint === 0x7f || + character.trim() === "" || + "~^:?*[\\".includes(character) + ); + }); + if (workflowRef.length === 0 || workflowRef.length > 255 || hasForbiddenCharacter) { + throw new Error(`${label} is invalid`); + } + return workflowRef; +} + +function normalizeSha(value, label) { + const sha = String(value ?? ""); + if (!/^[a-f0-9]{40}$/u.test(sha)) { + throw new Error(`${label} is invalid`); + } + return sha; +} + +function normalizePositiveInteger(value, label) { + const number = Number(value); + if (!Number.isSafeInteger(number) || number < 1) { + throw new Error(`${label} must be a positive integer`); + } + return number; +} + +function normalizeJsonObject(value, label) { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error(`${label} is invalid`); + } + return value; +} + +function normalizeExpectedRunAttempts(value) { + if (value === undefined) { + return undefined; + } + const entries = Object.entries(normalizeJsonObject(value, "expected run attempts")); + if (entries.length === 0 || entries.length > MAX_EXPECTED_RUN_ATTEMPTS) { + throw new Error(`expected run attempts must contain 1-${MAX_EXPECTED_RUN_ATTEMPTS} run IDs`); + } + return new Map( + entries.map(([runId, runAttempt]) => { + if (typeof runAttempt !== "number") { + throw new Error(`expected run ${runId} attempt must be a positive integer`); + } + return [ + normalizeRequiredRunId(runId, "expected run ID"), + normalizePositiveInteger(runAttempt, `expected run ${runId} attempt`), + ]; + }), + ); +} + +function consumeExpectedRunAttempt(expectedRunAttempts, runId, runAttempt, label) { + if (expectedRunAttempts === undefined) { + return; + } + const expected = expectedRunAttempts.get(runId); + if (expected === undefined) { + throw new Error(`expected run attempts omitted ${label} run ID: ${runId}`); + } + expectedRunAttempts.delete(runId); + if (runAttempt !== expected) { + throw new Error( + `${label} run attempt changed: ${runId} expected ${expected}, observed ${runAttempt}`, + ); + } +} + +function normalizeManifestChildEvidence(value) { + if (value === undefined) { + return undefined; + } + const evidence = normalizeJsonObject(value, "release validation manifest child evidence"); + return Object.fromEntries( + Object.entries(evidence) + .toSorted(([left], [right]) => left.localeCompare(right)) + .map(([key, raw]) => { + const child = normalizeJsonObject(raw, `release validation child evidence ${key}`); + const plannedRunAttempt = normalizePositiveInteger( + child.plannedRunAttempt, + `${key} planned run attempt`, + ); + const effectiveRunAttempt = normalizePositiveInteger( + child.effectiveRunAttempt, + `${key} effective run attempt`, + ); + if (effectiveRunAttempt < plannedRunAttempt) { + throw new Error(`release validation child attempt regressed: ${key}`); + } + const observedRunAttempts = Array.isArray(child.observedRunAttempts) + ? child.observedRunAttempts.map((attempt) => + normalizePositiveInteger(attempt, `${key} observed run attempt`), + ) + : []; + const expectedAttempts = Array.from( + { length: effectiveRunAttempt - plannedRunAttempt + 1 }, + (_, index) => plannedRunAttempt + index, + ); + if (JSON.stringify(observedRunAttempts) !== JSON.stringify(expectedAttempts)) { + throw new Error(`release validation child attempt evidence is gapped: ${key}`); + } + if (!Array.isArray(child.jobs) || child.jobs.length === 0) { + throw new Error(`release validation child jobs are missing: ${key}`); + } + const jobs = child.jobs.map((rawJob) => { + const job = normalizeJsonObject(rawJob, `release validation child job ${key}`); + const acceptedRunAttempt = normalizePositiveInteger( + job.acceptedRunAttempt, + `${key} accepted run attempt`, + ); + if (acceptedRunAttempt < plannedRunAttempt || acceptedRunAttempt > effectiveRunAttempt) { + throw new Error(`release validation child job attempt is invalid: ${key}`); + } + const name = String(job.name ?? ""); + if (!name) { + throw new Error(`release validation child job identity is invalid: ${key}`); + } + return { + acceptedRunAttempt, + completedAt: String(job.completedAt ?? ""), + conclusion: String(job.conclusion ?? ""), + name, + startedAt: String(job.startedAt ?? ""), + status: String(job.status ?? ""), + url: String(job.url ?? ""), + }; + }); + if ( + new Set(jobs.map((job) => job.name)).size !== jobs.length || + jobs.some( + (job, index) => index > 0 && compareReleaseJobsByName(jobs[index - 1], job) >= 0, + ) + ) { + throw new Error(`release validation child job identity is duplicated: ${key}`); + } + const composite = { effectiveRunAttempt, jobs, plannedRunAttempt }; + const compositeJobsSha256 = String(child.compositeJobsSha256 ?? ""); + if ( + !/^[a-f0-9]{64}$/u.test(compositeJobsSha256) || + releaseCompositeJobsSha256(composite) !== compositeJobsSha256 + ) { + throw new Error(`release validation child composite digest is invalid: ${key}`); + } + const dispatchActor = String(child.dispatchActor ?? ""); + const triggeringActor = String(child.triggeringActor ?? ""); + const repository = String(child.repository ?? ""); + if ( + dispatchActor !== "github-actions[bot]" || + !triggeringActor || + !/^[^/]+\/[^/]+$/u.test(repository) || + (effectiveRunAttempt === plannedRunAttempt && triggeringActor !== "github-actions[bot]") + ) { + throw new Error(`release validation child rerun provenance is invalid: ${key}`); + } + return [ + key, + { + ...composite, + compositeJobsSha256, + dispatchActor, + observedRunAttempts, + repository, + runId: normalizeRequiredRunId(child.runId, `${key} run ID`), + triggeringActor, + }, + ]; + }), + ); +} + +export function releaseAdvisoryJobEvidence(childEvidence, releaseProfile, workflowRef) { + return Object.entries(childEvidence ?? {}) + .toSorted(([left], [right]) => left.localeCompare(right)) + .flatMap(([child, evidence]) => + /^releaseChecks(?:Independent|Candidate)?$/u.test(child) + ? evidence.jobs + .filter((job) => + isReleaseCheckJobAdvisory({ jobName: job.name, releaseProfile, workflowRef }), + ) + .toSorted(compareReleaseJobsByName) + .map((job) => ({ + child, + job: job.name, + status: job.status, + conclusion: job.conclusion, + policy: "advisory", + })) + : [], + ); +} + +function manifestEvidenceIdentity(manifest) { + return sortReleaseJsonValueKeys({ + sourceAdmission: publicationSourceReuseIdentity(manifest.sourceAdmission) ?? null, + childRunIds: manifest.childRunIds, + controls: manifest.controls, + releaseProfile: manifest.releaseProfile, + rerunGroup: manifest.rerunGroup, + runReleaseSoak: manifest.runReleaseSoak, + validationInputs: manifest.validationInputs, + }); +} + +export function validateParentManifest(value, expected) { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error("release validation manifest must be an object"); + } + if (![2, 3, 4].includes(value.version) || value.workflowName !== "Full Release Validation") { + throw new Error("release validation manifest schema is unsupported"); + } + if (String(value.runId) !== String(expected.runId)) { + throw new Error("release validation manifest run ID mismatch"); + } + if ( + !/^[1-9][0-9]*$/u.test(String(value.runAttempt)) || + (expected.runAttempt !== undefined && Number(value.runAttempt) !== Number(expected.runAttempt)) + ) { + throw new Error("release validation manifest run attempt mismatch"); + } + const targetSha = normalizeSha(value.targetSha, "release validation manifest target SHA"); + if (typeof value.workflowRef !== "string" || value.workflowRef.length === 0) { + throw new Error("release validation manifest workflow ref is invalid"); + } + if (expected.workflowRef !== undefined && value.workflowRef !== expected.workflowRef) { + throw new Error("release validation manifest workflow ref mismatch"); + } + let workflowSha; + let workflowFullRef; + let workflowRefType; + if (value.version >= 3) { + workflowSha = normalizeSha(value.workflowSha, "release validation manifest workflow SHA"); + if (expected.workflowSha !== undefined && workflowSha !== expected.workflowSha) { + throw new Error("release validation manifest workflow SHA mismatch"); + } + workflowFullRef = String(value.workflowFullRef ?? ""); + workflowRefType = String(value.workflowRefType ?? ""); + if ( + !["branch", "tag"].includes(workflowRefType) || + workflowFullRef !== + `refs/${workflowRefType === "branch" ? "heads" : "tags"}/${value.workflowRef}` + ) { + throw new Error("release validation manifest workflow full ref is invalid"); + } + } else if (expected.workflowSha !== undefined) { + workflowSha = normalizeSha(expected.workflowSha, "release validation workflow SHA"); + } + const rerunGroup = String(value.rerunGroup ?? ""); + requiredChildKeysForManifest({ rerunGroup, version: value.version }); + const releaseProfile = String(value.releaseProfile ?? ""); + if (!["beta", "stable", "full"].includes(releaseProfile)) { + throw new Error("release validation manifest release profile is invalid"); + } + const candidateBinding = + value.candidateBinding === undefined || value.candidateBinding === null + ? null + : validateFullReleaseCandidateBinding(value.candidateBinding); + if ( + candidateBinding !== null && + (candidateBinding.request.targetSha !== targetSha || + candidateBinding.request.toolingSha !== workflowSha || + candidateBinding.request.releaseProfile !== releaseProfile || + (expected.repository !== undefined && + candidateBinding.request.repository !== expected.repository)) + ) { + throw new Error("release validation manifest candidate binding is invalid"); + } + if ( + Object.hasOwn(expected, "candidateBinding") && + JSON.stringify(sortReleaseJsonValueKeys(candidateBinding)) !== + JSON.stringify( + sortReleaseJsonValueKeys( + expected.candidateBinding === null + ? null + : validateFullReleaseCandidateBinding(expected.candidateBinding), + ), + ) + ) { + throw new Error("release validation manifest candidate differs from the immutable plan"); + } + const runReleaseSoak = String(value.runReleaseSoak ?? ""); + if (!["true", "false"].includes(runReleaseSoak)) { + throw new Error("release validation manifest release soak value is invalid"); + } + const controls = normalizeJsonObject(value.controls, "release validation manifest controls"); + if (value.version >= 3 && controls.performanceReportPublication !== "artifact-only") { + throw new Error("release validation manifest performance report publication mode is invalid"); + } + const validationInputs = + value.validationInputs === undefined + ? undefined + : normalizeJsonObject( + value.validationInputs, + "release validation manifest validation inputs", + ); + const sourceAdmission = validatePublicationSourceBinding(value, expected); + const publicationAdmission = validatePublicationAdmissionBinding(value, expected); + normalizeReleaseTelegramWaiver({ + ...validationInputs, + candidateVersion: candidateBinding?.package.version, + releaseProfile, + rerunGroup: value.rerunGroup, + }); + if (validationInputs?.coveragePolicy !== undefined && value.version !== 4) { + throw new Error("release coverage policy requires a version 4 manifest"); + } + const coveragePolicy = normalizeReleaseCoveragePolicy({ + ...validationInputs, + candidateVersion: candidateBinding?.package.version, + releaseProfile, + rerunGroup, + runReleaseSoak, + }); + if ( + coveragePolicy === "npm-stable-v1" && + (!resolveReleaseContextIdentity( + validationInputs.targetContextRef || String(value.targetRef ?? ""), + validationInputs.targetVersion, + ) || + controls.performanceBlocking !== true || + controls.stableSoakRequired !== true) + ) { + throw new Error( + "npm stable coverage policy requires release context, blocking performance, and stable soak", + ); + } + const childEvidence = normalizeManifestChildEvidence(value.childEvidence); + const advisoryJobs = releaseAdvisoryJobEvidence(childEvidence, releaseProfile, value.workflowRef); + if ( + value.advisoryJobs !== undefined && + JSON.stringify(sortReleaseJsonValueKeys(value.advisoryJobs)) !== + JSON.stringify(sortReleaseJsonValueKeys(advisoryJobs)) + ) { + throw new Error("release validation advisory jobs differ from canonical policy evidence"); + } + const childRuns = value.childRuns; + if (!childRuns || typeof childRuns !== "object" || Array.isArray(childRuns)) { + throw new Error("release validation manifest childRuns is invalid"); + } + const childRunIds = + value.version === 4 + ? { + normalCi: normalizeOptionalRunId(childRuns.normalCi, "normal CI run ID"), + npmTelegram: normalizeOptionalRunId(childRuns.npmTelegram, "npm Telegram run ID"), + pluginPrereleaseIndependent: normalizeOptionalRunId( + childRuns.pluginPrereleaseIndependent, + "plugin prerelease independent run ID", + ), + pluginPrereleaseCandidate: normalizeOptionalRunId( + childRuns.pluginPrereleaseCandidate, + "plugin prerelease candidate run ID", + ), + productPerformance: normalizeOptionalRunId( + childRuns.productPerformance?.runId ?? "", + "performance run ID", + ), + releaseChecksIndependent: normalizeOptionalRunId( + childRuns.releaseChecksIndependent, + "release checks independent run ID", + ), + releaseChecksCandidate: normalizeOptionalRunId( + childRuns.releaseChecksCandidate, + "release checks candidate run ID", + ), + } + : { + normalCi: normalizeOptionalRunId(childRuns.normalCi, "normal CI run ID"), + npmTelegram: normalizeOptionalRunId(childRuns.npmTelegram, "npm Telegram run ID"), + pluginPrerelease: normalizeOptionalRunId( + childRuns.pluginPrerelease, + "plugin prerelease run ID", + ), + productPerformance: normalizeOptionalRunId( + childRuns.productPerformance?.runId ?? "", + "performance run ID", + ), + releaseChecks: normalizeOptionalRunId(childRuns.releaseChecks, "release checks run ID"), + }; + if ( + coveragePolicy === "npm-beta-v1" && + (childRunIds.productPerformance || + childRunIds.npmTelegram || + controls.performanceBlocking !== false || + validationInputs.skipPackageTelegramE2e !== "true") + ) { + throw new Error("npm beta coverage policy requires deferred confidence children"); + } + let evidenceReuse; + if (value.evidenceReuse !== undefined) { + const reuse = normalizeJsonObject( + value.evidenceReuse, + "release validation manifest evidence reuse", + ); + if (!EVIDENCE_REUSE_POLICIES.has(reuse.policy)) { + throw new Error("release validation manifest evidence reuse policy is invalid"); + } + if (!/^[a-f0-9]{40}$/u.test(String(reuse.evidenceSha))) { + throw new Error("release validation manifest evidence SHA is invalid"); + } + if ( + !Array.isArray(reuse.changedPaths) || + reuse.changedPaths.some( + (changedPath) => typeof changedPath !== "string" || changedPath.length === 0, + ) || + new Set(reuse.changedPaths).size !== reuse.changedPaths.length + ) { + throw new Error("release validation manifest evidence changed paths are invalid"); + } + evidenceReuse = { + changedPaths: reuse.changedPaths, + evidenceSha: String(reuse.evidenceSha), + policy: reuse.policy, + runId: normalizeRequiredRunId(reuse.runId, "evidence reuse root run ID"), + selectedRunId: normalizeRequiredRunId(reuse.selectedRunId, "evidence reuse selected run ID"), + }; + if (reuse.publication !== undefined) { + const publication = normalizeJsonObject(reuse.publication, "retained root publication"); + if ( + Object.keys(publication).toSorted().join(",") !== + "publicationAdmission,publicationAdmissionContract,sourceAdmission,sourceAdmissionContract" + ) { + throw new Error("retained root publication fields are invalid"); + } + validatePublicationAdmissionBinding(publication, { publicationAdmissionContract: "1" }); + if ( + publication.sourceAdmission.runId !== evidenceReuse.runId || + publication.sourceAdmission.candidateSha !== evidenceReuse.evidenceSha + ) { + throw new Error("retained root publication identity mismatch"); + } + evidenceReuse.publication = publication; + } + } + return { + advisoryJobs, + ...(value.publicationAdmissionContract !== undefined + ? { publicationAdmissionContract: value.publicationAdmissionContract, publicationAdmission } + : {}), + ...(sourceAdmission + ? { + sourceAdmissionContract: value.sourceAdmissionContract, + sourceAdmission, + trustedWorkflow: value.trustedWorkflow, + sourceParentRunAttempt: value.sourceParentRunAttempt, + } + : {}), + candidateBinding, + childEvidence, + childRunIds, + controls, + evidenceReuse, + releaseProfile, + rerunGroup, + runAttempt: Number(value.runAttempt), + runId: String(value.runId), + runReleaseSoak, + targetRef: String(value.targetRef ?? ""), + targetSha, + validationInputs, + version: value.version, + workflowFullRef, + workflowSha, + workflowRef: value.workflowRef, + workflowRefType, + }; +} + +export function validateEvidenceReuseChain( + currentManifest, + selectedManifest, + rootManifest, + compareCommits, +) { + const reuse = currentManifest.evidenceReuse; + if (!reuse) { + throw new Error("release validation manifest does not authorize evidence reuse"); + } + if (rootManifest.evidenceReuse || selectedManifest.evidenceReuse) { + throw new Error("evidence reuse must select a root execution manifest"); + } + if ( + !currentManifest.validationInputs || + !selectedManifest.validationInputs || + !rootManifest.validationInputs + ) { + throw new Error("evidence reuse manifests must record validation inputs"); + } + if (rootManifest.runId !== reuse.runId) { + throw new Error("evidence reuse root manifest run ID mismatch"); + } + if (selectedManifest.runId !== reuse.selectedRunId) { + throw new Error("evidence reuse selected manifest run ID mismatch"); + } + if (selectedManifest.targetSha !== reuse.evidenceSha) { + throw new Error("evidence reuse selected manifest SHA mismatch"); + } + if (rootManifest.targetSha !== reuse.evidenceSha) { + throw new Error("full release evidence reuse root SHA mismatch"); + } + if (selectedManifest.runId !== rootManifest.runId) { + throw new Error("evidence reuse selected manifest is not the chain root"); + } + const rootPublication = + rootManifest.publicationAdmissionContract === "1" + ? { + sourceAdmissionContract: rootManifest.sourceAdmissionContract, + sourceAdmission: rootManifest.sourceAdmission, + publicationAdmissionContract: rootManifest.publicationAdmissionContract, + publicationAdmission: rootManifest.publicationAdmission, + } + : null; + if ( + publicationObservationJson(reuse.publication ?? null) !== + publicationObservationJson(rootPublication) + ) { + throw new Error("retained root publication differs from authenticated root evidence"); + } + if (reuse.policy === EXACT_TARGET_EVIDENCE_REUSE_POLICY) { + if (reuse.changedPaths.length !== 0 || currentManifest.targetSha !== reuse.evidenceSha) { + throw new Error("exact-target release evidence reuse requires no changed paths"); + } + } else if ( + reuse.policy === CHANGELOG_ONLY_EVIDENCE_REUSE_POLICY || + reuse.policy === SPLIT_CHANGELOG_EVIDENCE_REUSE_POLICY + ) { + const split = reuse.policy === SPLIT_CHANGELOG_EVIDENCE_REUSE_POLICY; + const version = + currentManifest.candidateBinding?.package?.version ?? + currentManifest.validationInputs?.targetVersion; + if ( + (split + ? !isSplitChangelogEvidenceDelta(reuse.changedPaths, version) + : reuse.changedPaths.length !== 1 || reuse.changedPaths[0] !== "CHANGELOG.md") || + currentManifest.targetSha === reuse.evidenceSha + ) { + throw new Error("changelog-only release evidence reuse has an invalid target delta"); + } + if (typeof compareCommits !== "function") { + throw new Error("changelog-only release evidence reuse requires commit comparison"); + } + const comparison = compareCommits(reuse.evidenceSha, currentManifest.targetSha); + const verified = classifyReleaseChangelogEvidenceComparison(comparison, { + baseSha: reuse.evidenceSha, + version, + }); + if ( + verified.policy !== reuse.policy || + verified.changedPaths.length !== reuse.changedPaths.length || + verified.changedPaths.some((name) => !reuse.changedPaths.includes(name)) + ) { + throw new Error("changelog-only release evidence reuse failed commit comparison"); + } + } else { + throw new Error("release validation manifest evidence reuse policy is invalid"); + } + + const rootIdentity = JSON.stringify(manifestEvidenceIdentity(rootManifest)); + for (const [label, manifest] of [ + ["selected", selectedManifest], + ["current", currentManifest], + ]) { + if (JSON.stringify(manifestEvidenceIdentity(manifest)) !== rootIdentity) { + throw new Error(`evidence reuse ${label} manifest policy differs from the chain root`); + } + } + return rootManifest.targetSha; +} + +export function validateRequestedEvidenceReuse( + currentManifest, + selectedManifest, + rootManifest, + { + expectedChangedPaths, + expectedEvidencePolicy, + expectedEvidenceSha, + expectedRootRunId, + expectedSelectedRunId, + expectedTargetSha, + }, + compareCommits, +) { + if ( + !Array.isArray(expectedChangedPaths) || + expectedChangedPaths.some( + (changedPath) => typeof changedPath !== "string" || changedPath.length === 0, + ) || + new Set(expectedChangedPaths).size !== expectedChangedPaths.length + ) { + throw new Error("expected evidence changed paths are invalid"); + } + const requested = { + changedPaths: expectedChangedPaths, + evidenceSha: normalizeSha(expectedEvidenceSha, "expected evidence SHA"), + policy: String(expectedEvidencePolicy ?? ""), + runId: normalizeRequiredRunId(expectedRootRunId, "expected evidence root run ID"), + selectedRunId: normalizeRequiredRunId( + expectedSelectedRunId, + "expected evidence selected run ID", + ), + }; + const expectedTarget = normalizeSha(expectedTargetSha, "expected target SHA"); + const reuse = currentManifest.evidenceReuse; + if (!reuse) { + if ( + currentManifest.runId !== requested.selectedRunId || + selectedManifest.runId !== requested.selectedRunId || + rootManifest.runId !== requested.runId + ) { + throw new Error("reused release evidence no longer matches the requested validation"); + } + validateEvidenceReuseChain( + { ...currentManifest, evidenceReuse: requested, targetSha: expectedTarget }, + selectedManifest, + rootManifest, + compareCommits, + ); + return; + } + if ( + currentManifest.targetSha !== expectedTarget || + selectedManifest.runId !== requested.selectedRunId || + rootManifest.runId !== requested.runId || + rootManifest.targetSha !== requested.evidenceSha || + reuse.evidenceSha !== requested.evidenceSha || + reuse.policy !== requested.policy || + reuse.runId !== requested.runId || + reuse.selectedRunId !== requested.selectedRunId || + JSON.stringify(reuse.changedPaths) !== JSON.stringify(requested.changedPaths) + ) { + throw new Error("reused release evidence no longer matches the requested validation"); + } +} + +function hasRequestedEvidenceReuse(options) { + return [ + options.expectedTargetSha, + options.expectedEvidencePolicy, + options.expectedEvidenceSha, + options.expectedChangedPaths, + options.expectedRootRunId, + options.expectedSelectedRunId, + ].some((value) => value !== undefined); +} + +export function selectedChildKeys(parentJobs) { + return new Set( + [...LEGACY_CHILD_DISPATCHES, ...PHASED_CHILD_DISPATCHES] + .filter((child) => { + const parentJob = parentJobs.find((job) => job.name === child.parentJobName); + return parentJob && parentJob.conclusion !== "skipped"; + }) + .map((child) => child.manifestKey), + ); +} + +/** + * @template {{ manifestKey: string, name: string }} Child + * @param {{ childRunIds: Partial<Record<string, string>> }} manifest + * @param {Child[]} children + * @param {Set<string>} selectedKeys + * @returns {Array<{ child: Child, runId: string }>} + */ +export function manifestChildEntries(manifest, children, selectedKeys) { + return children.flatMap((child) => { + const runId = manifest.childRunIds[child.manifestKey]; + if (!runId) { + if (selectedKeys.has(child.manifestKey)) { + throw new Error(`selected child is missing from manifest: ${child.name}`); + } + return []; + } + return [{ child, runId }]; + }); +} + +function childDispatchAttempt(displayTitle, child, parentRunId, parentRunAttempt) { + const prefix = `${child.name} full-release-validation-${parentRunId}-`; + if (!displayTitle.startsWith(prefix) || !displayTitle.endsWith(child.suffix)) { + return undefined; + } + const attemptEnd = child.suffix ? -child.suffix.length : undefined; + const attemptText = displayTitle.slice(prefix.length, attemptEnd); + if (!/^[1-9][0-9]*$/u.test(attemptText)) { + return undefined; + } + const attempt = Number(attemptText); + if (!Number.isSafeInteger(attempt) || attempt > parentRunAttempt) { + return undefined; + } + return attempt; +} + +function parentJobExecutionFingerprint(job) { + return sortReleaseJsonValueKeys({ + completedAt: job.completed_at, + conclusion: job.conclusion, + name: job.name, + startedAt: job.started_at, + status: job.status, + steps: (job.steps ?? []).map((step) => ({ + completedAt: step.completed_at, + conclusion: step.conclusion, + name: step.name, + number: step.number, + startedAt: step.started_at, + status: step.status, + })), + }); +} + +function selectedAttemptParentJob(parentJobs, child, parentManifest) { + const slotJobs = parentJobs.filter((job) => job.name === child.parentJobName); + if (slotJobs.length === 0) { + throw new Error(`manifest parent job is missing: ${child.name}`); + } + const latestAttempt = Math.max(...slotJobs.map((job) => Number(job.run_attempt))); + if (latestAttempt !== parentManifest.runAttempt) { + throw new Error(`manifest parent job latest attempt mismatch: ${child.name}`); + } + const currentJobs = slotJobs.filter( + (job) => Number(job.run_attempt) === parentManifest.runAttempt, + ); + if (currentJobs.length !== 1) { + throw new Error(`manifest parent job is not unique at the selected attempt: ${child.name}`); + } + return { currentJob: currentJobs[0], slotJobs }; +} + +export function resolveManifestChildOriginAttempt(run, child, parentManifest, parentJobs) { + const correlatedAttempt = childDispatchAttempt( + String(run.display_title ?? ""), + child, + parentManifest.runId, + parentManifest.runAttempt, + ); + if (correlatedAttempt !== undefined) { + return correlatedAttempt; + } + if (run.display_title !== child.name) { + return undefined; + } + + const { currentJob, slotJobs } = selectedAttemptParentJob(parentJobs, child, parentManifest); + if (currentJob.status !== "completed" || currentJob.conclusion !== "success") { + throw new Error(`manifest parent job is not completed/success: ${child.name}`); + } + const currentFingerprint = JSON.stringify(parentJobExecutionFingerprint(currentJob)); + const carriedOriginAttempts = slotJobs + .filter( + (job) => + Number(job.run_attempt) < parentManifest.runAttempt && + job.status === "completed" && + job.conclusion === "success" && + JSON.stringify(parentJobExecutionFingerprint(job)) === currentFingerprint, + ) + .map((job) => Number(job.run_attempt)); + return carriedOriginAttempts.length > 0 + ? Math.min(...carriedOriginAttempts) + : parentManifest.runAttempt; +} + +export function selectManifestParentJob( + parentJobs, + child, + parentManifest, + originAttempt, + options = {}, +) { + const { currentJob, slotJobs } = selectedAttemptParentJob(parentJobs, child, parentManifest); + const originJobs = slotJobs.filter((job) => Number(job.run_attempt) === originAttempt); + if (originJobs.length !== 1) { + throw new Error(`manifest parent job origin is not unique: ${child.name}`); + } + const originJob = originJobs[0]; + if (originJob.status !== "completed" || originJob.conclusion !== "success") { + throw new Error(`manifest parent job origin is not completed/success: ${child.name}`); + } + if (originAttempt === parentManifest.runAttempt) { + return originJob; + } + if (originAttempt > parentManifest.runAttempt) { + throw new Error(`manifest parent job origin attempt is invalid: ${child.name}`); + } + if (options.requireSkippedCarryForward === true) { + for (let attempt = originAttempt + 1; attempt <= parentManifest.runAttempt; attempt += 1) { + const carriedJobs = slotJobs.filter((job) => Number(job.run_attempt) === attempt); + if (carriedJobs.length !== 1) { + throw new Error(`manifest parent job carry-forward is not unique: ${child.name}`); + } + const carriedJob = carriedJobs[0]; + if (carriedJob.status !== "completed" || carriedJob.conclusion !== "skipped") { + throw new Error(`manifest parent job was redispatched during recovery: ${child.name}`); + } + } + return originJob; + } + if (currentJob.status !== "completed" || currentJob.conclusion !== "success") { + throw new Error(`manifest parent job is not completed/success: ${child.name}`); + } + if ( + JSON.stringify(parentJobExecutionFingerprint(currentJob)) !== + JSON.stringify(parentJobExecutionFingerprint(originJob)) + ) { + throw new Error(`manifest parent job carry-forward fingerprint mismatch: ${child.name}`); + } + return currentJob; +} + +export function validateManifestChildRun( + run, + child, + runId, + parentManifest, + parentJobs, + selectedParentJobLog, + repository, + plannedRunAttempt, + requireSkippedCarryForward = false, +) { + const targetRepository = repository ?? DEFAULT_REPO; + if (String(run.id) !== String(runId)) { + throw new Error(`manifest child run ID mismatch: ${child.name}`); + } + const originAttempt = resolveManifestChildOriginAttempt(run, child, parentManifest, parentJobs); + if (plannedRunAttempt !== undefined) { + validateReleaseChildRunProvenance(run, { + displayTitle: child.displayTitle, + key: child.manifestKey, + plannedRunAttempt, + repository: targetRepository, + runId, + workflow: child.workflow, + workflowRef: child.headBranch, + workflowSha: parentManifest.workflowSha, + }); + } else if ( + run.event !== "workflow_dispatch" || + run.head_branch !== child.headBranch || + (child.trustedRef === "parent" && run.head_sha !== parentManifest.workflowSha) || + !/^[a-f0-9]{40}$/u.test(String(run.head_sha)) || + run.actor?.login !== "github-actions[bot]" || + run.triggering_actor?.login !== "github-actions[bot]" || + !Number.isSafeInteger(Number(run.run_attempt)) || + Number(run.run_attempt) < 1 + ) { + throw new Error(`manifest child dispatch tuple mismatch: ${child.name}`); + } + if (originAttempt === undefined) { + throw new Error(`manifest child dispatch tuple mismatch: ${child.name}`); + } + const childWorkflowPath = String(run.path ?? "").split("@", 1)[0]; + if (childWorkflowPath !== `.github/workflows/${child.workflow}`) { + throw new Error(`manifest child workflow mismatch: ${child.name}`); + } + selectManifestParentJob(parentJobs, child, parentManifest, originAttempt, { + requireSkippedCarryForward, + }); + validateReleaseChildDispatchBinding({ + child: { + key: child.manifestKey, + runId, + }, + log: selectedParentJobLog, + coveragePolicy: parentManifest.validationInputs?.coveragePolicy, + plannedRunAttempt: plannedRunAttempt ?? run.run_attempt, + repository: targetRepository, + targetSha: parentManifest.targetSha, + }); + return run; +} + +export function validatePerformanceArtifactOnlyJobs(jobs, runAttempt) { + const normalizedRunAttempt = normalizePositiveInteger(runAttempt, "performance run attempt"); + const currentJobs = jobs.filter((job) => Number(job.run_attempt) === normalizedRunAttempt); + const guards = currentJobs.filter((job) => job.name === "Verify artifact-only report mode"); + if ( + guards.length !== 1 || + guards[0].status !== "completed" || + guards[0].conclusion !== "success" + ) { + throw new Error("performance artifact-only guard is missing or unsuccessful"); + } + const unsafePublisher = currentJobs.find( + (job) => + String(job.name ?? "").startsWith("Publish ") && + String(job.name ?? "").endsWith(" report") && + job.conclusion !== "skipped", + ); + if (unsafePublisher) { + throw new Error(`performance report publisher was not skipped: ${unsafePublisher.name}`); + } + return guards[0]; +} + +function manifestArtifactName(runId, runAttempt) { + const normalizedRunId = normalizeRequiredRunId(runId, "full release run ID"); + const normalizedRunAttempt = normalizePositiveInteger(runAttempt, "full release run attempt"); + return `full-release-validation-${normalizedRunId}-${normalizedRunAttempt}`; +} + +function legacyManifestArtifactName(runId) { + return `full-release-validation-${normalizeRequiredRunId(runId, "full release run ID")}`; +} + +export function validateManifestArtifactIdentity( + artifact, + { artifactDigest, artifactId, runAttempt, runId }, +) { + const normalizedArtifactId = normalizeRequiredRunId(artifactId, "manifest artifact ID"); + const normalizedRunId = normalizeRequiredRunId(runId, "full release run ID"); + const normalizedRunAttempt = normalizePositiveInteger(runAttempt, "full release run attempt"); + const normalizedDigest = String(artifactDigest ?? ""); + if (!/^sha256:[a-f0-9]{64}$/u.test(normalizedDigest)) { + throw new Error(`release validation manifest artifact digest is invalid: ${normalizedRunId}`); + } + const canonicalName = manifestArtifactName(normalizedRunId, normalizedRunAttempt); + const legacyName = legacyManifestArtifactName(normalizedRunId); + const validName = + artifact.name === canonicalName || (normalizedRunAttempt === 1 && artifact.name === legacyName); + if ( + String(artifact.id) !== normalizedArtifactId || + !validName || + artifact.digest !== normalizedDigest || + artifact.expired !== false || + String(artifact.workflow_run?.id) !== normalizedRunId || + !Number.isSafeInteger(Number(artifact.size_in_bytes)) || + Number(artifact.size_in_bytes) < 1 + ) { + throw new Error(`release validation manifest artifact identity mismatch: ${normalizedRunId}`); + } + return artifact; +} + +export function selectManifestArtifact(artifacts, runId, runAttempt) { + const expectedName = manifestArtifactName(runId, runAttempt); + const canonicalMatches = artifacts.filter( + (artifact) => + artifact.name === expectedName && + artifact.expired === false && + String(artifact.workflow_run?.id) === String(runId), + ); + if (canonicalMatches.length > 1) { + throw new Error(`multiple release validation manifest artifacts found: ${runId}`); + } + const canonicalArtifact = canonicalMatches[0]; + if (canonicalArtifact) { + return validateManifestArtifactIdentity(canonicalArtifact, { + artifactDigest: canonicalArtifact.digest, + artifactId: canonicalArtifact.id, + runAttempt, + runId, + }); + } + + const legacyName = legacyManifestArtifactName(runId); + const legacyMatches = artifacts.filter( + (artifact) => + artifact.name === legacyName && + artifact.expired === false && + String(artifact.workflow_run?.id) === String(runId), + ); + if (legacyMatches.length > 1) { + throw new Error(`multiple legacy release validation manifest artifacts found: ${runId}`); + } + const legacyArtifact = legacyMatches[0]; + if (!legacyArtifact) { + return undefined; + } + if (Number(runAttempt) !== 1) { + throw new Error(`legacy release validation manifest requires run attempt 1: ${runId}`); + } + return validateManifestArtifactIdentity(legacyArtifact, { + artifactDigest: legacyArtifact.digest, + artifactId: legacyArtifact.id, + runAttempt, + runId, + }); +} + +export function validateManifestArtifactCompatibility(artifact, manifest, runId, runAttempt) { + if (artifact.name === manifestArtifactName(runId, runAttempt)) { + return artifact; + } + if ( + Number(runAttempt) === 1 && + artifact.name === legacyManifestArtifactName(runId) && + manifest?.version === 2 + ) { + return artifact; + } + throw new Error(`legacy release validation manifest artifact is not compatible: ${runId}`); +} + +export function readManifestArtifactArchive(archivePath, expectedDigest) { + const archiveSize = statSync(archivePath).size; + if ( + !Number.isSafeInteger(archiveSize) || + archiveSize < 1 || + archiveSize > MAX_MANIFEST_ARTIFACT_ZIP_BYTES + ) { + throw new Error("release validation manifest artifact compressed size is invalid"); + } + const archiveBytes = readFileSync(archivePath); + if (archiveBytes.byteLength !== archiveSize) { + throw new Error("release validation manifest artifact changed while being verified"); + } + const actualDigest = `sha256:${createHash("sha256").update(archiveBytes).digest("hex")}`; + if (actualDigest !== expectedDigest) { + throw new Error("release validation manifest artifact digest mismatch"); + } + + let entryList; + try { + entryList = execFileSync("unzip", ["-Z", "-1", archivePath], { + encoding: "utf8", + maxBuffer: MAX_MANIFEST_ENTRY_LIST_BYTES, + stdio: ["ignore", "pipe", "pipe"], + }); + } catch { + throw new Error("release validation manifest artifact entry list is invalid"); + } + const entries = entryList.split(/\r?\n/u).filter((entry) => entry.length > 0); + if (entries.length !== 1 || entries[0] !== MANIFEST_ARTIFACT_ENTRY) { + throw new Error( + `release validation manifest artifact must contain only ${MANIFEST_ARTIFACT_ENTRY}`, + ); + } + + let manifestBytes; + try { + manifestBytes = execFileSync("unzip", ["-p", archivePath, MANIFEST_ARTIFACT_ENTRY], { + maxBuffer: MAX_RELEASE_ARTIFACT_BYTES + 1, + stdio: ["ignore", "pipe", "pipe"], + }); + } catch { + throw new Error("release validation manifest artifact entry could not be read safely"); + } + if (manifestBytes.byteLength < 1 || manifestBytes.byteLength > MAX_RELEASE_ARTIFACT_BYTES) { + throw new Error("release validation manifest artifact entry size is invalid"); + } + return JSON.parse(manifestBytes.toString("utf8")); +} + +function downloadParentManifestEvidence(runId, runAttempt, repository, manifestPath) { + const targetRepository = repository ?? DEFAULT_REPO; + const artifacts = []; + for (let page = 1; page <= 10; page += 1) { + const pageArtifacts = + githubRestJson(`actions/runs/${runId}/artifacts?per_page=100&page=${page}`, targetRepository) + .artifacts ?? []; + artifacts.push(...pageArtifacts); + if (pageArtifacts.length < 100) { + break; + } + } + const listedArtifact = selectManifestArtifact(artifacts, runId, runAttempt); + if (!listedArtifact) { + return undefined; + } + const artifact = validateManifestArtifactIdentity( + githubRestJson(`actions/artifacts/${listedArtifact.id}`, targetRepository), + { + artifactDigest: listedArtifact.digest, + artifactId: listedArtifact.id, + runAttempt, + runId, + }, + ); + const downloadDir = mkdtempSync(join(tmpdir(), "openclaw-release-ci-summary-")); + try { + const archivePath = join(downloadDir, "manifest.zip"); + downloadArtifactZip(String(artifact.id), archivePath, artifact.size_in_bytes, targetRepository); + const manifest = readManifestArtifactArchive(archivePath, artifact.digest); + validateManifestArtifactCompatibility(artifact, manifest, runId, runAttempt); + if (manifestPath) { + const providedManifest = JSON.parse(readFileSync(resolve(manifestPath), "utf8")); + if ( + JSON.stringify(sortReleaseJsonValueKeys(providedManifest)) !== + JSON.stringify(sortReleaseJsonValueKeys(manifest)) + ) { + throw new Error("provided release validation manifest differs from the run artifact"); + } + } + return { artifact, manifest }; + } finally { + rmSync(downloadDir, { force: true, recursive: true }); + } +} + +function tryDownloadParentManifest(runId, runAttempt, repository = DEFAULT_REPO) { + return downloadParentManifestEvidence(runId, runAttempt, repository)?.manifest; +} + +function workflowPath(run) { + return String(run.path ?? "").split("@", 1)[0]; +} + +function normalizedManifestArtifact(artifact, runAttempt) { + return { + digest: artifact.digest, + id: String(artifact.id), + name: artifact.name, + runAttempt, + sizeInBytes: Number(artifact.size_in_bytes), + }; +} + +function validateManifestArtifactBinding(artifact, manifest, parentRun, runId) { + validateManifestArtifactCompatibility(artifact, manifest, runId, parentRun.run_attempt); + if ( + String(artifact.workflow_run?.id) !== String(runId) || + artifact.workflow_run?.head_branch !== parentRun.head_branch || + artifact.workflow_run?.head_sha !== parentRun.head_sha + ) { + throw new Error(`release validation manifest artifact binding mismatch: ${runId}`); + } +} + +function validateCompletedParentRun(parentView, parentRest, repository, runId) { + validateParentRunBinding(parentView, parentRest, runId); + if ( + parentView.status !== "completed" || + parentView.conclusion !== "success" || + parentRest.status !== "completed" || + parentRest.conclusion !== "success" || + parentRest.repository?.full_name !== repository + ) { + throw new Error(`full release parent run is not completed/success: ${runId}`); + } +} + +export function createReleaseEvidenceClient(repository = DEFAULT_REPO) { + const normalizedRepository = normalizeRepository(repository); + return { + getWorkflowSource(sha) { + const exactSha = normalizeSha(sha, "source admission workflow SHA"); + const payload = githubRestJson( + `contents/.github/workflows/full-release-validation.yml?ref=${exactSha}`, + normalizedRepository, + ); + if ( + payload?.type !== "file" || + payload.encoding !== "base64" || + payload.path !== ".github/workflows/full-release-validation.yml" || + !Number.isSafeInteger(payload.size) || + payload.size < 1 || + payload.size > 1024 * 1024 || + typeof payload.content !== "string" || + payload.content.length > 2 * 1024 * 1024 + ) { + throw new Error("invalid immutable source-admission workflow response"); + } + const bytes = Buffer.from(payload.content, "base64"); + const blob = createHash("sha1").update(`blob ${bytes.length}\0`).update(bytes).digest("hex"); + if (bytes.length !== payload.size || blob !== payload.sha) { + throw new Error("source-admission workflow blob mismatch"); + } + return new TextDecoder("utf-8", { fatal: true }).decode(bytes); + }, + compareCommitLineage(base, head) { + return githubRestJson(`compare/${base}...${head}?per_page=1&page=2`, normalizedRepository); + }, + compareCommits(base, head) { + return githubRestJson(`compare/${base}...${head}`, normalizedRepository); + }, + getJobLog(jobId) { + return parentJobLog(jobId, normalizedRepository); + }, + getParentJobs(runId) { + return findParentJobsAll(runId, normalizedRepository); + }, + getRunAttemptJobs(runId, runAttempt, { requireComplete = false } = {}) { + return findRunAttemptJobsAll(runId, runAttempt, normalizedRepository, requireComplete); + }, + getRunAttempt(runId, runAttempt) { + return githubRestJson(`actions/runs/${runId}/attempts/${runAttempt}`, normalizedRepository); + }, + getArtifact(artifactId) { + if (!/^[1-9][0-9]{0,19}$/u.test(String(artifactId))) { + throw new Error("invalid publication artifact ID"); + } + return githubRestJson(`actions/artifacts/${artifactId}`, normalizedRepository); + }, + getRef(fullRef) { + const refPath = String(fullRef) + .replace(/^refs\//u, "") + .split("/") + .map(encodeURIComponent) + .join("/"); + return githubRestJson(`git/ref/${refPath}`, normalizedRepository); + }, + getRun(runId) { + return githubRestJsonAsync(`actions/runs/${runId}`, normalizedRepository); + }, + getRunView(runId) { + return jsonGh([ + "run", + "view", + String(runId), + "--repo", + normalizedRepository, + "--json", + "status,conclusion,attempt,headBranch,headSha,url", + ]); + }, + loadManifest(runId, runAttempt, manifestPath) { + return downloadParentManifestEvidence(runId, runAttempt, normalizedRepository, manifestPath); + }, + loadExecutionPlan(runId) { + return tryDownloadExecutionPlan(runId, normalizedRepository); + }, + loadExecutionPlanEvidence(runId) { + return readExecutionPlanEvidence( + normalizeRequiredRunId(runId, "publication parent run ID"), + normalizedRepository, + ); + }, + }; +} + +async function loadValidatedParentEvidence({ + client, + expectedRunAttempts, + manifestPath, + repository, + runId, +}) { + const parentView = client.getRunView(runId); + const parentRun = await client.getRun(runId); + const parentRunAttempt = normalizePositiveInteger( + parentRun.run_attempt, + `full release parent ${runId} run attempt`, + ); + consumeExpectedRunAttempt(expectedRunAttempts, runId, parentRunAttempt, "parent"); + validateCompletedParentRun(parentView, parentRun, repository, runId); + + const manifestEvidence = client.loadManifest(runId, parentRunAttempt, manifestPath); + if (!manifestEvidence) { + throw new ReleaseEvidenceRefreshRequiredError( + `successful parent run is missing its release validation manifest: ${runId}`, + ); + } + const manifest = validateParentManifest(manifestEvidence.manifest, { + runAttempt: parentRun.run_attempt, + runId, + workflowRef: parentRun.head_branch, + workflowSha: parentRun.head_sha, + }); + validateManifestArtifactBinding(manifestEvidence.artifact, manifest, parentRun, runId); + + return { + artifact: manifestEvidence.artifact, + manifest, + manifestJson: sortReleaseJsonValueKeys(manifestEvidence.manifest), + parentRun, + parentView, + }; +} + +function resolveTrustedWorkflowIdentity(workflowRef, workflowFullRef, workflowSha) { + const fullRef = workflowFullRef ?? `refs/heads/${workflowRef}`; + const protectedTag = TRUSTED_RELEASE_PUBLISH_TAG_PATTERN.exec(fullRef); + if (protectedTag) { + if (workflowRef !== fullRef.slice("refs/tags/".length)) { + throw new Error("trusted workflow tag name does not match its full ref"); + } + const sha = normalizeSha(workflowSha, "trusted workflow SHA"); + if (sha.slice(0, 12) !== protectedTag[1]) { + throw new Error("trusted workflow tag does not match its workflow SHA"); + } + return { fullRef, ref: workflowRef, sha, type: "tag" }; + } + if (fullRef !== `refs/heads/${workflowRef}`) { + throw new Error("trusted workflow full ref does not match its ref"); + } + if (workflowRef.startsWith("release-publish/")) { + throw new Error("trusted release-publish workflow ref must be a protected tag"); + } + return { fullRef, ref: workflowRef, sha: undefined, type: "branch" }; +} + +function normalizeWorkflowPathRef(ref) { + if (!ref || ref.startsWith("refs/")) { + return ref; + } + return `refs/heads/${ref}`; +} + +export function validateTrustedProducerIdentity( + evidence, + client, + verifier, + trustedWorkflowRef, + trustedWorkflowFullRef, + trustedWorkflowSha, +) { + const { manifest, parentRun } = evidence; + const trustedIdentity = resolveTrustedWorkflowIdentity( + trustedWorkflowRef, + trustedWorkflowFullRef, + trustedWorkflowSha, + ); + const shaPinned = SHA_PINNED_BRANCH_PATTERN.test(manifest.workflowRef ?? ""); + const protectedTagRoute = trustedIdentity.type === "tag"; + let protectedTagWorkflowRefProof = "manifest-v3-protected-tag-exact-sha"; + if (protectedTagRoute) { + let liveTag; + try { + liveTag = client.getRef(trustedIdentity.fullRef); + } catch (error) { + throw new Error( + `protected tooling tag is unavailable: ${ + error instanceof Error ? error.message : String(error) + }`, + { cause: error }, + ); + } + if (liveTag?.object?.sha !== trustedIdentity.sha) { + throw new Error("protected tooling tag moved after release validation was sealed"); + } + if (!shaPinned) { + throw new Error("protected-tag release evidence must use a canonical release-ci branch"); + } + if (manifest.workflowSha !== trustedIdentity.sha) { + const comparison = client.compareCommitLineage(manifest.workflowSha, trustedIdentity.sha); + if ( + !["ahead", "identical"].includes(String(comparison.status)) || + comparison.merge_base_commit?.sha !== manifest.workflowSha + ) { + throw new Error( + "protected-tag release evidence producer is not on the trusted tooling lineage", + ); + } + protectedTagWorkflowRefProof = "manifest-v3-protected-tag-tooling-lineage"; + } + } else if (manifest.workflowRef !== trustedWorkflowRef && !shaPinned) { + throw new Error( + `release evidence producer must run from trusted workflow ref: ${trustedWorkflowRef}`, + ); + } + if (shaPinned) { + if (manifest.version < 3) { + throw new Error("SHA-pinned release evidence requires a v3+ manifest"); + } + if (!manifest.workflowRef.startsWith(`release-ci/${manifest.workflowSha.slice(0, 12)}-`)) { + throw new Error("SHA-pinned release evidence branch does not match its workflow SHA"); + } + if (manifest.targetRef !== manifest.targetSha) { + throw new Error("SHA-pinned release evidence target ref must equal its target SHA"); + } + } + const expectedFullRef = `refs/heads/${manifest.workflowRef}`; + const runPath = String(parentRun.path ?? ""); + const [runWorkflowPath, runWorkflowFullRef] = runPath.split("@", 2); + if (runWorkflowPath !== ".github/workflows/full-release-validation.yml") { + throw new Error("release evidence producer workflow path is not trusted"); + } + if (runWorkflowFullRef && normalizeWorkflowPathRef(runWorkflowFullRef) !== expectedFullRef) { + throw new Error("release evidence producer workflow full ref is not trusted"); + } + + let workflowRefProof = "legacy-v2-main-ancestry"; + if (manifest.version >= 3) { + if (manifest.workflowRefType !== "branch" || manifest.workflowFullRef !== expectedFullRef) { + throw new Error("release evidence producer workflow full ref is not trusted"); + } + workflowRefProof = protectedTagRoute + ? protectedTagWorkflowRefProof + : shaPinned + ? "manifest-v3-sha-pinned-main-ancestry" + : "manifest-v3-branch"; + } + + if (!protectedTagRoute) { + const comparison = client.compareCommitLineage(manifest.workflowSha, verifier.sourceSha); + if ( + !["ahead", "identical"].includes(String(comparison.status)) || + comparison.merge_base_commit?.sha !== manifest.workflowSha + ) { + throw new Error("release evidence producer is not on the trusted main verifier lineage"); + } + } + + return { + producerOnTrustedMainLineage: !protectedTagRoute, + workflowFullRef: expectedFullRef, + workflowQualifiedPath: `${runWorkflowPath}@${expectedFullRef}`, + workflowRefProof, + workflowRefType: "branch", + workflowRunPath: runPath, + }; +} + +function normalizedParentTuple(evidence, identity) { + const { manifest, parentRun } = evidence; + return { + artifact: normalizedManifestArtifact(evidence.artifact, manifest.runAttempt), + conclusion: parentRun.conclusion, + manifest: evidence.manifestJson, + manifestVersion: manifest.version, + runAttempt: manifest.runAttempt, + runId: manifest.runId, + status: parentRun.status, + targetSha: manifest.targetSha, + url: parentRun.html_url ?? evidence.parentView.url, + ...identity, + workflowPath: workflowPath(parentRun), + workflowRef: manifest.workflowRef, + workflowSha: manifest.workflowSha, + }; +} + +export function resolveVerifierIdentity( + sourceSha, + verifierSourceContent, + repositoryRoot = RELEASE_EVIDENCE_REPO_ROOT, +) { + let normalizedSourceSha = sourceSha ?? process.env.GITHUB_SHA; + if (!/^[a-f0-9]{40}$/u.test(String(normalizedSourceSha ?? ""))) { + try { + normalizedSourceSha = execFileSync("git", ["-C", repositoryRoot, "rev-parse", "HEAD"], { + encoding: "utf8", + stdio: ["ignore", "pipe", "ignore"], + }).trim(); + } catch { + normalizedSourceSha = null; + } + } + if (!/^[a-f0-9]{40}$/u.test(String(normalizedSourceSha ?? ""))) { + throw new Error("release evidence verifier source SHA is unavailable"); + } + const script = readFileSync(RELEASE_EVIDENCE_FILE); + const scriptSha256 = createHash("sha256").update(script).digest("hex"); + let sourceScript; + if (verifierSourceContent !== undefined) { + sourceScript = Buffer.from(verifierSourceContent); + } else { + try { + sourceScript = execFileSync( + "git", + ["-C", repositoryRoot, "show", `${normalizedSourceSha}:${RELEASE_EVIDENCE_SCRIPT}`], + { + // Evidence verification must stay local-deterministic: in a partial + // clone a missing blob would otherwise trigger a promisor network + // fetch (hang/minutes) inside this security check. + env: { ...process.env, GIT_NO_LAZY_FETCH: "1" }, + maxBuffer: 16 * 1024 * 1024, + stdio: ["ignore", "pipe", "pipe"], + }, + ); + } catch { + throw new Error("release evidence verifier source blob is unavailable"); + } + } + const sourceScriptSha256 = createHash("sha256").update(sourceScript).digest("hex"); + if (scriptSha256 !== sourceScriptSha256) { + throw new Error("release evidence verifier script differs from its source SHA"); + } + return { + schemaVersion: 3, + script: RELEASE_EVIDENCE_SCRIPT, + scriptSha256, + sourceSha: normalizedSourceSha, + }; +} + +async function validateStrictChildRun({ + child, + childEvidence, + client, + parentEvidence, + parentJobs, + plannedChild, + releaseProfile, + repository, + runId, + expectedRunAttempts, +}) { + const run = await client.getRun(runId); + const effectiveRunAttempt = normalizePositiveInteger( + run.run_attempt, + `${child.name} run attempt`, + ); + consumeExpectedRunAttempt(expectedRunAttempts, runId, effectiveRunAttempt, "child"); + if (plannedChild) { + try { + validateReleaseChildRunProvenance(run, { + ...plannedChild, + plannedRunAttempt: plannedChild.runAttempt, + repository, + }); + } catch { + throw new Error(`execution plan child dispatch tuple mismatch: ${child.name}`); + } + } + const originAttempt = resolveManifestChildOriginAttempt( + run, + child, + parentEvidence.manifest, + parentJobs, + ); + if (originAttempt === undefined) { + throw new Error(`manifest child dispatch tuple mismatch: ${child.name}`); + } + const parentJob = selectManifestParentJob( + parentJobs, + child, + parentEvidence.manifest, + originAttempt, + { requireSkippedCarryForward: plannedChild !== undefined }, + ); + validateManifestChildRun( + run, + child, + runId, + parentEvidence.manifest, + parentJobs, + await client.getJobLog(parentJob.id), + repository, + plannedChild?.runAttempt, + plannedChild !== undefined, + ); + let jobs; + let composite; + let currentAttemptJobs; + if (plannedChild && childEvidence) { + if (childEvidence.effectiveRunAttempt > effectiveRunAttempt) { + throw new Error(`manifest child composite evidence mismatch: ${child.name}`); + } + const attempts = []; + for ( + let runAttempt = plannedChild.runAttempt; + runAttempt <= childEvidence.effectiveRunAttempt; + runAttempt += 1 + ) { + currentAttemptJobs = await client.getRunAttemptJobs(runId, runAttempt); + attempts.push({ jobs: currentAttemptJobs, runAttempt }); + } + const evidence = composeReleaseChildAttemptEvidence({ + attempts, + expected: { + ...plannedChild, + plannedRunAttempt: plannedChild.runAttempt, + repository, + }, + run: + childEvidence.effectiveRunAttempt === effectiveRunAttempt + ? run + : { + ...run, + run_attempt: childEvidence.effectiveRunAttempt, + triggering_actor: { login: childEvidence.triggeringActor }, + }, + }); + const expectedEvidence = { + ...evidence, + }; + if ( + JSON.stringify(sortReleaseJsonValueKeys(childEvidence)) !== + JSON.stringify(sortReleaseJsonValueKeys(expectedEvidence)) + ) { + throw new Error(`manifest child composite evidence mismatch: ${child.name}`); + } + if (childEvidence.effectiveRunAttempt < effectiveRunAttempt) { + throw new ReleaseEvidenceRefreshRequiredError( + `successful parent manifest predates ${child.name} attempt ${effectiveRunAttempt}`, + ); + } + composite = { + effectiveRunAttempt: evidence.effectiveRunAttempt, + jobs: evidence.jobs, + plannedRunAttempt: evidence.plannedRunAttempt, + sha256: evidence.compositeJobsSha256, + }; + jobs = evidence.jobs; + } else { + jobs = + run.conclusion === "success" && child.manifestKey !== "productPerformance" + ? [] + : await client.getParentJobs(runId); + } + if ( + run.repository?.full_name !== repository || + run.head_sha !== (plannedChild?.workflowSha ?? parentEvidence.manifest.workflowSha) || + !terminalPolicyPass( + { + conclusion: run.conclusion, + jobs, + key: child.manifestKey, + status: run.status, + }, + releaseProfile, + parentEvidence.manifest.workflowRef, + ) + ) { + throw new Error(`manifest child run does not pass release policy: ${child.name}`); + } + if (child.manifestKey === "productPerformance") { + // A composite may carry earlier successes; the publication guard must pass + // in the current raw attempt, already fetched while composing the evidence. + validatePerformanceArtifactOnlyJobs( + composite + ? currentAttemptJobs.map((job) => + Object.assign({}, job, { run_attempt: effectiveRunAttempt }), + ) + : jobs, + effectiveRunAttempt, + ); + } + + return { + advisoryJobs: releaseAdvisoryJobEvidence( + { [child.manifestKey]: { jobs } }, + releaseProfile, + parentEvidence.manifest.workflowRef, + ), + conclusion: run.conclusion, + dispatchNonce: `full-release-validation-${parentEvidence.manifest.runId}-${originAttempt}${child.suffix}`, + displayTitle: run.display_title, + event: run.event, + headBranch: run.head_branch, + parentJobId: String(parentJob.id), + path: workflowPath(run), + policyPassed: true, + role: child.manifestKey, + ...(composite + ? { + compositeJobsSha256: composite.sha256, + dispatchActor: run.actor.login, + plannedRunAttempt: plannedChild.runAttempt, + triggeringActor: run.triggering_actor.login, + } + : {}), + runAttempt: effectiveRunAttempt, + runId: String(run.id), + sourceParentAttempt: originAttempt, + sourceParentRunId: parentEvidence.manifest.runId, + status: run.status, + url: run.html_url, + workflowSha: run.head_sha, + ...(child.manifestKey === "productPerformance" ? { reportPublication: "artifact-only" } : {}), + }; +} + +/** + * @param {{ + * manifestPath?: string, + * repository?: string, + * reuseRequest?: { releaseProfile: string, runReleaseSoak: string, targetSha: string, validationInputs: Record<string, unknown> }, + * runId: string, + * expectedChangedPaths?: string[], + * expectedEvidencePolicy?: string, + * expectedEvidenceSha?: string, + * expectedRootRunId?: string, + * expectedRunAttempts?: Record<string, number>, + * expectedSelectedRunId?: string, + * expectedTargetSha?: string, + * trustedWorkflowFullRef?: string, + * trustedWorkflowRef?: string, + * trustedWorkflowSha?: string, + * verifierSourceContent?: string | Uint8Array, + * verifierSourceSha: string, + * }} options + */ +export async function validateReleaseRunEvidence( + { + manifestPath, + repository = DEFAULT_REPO, + reuseRequest, + runId, + expectedChangedPaths, + expectedEvidencePolicy, + expectedEvidenceSha, + expectedRootRunId, + expectedRunAttempts, + expectedSelectedRunId, + expectedTargetSha, + trustedWorkflowFullRef, + trustedWorkflowRef = "main", + trustedWorkflowSha, + verifierSourceContent, + verifierSourceSha, + }, + client, +) { + const normalizedRepository = normalizeRepository(repository); + const normalizedRunId = normalizeRequiredRunId(runId, "full release run ID"); + const remainingExpectedRunAttempts = normalizeExpectedRunAttempts(expectedRunAttempts); + const normalizedTrustedWorkflowRef = normalizeWorkflowRef( + trustedWorkflowRef, + "trusted workflow ref", + ); + const trustedIdentity = resolveTrustedWorkflowIdentity( + normalizedTrustedWorkflowRef, + trustedWorkflowFullRef, + trustedWorkflowSha, + ); + const evidenceClient = client ?? createReleaseEvidenceClient(normalizedRepository); + const verifier = resolveVerifierIdentity(verifierSourceSha, verifierSourceContent); + const currentEvidence = await loadValidatedParentEvidence({ + client: evidenceClient, + expectedRunAttempts: remainingExpectedRunAttempts, + manifestPath, + repository: normalizedRepository, + runId: normalizedRunId, + }); + const requestedReuse = { + expectedChangedPaths, + expectedEvidencePolicy, + expectedEvidenceSha, + expectedRootRunId, + expectedSelectedRunId, + expectedTargetSha, + }; + if (reuseRequest !== undefined) { + // Reject mismatched searches before fetching root/child evidence. Matching + // metadata only admits a candidate to the full verifier below; it never passes it. + const manifest = currentEvidence.manifest; + if ( + manifest.version !== 4 || + manifest.evidenceReuse || + manifest.rerunGroup !== "all" || + manifest.releaseProfile !== reuseRequest.releaseProfile || + manifest.runReleaseSoak !== reuseRequest.runReleaseSoak || + JSON.stringify(sortReleaseJsonValueKeys(manifest.validationInputs)) !== + JSON.stringify(sortReleaseJsonValueKeys(reuseRequest.validationInputs)) + ) { + throw new Error( + "ineligible reuse candidate: requires a direct full run with matching profile, soak, and inputs", + ); + } + const exactTarget = manifest.targetSha === reuseRequest.targetSha; + const comparison = exactTarget + ? null + : evidenceClient.compareCommits(manifest.targetSha, reuseRequest.targetSha); + const delta = exactTarget + ? { changedPaths: [], policy: EXACT_TARGET_EVIDENCE_REUSE_POLICY } + : classifyReleaseChangelogEvidenceComparison(comparison, { + baseSha: manifest.targetSha, + version: + manifest.candidateBinding?.package?.version ?? manifest.validationInputs?.targetVersion, + }); + validateRequestedEvidenceReuse( + manifest, + manifest, + manifest, + { + expectedChangedPaths: delta.changedPaths, + expectedEvidencePolicy: delta.policy, + expectedEvidenceSha: manifest.targetSha, + expectedRootRunId: manifest.runId, + expectedSelectedRunId: manifest.runId, + expectedTargetSha: reuseRequest.targetSha, + }, + () => comparison, + ); + } + const producerIdentities = new Map([ + [ + currentEvidence.manifest.runId, + validateTrustedProducerIdentity( + currentEvidence, + evidenceClient, + verifier, + normalizedTrustedWorkflowRef, + trustedIdentity.fullRef, + trustedIdentity.sha, + ), + ], + ]); + + let rootEvidence = currentEvidence; + let selectedEvidence = currentEvidence; + const reuse = currentEvidence.manifest.evidenceReuse; + if (reuse) { + rootEvidence = await loadValidatedParentEvidence({ + client: evidenceClient, + expectedRunAttempts: remainingExpectedRunAttempts, + repository: normalizedRepository, + runId: reuse.runId, + }); + selectedEvidence = + reuse.selectedRunId === reuse.runId + ? rootEvidence + : await loadValidatedParentEvidence({ + client: evidenceClient, + expectedRunAttempts: remainingExpectedRunAttempts, + repository: normalizedRepository, + runId: reuse.selectedRunId, + }); + validateEvidenceReuseChain( + currentEvidence.manifest, + selectedEvidence.manifest, + rootEvidence.manifest, + (base, head) => evidenceClient.compareCommits(base, head), + ); + } + if (hasRequestedEvidenceReuse(requestedReuse)) { + validateRequestedEvidenceReuse( + currentEvidence.manifest, + selectedEvidence.manifest, + rootEvidence.manifest, + requestedReuse, + (base, head) => evidenceClient.compareCommits(base, head), + ); + } + + const sourcePlans = new Map(); + const sourceContracts = new Map(); + for (const evidence of [currentEvidence, selectedEvidence, rootEvidence]) { + if (!producerIdentities.has(evidence.manifest.runId)) { + producerIdentities.set( + evidence.manifest.runId, + validateTrustedProducerIdentity( + evidence, + evidenceClient, + verifier, + normalizedTrustedWorkflowRef, + trustedIdentity.fullRef, + trustedIdentity.sha, + ), + ); + } + const manifest = evidence.manifest; + if (!sourceContracts.has(manifest.workflowSha)) { + const workflow = evidenceClient.getWorkflowSource(manifest.workflowSha); + sourceContracts.set(manifest.workflowSha, { + workflow, + source: publicationSourceContract(workflow), + publication: publicationAdmissionContract(workflow), + }); + } + const { source: contract, publication: registryContract } = sourceContracts.get( + manifest.workflowSha, + ); + if (manifest.sourceAdmissionContract !== contract) { + throw new Error("source admission differs from the exact trusted workflow contract"); + } + if (manifest.publicationAdmissionContract !== registryContract) { + throw new Error("publication admission differs from the exact trusted workflow contract"); + } + validatePublicationSourceBinding(manifest, { sourceAdmissionContract: contract }); + validatePublicationAdmissionBinding(manifest, { + publicationAdmissionContract: registryContract, + }); + if (contract && !sourcePlans.has(manifest.runId)) { + const original = + registryContract && manifest.sourceAdmission.validationPurpose === "publish" + ? await restoreOriginalPublicationAdmission({ + request: manifest.sourceAdmission, + client: { + ...evidenceClient, + getWorkflowSource: (sha) => + sourceContracts.get(sha)?.workflow ?? evidenceClient.getWorkflowSource(sha), + }, + }) + : undefined; + const plan = validateReleaseExecutionPlanArtifact( + original?.plan ?? evidenceClient.loadExecutionPlan(manifest.runId), + { + sourceAdmissionContract: contract, + publicationAdmissionContract: registryContract, + parentRunId: manifest.runId, + repository: normalizedRepository, + targetSha: manifest.targetSha, + workflowRef: manifest.workflowRef, + workflowSha: manifest.workflowSha, + releaseProfile: manifest.releaseProfile, + rerunGroup: manifest.rerunGroup, + }, + ); + if ( + publicationSourceJson(plan.sourceAdmission) !== + publicationSourceJson(manifest.sourceAdmission) || + evidence.manifestJson.executionPlanSha256 !== plan.sha256 || + Number(evidence.manifestJson.sourceParentRunAttempt) !== plan.parentRunAttempt + ) { + throw new Error("source admission manifest differs from its immutable execution plan"); + } + if ( + registryContract && + publicationObservationJson(plan.publicationAdmission) !== + publicationObservationJson(manifest.publicationAdmission) + ) { + throw new Error("publication admission manifest differs from its original execution plan"); + } + sourcePlans.set(manifest.runId, plan); + } + } + const selectedKeys = requiredChildKeysForManifest(rootEvidence.manifest); + const executionPlanPayload = + sourcePlans.get(rootEvidence.manifest.runId) ?? + evidenceClient.loadExecutionPlan?.(rootEvidence.manifest.runId); + const executionPlan = executionPlanPayload + ? validateReleaseExecutionPlanArtifact(executionPlanPayload, { + parentRunId: rootEvidence.manifest.runId, + repository: normalizedRepository, + releaseProfile: rootEvidence.manifest.releaseProfile, + rerunGroup: rootEvidence.manifest.rerunGroup, + targetSha: rootEvidence.manifest.targetSha, + workflowRef: rootEvidence.manifest.workflowRef, + workflowSha: rootEvidence.manifest.workflowSha, + }) + : undefined; + validateReleaseTelegramWaiverBinding(executionPlan, rootEvidence.manifest.validationInputs); + validateReleaseCoveragePolicyBinding(executionPlan, rootEvidence.manifest.validationInputs); + const plannedByKey = new Map( + (executionPlan?.children ?? []).map((plannedChild) => [plannedChild.key, plannedChild]), + ); + if (executionPlan?.attemptEvidenceVersion !== undefined) { + if ( + rootEvidence.manifestJson.executionPlanSha256 !== executionPlan.sha256 || + Number(rootEvidence.manifestJson.sourceParentRunAttempt) !== executionPlan.parentRunAttempt || + JSON.stringify(sortReleaseJsonValueKeys(rootEvidence.manifest.candidateBinding)) !== + JSON.stringify(sortReleaseJsonValueKeys(executionPlan.candidate)) + ) { + throw new Error("release validation manifest differs from its immutable execution plan"); + } + if (!rootEvidence.manifest.childEvidence) { + throw new Error("release validation manifest omitted composite child evidence"); + } + if ( + executionPlan.coveragePolicy && + JSON.stringify( + executionPlan.children + .filter((child) => child.selected) + .map((child) => child.key) + .toSorted(), + ) !== JSON.stringify([...selectedKeys].toSorted()) + ) { + throw new Error( + "release validation selected child set differs from its immutable execution plan", + ); + } + } + const expectedChildren = executionPlan + ? childDispatchesForPhaseVersion(executionPlan.attemptEvidenceVersion === 3 ? 3 : 2) + .filter((child) => selectedKeys.has(child.manifestKey)) + .map((child) => { + const plannedChild = plannedByKey.get(child.manifestKey); + if ( + !plannedChild?.selected || + !plannedChild.required || + !plannedChild.runId || + !plannedChild.runAttempt + ) { + throw new Error(`execution plan omits required child: ${child.name}`); + } + return Object.assign({}, child, { + displayTitle: plannedChild.displayTitle, + headBranch: plannedChild.workflowRef, + plannedChild, + }); + }) + : expectedSelectedChildDispatches( + rootEvidence.manifest.runId, + rootEvidence.manifest.runAttempt, + rootEvidence.manifest.workflowRef, + selectedKeys, + rootEvidence.manifest.version === 4 ? 3 : 2, + ); + if ( + executionPlan?.attemptEvidenceVersion !== undefined && + JSON.stringify(Object.keys(rootEvidence.manifest.childEvidence).toSorted()) !== + JSON.stringify([...selectedKeys].toSorted()) + ) { + throw new Error("release validation manifest composite child set is invalid"); + } + const dispatchEvidence = rootEvidence; + const parentJobs = await evidenceClient.getParentJobs(dispatchEvidence.manifest.runId); + const childEntries = executionPlan + ? expectedChildren.map((child) => { + const manifestRunId = rootEvidence.manifest.childRunIds[child.manifestKey]; + if (manifestRunId !== child.plannedChild.runId) { + throw new Error(`execution plan and manifest child identity differ: ${child.name}`); + } + return { child, runId: child.plannedChild.runId }; + }) + : manifestChildEntries(rootEvidence.manifest, expectedChildren, selectedKeys); + // The fixed child set bounds concurrent reads (at most seven). Each child + // walks its attempts/pages serially; drain all reads before returning or failing. + const childResults = await Promise.allSettled( + childEntries.map(({ child, runId: childRunId }) => + validateStrictChildRun({ + child, + childEvidence: rootEvidence.manifest.childEvidence?.[child.manifestKey], + client: evidenceClient, + parentEvidence: dispatchEvidence, + parentJobs, + plannedChild: child.plannedChild, + releaseProfile: rootEvidence.manifest.releaseProfile, + repository: normalizedRepository, + runId: childRunId, + expectedRunAttempts: remainingExpectedRunAttempts, + }), + ), + ); + const children = childResults.map((result) => { + if (result.status === "rejected") { + throw result.reason; + } + return result.value; + }); + if (remainingExpectedRunAttempts?.size) { + throw new Error( + `expected run attempts contain unvalidated run IDs: ${[...remainingExpectedRunAttempts.keys()].join(", ")}`, + ); + } + + const current = normalizedParentTuple( + currentEvidence, + producerIdentities.get(currentEvidence.manifest.runId), + ); + const root = normalizedParentTuple( + rootEvidence, + producerIdentities.get(rootEvidence.manifest.runId), + ); + const childConclusions = Object.fromEntries( + children.map((child) => [child.role, child.conclusion]), + ); + return sortReleaseJsonValueKeys({ + children, + conclusions: { + allRequiredSucceeded: children.every((child) => child.policyPassed), + children: childConclusions, + current: current.conclusion, + root: root.conclusion, + }, + controls: rootEvidence.manifest.controls, + current, + directRoot: !reuse, + evidenceReuse: reuse + ? { + changedPaths: reuse.changedPaths, + evidenceSha: reuse.evidenceSha, + policy: reuse.policy, + rootRunId: reuse.runId, + selectedRunId: reuse.selectedRunId, + } + : null, + executionPlan: executionPlan + ? { + parentRunAttempt: executionPlan.parentRunAttempt, + sha256: executionPlan.sha256, + } + : null, + manifest: rootEvidence.manifestJson, + releaseProfile: rootEvidence.manifest.releaseProfile, + repository: normalizedRepository, + rerunGroup: rootEvidence.manifest.rerunGroup, + root, + runReleaseSoak: rootEvidence.manifest.runReleaseSoak === "true", + schema: + rootEvidence.manifest.version === 4 + ? PHASED_RELEASE_EVIDENCE_SCHEMA + : RELEASE_EVIDENCE_SCHEMA, + producerOnTrustedMainLineage: trustedIdentity.type === "branch", + trustedWorkflowFullRef: trustedIdentity.fullRef, + trustedWorkflowRef: normalizedTrustedWorkflowRef, + valid: true, + validationInputs: rootEvidence.manifest.validationInputs ?? null, + verifier, + }); +} + +function parseReleaseCiSummaryArgs(argv) { + const options = { + intervalMs: 30_000, + expectedChangedPaths: undefined, + expectedEvidencePolicy: undefined, + expectedEvidenceSha: undefined, + expectedRootRunId: undefined, + expectedRunAttempts: undefined, + expectedSelectedRunId: undefined, + expectedTargetSha: undefined, + json: false, + manifestPath: undefined, + repository: DEFAULT_REPO, + reuseRequest: undefined, + runId: undefined, + trustedWorkflowFullRef: undefined, + trustedWorkflowRef: "main", + trustedWorkflowSha: undefined, + validate: false, + verifierSourceFile: undefined, + verifierSourceSha: undefined, + watch: false, + }; + for (let index = 0; index < argv.length; index += 1) { + const argument = argv[index]; + if (argument === "--validate-run") { + options.validate = true; + options.runId = argv[++index]; + } else if (argument === "--repo") { + options.repository = argv[++index]; + } else if (argument === "--manifest") { + options.manifestPath = argv[++index]; + } else if (argument === "--reuse-request-json") { + options.reuseRequest = normalizeJsonObject(JSON.parse(argv[++index]), "reuse request"); + } else if (argument === "--trusted-workflow-ref") { + options.trustedWorkflowRef = argv[++index]; + } else if (argument === "--trusted-workflow-full-ref") { + options.trustedWorkflowFullRef = argv[++index]; + } else if (argument === "--trusted-workflow-sha") { + options.trustedWorkflowSha = argv[++index]; + } else if (argument === "--verifier-source-sha") { + options.verifierSourceSha = argv[++index]; + } else if (argument === "--verifier-source-file") { + options.verifierSourceFile = argv[++index]; + } else if (argument === "--expected-target-sha") { + options.expectedTargetSha = argv[++index]; + } else if (argument === "--expected-evidence-policy") { + options.expectedEvidencePolicy = argv[++index]; + } else if (argument === "--expected-evidence-sha") { + options.expectedEvidenceSha = argv[++index]; + } else if (argument === "--expected-root-run-id") { + options.expectedRootRunId = argv[++index]; + } else if (argument === "--expected-run-attempts-json") { + const value = argv[++index]; + if (!value || Buffer.byteLength(value, "utf8") > MAX_EXPECTED_RUN_ATTEMPTS_JSON_BYTES) { + throw new Error("--expected-run-attempts-json requires a bounded JSON object"); + } + try { + options.expectedRunAttempts = JSON.parse(value); + } catch { + throw new Error("--expected-run-attempts-json requires a JSON object"); + } + if (!options.expectedRunAttempts || Array.isArray(options.expectedRunAttempts)) { + throw new Error("--expected-run-attempts-json requires a JSON object"); + } + normalizeExpectedRunAttempts(options.expectedRunAttempts); + } else if (argument === "--expected-selected-run-id") { + options.expectedSelectedRunId = argv[++index]; + } else if (argument === "--expected-changed-paths-json") { + const value = argv[++index]; + try { + options.expectedChangedPaths = JSON.parse(value); + } catch { + throw new Error("--expected-changed-paths-json requires a JSON array"); + } + if ( + !Array.isArray(options.expectedChangedPaths) || + options.expectedChangedPaths.some((entry) => typeof entry !== "string") + ) { + throw new Error("--expected-changed-paths-json requires a JSON array"); + } + } else if (argument === "--json") { + options.json = true; + } else if (argument === "--watch") { + options.watch = true; + } else if (argument === "--interval") { + const seconds = argv[++index]; + if (!/^[1-9][0-9]*$/u.test(seconds ?? "")) { + throw new Error("--interval requires a positive number of seconds"); + } + options.intervalMs = Number(seconds) * 1000; + } else if (!argument.startsWith("-") && !options.runId && !options.validate) { + options.runId = argument; + } else { + throw new Error(`unknown or incomplete argument: ${argument}`); + } + } + if (!options.validate && options.manifestPath) { + throw new Error("--manifest requires --validate-run"); + } + if (!options.validate && options.reuseRequest !== undefined) { + throw new Error("--reuse-request-json requires --validate-run"); + } + if (!options.validate && options.expectedRunAttempts !== undefined) { + throw new Error("--expected-run-attempts-json requires --validate-run"); + } + if (options.validate && options.watch) { + throw new Error("--watch cannot be combined with --validate-run"); + } + if (options.verifierSourceFile && !options.verifierSourceSha) { + throw new Error("--verifier-source-file requires --verifier-source-sha"); + } + if (!options.runId) { + throw new Error("full release run ID is required"); + } + return options; +} + +function printUsage() { + console.error( + [ + "usage: release-ci-summary.mjs <full-release-run-id>", + " release-ci-summary.mjs <full-release-run-id> --watch [--interval seconds]", + " release-ci-summary.mjs --validate-run <id> [--repo owner/name] [--trusted-workflow-ref main --trusted-workflow-full-ref refs/heads/main] [--trusted-workflow-sha sha] [--manifest path] [--verifier-source-sha sha --verifier-source-file path] [--expected-target-sha sha --expected-evidence-sha sha --expected-evidence-policy policy --expected-root-run-id id --expected-selected-run-id id --expected-changed-paths-json json] [--expected-run-attempts-json json] [--reuse-request-json json] --json", + ].join("\n"), + ); +} + +function releaseCiWatchFingerprint(parent) { + return JSON.stringify({ + attempt: parent.attempt, + conclusion: parent.conclusion ?? "", + jobs: (parent.jobs ?? []) + .map((job) => ({ + conclusion: job.conclusion ?? "", + name: job.name, + status: job.status, + })) + .toSorted((left, right) => left.name.localeCompare(right.name)), + status: parent.status, + }); +} + +function terminalParentJobFailures(parent) { + return (parent.jobs ?? []) + .filter( + (job) => + job.status === "completed" && + !SUCCESSFUL_PARENT_JOB_CONCLUSIONS.has(String(job.conclusion ?? "")), + ) + .map((job) => String(job.name || "unnamed parent job")); +} + +export function tryReadReleaseDecisionArtifact( + parent, + runId, + repository, + runReleaseCiGhImpl = runReleaseCiGh, +) { + const artifactName = `full-release-decision-${runId}-${parent.attempt}`; + const downloadDir = mkdtempSync(join(tmpdir(), "openclaw-release-decision-watch-")); + try { + try { + runReleaseCiGhImpl( + [ + "run", + "download", + String(runId), + "--repo", + repository, + "--name", + artifactName, + "--dir", + downloadDir, + ], + { stdio: ["ignore", "ignore", "pipe"] }, + ); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + if (isReleaseGhArtifactMissingError(error)) { + return undefined; + } + if (classifyReleaseGhTransportError(error) === "transient") { + console.warn(`release decision artifact unavailable this poll; retrying: ${message}`); + return undefined; + } + throw new Error(`release decision artifact read failed: ${message}`, { cause: error }); + } + const path = join(downloadDir, "full-release-decision.json"); + if (!statSync(path, { throwIfNoEntry: false })) { + throw new Error(`release decision artifact ${artifactName} omitted its manifest`); + } + if (statSync(path).size > MAX_RELEASE_ARTIFACT_BYTES) { + throw new Error(`release decision artifact ${artifactName} exceeds the size limit`); + } + return validateReleaseStateArtifact( + JSON.parse(readFileSync(path, "utf8")), + { + parentRunAttempt: parent.attempt, + parentRunId: String(runId), + workflowSha: parent.headSha, + }, + "decision", + ); + } finally { + rmSync(downloadDir, { force: true, recursive: true }); + } +} + +function releaseDecisionBlockedDuringDrain(parent, runId, repository) { + const jobs = parent.jobs ?? []; + const decision = jobs.find((job) => job.name === "Release Decision"); + const drain = jobs.find((job) => job.name === "Diagnostic Drain"); + if ( + decision?.status !== "completed" || + SUCCESSFUL_PARENT_JOB_CONCLUSIONS.has(String(decision.conclusion ?? "")) || + !drain || + drain.status === "completed" + ) { + return undefined; + } + return tryReadReleaseDecisionArtifact(parent, runId, repository); +} + +function summarizeReleaseCiRun(options) { + execFileSync( + process.execPath, + [ + RELEASE_EVIDENCE_FILE, + options.runId, + "--repo", + options.repository, + "--trusted-workflow-ref", + options.trustedWorkflowRef, + ], + { stdio: "inherit" }, + ); +} + +async function watchReleaseCiRun(options) { + let previousFingerprint; + while (true) { + const parent = jsonGh([ + "run", + "view", + options.runId, + "--repo", + options.repository, + "--json", + "status,conclusion,attempt,headSha,jobs", + ]); + const fingerprint = releaseCiWatchFingerprint(parent); + if (fingerprint !== previousFingerprint) { + summarizeReleaseCiRun(options); + previousFingerprint = fingerprint; + } + const blockedDuringDrain = releaseDecisionBlockedDuringDrain( + parent, + options.runId, + options.repository, + ); + if (blockedDuringDrain) { + throw new Error( + `full release run ${options.runId} stopped at Release Decision:\n${formatReleaseStateOutcome(blockedDuringDrain)}`, + ); + } + const failedJobs = terminalParentJobFailures(parent); + if (failedJobs.length > 0) { + throw new Error( + `full release run ${options.runId} has terminal parent job failure(s): ${failedJobs.join(", ")}`, + ); + } + if (parent.status === "completed") { + if (parent.conclusion !== "success") { + throw new Error( + `full release run ${options.runId} completed with ${parent.conclusion || "no conclusion"}`, + ); + } + return; + } + await new Promise((complete) => { + setTimeout(complete, options.intervalMs); + }); + } +} + +async function main() { + let options; + try { + options = parseReleaseCiSummaryArgs(process.argv.slice(2)); + } catch (error) { + printUsage(); + console.error(error instanceof Error ? error.message : String(error)); + process.exit(2); + } + const { repository, runId } = options; + + if (options.validate) { + try { + const evidence = await validateReleaseRunEvidence({ + expectedChangedPaths: options.expectedChangedPaths, + expectedEvidencePolicy: options.expectedEvidencePolicy, + expectedEvidenceSha: options.expectedEvidenceSha, + expectedRootRunId: options.expectedRootRunId, + expectedRunAttempts: options.expectedRunAttempts, + expectedSelectedRunId: options.expectedSelectedRunId, + expectedTargetSha: options.expectedTargetSha, + manifestPath: options.manifestPath, + repository, + reuseRequest: options.reuseRequest, + runId, + trustedWorkflowFullRef: options.trustedWorkflowFullRef, + trustedWorkflowRef: options.trustedWorkflowRef, + trustedWorkflowSha: options.trustedWorkflowSha, + verifierSourceContent: options.verifierSourceFile + ? readFileSync(options.verifierSourceFile) + : undefined, + verifierSourceSha: options.verifierSourceSha, + }); + console.log(JSON.stringify(evidence, null, options.json ? 2 : 0)); + } catch (error) { + const failure = { + error: error instanceof Error ? error.message : String(error), + ...(error instanceof ReleaseEvidenceRefreshRequiredError ? { refreshable: true } : {}), + schema: RELEASE_EVIDENCE_SCHEMA, + valid: false, + }; + if (options.json) { + console.log(JSON.stringify(failure, null, 2)); + } else { + console.error(failure.error); + } + process.exit(1); + } + return; + } + if (options.watch) { + await watchReleaseCiRun(options); + return; + } + + const core = rate(); + if (core) { + const reset = new Date(core.reset * 1000).toISOString(); + console.log(`rate: remaining=${core.remaining}/${core.limit} reset=${reset}`); + if (core.remaining < 20) { + console.error("rate too low for CI summary; wait for reset before polling"); + process.exit(3); + } + } + + const parent = jsonGh([ + "run", + "view", + runId, + "--repo", + repository, + "--json", + "status,conclusion,attempt,headBranch,headSha,url,jobs", + ]); + validateParentRunBinding(parent, githubRestJson(`actions/runs/${runId}`, repository), runId); + + console.log(`parent: ${runId} ${parent.status}/${parent.conclusion || "none"}`); + console.log(`workflow-ref: ${parent.headBranch}`); + console.log(`workflow-sha: ${parent.headSha}`); + console.log(`url: ${parent.url}`); + + for (const job of parent.jobs ?? []) { + const marker = job.conclusion || job.status; + console.log(`parent-job: ${marker} ${job.name}`); + } + + const currentManifestRaw = tryDownloadParentManifest(runId, parent.attempt, repository); + let children; + if (currentManifestRaw) { + const currentManifest = validateParentManifest(currentManifestRaw, { + runAttempt: parent.attempt, + runId, + workflowRef: parent.headBranch, + workflowSha: parent.headSha, + }); + console.log(`candidate-sha: ${currentManifest.targetSha}`); + console.log(`manifest-run: ${currentManifest.runId}/${currentManifest.runAttempt}`); + + let sourceManifest = currentManifest; + let sourceParent = parent; + if (currentManifest.evidenceReuse) { + const rootRunId = currentManifest.evidenceReuse.runId; + const rootParent = jsonGh([ + "run", + "view", + rootRunId, + "--repo", + repository, + "--json", + "status,conclusion,attempt,headBranch,headSha,url,jobs", + ]); + validateParentRunBinding( + rootParent, + githubRestJson(`actions/runs/${rootRunId}`, repository), + rootRunId, + ); + if (rootParent.status !== "completed" || rootParent.conclusion !== "success") { + throw new Error(`evidence root run is not completed/success: ${rootRunId}`); + } + const rootManifestRaw = tryDownloadParentManifest(rootRunId, rootParent.attempt, repository); + if (!rootManifestRaw) { + throw new Error(`evidence root manifest is unavailable: ${rootRunId}`); + } + const rootManifest = validateParentManifest(rootManifestRaw, { + runAttempt: rootParent.attempt, + runId: rootRunId, + workflowRef: rootParent.headBranch, + workflowSha: rootParent.headSha, + }); + + const selectedRunId = currentManifest.evidenceReuse.selectedRunId; + let selectedManifest = rootManifest; + if (selectedRunId !== rootRunId) { + const selectedParent = jsonGh([ + "run", + "view", + selectedRunId, + "--repo", + repository, + "--json", + "status,conclusion,attempt,headBranch,headSha,url,jobs", + ]); + validateParentRunBinding( + selectedParent, + githubRestJson(`actions/runs/${selectedRunId}`, repository), + selectedRunId, + ); + if (selectedParent.status !== "completed" || selectedParent.conclusion !== "success") { + throw new Error(`selected evidence run is not completed/success: ${selectedRunId}`); + } + const selectedManifestRaw = tryDownloadParentManifest( + selectedRunId, + selectedParent.attempt, + repository, + ); + if (!selectedManifestRaw) { + throw new Error(`selected evidence manifest is unavailable: ${selectedRunId}`); + } + selectedManifest = validateParentManifest(selectedManifestRaw, { + runAttempt: selectedParent.attempt, + runId: selectedRunId, + workflowRef: selectedParent.headBranch, + workflowSha: selectedParent.headSha, + }); + } + + const evidenceSha = validateEvidenceReuseChain( + currentManifest, + selectedManifest, + rootManifest, + (base, head) => githubRestJson(`compare/${base}...${head}`, repository), + ); + sourceManifest = rootManifest; + sourceParent = rootParent; + console.log(`evidence-selected-run: ${selectedRunId}`); + console.log(`evidence-root-run: ${rootRunId}`); + console.log(`evidence-sha: ${evidenceSha}`); + console.log(`evidence-policy: ${currentManifest.evidenceReuse.policy}`); + console.log( + `evidence-changed-paths: ${JSON.stringify(currentManifest.evidenceReuse.changedPaths)}`, + ); + } + + const selectedKeys = requiredChildKeysForManifest(sourceManifest); + for (const job of sourceManifest.advisoryJobs) { + console.log(`advisory: ${job.child} ${job.status}/${job.conclusion || "none"} ${job.job}`); + } + const expectedChildren = expectedSelectedChildDispatches( + sourceManifest.runId, + sourceManifest.runAttempt, + sourceManifest.workflowRef, + selectedKeys, + sourceManifest.version === 4 ? 3 : 2, + ); + const sourceParentJobs = await findParentJobsAll(sourceManifest.runId, repository); + children = []; + for (const { child, runId: childRunId } of manifestChildEntries( + sourceManifest, + expectedChildren, + selectedKeys, + )) { + const run = githubRestJson(`actions/runs/${childRunId}`, repository); + const originAttempt = resolveManifestChildOriginAttempt( + run, + child, + sourceManifest, + sourceParentJobs, + ); + if (originAttempt === undefined) { + throw new Error(`manifest child dispatch tuple mismatch: ${child.name}`); + } + const parentJob = selectManifestParentJob( + sourceParentJobs, + child, + sourceManifest, + originAttempt, + ); + const validatedRun = validateManifestChildRun( + run, + child, + childRunId, + { ...sourceManifest, workflowSha: sourceParent.headSha }, + sourceParentJobs, + await parentJobLog(parentJob.id, repository), + repository, + ); + if (child.manifestKey === "productPerformance") { + validatePerformanceArtifactOnlyJobs( + await findParentJobsAll(childRunId, repository), + run.run_attempt, + ); + } + children.push({ child, run: validatedRun }); + } + } else { + console.log("candidate-sha: unavailable (release validation manifest not uploaded)"); + if (parent.status === "completed" && parent.conclusion === "success") { + throw new Error("successful parent run is missing its release validation manifest"); + } + const selectedKeys = selectedChildKeys(parent.jobs ?? []); + children = expectedSelectedChildDispatches( + runId, + parent.attempt, + parent.headBranch, + selectedKeys, + selectedKeys.has("releaseChecksCandidate") ? 3 : 2, + ) + .map((child) => { + const run = findExactChildRun(child, repository); + if (!run) { + console.log( + `child-missing: ${child.name} title=${child.displayTitle} branch=${child.headBranch}`, + ); + } + return { child, run }; + }) + .filter((entry) => entry.run); + } + if (children.length === 0) { + console.log("children: none found yet"); + return; + } + + console.log("children:"); + for (const { child, run } of children) { + console.log( + `child: ${run.id} ${child.name} ${run.status}/${run.conclusion || "none"} branch=${run.head_branch} workflow_sha=${run.head_sha}`, + ); + console.log(`child-url: ${run.html_url}`); + } +} + +if (process.argv[1]?.endsWith("release-ci-summary.mjs")) { + await main().catch( + /** @param {unknown} error */ (error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); + }, + ); +} diff --git a/scripts/release-fast-pretag-check.sh b/scripts/release-fast-pretag-check.sh new file mode 100644 index 0000000000000000000000000000000000000000..a11f5672a23002d19b9588486fec3e76138bf5c5 --- /dev/null +++ b/scripts/release-fast-pretag-check.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -euo pipefail + +version="$(node -p 'require("./package.json").version')" +release_tag="${RELEASE_TAG:-v${version}}" +export RELEASE_TAG="${release_tag}" + +echo "release tag: ${RELEASE_TAG}" +git diff --check +pnpm check:temp-path-guardrails +pnpm plugins:sync:check +pnpm release:generated:check +pnpm release:plugins:npm:check -- --selection-mode all-publishable +pnpm release:plugins:clawhub:check -- --selection-mode all-publishable +node --import tsx scripts/plugin-release-pretag-pack-check.ts +pnpm build +OPENCLAW_CONTROL_UI_RELEASE_BUILD=1 pnpm ui:build +pnpm release:openclaw:npm:check diff --git a/scripts/release-plan-producer.mts b/scripts/release-plan-producer.mts new file mode 100644 index 0000000000000000000000000000000000000000..ab871af6b194e9d2deef65155443231901321ba7 --- /dev/null +++ b/scripts/release-plan-producer.mts @@ -0,0 +1,695 @@ +#!/usr/bin/env node +import { execFileSync } from "node:child_process"; +import * as fs from "node:fs"; +import { isBuiltin } from "node:module"; +import { delimiter, join, posix, resolve } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +export type ReleasePlanIntent = + | "publish" + | "diagnostic" + | "postpublish-confidence" + | "main-qualification"; +export type MainQualificationValidationIntent = "main-daily" | "main-weekly"; +type RunGh = (args: string[]) => string; +export type ReleaseInventorySource = { + repoRoot?: string; + candidateSha: string; + toolingSha: string; + toolingFullRef: string; + runGh?: RunGh; +}; +type ReleasePlanSourceBase = ReleaseInventorySource & { candidateRef: string }; +export type ReleasePlanSource = + | (ReleasePlanSourceBase & { + intent: "main-qualification"; + validationIntent: MainQualificationValidationIntent; + }) + | (ReleasePlanSourceBase & { + intent: Exclude<ReleasePlanIntent, "main-qualification">; + validationIntent?: never; + }); +type ReleasePlan = { + schema: string; + release_id: string; + version: string; + tag: string | null; + candidate_sha: string; + target_context_ref: string; + purpose: string; + tooling: { repository: string; workflow_path: string; ref: string; sha: string }; + validation: { + intent: string; + profile: string; + soak: boolean; + allowed_groups: string[]; + }; + inventory: { + packages: Array<{ name: string; version: string; targets: string[] }>; + platforms: Array<{ id: string; source: string }>; + }; +}; +type ReleasePlanLock = Record<"schema" | "digest", string> & { plan: ReleasePlan }; +export type VerifiedReleaseInventory = { + candidateSha: string; + tooling: ReleasePlan["tooling"]; + version: string; + inventory: ReleasePlan["inventory"]; +}; + +const REPOSITORY = "openclaw/openclaw"; +const EXECUTION_ROOT = fileURLToPath(new URL("..", import.meta.url)); +const BOOTSTRAP_PATH = "scripts/release-plan-producer.mts", + CORE_PATH = "scripts/release-plan-producer-core.mts"; +const TOOLING_MODULE_PATHS = [ + "packages/normalization-core/src/record-coerce.ts", + "packages/normalization-core/src/string-coerce.ts", + "packages/plugin-package-contract/src/categories.ts", + "packages/plugin-package-contract/src/index.ts", + "scripts/lib/bounded-response.mjs", + "scripts/lib/canonical-json.mjs", + "scripts/lib/npm-publish-plan.mjs", + "scripts/lib/plugin-publication-candidates.ts", + "scripts/lib/plugin-publication-collector.ts", + "scripts/lib/plugin-publication-target.mjs", + "scripts/lib/pnpm-lockfile-documents.mjs", + "scripts/lib/record-shared.mjs", + "scripts/lib/release-version.mjs", + CORE_PATH, + "scripts/release-plan-contract.mjs", + "scripts/release-tooling-identity.mjs", + "scripts/release-validation-intent.mjs", +] as const; +const PROTECTED_TAG_PATTERN = /^release-publish\/([a-f0-9]{12})-([1-9][0-9]*)$/u; +const MAX_TOOLING_FILE_BYTES = 512 * 1024, + MAX_TOOLING_BYTES = 2 * 1024 * 1024; +const YAML_PACKAGE_TREE_SHA256 = "610ccacfe592d226ac1eb04842d1f591c5381f2a68b9f785643101d10db52c27"; +const YAML_PACKAGE_MAX_FILES = 512; +const YAML_PACKAGE_MAX_ENTRIES = 1024; +const YAML_PACKAGE_MAX_BYTES = 4 * 1024 * 1024; +// Keep both comparators local: this one precedes tooling verification, and CHILD_RUNNER's precedes +// loader-hook registration. Importing either would execute code before its integrity boundary. +const compareAscii = (left: string, right: string) => (left < right ? -1 : left > right ? 1 : 0); + +function resolveReleasePlanNodeExecutable(): string | undefined { + if (!process.versions.bun && process.allowedNodeEnvironmentFlags.has("--input-type")) { + return process.execPath; + } + + const names = + process.platform === "win32" + ? (process.env.PATHEXT ?? ".EXE;.CMD;.BAT;.COM") + .split(";") + .filter(Boolean) + .map((extension) => `node${extension.toLowerCase()}`) + : ["node"]; + const candidates = (process.env.PATH ?? "") + .split(delimiter) + .filter(Boolean) + .flatMap((directory) => names.map((name) => join(directory, name))); + if (process.platform === "darwin") { + candidates.push( + "/opt/homebrew/bin/node", + "/opt/homebrew/opt/node/bin/node", + "/usr/local/bin/node", + "/usr/local/opt/node/bin/node", + "/usr/bin/node", + ); + } else if (process.platform === "linux") { + candidates.push("/usr/local/bin/node", "/usr/bin/node"); + } + + const probeEnv: NodeJS.ProcessEnv = {}; + for (const key of ["SystemRoot", "SYSTEMROOT", "WINDIR", "TEMP", "TMP", "TMPDIR"]) { + const value = process.env[key]; + if (value) { + probeEnv[key] = value; + } + } + const probeSource = + 'process.stdout.write(!process.versions.bun&&process.allowedNodeEnvironmentFlags.has("--input-type")?process.execPath:"")'; + for (const candidate of new Set(candidates)) { + try { + const nodePath = execFileSync(candidate, ["--eval", probeSource], { + encoding: "utf8", + env: probeEnv, + stdio: ["ignore", "pipe", "ignore"], + timeout: 5_000, + }).trim(); + if (nodePath) { + return nodePath; + } + } catch { + // Missing, non-executable, incompatible, and Bun shim candidates are skipped. + } + } + return undefined; +} + +type ToolingModule = { path: string; bytes: Buffer; imports: Array<[string, string]> }; +type YamlEntry = + | { kind: "directory"; path: string } + | { kind: "file"; path: string; bytes: Buffer }; +type SerializableSource = Omit<ReleasePlanSourceBase, "runGh"> & Record<string, unknown>; +type ProducerRequest = + | { operation: "produce" | "produce-lock"; params: SerializableSource } + | { operation: "produce-inventory"; params: Omit<ReleaseInventorySource, "runGh"> } + | { operation: "verify-lock"; lockJson: string; params: SerializableSource }; + +const CHILD_RUNNER = String.raw` +import { createHash } from "node:crypto"; import { readFileSync } from "node:fs"; +import { createRequire, isBuiltin, registerHooks } from "node:module"; +const TOOLING_ROOT = "file:///__openclaw_verified_tooling__/", YAML_ROOT = "file:///__openclaw_verified_yaml__/"; +const YAML_ABSOLUTE_ROOT = "/__openclaw_verified_yaml__", CORE_PATH = ${JSON.stringify(CORE_PATH)}; +const EXPECTED_TOOLING_PATHS = ${JSON.stringify(TOOLING_MODULE_PATHS)}; +const compareAscii = (left, right) => left < right ? -1 : left > right ? 1 : 0; +const fail = message => { throw new Error(message); }; +const decodeBase64 = value => { + if (typeof value !== "string" || value.length % 4 !== 0 || !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(value)) fail("verified retained bytes are not canonical base64"); + const bytes = Buffer.from(value, "base64"); + if (bytes.toString("base64") !== value) fail("verified retained bytes are not canonical base64"); + return bytes; +}; +const toolingUrl = path => new URL(path, TOOLING_ROOT).href, yamlUrl = path => new URL(path, YAML_ROOT).href; +const safePath = path => typeof path === "string" && path.length > 0 && + /^[\x20-\x7e]+$/.test(path) && !path.includes("\\") && !path.startsWith("/") && + !path.split("/").some(part => part === "." || part === ".."); +try { + const payload = JSON.parse(readFileSync(0, "utf8")), expectedPaths = [...EXPECTED_TOOLING_PATHS].sort(compareAscii); + if (!Array.isArray(payload.toolingModules) || payload.toolingModules.length !== expectedPaths.length) fail("verified tooling module set is incomplete"); + const toolingModules = new Map(); + for (const record of payload.toolingModules) { + if (!record || !expectedPaths.includes(record.path) || toolingModules.has(record.path) || !Array.isArray(record.imports)) fail("verified tooling module record is invalid"); + toolingModules.set(record.path, { bytes: decodeBase64(record.bytesBase64), + format: record.path.endsWith(".mjs") ? "module" : "module-typescript", + imports: new Map(record.imports) }); + } + if ([...toolingModules.keys()].sort(compareAscii).some((path, index) => path !== expectedPaths[index])) fail("verified tooling module paths do not match the allowlist"); + if (!Array.isArray(payload.yamlEntries) || payload.yamlEntries.length > ${YAML_PACKAGE_MAX_ENTRIES}) fail("verified yaml retained tree has too many entries"); + const yamlModules = new Map(), yamlRecords = []; let yamlFiles = 0, yamlBytes = 0; + for (const entry of payload.yamlEntries) { + if (!entry || !safePath(entry.path)) fail("verified yaml retained tree contains an unsafe path"); + if (entry.kind === "directory") { + yamlRecords.push(JSON.stringify(["directory", entry.path])); continue; + } + if (entry.kind !== "file" || yamlModules.has(entry.path)) fail("verified yaml retained tree contains an invalid entry"); + const bytes = decodeBase64(entry.bytesBase64); yamlFiles += 1; yamlBytes += bytes.byteLength; + if (yamlFiles > ${YAML_PACKAGE_MAX_FILES} || yamlBytes > ${YAML_PACKAGE_MAX_BYTES}) fail("verified yaml retained tree exceeds its bounds"); + yamlModules.set(entry.path, bytes); + yamlRecords.push(JSON.stringify(["file", entry.path, bytes.byteLength, createHash("sha256").update(bytes).digest("hex")])); + } + const yamlManifest = yamlRecords.sort(compareAscii).join("\n") + "\n"; + if (createHash("sha256").update(yamlManifest, "ascii").digest("hex") !== ${JSON.stringify(YAML_PACKAGE_TREE_SHA256)}) fail("verified yaml retained tree digest mismatch"); + const packageBytes = yamlModules.get("package.json"); + if (!packageBytes) fail("verified yaml retained package.json is missing"); + const yamlPackage = JSON.parse(packageBytes.toString("utf8")); + if (yamlPackage.name !== "yaml" || yamlPackage.version !== "2.9.0") fail("verified yaml retained package identity mismatch"); + registerHooks({ + resolve(specifier, context, nextResolve) { + if (isBuiltin(specifier)) return nextResolve(specifier, context); + if (context.parentURL?.startsWith(TOOLING_ROOT)) { + const parentPath = context.parentURL.slice(TOOLING_ROOT.length); + const targetPath = toolingModules.get(parentPath)?.imports.get(specifier); + if (!targetPath) fail("verified tooling import is not allowlisted"); + return { url: toolingUrl(targetPath), format: toolingModules.get(targetPath).format, shortCircuit: true }; + } + if (specifier === toolingUrl(CORE_PATH)) return { url: specifier, format: toolingModules.get(CORE_PATH).format, shortCircuit: true }; + const targetUrl = specifier.startsWith(YAML_ABSOLUTE_ROOT + "/") ? + yamlUrl(specifier.slice(YAML_ABSOLUTE_ROOT.length + 1)) : + context.parentURL?.startsWith(YAML_ROOT) && specifier.startsWith(".") ? + new URL(specifier, context.parentURL).href : undefined; + if (targetUrl) { + const targetPath = targetUrl.slice(YAML_ROOT.length); + if (!yamlModules.has(targetPath)) fail("verified yaml import is not retained"); + return { url: targetUrl, format: "commonjs", shortCircuit: true }; + } + fail("verified child rejected an external module import"); + }, + load(url, context, nextLoad) { + if (url.startsWith(TOOLING_ROOT)) { + const record = toolingModules.get(url.slice(TOOLING_ROOT.length)); + if (!record) fail("verified tooling module is not retained"); + return { format: record.format, source: record.bytes, shortCircuit: true }; + } + if (url.startsWith(YAML_ROOT)) { + const bytes = yamlModules.get(url.slice(YAML_ROOT.length)); + if (!bytes) fail("verified yaml module is not retained"); + return { format: "commonjs", source: bytes, shortCircuit: true }; + } + if (url.startsWith("node:")) return nextLoad(url, context); + fail("verified child rejected an external module load"); + }, + }); + const identityResponses = new Map(payload.identityResponses); + if (identityResponses.size !== 1) fail("verified identity response cache must contain exactly one request"); + const runGh = args => { + const key = JSON.stringify(args); + if (!identityResponses.has(key)) fail("verified child rejected an uncached GitHub request"); + return identityResponses.get(key); + }; + let parseYaml; + const parseYamlDocuments = sources => { + if (!Array.isArray(sources) || sources.length !== 3 || sources.some(value => typeof value !== "string")) fail("verified yaml parser input must contain three workflow strings"); + if (!parseYaml) { + const yaml = createRequire(import.meta.url)(YAML_ABSOLUTE_ROOT + "/dist/index.js"); + if (typeof yaml.parse !== "function") fail("verified yaml parser must export parse"); parseYaml = yaml.parse; + } + return sources.map(parseYaml); + }; + const core = await import(toolingUrl(CORE_PATH)), value = core.runReleasePlanProducerOperation(payload.request, { runGh, parseYamlDocuments }); + process.stdout.write(JSON.stringify({ ok: true, value })); +} catch (error) { + process.stdout.write(JSON.stringify({ ok: false, message: error instanceof Error ? error.message : String(error) })); +} +`; + +const gitBytes = (repoRoot: string, args: string[]) => + execFileSync("git", args, { + cwd: repoRoot, + encoding: null, + maxBuffer: 16 * 1024 * 1024, + stdio: ["ignore", "pipe", "pipe"], + }); + +function requireSha(value: string, label: string) { + if (!/^[a-f0-9]{40}$/u.test(value)) { + throw new Error(`${label} must be an exact lowercase 40-character commit SHA`); + } + return value; +} + +function defaultRunGh(args: string[]) { + return execFileSync("gh", args, { + encoding: "utf8", + killSignal: "SIGKILL", + maxBuffer: 1024 * 1024, + stdio: ["ignore", "pipe", "pipe"], + timeout: 60_000, + }); +} + +function verifyRemoteTooling( + params: ReleaseInventorySource | ReleasePlanSource, + runGh: RunGh, + inventoryOnly: boolean, +) { + const sha = requireSha(params.toolingSha, "tooling SHA"); + const tagRef = params.toolingFullRef.replace(/^refs\/tags\//u, ""); + const protectedMatch = PROTECTED_TAG_PATTERN.exec(tagRef); + // Acquisition is not admission: the verified child owns canonical branch policy. + // Preserve its literal GET argv while rejecting reserved and traversal operands. + const inventoryBranch = + inventoryOnly && + params.toolingFullRef !== "refs/heads/main" && + /^refs\/heads\/[A-Za-z0-9._/-]{1,256}$/u.test(params.toolingFullRef) && + params.toolingFullRef.trim() === params.toolingFullRef && + !params.toolingFullRef.includes("..") && + params.toolingFullRef.split("/").every((part) => part !== "" && part !== "."); + let args: string[], failure: string; + if (protectedMatch) { + if (protectedMatch[1] !== sha.slice(0, 12)) { + throw new Error("protected release tooling tag SHA prefix does not match the workflow SHA"); + } + args = ["api", `repos/${REPOSITORY}/git/ref/tags/${tagRef}`, "--method", "GET"]; + failure = "protected release tooling tag is missing or unreadable"; + } else if (params.toolingFullRef === "refs/heads/main") { + // Keep this bounded query identical to the verified child's cached identity request. + args = [ + "api", + `repos/${REPOSITORY}/compare/${sha}...main`, + "--method", + "GET", + "--jq", + "{status}", + ]; + failure = "main release tooling ancestry could not be verified"; + } else if (inventoryBranch) { + args = [ + "api", + `repos/${REPOSITORY}/git/ref/heads/${params.toolingFullRef.slice("refs/heads/".length)}`, + "--method", + "GET", + ]; + failure = "inventory release tooling branch is missing or unreadable"; + } else { + throw new Error("release tooling identity must be trusted main or an exact protected tag"); + } + let raw: string; + try { + raw = runGh(args); + } catch (error) { + throw new Error(failure, { cause: error }); + } + const response = JSON.parse(raw) as { + ref?: unknown; + status?: unknown; + object?: { type?: unknown; sha?: unknown }; + }; + if ( + (protectedMatch || inventoryBranch) && + (response.ref !== params.toolingFullRef || + response.object?.type !== "commit" || + response.object.sha !== sha) + ) { + throw new Error( + `${inventoryBranch ? "inventory release tooling branch" : "protected release tooling tag"} is missing, moved, annotated, or bound to the wrong SHA`, + ); + } + if ( + !protectedMatch && + !inventoryBranch && + response.status !== "ahead" && + response.status !== "identical" + ) { + throw new Error("main release tooling SHA is not reachable from current main"); + } + if ( + !inventoryOnly && + "intent" in params && + params.intent !== "diagnostic" && + params.intent !== "main-qualification" && + !protectedMatch + ) { + throw new Error(`${params.intent} tooling must use a release-publish tag bound to its SHA`); + } + return [[JSON.stringify(args), raw]] as Array<[string, string]>; +} + +function readGitFile(repoRoot: string, sha: string, path: string) { + const entry = gitBytes(repoRoot, ["ls-tree", sha, "--", path]).toString("utf8").trim(); + if (!/^100(?:644|755) blob [a-f0-9]{40}\t/u.test(entry)) { + throw new Error(`tooling closure path must be a regular Git blob: ${path}`); + } + return gitBytes(repoRoot, ["show", `${sha}:${path}`]); +} + +function gitPathExists(repoRoot: string, sha: string, path: string) { + try { + execFileSync("git", ["cat-file", "-e", `${sha}:${path}`], { cwd: repoRoot, stdio: "ignore" }); + return true; + } catch { + return false; + } +} + +function collectLiteralImports(source: string) { + return [ + ...new Set( + [...source.matchAll(/\b(?:from\s+|import\s*(?:\(\s*)?)["']([^"']+)["']/gu)].map( + (match) => match[1]!, + ), + ), + ].toSorted(compareAscii); +} + +function resolveImport(repoRoot: string, sha: string, sourcePath: string, specifier: string) { + if (!specifier.startsWith(".") || specifier.startsWith("file:")) { + throw new Error(`tooling closure contains an unowned import: ${sourcePath} -> ${specifier}`); + } + const importedPath = posix.normalize(posix.join(posix.dirname(sourcePath), specifier)); + if (importedPath === ".." || importedPath.startsWith("../")) { + throw new Error(`tooling import escapes repository root: ${sourcePath} -> ${specifier}`); + } + const candidates = new Set([importedPath]); + if (importedPath.endsWith(".js")) { + candidates.add(`${importedPath.slice(0, -3)}.ts`); + } else if (importedPath.endsWith(".mjs")) { + candidates.add(`${importedPath.slice(0, -4)}.mts`); + } else if (!posix.extname(importedPath)) { + for (const suffix of [".ts", ".mts", ".mjs", "/index.ts"]) { + candidates.add(`${importedPath}${suffix}`); + } + } + const existing = [...candidates].filter((path) => gitPathExists(repoRoot, sha, path)); + if (existing.length !== 1) { + throw new Error( + `tooling import must resolve to one unambiguous file: ${sourcePath} -> ${specifier}`, + ); + } + const [target] = existing; + if (!target || !TOOLING_MODULE_PATHS.includes(target as (typeof TOOLING_MODULE_PATHS)[number])) { + throw new Error(`tooling import is outside the fixed closure: ${sourcePath} -> ${specifier}`); + } + return target; +} + +function retainToolingClosure(repoRoot: string, sha: string) { + const pending = [CORE_PATH]; + const modules = new Map<string, ToolingModule>(); + let totalBytes = 0; + while (pending.length > 0) { + const path = pending.pop(); + if (!path || modules.has(path)) { + continue; + } + const bytes = readGitFile(repoRoot, sha, path); + totalBytes += bytes.byteLength; + if (bytes.byteLength > MAX_TOOLING_FILE_BYTES || totalBytes > MAX_TOOLING_BYTES) { + throw new Error("tooling closure exceeds its retained-byte bounds"); + } + const imports: Array<[string, string]> = []; + for (const specifier of collectLiteralImports(bytes.toString("utf8"))) { + if (isBuiltin(specifier)) { + continue; + } + const target = resolveImport(repoRoot, sha, path, specifier); + imports.push([specifier, target]); + pending.push(target); + } + modules.set(path, { path, bytes, imports }); + } + const retained = [...modules.values()].toSorted((left, right) => + compareAscii(left.path, right.path), + ); + if ( + retained.length !== TOOLING_MODULE_PATHS.length || + retained.some( + (record, index) => record.path !== [...TOOLING_MODULE_PATHS].toSorted(compareAscii)[index], + ) + ) { + throw new Error("tooling closure does not match the fixed allowlist"); + } + return retained; +} + +function assertSafeYamlPath(path: string) { + const components = new Set(path.split("/")); + if ( + !path || + !/^[\x20-\x7e]+$/u.test(path) || + path.includes("\\") || + posix.isAbsolute(path) || + components.has(".") || + components.has("..") + ) { + throw new Error(`installed yaml package contains an unsafe path: ${JSON.stringify(path)}`); + } +} + +function retainYamlPackage() { + if (typeof fs.constants.O_NOFOLLOW !== "number") { + throw new Error("installed yaml package verification requires O_NOFOLLOW support"); + } + const packageRoot = fs.realpathSync(join(EXECUTION_ROOT, "node_modules", "yaml")); + if (!fs.lstatSync(packageRoot).isDirectory()) { + throw new Error("installed yaml package root must be a directory"); + } + const entries: YamlEntry[] = []; + let fileCount = 0; + let totalBytes = 0; + const walk = (directory: string, relativeDirectory = "") => { + for (const name of fs.readdirSync(directory).toSorted(compareAscii)) { + // Installer-created dependencies and bin shims are not package bytes or retained modules. + if (!relativeDirectory && name === "node_modules") { + continue; + } + const path = relativeDirectory ? `${relativeDirectory}/${name}` : name; + assertSafeYamlPath(path); + if (entries.length >= YAML_PACKAGE_MAX_ENTRIES) { + throw new Error(`installed yaml package exceeds ${YAML_PACKAGE_MAX_ENTRIES} entries`); + } + const absolutePath = join(directory, name); + const lstat = fs.lstatSync(absolutePath); + if (lstat.isSymbolicLink()) { + throw new Error(`installed yaml package must not contain symbolic links: ${path}`); + } + if (lstat.isDirectory()) { + entries.push({ kind: "directory", path }); + walk(absolutePath, path); + continue; + } + if (!lstat.isFile()) { + throw new Error("installed yaml package must contain only directories and files"); + } + fileCount += 1; + if (fileCount > YAML_PACKAGE_MAX_FILES) { + throw new Error(`installed yaml package exceeds ${YAML_PACKAGE_MAX_FILES} files`); + } + const descriptor = fs.openSync(absolutePath, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW); + let bytes: Buffer; + try { + const stat = fs.fstatSync(descriptor); + if (!stat.isFile()) { + throw new Error(`installed yaml package file changed type: ${path}`); + } + totalBytes += stat.size; + if (totalBytes > YAML_PACKAGE_MAX_BYTES) { + throw new Error(`installed yaml package exceeds ${YAML_PACKAGE_MAX_BYTES} bytes`); + } + bytes = fs.readFileSync(descriptor); + if (bytes.byteLength !== stat.size) { + throw new Error(`installed yaml package file changed while being read: ${path}`); + } + } finally { + fs.closeSync(descriptor); + } + entries.push({ kind: "file", path, bytes }); + } + }; + walk(packageRoot); + return entries; +} + +const serializableParams = ({ runGh: _runGh, ...source }: ReleasePlanSource) => + source as SerializableSource; + +function runOperation( + request: ProducerRequest, + params: ReleaseInventorySource | ReleasePlanSource, +) { + const repoRoot = resolve(params.repoRoot ?? "."); + const runGh = params.runGh ?? defaultRunGh; + const identityResponses = verifyRemoteTooling( + params, + runGh, + request.operation === "produce-inventory", + ); + const toolingSha = requireSha(params.toolingSha, "tooling SHA"); + const executionHead = gitBytes(EXECUTION_ROOT, ["rev-parse", "HEAD"]).toString("utf8").trim(); + if (executionHead !== toolingSha) { + throw new Error("tooling bootstrap checkout HEAD must equal tooling SHA"); + } + const bootstrapBytes = readGitFile(repoRoot, toolingSha, BOOTSTRAP_PATH); + if (!fs.readFileSync(fileURLToPath(import.meta.url)).equals(bootstrapBytes)) { + throw new Error(`tooling bootstrap differs from tooling SHA: ${BOOTSTRAP_PATH}`); + } + let stdout: string; + try { + const nodeExecPath = resolveReleasePlanNodeExecutable(); + if (!nodeExecPath) { + throw new Error("verified release plan child requires a Node executable"); + } + // The verified child always runs on Node, while its parent may run on Bun. + // JSON plus canonical base64 keeps this integrity boundary runtime-neutral. + const toolingModules = retainToolingClosure(repoRoot, toolingSha).map(({ bytes, ...record }) => + Object.assign(record, { bytesBase64: bytes.toString("base64") }), + ); + const yamlEntries = retainYamlPackage().map((entry) => + entry.kind === "file" + ? { kind: entry.kind, path: entry.path, bytesBase64: entry.bytes.toString("base64") } + : entry, + ); + stdout = execFileSync(nodeExecPath, ["--input-type=module", "-e", CHILD_RUNNER], { + cwd: repoRoot, + encoding: "utf8", + env: {}, + input: JSON.stringify({ + identityResponses, + request, + toolingModules, + yamlEntries, + }), + maxBuffer: 16 * 1024 * 1024, + stdio: ["pipe", "pipe", "pipe"], + }); + } catch (error) { + throw new Error("verified release plan child failed", { cause: error }); + } + const envelope = JSON.parse(stdout) as { ok?: unknown; value?: unknown; message?: unknown }; + if (envelope.ok !== true) { + throw new Error( + typeof envelope.message === "string" ? envelope.message : "verified child failed", + ); + } + return envelope.value; +} + +export function produceReleasePlan(params: ReleasePlanSource): ReleasePlan { + return runOperation( + { operation: "produce", params: serializableParams(params) }, + params, + ) as ReleasePlan; +} + +export function produceVerifiedReleaseInventory( + params: ReleaseInventorySource, +): VerifiedReleaseInventory { + const { repoRoot, candidateSha, toolingSha, toolingFullRef } = params; + return runOperation( + { + operation: "produce-inventory", + params: { repoRoot, candidateSha, toolingSha, toolingFullRef }, + }, + params, + ) as VerifiedReleaseInventory; +} + +export function verifyReleasePlanLock(lockJson: string, params: ReleasePlanSource) { + return runOperation( + { operation: "verify-lock", lockJson, params: serializableParams(params) }, + params, + ) as ReleasePlanLock; +} + +function requiredOption(args: string[], name: string) { + const index = args.indexOf(name); + const value = index >= 0 ? args[index + 1] : undefined; + if (!value || value.startsWith("-")) { + throw new Error(`${name} is required`); + } + return value; +} + +function main() { + const args = process.argv.slice(2); + const intent = requiredOption(args, "--intent") as ReleasePlanIntent; + if (!["publish", "diagnostic", "postpublish-confidence", "main-qualification"].includes(intent)) { + throw new Error( + "--intent must be publish, diagnostic, postpublish-confidence, or main-qualification", + ); + } + const source = { + candidateSha: requiredOption(args, "--candidate-sha"), + candidateRef: requiredOption(args, "--candidate-ref"), + toolingSha: requiredOption(args, "--tooling-sha"), + toolingFullRef: requiredOption(args, "--tooling-full-ref"), + }; + const params = { + ...source, + intent, + ...(intent === "main-qualification" + ? { + validationIntent: requiredOption( + args, + "--validation-intent", + ) as MainQualificationValidationIntent, + } + : {}), + } as ReleasePlanSource; + process.stdout.write( + runOperation( + { operation: "produce-lock", params: serializableParams(params) }, + params, + ) as string, + ); +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { + try { + main(); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + console.error("[release-plan-producer] FAILED (exit 1)"); + process.exitCode = 1; + } +} diff --git a/scripts/release-preflight.mjs b/scripts/release-preflight.mjs new file mode 100644 index 0000000000000000000000000000000000000000..6c13b10bdbb86ff9f5230365e3f4e7bc1af3b445 --- /dev/null +++ b/scripts/release-preflight.mjs @@ -0,0 +1,15 @@ +#!/usr/bin/env node +import { runTsxCliShim } from "./lib/tsx-cli-shim.mjs"; + +const args = process.argv.slice(2); +if (args.length === 1 && args[0] === "--macos-versions-only") { + // Evidence reuse runs before dependencies exist; only this file-read-only probe bypasses tsx. + const { writeFailedTrailer } = await import("./lib/failed-trailer.mts"); + process.once("exit", (exitCode) => writeFailedTrailer("release-preflight", exitCode)); + await import("./release-preflight.mts"); +} else { + await runTsxCliShim(import.meta.url, { + implementation: "./release-preflight.mts", + failureTool: "release-preflight", + }); +} diff --git a/scripts/release-prepare.ts b/scripts/release-prepare.ts new file mode 100644 index 0000000000000000000000000000000000000000..4b7ef3edc2b1a3bc70ce2a27bafe41d8c68984e4 --- /dev/null +++ b/scripts/release-prepare.ts @@ -0,0 +1,411 @@ +// Release Prepare provides one bounded entrypoint for version alignment and generated metadata. +import { spawnSync } from "node:child_process"; +import crypto from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import { expectDefined } from "../packages/normalization-core/src/expect.js"; +import { runManagedCommand } from "./lib/managed-child-process.mts"; +import { parseReleaseVersion } from "./lib/release-version.mjs"; + +type ReleasePrepareMode = "check" | "shadow" | "write"; + +type ReleasePrepareArgs = { + android: boolean; + help: boolean; + jobs: number; + json: boolean; + manifestPath: string | null; + mode: ReleasePrepareMode; + rootDir: string; + version: string | null; +}; + +type ReleasePrepareStep = { + args: string[]; + command: string; + id: "generated-version-metadata" | "release-version"; + name: string; +}; + +type ReleasePrepareStepResult = ReleasePrepareStep & { + durationMs: number; + status: "failed" | "passed" | "planned" | "skipped"; +}; + +type WorktreeState = { + changedFiles: string[]; + fingerprint: string; + head: string; + packageVersion: string; + status: string; +}; + +const DEFAULT_JOBS = 4; +const MAX_JOBS = 16; +const GIT_OUTPUT_MAX_BUFFER_BYTES = 64 * 1024 * 1024; + +export function parseReleasePrepareArgs(argv: string[]): ReleasePrepareArgs { + let android = false; + let help = false; + let jobs = DEFAULT_JOBS; + let json = false; + let manifestPath: string | null = null; + let mode: ReleasePrepareMode = "shadow"; + let modeFlag: string | null = null; + let rootDir = path.resolve("."); + let version: string | null = null; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + switch (arg) { + case "--": { + break; + } + case "--android": { + android = true; + break; + } + case "--check": + case "--shadow": + case "--write": { + if (modeFlag) { + throw new Error(`Use only one mode flag; received ${modeFlag} and ${arg}.`); + } + modeFlag = arg; + mode = arg.slice(2) as ReleasePrepareMode; + break; + } + case "--jobs": { + jobs = parseJobs(readOptionValue(argv, index, arg)); + index += 1; + break; + } + case "--json": { + json = true; + break; + } + case "--manifest": { + manifestPath = path.resolve(readOptionValue(argv, index, arg)); + index += 1; + break; + } + case "--root": { + rootDir = path.resolve(readOptionValue(argv, index, arg)); + index += 1; + break; + } + case "--version": { + version = readOptionValue(argv, index, arg); + index += 1; + break; + } + case "-h": + case "--help": { + help = true; + break; + } + default: { + throw new Error(`Unknown argument: ${arg}`); + } + } + } + + return { android, help, jobs, json, manifestPath, mode, rootDir, version }; +} + +export function createReleasePrepareSteps( + args: Pick<ReleasePrepareArgs, "android" | "jobs" | "mode" | "rootDir" | "version">, +): ReleasePrepareStep[] { + if (!args.version) { + throw new Error("Missing required --version."); + } + const parsedVersion = parseReleaseVersion(args.version); + if (!parsedVersion) { + throw new Error( + `Invalid release version '${args.version}'. Expected YYYY.M.PATCH, YYYY.M.PATCH-alpha.N, YYYY.M.PATCH-beta.N, or YYYY.M.PATCH-N.`, + ); + } + + const versionArgs = [ + "--import", + "tsx", + "scripts/release-version.ts", + "--root", + args.rootDir, + "--version", + parsedVersion.version, + ]; + if (args.android) { + versionArgs.push("--android"); + } + if (args.mode === "write") { + versionArgs.push("--write"); + } + + const preflightArgs = [ + "scripts/release-preflight.mjs", + args.mode === "write" ? "--fix" : "--check", + "--scope", + "version", + "--jobs", + String(args.jobs), + ]; + + return [ + { + args: versionArgs, + command: process.execPath, + id: "release-version", + name: "release version alignment", + }, + { + args: preflightArgs, + command: process.execPath, + id: "generated-version-metadata", + name: "version-owned generated metadata", + }, + ]; +} + +export function runReleasePrepareSteps(params: { + cwd: string; + json?: boolean; + mode: ReleasePrepareMode; + runStep?: (step: ReleasePrepareStep, cwd: string) => number; + steps: ReleasePrepareStep[]; +}): ReleasePrepareStepResult[] { + if (params.mode === "shadow") { + return params.steps.map((step) => ({ ...step, durationMs: 0, status: "planned" })); + } + + const runStep = + params.runStep ?? + ((step: ReleasePrepareStep, cwd: string) => + runReleasePrepareStep(step, cwd, { json: params.json ?? false })); + const results: ReleasePrepareStepResult[] = []; + let blocked = false; + for (const step of params.steps) { + if (blocked) { + results.push({ ...step, durationMs: 0, status: "skipped" }); + continue; + } + const startedAt = performance.now(); + const exitCode = runStep(step, params.cwd); + const status = exitCode === 0 ? "passed" : "failed"; + results.push({ + ...step, + durationMs: Math.round(performance.now() - startedAt), + status, + }); + blocked = status === "failed"; + } + return results; +} + +export function buildReleasePreparationManifest(params: { + after: WorktreeState; + before: WorktreeState; + mode: ReleasePrepareMode; + steps: ReleasePrepareStepResult[]; + version: string; +}) { + const status = params.steps.some((step) => step.status === "failed") + ? "failed" + : params.mode === "shadow" + ? "shadow" + : "passed"; + return { + schemaVersion: 1, + requestedVersion: params.version, + mode: params.mode, + status, + sourceHead: params.before.head, + candidateFingerprint: params.after.fingerprint, + before: params.before, + after: params.after, + steps: params.steps.map((step) => ({ + id: step.id, + name: step.name, + command: [step.command, ...step.args], + status: step.status, + durationMs: step.durationMs, + })), + }; +} + +export async function main(argv = process.argv.slice(2)): Promise<number> { + const args = parseReleasePrepareArgs(argv); + if (args.help) { + printUsage(); + return 0; + } + const steps = createReleasePrepareSteps(args); + const before = await readWorktreeState(args.rootDir); + const results = runReleasePrepareSteps({ + cwd: args.rootDir, + json: args.json, + mode: args.mode, + steps, + }); + const after = await readWorktreeState(args.rootDir); + const manifest = buildReleasePreparationManifest({ + after, + before, + mode: args.mode, + steps: results, + version: args.version!, + }); + const manifestPath = + args.manifestPath ?? defaultManifestPath(args.rootDir, args.version!, after.fingerprint); + writeJsonAtomic(manifestPath, manifest); + + if (args.json) { + process.stdout.write(`${JSON.stringify({ manifestPath, ...manifest }, null, 2)}\n`); + } else { + const verb = args.mode === "shadow" ? "planned" : manifest.status; + process.stdout.write( + `Release preparation ${verb} for ${args.version}; manifest: ${manifestPath}\n`, + ); + for (const step of results) { + process.stdout.write(`- ${step.name}: ${step.status} (${step.durationMs}ms)\n`); + } + } + return manifest.status === "failed" ? 1 : 0; +} + +export function runReleasePrepareStep( + step: ReleasePrepareStep, + cwd: string, + options: { json?: boolean } = {}, +): number { + const json = options.json ?? false; + const progressStream = json ? process.stderr : process.stdout; + progressStream.write(`\n[release-prepare] ${step.name}\n`); + const result = spawnSync(step.command, step.args, { + cwd, + env: process.env, + stdio: json ? ["ignore", process.stderr.fd, process.stderr.fd] : "inherit", + }); + if (result.error) { + throw result.error; + } + return result.status ?? 1; +} + +export async function readWorktreeState(rootDir: string): Promise<WorktreeState> { + const head = git(rootDir, ["rev-parse", "HEAD"]); + const status = git(rootDir, ["status", "--porcelain=v1", "--untracked-files=all"]); + const fingerprint = crypto.createHash("sha256").update(`${head}\0${status}\0`); + // Generated locale diffs can exceed buffered child-output limits. Hash every + // byte as it arrives, and publish the digest only after Git closes successfully. + const diffExit = await runManagedCommand({ + bin: "git", + args: ["diff", "--binary", "HEAD"], + cwd: rootDir, + stdio: ["ignore", "pipe", "inherit"], + onReady(child) { + expectDefined(child.stdout, "git diff output").on("data", (chunk: Buffer) => { + fingerprint.update(chunk); + }); + }, + }); + if (diffExit !== 0) { + throw new Error(`git diff --binary HEAD failed (exit ${diffExit})`); + } + const packageJson = JSON.parse(fs.readFileSync(path.join(rootDir, "package.json"), "utf8")) as { + version?: unknown; + }; + const packageVersion = typeof packageJson.version === "string" ? packageJson.version : ""; + const changedFiles = status + .split("\n") + .filter(Boolean) + .map((line) => line.slice(3)) + .toSorted(); + return { + changedFiles, + fingerprint: fingerprint.digest("hex"), + head, + packageVersion, + status, + }; +} + +function defaultManifestPath(rootDir: string, version: string, fingerprint: string): string { + const gitPath = git(rootDir, [ + "rev-parse", + "--git-path", + `openclaw-release-cache/candidates/${version}-${fingerprint.slice(0, 12)}.json`, + ]); + return path.resolve(rootDir, gitPath); +} + +function git(cwd: string, args: string[]): string { + const result = spawnSync("git", args, { + cwd, + encoding: "utf8", + env: process.env, + // Bound metadata capture; the larger generated diff is streamed separately. + maxBuffer: GIT_OUTPUT_MAX_BUFFER_BYTES, + stdio: ["ignore", "pipe", "pipe"], + }); + if (result.status !== 0) { + throw new Error(result.stderr.trim() || `git ${args.join(" ")} failed`); + } + return result.stdout.trimEnd(); +} + +function writeJsonAtomic(filePath: string, value: unknown): void { + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + const tempPath = `${filePath}.${process.pid}.tmp`; + try { + fs.writeFileSync(tempPath, `${JSON.stringify(value, null, 2)}\n`); + fs.renameSync(tempPath, filePath); + } finally { + fs.rmSync(tempPath, { force: true }); + } +} + +function parseJobs(raw: string): number { + const jobs = Number(raw); + if (!Number.isInteger(jobs) || jobs < 1 || jobs > MAX_JOBS) { + throw new Error(`Invalid --jobs value '${raw}'. Expected 1 through ${MAX_JOBS}.`); + } + return jobs; +} + +function readOptionValue(argv: string[], index: number, flag: string): string { + const value = argv[index + 1]; + if (!value || value.startsWith("-")) { + throw new Error(`Missing value for ${flag}.`); + } + return value; +} + +function printUsage(): void { + process.stdout.write( + [ + "Usage: node --import tsx scripts/release-prepare.ts --version <version> [mode] [options]", + "", + "Modes:", + " --shadow emit the exact preparation plan and candidate manifest without commands (default)", + " --check verify an already prepared candidate without writing", + " --write align versions and refresh version-owned generated metadata", + "", + "Options:", + " --android prepare shared mobile and Android release metadata", + " --jobs <count> preflight concurrency, 1 through 16 (default: 4)", + " --manifest <path> override the git-local candidate manifest path", + " --json emit machine-readable output", + "", + ].join("\n"), + ); +} + +if (import.meta.main) { + try { + process.exitCode = await main(); + } catch (error) { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + process.exitCode = 1; + } +} diff --git a/scripts/release-tooling-identity.mjs b/scripts/release-tooling-identity.mjs new file mode 100644 index 0000000000000000000000000000000000000000..fe5a89035609e9966d46a86baf3e5de428778d2b --- /dev/null +++ b/scripts/release-tooling-identity.mjs @@ -0,0 +1,691 @@ +#!/usr/bin/env node + +import { execFileSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import { isRecord } from "./lib/record-shared.mjs"; + +const SHA_PATTERN = /^[a-f0-9]{40}$/u; +const RELEASE_PUBLISH_REF_PATTERN = /^release-publish\/([a-f0-9]{12})-([1-9][0-9]*)$/u; +const RELEASE_CI_REF_PATTERN = /^release-ci\/([a-f0-9]{12})-([1-9][0-9]*)$/u; +const DIRECT_WORKFLOW_REF_PATTERN = + /^(?:main|release\/[0-9]{4}\.(?:[1-9]|1[0-2])\.[1-9][0-9]*|extended-stable\/[0-9]{4}\.(?:[1-9]|1[0-2])\.33|tideclaw\/alpha\/[0-9]{4}-[0-9]{2}-[0-9]{2}-[0-9]{4}Z)$/u; +const RELEASE_PUBLISH_PARENT_STATE_POLICIES = new Set([ + "active", + "active-or-failure", + "active-or-success", + "manual-recovery", +]); +const GH_COMMAND_TIMEOUT_MS = 60_000; + +function fail(message) { + throw new Error(message); +} + +function requiredString(value, label) { + if (typeof value !== "string" || value.trim().length === 0) { + fail(`${label} is required.`); + } + return value.trim(); +} + +function requiredSha(value, label) { + const sha = requiredString(value, label); + if (!SHA_PATTERN.test(sha)) { + fail(`${label} must be a lowercase 40-character commit SHA.`); + } + return sha; +} + +function requireRepository(value) { + const repository = requiredString(value, "release tooling repository"); + if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(repository)) { + fail("release tooling repository must be owner/name."); + } + return repository; +} + +function parseIdentityJson(value) { + const raw = requiredString(value, "requested release tooling identity"); + let identity; + try { + identity = JSON.parse(raw); + } catch (error) { + throw new Error("requested release tooling identity must be valid JSON.", { cause: error }); + } + if (!isRecord(identity)) { + fail("requested release tooling identity must be a JSON object."); + } + return { + fullRef: requiredString(identity.fullRef, "requested release tooling full ref"), + ref: requiredString(identity.ref, "requested release tooling ref"), + sha: requiredSha(identity.sha, "requested release tooling SHA"), + }; +} + +export function resolveReleaseToolingIdentity({ + requestedIdentityJson = "", + workflowContract, + workflowFullRef, + workflowRef, + workflowSha, +}) { + const contract = requiredString(workflowContract, "release tooling contract"); + if (contract !== "1" && contract !== "2") { + fail(`release tooling contract ${contract} is not supported.`); + } + const ref = requiredString(workflowRef, "workflow ref"); + const fullRef = requiredString(workflowFullRef, "workflow full ref"); + const sha = requiredSha(workflowSha, "workflow SHA"); + const directRoute = fullRef === `refs/heads/${ref}` && DIRECT_WORKFLOW_REF_PATTERN.test(ref); + const releaseCiMatch = fullRef === `refs/heads/${ref}` ? RELEASE_CI_REF_PATTERN.exec(ref) : null; + const protectedTagMatch = + fullRef === `refs/tags/${ref}` ? RELEASE_PUBLISH_REF_PATTERN.exec(ref) : null; + + if (releaseCiMatch && releaseCiMatch[1] !== sha.slice(0, 12)) { + fail("release-ci workflow ref does not match the workflow SHA."); + } + if (protectedTagMatch && protectedTagMatch[1] !== sha.slice(0, 12)) { + fail("protected workflow ref does not match the workflow SHA."); + } + if (!directRoute && !releaseCiMatch && !protectedTagMatch) { + fail("workflow ref is not a trusted direct, release-ci, or protected-tag route."); + } + + const requested = requestedIdentityJson.trim() + ? parseIdentityJson(requestedIdentityJson) + : undefined; + if (!requested) { + if (contract !== "1" && contract !== "2") { + fail(`release tooling contract ${contract} requires explicit trusted workflow identity.`); + } + if (!directRoute) { + fail("release-ci and protected-tag workflows require explicit trusted workflow identity."); + } + return { fullRef, ref, sha }; + } + + if (directRoute || protectedTagMatch) { + if (requested.ref !== ref || requested.fullRef !== fullRef || requested.sha !== sha) { + fail("direct workflow identity must match the executing workflow ref and SHA."); + } + return requested; + } + + const requestedProtectedTag = RELEASE_PUBLISH_REF_PATTERN.test(requested.ref); + const requestedMain = requested.ref === "main" && requested.fullRef === "refs/heads/main"; + if ( + requested.sha !== sha || + (!requestedMain && + (!requestedProtectedTag || requested.fullRef !== `refs/tags/${requested.ref}`)) + ) { + fail("release-ci workflow identity must be trusted main or an exact protected tag."); + } + return requested; +} + +function classifyIdentity({ allowPrevalidatedRef, workflowFullRef, workflowRef, workflowSha }) { + const ref = requiredString(workflowRef, "release tooling ref"); + const fullRef = requiredString(workflowFullRef, "release tooling full ref"); + const sha = requiredSha(workflowSha, "release tooling SHA"); + const protectedMatch = RELEASE_PUBLISH_REF_PATTERN.exec(ref); + + if (protectedMatch) { + if (fullRef !== `refs/tags/${ref}`) { + fail("protected release tooling identity must use the exact tag full ref."); + } + if (sha.slice(0, 12) !== protectedMatch[1]) { + fail("protected release tooling tag SHA prefix does not match the workflow SHA."); + } + return { fullRef, ref, route: "protected-tag", sha }; + } + + if ( + ref.startsWith("release-publish/") || + fullRef.startsWith("refs/tags/release-publish/") || + fullRef.startsWith("refs/heads/release-publish/") + ) { + fail("release-publish tooling identity must be an exact protected tag."); + } + + if (ref === "main" || fullRef === "refs/heads/main") { + if (ref !== "main" || fullRef !== "refs/heads/main") { + fail("main release tooling identity must use ref main and full ref refs/heads/main."); + } + return { fullRef, ref, route: "main", sha }; + } + + if (allowPrevalidatedRef !== true || fullRef !== `refs/heads/${ref}`) { + fail( + "release tooling identity is not trusted main, a protected tag, or a prevalidated branch.", + ); + } + return { fullRef, ref, route: "prevalidated-branch", sha }; +} + +export function validateReleaseToolingIdentity({ + allowPrevalidatedRef = false, + branchRef, + mainComparisonStatus, + tagRef, + workflowFullRef, + workflowRef, + workflowSha, +}) { + const identity = classifyIdentity({ + allowPrevalidatedRef, + workflowFullRef, + workflowRef, + workflowSha, + }); + + if (identity.route === "protected-tag") { + if ( + !isRecord(tagRef) || + tagRef.ref !== identity.fullRef || + !isRecord(tagRef.object) || + tagRef.object.type !== "commit" || + tagRef.object.sha !== identity.sha + ) { + fail( + "protected release tooling tag is missing, moved, annotated, or bound to the wrong SHA.", + ); + } + } else if (identity.route === "main") { + if (mainComparisonStatus !== "ahead" && mainComparisonStatus !== "identical") { + fail("main release tooling SHA is not reachable from current main."); + } + } else if ( + !isRecord(branchRef) || + branchRef.ref !== identity.fullRef || + !isRecord(branchRef.object) || + branchRef.object.type !== "commit" || + branchRef.object.sha !== identity.sha + ) { + fail("prevalidated release tooling branch is missing or moved from the workflow SHA."); + } + + return identity; +} + +export function validateReleasePublishParentRun({ + identity, + releasePublishFullRef, + releasePublishParentStatePolicy, + releasePublishRef, + releasePublishRunAttempt, + releasePublishRunId, + repository, + run, +}) { + const runId = requiredString(releasePublishRunId, "release publish run id"); + const runAttempt = requiredString(releasePublishRunAttempt, "release publish run attempt"); + if (!/^[1-9][0-9]*$/u.test(runId) || !/^[1-9][0-9]*$/u.test(runAttempt)) { + fail("release publish run id and attempt must be positive integers."); + } + const parentStatePolicy = requiredString( + releasePublishParentStatePolicy, + "release publish parent state policy", + ); + if (!RELEASE_PUBLISH_PARENT_STATE_POLICIES.has(parentStatePolicy)) { + fail(`release publish parent state policy ${parentStatePolicy} is not supported.`); + } + const parentRef = requiredString(releasePublishRef, "release publish ref"); + const parentFullRef = requiredString(releasePublishFullRef, "release publish full ref"); + const parentProtectedMatch = RELEASE_PUBLISH_REF_PATTERN.exec(parentRef); + const parentDirectRoute = + parentFullRef === `refs/heads/${parentRef}` && DIRECT_WORKFLOW_REF_PATTERN.test(parentRef); + const parentProtectedRoute = + parentFullRef === `refs/tags/${parentRef}` && + parentProtectedMatch?.[1] === identity.sha.slice(0, 12); + if (!parentDirectRoute && !parentProtectedRoute) { + fail("release publish parent ref is not a trusted direct or exact protected-tag route."); + } + const normalizedRepository = requireRepository(repository); + const [workflowPath, workflowFullRef] = String(run?.path ?? "").split("@", 2); + const expected = { + event: "workflow_dispatch", + headBranch: parentRef, + headSha: identity.sha, + repository: normalizedRepository, + runAttempt: Number(runAttempt), + runId: Number(runId), + workflowPath: ".github/workflows/openclaw-release-publish.yml", + }; + const actual = { + event: run?.event, + headBranch: run?.head_branch, + headSha: run?.head_sha, + repository: run?.repository?.full_name, + runAttempt: run?.run_attempt, + runId: run?.id, + workflowPath, + }; + for (const key of Object.keys(expected)) { + if (actual[key] !== expected[key]) { + fail(`release publish parent run ${key} does not match the trusted tooling identity.`); + } + } + if (workflowFullRef && workflowFullRef !== parentFullRef) { + fail("release publish parent run workflow full ref does not match trusted tooling."); + } + const active = run?.status === "in_progress" && !run?.conclusion; + const completedSuccess = run?.status === "completed" && run?.conclusion === "success"; + const completedFailure = run?.status === "completed" && run?.conclusion === "failure"; + if ( + !active && + !(parentStatePolicy === "active-or-failure" && completedFailure) && + !(parentStatePolicy === "active-or-success" && completedSuccess) && + !(parentStatePolicy === "manual-recovery" && (completedSuccess || completedFailure)) + ) { + fail( + `release publish parent run state is not allowed by ${parentStatePolicy}: status=${run?.status ?? "<missing>"} conclusion=${run?.conclusion ?? "<missing>"}.`, + ); + } +} + +function parseJson(raw, label) { + try { + return JSON.parse(raw); + } catch (error) { + throw new Error(`${label} returned invalid JSON.`, { cause: error }); + } +} + +export function runReleaseToolingGh(args) { + return execFileSync("gh", args, { + encoding: "utf8", + killSignal: "SIGKILL", + maxBuffer: 1024 * 1024, + stdio: ["ignore", "pipe", "pipe"], + timeout: GH_COMMAND_TIMEOUT_MS, + }); +} + +export function verifyReleaseWorkflowRun({ + repository, + workflowFullRef, + workflowRef, + workflowSha, + runId, + runAttempt, + workflowPath, + workflowEvent, + runStatePolicy, + runGh = runReleaseToolingGh, +}) { + const normalizedRepository = requireRepository(repository); + const ref = requiredString(workflowRef, "release workflow ref"); + const fullRef = requiredString(workflowFullRef, "release workflow full ref"); + const sha = requiredSha(workflowSha, "release workflow SHA"); + const id = requiredString(runId, "release workflow run id"); + const attempt = requiredString(runAttempt, "release workflow run attempt"); + const path = requiredString(workflowPath, "release workflow path"); + const event = requiredString(workflowEvent, "release workflow event"); + if ( + !/^[1-9][0-9]*$/u.test(id) || + !/^[1-9][0-9]*$/u.test(attempt) || + !Number.isSafeInteger(Number(id)) || + !Number.isSafeInteger(Number(attempt)) + ) { + fail("release workflow run id and attempt must be positive safe integers."); + } + if ( + (fullRef !== `refs/heads/${ref}` && fullRef !== `refs/tags/${ref}`) || + !/^\.github\/workflows\/[^/@]+\.ya?ml$/u.test(path) + ) { + fail("release workflow path and full ref must name the exact executing workflow."); + } + if (runStatePolicy !== "active" && runStatePolicy !== "success") { + fail("release workflow run state policy must be active or success."); + } + const run = parseJson( + runGh(["api", `repos/${normalizedRepository}/actions/runs/${id}`, "--method", "GET"]), + "release workflow run", + ); + if (!isRecord(run)) { + fail("release workflow run must be a JSON object."); + } + const observedPath = typeof run.path === "string" ? run.path : ""; + const refSeparator = observedPath.indexOf("@"); + const expected = { + repository: normalizedRepository, + runId: Number(id), + runAttempt: Number(attempt), + headSha: sha, + headBranch: ref, + workflowPath: path, + event, + status: runStatePolicy === "active" ? "in_progress" : "completed", + conclusion: runStatePolicy === "active" ? null : "success", + }; + const actual = { + repository: isRecord(run.repository) ? run.repository.full_name : undefined, + runId: run.id, + runAttempt: run.run_attempt, + headSha: run.head_sha, + headBranch: run.head_branch, + workflowPath: refSeparator === -1 ? observedPath : observedPath.slice(0, refSeparator), + event: run.event, + status: run.status, + conclusion: run.conclusion, + }; + for (const key of Object.keys(expected)) { + if (actual[key] !== expected[key]) { + fail(`release workflow run ${key} does not match the authorized workflow identity.`); + } + } + if (refSeparator !== -1 && observedPath.slice(refSeparator + 1) !== fullRef) { + fail("release workflow run full ref does not match the authorized workflow identity."); + } + return run; +} + +export function verifyReleaseToolingIdentity({ + allowPrevalidatedRef = false, + releasePublishFullRef, + releasePublishParentStatePolicy, + releasePublishRef, + releasePublishRunAttempt, + releasePublishRunId, + repository, + runGh = runReleaseToolingGh, + workflowFullRef, + workflowRef, + workflowSha, +}) { + const normalizedRepository = requireRepository(repository); + const identity = classifyIdentity({ + allowPrevalidatedRef, + workflowFullRef, + workflowRef, + workflowSha, + }); + + if (identity.route === "protected-tag") { + let tagRef; + try { + tagRef = parseJson( + runGh([ + "api", + `repos/${normalizedRepository}/git/ref/tags/${identity.ref}`, + "--method", + "GET", + ]), + "protected release tooling tag", + ); + } catch (error) { + throw new Error("protected release tooling tag is missing or unreadable.", { cause: error }); + } + const validated = validateReleaseToolingIdentity({ + allowPrevalidatedRef, + tagRef, + workflowFullRef, + workflowRef, + workflowSha, + }); + validateParentRunIfRequested({ + identity: validated, + releasePublishFullRef, + releasePublishParentStatePolicy, + releasePublishRef, + releasePublishRunAttempt, + releasePublishRunId, + repository: normalizedRepository, + runGh, + }); + return validated; + } + + if (identity.route === "main") { + let comparison; + try { + comparison = parseJson( + runGh([ + "api", + `repos/${normalizedRepository}/compare/${identity.sha}...main`, + "--method", + "GET", + // Full comparison patches can exceed the subprocess buffer; only ancestry status is used. + "--jq", + "{status}", + ]), + "main release tooling comparison", + ); + } catch (error) { + throw new Error("main release tooling ancestry could not be verified.", { cause: error }); + } + const validated = validateReleaseToolingIdentity({ + allowPrevalidatedRef, + mainComparisonStatus: isRecord(comparison) ? comparison.status : undefined, + workflowFullRef, + workflowRef, + workflowSha, + }); + validateParentRunIfRequested({ + identity: validated, + releasePublishFullRef, + releasePublishParentStatePolicy, + releasePublishRef, + releasePublishRunAttempt, + releasePublishRunId, + repository: normalizedRepository, + runGh, + }); + return validated; + } + + let branchRef; + try { + branchRef = parseJson( + runGh([ + "api", + `repos/${normalizedRepository}/git/ref/heads/${identity.ref}`, + "--method", + "GET", + ]), + "prevalidated release tooling branch", + ); + } catch (error) { + throw new Error("prevalidated release tooling branch is missing or unreadable.", { + cause: error, + }); + } + const validated = validateReleaseToolingIdentity({ + allowPrevalidatedRef, + branchRef, + workflowFullRef, + workflowRef, + workflowSha, + }); + validateParentRunIfRequested({ + identity: validated, + releasePublishFullRef, + releasePublishParentStatePolicy, + releasePublishRef, + releasePublishRunAttempt, + releasePublishRunId, + repository: normalizedRepository, + runGh, + }); + return validated; +} + +function validateParentRunIfRequested({ + identity, + releasePublishFullRef, + releasePublishParentStatePolicy, + releasePublishRef, + releasePublishRunAttempt, + releasePublishRunId, + repository, + runGh, +}) { + if ( + !releasePublishRunId && + !releasePublishRunAttempt && + !releasePublishParentStatePolicy && + !releasePublishRef && + !releasePublishFullRef + ) { + return; + } + if ( + !releasePublishRunId || + !releasePublishRunAttempt || + !releasePublishParentStatePolicy || + !releasePublishRef || + !releasePublishFullRef + ) { + fail( + "release publish run id, attempt, ref, full ref, and parent state policy must be provided together.", + ); + } + let run; + try { + run = parseJson( + runGh(["api", `repos/${repository}/actions/runs/${releasePublishRunId}`, "--method", "GET"]), + "release publish parent run", + ); + } catch (error) { + throw new Error("release publish parent run is missing or unreadable.", { cause: error }); + } + validateReleasePublishParentRun({ + identity, + releasePublishFullRef, + releasePublishParentStatePolicy, + releasePublishRef, + releasePublishRunAttempt, + releasePublishRunId, + repository, + run, + }); +} + +function parseArgs(argv) { + const options = { + allowPrevalidatedRef: false, + command: "", + releasePublishRunAttempt: "", + releasePublishRunId: "", + releasePublishRef: "", + releasePublishFullRef: "", + releasePublishParentStatePolicy: "", + repository: "", + requestedIdentityJson: "", + workflowContract: "", + workflowFullRef: "", + workflowRef: "", + workflowSha: "", + writerRunId: "", + writerRunAttempt: "", + writerWorkflowPath: "", + writerWorkflowEvent: "", + }; + let writerRequested = false; + options.command = argv.shift() ?? ""; + if (options.command !== "verify" && options.command !== "resolve") { + fail("usage: release-tooling-identity.mjs <verify|resolve> [options]"); + } + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === "--allow-prevalidated-ref") { + options.allowPrevalidatedRef = true; + continue; + } + const value = argv[(index += 1)] ?? ""; + if (arg.startsWith("--writer-")) { + writerRequested = true; + } + if (arg === "--release-publish-run-id") { + options.releasePublishRunId = value; + } else if (arg === "--release-publish-run-attempt") { + options.releasePublishRunAttempt = value; + } else if (arg === "--release-publish-ref") { + options.releasePublishRef = value; + } else if (arg === "--release-publish-full-ref") { + options.releasePublishFullRef = value; + } else if (arg === "--release-publish-parent-state-policy") { + options.releasePublishParentStatePolicy = value; + } else if (arg === "--repository") { + options.repository = value; + } else if (arg === "--requested-identity-json") { + options.requestedIdentityJson = value; + } else if (arg === "--workflow-contract") { + options.workflowContract = value; + } else if (arg === "--workflow-full-ref") { + options.workflowFullRef = value; + } else if (arg === "--workflow-ref") { + options.workflowRef = value; + } else if (arg === "--workflow-sha") { + options.workflowSha = value; + } else if (arg === "--writer-run-id") { + options.writerRunId = value; + } else if (arg === "--writer-run-attempt") { + options.writerRunAttempt = value; + } else if (arg === "--writer-workflow-path") { + options.writerWorkflowPath = value; + } else if (arg === "--writer-workflow-event") { + options.writerWorkflowEvent = value; + } else { + fail(`unknown release tooling identity argument: ${arg}`); + } + } + if ( + writerRequested && + [ + options.writerRunId, + options.writerRunAttempt, + options.writerWorkflowPath, + options.writerWorkflowEvent, + ].some((value) => !value) + ) { + fail("writer run id, attempt, workflow path, and workflow event must be provided together."); + } + return options; +} + +function main(argv = process.argv.slice(2)) { + const options = parseArgs([...argv]); + let identity; + if (options.command === "resolve") { + identity = resolveReleaseToolingIdentity(options); + const protectedMatch = RELEASE_PUBLISH_REF_PATTERN.exec(identity.ref); + verifyReleaseToolingIdentity({ + allowPrevalidatedRef: identity.ref !== "main" && !protectedMatch, + releasePublishFullRef: options.releasePublishFullRef, + releasePublishParentStatePolicy: options.releasePublishParentStatePolicy, + releasePublishRef: options.releasePublishRef, + releasePublishRunAttempt: options.releasePublishRunAttempt, + releasePublishRunId: options.releasePublishRunId, + repository: options.repository, + workflowFullRef: identity.fullRef, + workflowRef: identity.ref, + workflowSha: identity.sha, + }); + } else { + identity = verifyReleaseToolingIdentity(options); + } + if (options.writerRunId) { + verifyReleaseWorkflowRun({ + repository: options.repository, + workflowFullRef: options.workflowFullRef, + workflowRef: options.workflowRef, + workflowSha: options.workflowSha, + runId: options.writerRunId, + runAttempt: options.writerRunAttempt, + workflowPath: options.writerWorkflowPath, + workflowEvent: options.writerWorkflowEvent, + runStatePolicy: "active", + }); + } + process.stdout.write(`${JSON.stringify(identity)}\n`); +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) { + try { + main(); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); + } +} diff --git a/scripts/release-version.ts b/scripts/release-version.ts new file mode 100644 index 0000000000000000000000000000000000000000..6df14de64bb20b96e8e6a6c9b6c5f93119faeb2d --- /dev/null +++ b/scripts/release-version.ts @@ -0,0 +1,256 @@ +// Release Version keeps the core and explicitly selected native release trains aligned. +import fs from "node:fs"; +import path from "node:path"; +import { expectDefined } from "../packages/normalization-core/src/expect.js"; +import { parseReleaseVersion } from "./lib/release-version.mjs"; +import { planMobileRelease } from "./mobile-release-version.ts"; + +const MACOS_INFO_PLIST = "apps/macos/Sources/OpenClaw/Resources/Info.plist"; + +type ReleaseVersionMode = "check" | "write"; + +type ReleaseVersionArgs = { + android: boolean; + help: boolean; + mode: ReleaseVersionMode; + rootDir: string; + version: string | null; +}; + +type ReleaseVersionChange = { + currentContent: string; + nextContent: string; + path: string; +}; + +type ReleaseVersionPlan = { + changes: ReleaseVersionChange[]; + version: string; +}; + +export function parseReleaseVersionArgs(argv: string[]): ReleaseVersionArgs { + let android = false; + let help = false; + let mode: ReleaseVersionMode = "check"; + let rootDir = path.resolve("."); + let version: string | null = null; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + switch (arg) { + case "--": { + break; + } + case "--android": { + android = true; + break; + } + case "--check": { + mode = "check"; + break; + } + case "--root": { + rootDir = path.resolve(readOptionValue(argv, index, arg)); + index += 1; + break; + } + case "--version": { + version = readOptionValue(argv, index, arg); + index += 1; + break; + } + case "--write": { + mode = "write"; + break; + } + case "-h": + case "--help": { + help = true; + break; + } + default: { + throw new Error(`Unknown argument: ${arg}`); + } + } + } + + return { android, help, mode, rootDir, version }; +} + +export function planReleaseVersion(params: { + android?: boolean; + rootDir?: string; + version: string; +}): ReleaseVersionPlan { + const rootDir = path.resolve(params.rootDir ?? "."); + const parsedVersion = parseReleaseVersion(params.version); + if (!parsedVersion) { + throw new Error( + `Invalid release version '${params.version}'. Expected YYYY.M.PATCH, YYYY.M.PATCH-alpha.N, YYYY.M.PATCH-beta.N, or YYYY.M.PATCH-N.`, + ); + } + + const packageVersion = + parsedVersion.correctionNumber === undefined + ? parsedVersion.version + : parsedVersion.baseVersion; + const changes = [ + planPackageJson(rootDir, packageVersion), + planMacosInfoPlist(rootDir, parsedVersion), + ]; + if (params.android) { + changes.push( + ...planMobileRelease({ + gatewayVersion: parsedVersion.baseVersion, + phase: "prepare", + rootDir, + }).changes, + ); + } + + return { + changes: changes.filter((change) => change.currentContent !== change.nextContent), + version: parsedVersion.version, + }; +} + +export function applyReleaseVersionPlan(plan: ReleaseVersionPlan): void { + const tempPaths: string[] = []; + try { + for (const [index, change] of plan.changes.entries()) { + const tempPath = `${change.path}.release-version-${process.pid}-${index}.tmp`; + fs.writeFileSync(tempPath, change.nextContent, "utf8"); + tempPaths.push(tempPath); + } + for (const [index, change] of plan.changes.entries()) { + fs.renameSync( + expectDefined(tempPaths[index], `release temp path at index ${index}`), + change.path, + ); + } + } finally { + for (const tempPath of tempPaths) { + fs.rmSync(tempPath, { force: true }); + } + } +} + +export function main(argv = process.argv.slice(2)): number { + const args = parseReleaseVersionArgs(argv); + if (args.help) { + printUsage(); + return 0; + } + if (!args.version) { + throw new Error("Missing required --version."); + } + + const plan = planReleaseVersion({ + android: args.android, + rootDir: args.rootDir, + version: args.version, + }); + if (plan.changes.length === 0) { + process.stdout.write(`Release version ${plan.version} is already aligned.\n`); + return 0; + } + + const relativePaths = plan.changes.map((change) => path.relative(args.rootDir, change.path)); + if (args.mode === "check") { + process.stderr.write( + `Release version ${plan.version} requires updates:\n- ${relativePaths.join("\n- ")}\n`, + ); + return 1; + } + + applyReleaseVersionPlan(plan); + process.stdout.write( + `Updated release version ${plan.version}:\n- ${relativePaths.join("\n- ")}\n`, + ); + return 0; +} + +function planPackageJson(rootDir: string, version: string): ReleaseVersionChange { + const filePath = path.join(rootDir, "package.json"); + const currentContent = fs.readFileSync(filePath, "utf8"); + const packageJson = JSON.parse(currentContent) as Record<string, unknown>; + if (typeof packageJson.version !== "string" || !packageJson.version.trim()) { + throw new Error(`Missing package.json version in ${filePath}.`); + } + packageJson.version = version; + return { + currentContent, + nextContent: `${JSON.stringify(packageJson, null, 2)}\n`, + path: filePath, + }; +} + +function planMacosInfoPlist( + rootDir: string, + releaseVersion: NonNullable<ReturnType<typeof parseReleaseVersion>>, +): ReleaseVersionChange { + const filePath = path.join(rootDir, MACOS_INFO_PLIST); + const currentContent = fs.readFileSync(filePath, "utf8"); + const buildVersion = [ + String(releaseVersion.year), + String(releaseVersion.month).padStart(2, "0"), + String(releaseVersion.patch).padStart(2, "0"), + "00", + ].join(""); + const shortVersionContent = replacePlistString( + currentContent, + "CFBundleShortVersionString", + releaseVersion.baseVersion, + MACOS_INFO_PLIST, + ); + const nextContent = replacePlistString( + shortVersionContent, + "CFBundleVersion", + buildVersion, + MACOS_INFO_PLIST, + ); + return { currentContent, nextContent, path: filePath }; +} + +function replacePlistString(content: string, key: string, value: string, filePath: string): string { + const escapedKey = key.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + const pattern = new RegExp( + `(<key>\\s*${escapedKey}\\s*</key>\\s*<string>)([^<]*)(</string>)`, + "gu", + ); + const matches = [...content.matchAll(pattern)]; + if (matches.length !== 1) { + throw new Error(`${filePath} must contain exactly one string value for ${key}.`); + } + return content.replace(pattern, `$1${value}$3`); +} + +function readOptionValue(argv: string[], index: number, flag: string): string { + const value = argv[index + 1]; + if (!value || value.startsWith("-")) { + throw new Error(`Missing value for ${flag}.`); + } + return value; +} + +function printUsage(): void { + process.stdout.write( + [ + "Usage: node --import tsx scripts/release-version.ts --version <version> [--check|--write] [--android] [--root dir]", + "", + " --check report release version drift without writing (default)", + " --write update all selected version files after validating them", + " --android also prepare shared mobile and Android release metadata", + "", + ].join("\n"), + ); +} + +if (import.meta.main) { + try { + process.exitCode = main(); + } catch (error) { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + process.exitCode = 1; + } +} diff --git a/scripts/render-release-changelog.mjs b/scripts/render-release-changelog.mjs new file mode 100644 index 0000000000000000000000000000000000000000..fc35d9aca6016fd1170df86859a9326ccc36ab0d --- /dev/null +++ b/scripts/render-release-changelog.mjs @@ -0,0 +1,59 @@ +#!/usr/bin/env node +import fs from "node:fs"; +import path from "node:path"; +import { parseArgs } from "node:util"; +import { checkReleaseDocsMirrors, renderReleaseDocsMirror } from "./lib/release-docs-mirror.mjs"; + +try { + const { values } = parseArgs({ + options: { + root: { type: "string", default: process.cwd() }, + version: { type: "string" }, + source: { type: "string", multiple: true }, + output: { type: "string" }, + check: { type: "boolean", default: false }, + }, + }); + const rootDir = path.resolve(values.root); + if (values.check) { + if (values.source || values.output) { + throw new Error("--check cannot be combined with --source or --output"); + } + const checked = checkReleaseDocsMirrors({ rootDir, version: values.version }); + process.stdout.write( + `Checked ${checked.length} docs mirror(s)${checked.length ? `: ${checked.join(", ")}` : ""}\n`, + ); + } else { + const rendered = renderReleaseDocsMirror({ + rootDir, + version: values.version, + sources: values.source, + }); + if (values.output) { + const output = path.resolve(values.output); + const physicalOutput = fs.existsSync(output) + ? fs.realpathSync(output) + : path.join(fs.realpathSync(path.dirname(output)), path.basename(output)); + const relative = path + .relative(fs.realpathSync(rootDir), physicalOutput) + .split(path.sep) + .join("/"); + if ( + relative === "CHANGELOG/records" || + relative.startsWith("CHANGELOG/records/") || + relative === "docs" || + relative.startsWith("docs/") + ) { + throw new Error( + "Mirror output must not overwrite docs sources or frozen contribution records", + ); + } + fs.writeFileSync(output, rendered); + } else { + process.stdout.write(rendered); + } + } +} catch (error) { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + process.exitCode = 1; +} diff --git a/scripts/resolve-frozen-codex-live-suite.mjs b/scripts/resolve-frozen-codex-live-suite.mjs new file mode 100644 index 0000000000000000000000000000000000000000..9345d7c94ec2409a5a8307ffa66a98114c8cced1 --- /dev/null +++ b/scripts/resolve-frozen-codex-live-suite.mjs @@ -0,0 +1,140 @@ +#!/usr/bin/env node + +import { appendFileSync, existsSync, readFileSync, realpathSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const CODEX_SUITE_PREFIX = "live-codex-harness"; +const GENERIC_CODEX_SUITE = "live-codex-harness-docker"; +const GPT_56_SUITE_PREFIX = "live-codex-harness-gpt56-"; + +function requireEnv(name) { + const value = process.env[name]?.trim(); + if (!value) { + throw new Error(`${name} is required`); + } + return value; +} + +function appendLine(file, line) { + appendFileSync(file, `${line}\n`, "utf8"); +} + +function readTargetModelIds(readSource) { + const catalogPath = "extensions/codex/provider-catalog.ts"; + const catalog = readSource(catalogPath); + if (catalog === null) { + const harnessPath = "scripts/test-live-codex-harness-docker.sh"; + const source = readSource(harnessPath); + if (source === null) { + throw new Error(`missing frozen Codex harness: ${harnessPath}`); + } + const defaults = new Set( + [...source.matchAll(/\$\{OPENCLAW_LIVE_CODEX_HARNESS_MODEL:-[^/}]+\/([^}\s]+)\}/gu)].map( + (match) => match[1], + ), + ); + if (defaults.size !== 1) { + throw new Error(`cannot read one frozen Codex harness default from ${harnessPath}`); + } + // The harness default proves only the generic lane's model. Specialized + // lanes need the explicit cohort declaration from the historical catalog. + return { modelIds: new Set(defaults), supportsGpt56Cohort: false }; + } + const source = catalog; + const start = source.indexOf("export const FALLBACK_CODEX_MODELS = ["); + const end = source.indexOf("] satisfies", start); + if (start < 0 || end < 0) { + throw new Error(`cannot read the frozen Codex fallback catalog from ${catalogPath}`); + } + const modelIds = new Set( + [...source.slice(start, end).matchAll(/\bid:\s*["']([^"']+)["']/gu)].map((match) => match[1]), + ); + const hasSol = modelIds.has("gpt-5.6-sol"); + const hasLuna = modelIds.has("gpt-5.6-luna"); + if (hasSol !== hasLuna) { + throw new Error("frozen Codex GPT-5.6 capability marker is incomplete; refusing to guess"); + } + return { modelIds, supportsGpt56Cohort: hasSol && hasLuna }; +} + +export function resolveFrozenCodexCompatibility({ suiteId, readSource }) { + const { modelIds, supportsGpt56Cohort } = readTargetModelIds(readSource); + + if (suiteId === GENERIC_CODEX_SUITE) { + const model = modelIds.has("gpt-5.6-luna") + ? "openai/gpt-5.6-luna" + : modelIds.has("gpt-5.5") + ? "openai/gpt-5.5" + : modelIds.has("gpt-5.4-mini") + ? "openai/gpt-5.4-mini" + : undefined; + if (!model) { + throw new Error("frozen Codex catalog has no supported generic live-harness model"); + } + return { model, runLane: true }; + } + + if (suiteId.startsWith(GPT_56_SUITE_PREFIX)) { + return { runLane: supportsGpt56Cohort }; + } + return { runLane: true }; +} + +function main() { + const outputFile = requireEnv("GITHUB_OUTPUT"); + const suiteId = requireEnv("OPENCLAW_FROZEN_CODEX_SUITE_ID"); + const selectedSha = requireEnv("OPENCLAW_SELECTED_SHA"); + const workflowSha = requireEnv("OPENCLAW_WORKFLOW_SHA"); + const isFrozenTarget = selectedSha !== workflowSha; + const omissionsAllowed = process.env.OPENCLAW_ALLOW_FROZEN_TARGET_SCENARIO_OMISSIONS === "1"; + + if (!suiteId.startsWith(CODEX_SUITE_PREFIX) || !isFrozenTarget || !omissionsAllowed) { + appendLine(outputFile, "run_lane=true"); + return; + } + + const targetRoot = requireEnv("OPENCLAW_FROZEN_TARGET_ROOT"); + const result = resolveFrozenCodexCompatibility({ + suiteId, + readSource: (relativePath) => { + const file = path.join(targetRoot, relativePath); + return existsSync(file) ? readFileSync(file, "utf8") : null; + }, + }); + appendLine(outputFile, `run_lane=${result.runLane}`); + + const summaryFile = requireEnv("GITHUB_STEP_SUMMARY"); + if (result.model) { + appendLine(requireEnv("GITHUB_ENV"), `OPENCLAW_LIVE_CODEX_HARNESS_MODEL=${result.model}`); + appendLine( + summaryFile, + `Frozen Codex target \`${selectedSha}\`: \`${suiteId}\` uses \`${result.model}\`.`, + ); + } else if (!result.runLane) { + appendLine( + summaryFile, + `Frozen Codex target \`${selectedSha}\`: omitted unsupported current-only suite \`${suiteId}\`.`, + ); + console.log(`::notice::Omitting unsupported frozen-target Codex suite ${suiteId}`); + } +} + +let invokedAsMain = false; +if (process.argv[1]) { + try { + invokedAsMain = + realpathSync.native(fileURLToPath(import.meta.url)) === realpathSync.native(process.argv[1]); + } catch { + // Inline and stdin importers need not have a filesystem entrypoint. + } +} + +if (invokedAsMain) { + try { + main(); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + } +} diff --git a/scripts/resolve-fs-safe-native-contract.mjs b/scripts/resolve-fs-safe-native-contract.mjs new file mode 100644 index 0000000000000000000000000000000000000000..423d7e60bc591032e1eed78e48a4ed2b7fb0f9da --- /dev/null +++ b/scripts/resolve-fs-safe-native-contract.mjs @@ -0,0 +1,111 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { realpathSync } from "node:fs"; +import { fileURLToPath } from "node:url"; + +const isSha = (value) => /^[0-9a-f]{40}$/u.test(value ?? ""); +const NATIVE_MARKER = + /\b(?:configureFsSafeNative|getFsSafeNativeConfig|getNativeBinding)\b|@openclaw\/fs-safe\/native/u; +const LEGACY_PYTHON_ONLY_CONTRACTS = new Set([ + // 2026.6.33 and earlier frozen lines selected the Python-only fs-safe 0.3 API. + "*:0.3.0", + // 2026.7.33 upgraded the package without adopting the native durability API. + "2026.7.33:0.4.1", +]); + +function listContainingBranches(ref) { + try { + const branches = execFileSync( + "git", + [ + "for-each-ref", + "--format=%(refname:short)", + "--contains", + ref, + "refs/remotes/origin/extended-stable", + ], + { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }, + ); + return branches.split("\n"); + } catch { + return []; + } +} + +export function resolveFsSafeNativeContract({ + selectedSha, + workflowSha, + allowFrozenSource, + containingBranches, + readSource, +}) { + assert.ok(isSha(selectedSha), "ref must be a full lowercase commit SHA"); + assert.ok(isSha(workflowSha), "workflow SHA must be a full lowercase commit SHA"); + if ( + !allowFrozenSource || + selectedSha === workflowSha || + !containingBranches().some((branch) => + /^origin\/extended-stable\/\d{4}\.(?:[1-9]|1[0-2])\.33$/u.test(branch), + ) + ) { + return "required"; + } + const packageSource = readSource("package.json"); + if (packageSource === null) { + throw new Error("missing fs-safe package source"); + } + const packageJson = JSON.parse(packageSource); + const fsSafeVersion = packageJson.dependencies?.["@openclaw/fs-safe"]; + const contractKey = `${packageJson.version ?? ""}:${fsSafeVersion ?? ""}`; + if ( + !LEGACY_PYTHON_ONLY_CONTRACTS.has(`*:${fsSafeVersion ?? ""}`) && + !LEGACY_PYTHON_ONLY_CONTRACTS.has(contractKey) + ) { + return "required"; + } + const defaults = readSource("src/infra/fs-safe-defaults.ts"); + if (defaults === null) { + throw new Error("missing fs-safe defaults source"); + } + // fs-safe 0.3.0's public config module exports Python/lock controls only; + // a built package on this exact dependency cannot consume native controls. + return defaults.includes('import { configureFsSafePython } from "@openclaw/fs-safe/config";') && + !NATIVE_MARKER.test(defaults) + ? "not-applicable" + : "required"; +} + +let invokedAsMain = false; +if (process.argv[1]) { + try { + invokedAsMain = + realpathSync.native(fileURLToPath(import.meta.url)) === realpathSync.native(process.argv[1]); + } catch { + // Inline and stdin importers need not have a filesystem entrypoint. + } +} + +// A canonical frozen release source may omit a proof for functionality it cannot use. +// Current, unknown, and native-consuming sources always verify the installed package. +if (invokedAsMain) { + const [ref, workflowSha, allowFrozenSource] = process.argv.slice(2); + assert.ok(isSha(ref), "ref must be a full lowercase commit SHA"); + assert.ok(isSha(workflowSha), "workflow SHA must be a full lowercase commit SHA"); + let contract = "required"; + try { + contract = resolveFsSafeNativeContract({ + selectedSha: ref, + workflowSha, + allowFrozenSource: allowFrozenSource === "1", + containingBranches: () => listContainingBranches(ref), + readSource: (path) => + execFileSync("git", ["show", `${ref}:${path}`], { + encoding: "utf8", + stdio: ["ignore", "pipe", "ignore"], + }), + }); + } catch { + // Standalone checks remain strict on unknown source; admission propagates errors. + } + process.stdout.write(`${contract}\n`); +} diff --git a/scripts/resolve-openclaw-package-candidate.mts b/scripts/resolve-openclaw-package-candidate.mts new file mode 100644 index 0000000000000000000000000000000000000000..63ecf40afc7cf6c9fa968ac6254ccef45219e539 --- /dev/null +++ b/scripts/resolve-openclaw-package-candidate.mts @@ -0,0 +1,1900 @@ +#!/usr/bin/env node +// Normalizes package-acceptance inputs into the tarball shape consumed by Docker E2E. +import { Buffer } from "node:buffer"; +import { spawn } from "node:child_process"; +import type { ChildProcess, SpawnOptions } from "node:child_process"; +import { createHash } from "node:crypto"; +import { lookup as dnsLookupCb } from "node:dns"; +import type { LookupAddress } from "node:dns"; +import { lookup as dnsLookup } from "node:dns/promises"; +import { once } from "node:events"; +import { createWriteStream } from "node:fs"; +import fs from "node:fs/promises"; +import { request as httpsRequest } from "node:https"; +import { isIP } from "node:net"; +import type { LookupFunction } from "node:net"; +import os from "node:os"; +import path from "node:path"; +import { pipeline } from "node:stream/promises"; +import { fileURLToPath } from "node:url"; +import { isRecord as isJsonRecord } from "../packages/normalization-core/src/record-coerce.ts"; +import { booleanFlag, parseFlagArgs, stringFlag } from "./lib/arg-utils.mts"; +import { appendBoundedTail } from "./lib/bounded-output-tail.mjs"; +import { toErrorObject } from "./lib/error-format.mts"; +import { terminateManagedChild } from "./lib/managed-child-process.mts"; +import { resolveNpmJsonEntries } from "./lib/npm-json-output.mts"; +import { resolveRepoRoot } from "./lib/repo-root.mjs"; +import { resolveNpmRunner } from "./npm-runner.mts"; +import { validatePackageSourceDir } from "./package-source-preflight.mjs"; +import { createPrepublishPluginRegistryArtifact } from "./prepublish-plugin-registry-artifact.mjs"; + +const ROOT_DIR = resolveRepoRoot(import.meta.url); +const DEFAULT_OUTPUT_NAME = "openclaw-current.tgz"; +const PACKAGE_URL_DOWNLOAD_TIMEOUT_MS = 60_000; +const PACKAGE_URL_MAX_BYTES = 250 * 1024 * 1024; +const PACKAGE_URL_MAX_REDIRECTS = 5; +export const ARTIFACT_TARBALL_SCAN_MAX_ENTRIES = 10_000; +const COMMAND_STDOUT_CAPTURE_MAX_CHARS = 8 * 1024 * 1024; +const COMMAND_STDERR_CAPTURE_MAX_CHARS = 128 * 1024; +const COMMAND_TIMEOUT_KILL_AFTER_MS = 5_000; +const FORWARDED_SIGNAL_KILL_AFTER_MS = 250; +const COMMAND_PROCESS_TREE_EXIT_POLL_MS = 50; +const MAX_TIMER_TIMEOUT_MS = 2_147_000_000; +type ChildSignal = ChildProcess["signalCode"]; +type TimerHandle = ReturnType<typeof setTimeout>; +type ChildKiller = (signal: NodeJS.Signals) => void; +type ProcessTreeChild = Pick<ChildProcess, "exitCode" | "kill" | "pid" | "signalCode">; +type CommandOutputBuffer = { + text: string; + truncatedChars: number; +}; + +type RunOptions = { + capture?: boolean; + cwd?: string; + env?: NodeJS.ProcessEnv; + killAfterMs?: unknown; + shell?: boolean | string; + timeoutMs?: unknown; + windowsVerbatimArguments?: boolean; +}; + +type RunCommand = ( + command: string, + args: readonly string[], + options?: RunOptions, +) => Promise<unknown>; + +type NpmPackageCandidatePackRunner = { + args: string[]; + command: string; + env?: NodeJS.ProcessEnv; + shell: boolean; + windowsVerbatimArguments?: boolean; +}; + +type PackageCandidateOptions = { + artifactDir: string; + githubOutput: string; + metadata: string; + outputDir: string; + outputName: string; + packageRef: string; + packageSha256: string; + packageSpec: string; + packageUrl: string; + pluginRegistryOutputDir: string; + requiredPluginPackagesJson: string; + source: string; + trustedSourceId: string; + trustedSourcePolicy: string; + help?: true; +}; + +type Ipv4Octets = [number, number, number, number]; + +type TrustedPackageSource = { + allowPrivateNetwork: boolean; + auth?: { type: string }; + hosts: string[]; + id: string; + pathPrefixes: string[]; + ports: number[]; + redirectHosts: string[]; +}; + +type PackageLookupHost = (hostname: string) => Promise<unknown>; + +type WebResponseBody = { + cancel(reason?: unknown): Promise<unknown>; + getReader(): { + cancel(reason?: unknown): Promise<unknown>; + read(): Promise<ReadableStreamReadResult<string | Uint8Array>>; + releaseLock(): void; + }; +}; + +type NodeResponseBody = { + destroy(error?: Error): void; + [Symbol.asyncIterator](): AsyncIterator<string | Uint8Array>; +}; +type PackageResponseBody = WebResponseBody | NodeResponseBody; +type PackageDownloadResponse = { + body: PackageResponseBody | null; + headers: { get(name: string): string | null }; + status: number; +}; +type PackageFetch = ( + input: URL, + init?: RequestInit, +) => PackageDownloadResponse | Promise<PackageDownloadResponse>; +type PackageDownloadOptions = { + addresses?: string[]; + fetchImpl?: PackageFetch; + headers?: Record<string, string>; + lookupHost?: PackageLookupHost; + maxBytes?: number; + maxRedirects?: number; + timeoutMs?: unknown; + trustedSource?: TrustedPackageSource; +}; +type ArtifactMetadata = Record<string, unknown>; + +const ACTIVE_CHILD_KILLERS = new Set<ChildKiller>(); +const SIGNAL_EXIT_CODES = { + SIGHUP: 129, + SIGINT: 130, + SIGTERM: 143, +} as const; +type ForwardedSignal = keyof typeof SIGNAL_EXIT_CODES; +const TRUSTED_PACKAGE_SOURCE_POLICY = ".github/package-trusted-sources.json"; +const TRUSTED_PACKAGE_SOURCE_TOKEN_ENV = "OPENCLAW_TRUSTED_PACKAGE_TOKEN"; +const BLOCKED_PACKAGE_HOSTNAMES = new Set([ + "localhost", + "localhost.localdomain", + "metadata.google.internal", +]); +let forwardedSignalExitCode: number | undefined; +let forwardedSignalForceKillTimer: TimerHandle | undefined; + +for (const signal of Object.keys(SIGNAL_EXIT_CODES) as ForwardedSignal[]) { + process.on(signal, () => { + forwardedSignalExitCode ??= SIGNAL_EXIT_CODES[signal]; + if (ACTIVE_CHILD_KILLERS.size === 0) { + process.exit(forwardedSignalExitCode); + } + const activeKillers = Array.from(ACTIVE_CHILD_KILLERS); + for (const killChild of activeKillers) { + killChild(signal); + } + forwardedSignalForceKillTimer ??= setTimeout(() => { + for (const killChild of activeKillers) { + killChild("SIGKILL"); + } + process.exit(forwardedSignalExitCode); + }, FORWARDED_SIGNAL_KILL_AFTER_MS); + }); +} +export const OPENCLAW_PACKAGE_SPEC_RE = + /^openclaw@(alpha|beta|extended-stable|latest|[0-9]{4}\.[1-9][0-9]*\.[1-9][0-9]*(-[1-9][0-9]*|-(alpha|beta)\.[1-9][0-9]*)?)$/u; + +function usage() { + return `Usage: node --import tsx scripts/resolve-openclaw-package-candidate.mts --source <ref|npm|url|trusted-url|artifact> --output-dir <dir> [options] + +Options: + --package-spec <spec> Published npm spec for source=npm. + --package-ref <ref> Trusted repo ref for source=ref or npm companion packaging. + --package-url <url> HTTPS tarball URL for source=url or source=trusted-url. + --package-sha256 <sha256> Expected tarball SHA-256 for source=url, source=trusted-url, or source=artifact. + --trusted-source-id <id> Named trusted URL policy for source=trusted-url. + --trusted-source-policy <file> + Repo-controlled trusted URL source policy. Default: ${TRUSTED_PACKAGE_SOURCE_POLICY} + --artifact-dir <dir> Directory containing exactly one .tgz for source=artifact. + --output-name <name> Output tarball filename. Default: ${DEFAULT_OUTPUT_NAME} + --metadata <file> Write package metadata JSON. + --plugin-registry-output-dir <dir> + Build an immutable registry for source=ref, npm, or artifact. + --required-plugin-packages-json <json> + Scoped package names to include in that registry. + --github-output <file> Append tarball, sha256, package name/version outputs.`; +} + +export function parseArgs(argv: readonly string[]) { + const options: PackageCandidateOptions = { + artifactDir: "", + githubOutput: "", + metadata: "", + outputDir: "", + outputName: DEFAULT_OUTPUT_NAME, + packageRef: "", + packageSha256: "", + packageSpec: "", + packageUrl: "", + pluginRegistryOutputDir: "", + requiredPluginPackagesJson: "[]", + source: "", + trustedSourceId: "", + trustedSourcePolicy: TRUSTED_PACKAGE_SOURCE_POLICY, + }; + parseFlagArgs( + argv, + options, + [ + ...( + [ + ["--artifact-dir", "artifactDir"], + ["--github-output", "githubOutput"], + ["--metadata", "metadata"], + ["--output-dir", "outputDir"], + ["--output-name", "outputName"], + ["--plugin-registry-output-dir", "pluginRegistryOutputDir"], + ["--required-plugin-packages-json", "requiredPluginPackagesJson"], + ["--source", "source"], + ["--trusted-source-policy", "trustedSourcePolicy"], + ] as const + ).map(([flag, key]) => + stringFlag(flag, key, { allowInline: false, rejectShortOptions: true }), + ), + ...( + [ + ["--package-ref", "packageRef"], + ["--package-spec", "packageSpec"], + ["--package-url", "packageUrl"], + ["--trusted-source-id", "trustedSourceId"], + ] as const + ).map(([flag, key]) => + stringFlag(flag, key, { + allowEmpty: true, + allowInline: false, + rejectShortOptions: true, + }), + ), + stringFlag("--package-sha256", "packageSha256", { + allowEmpty: true, + allowInline: false, + rejectShortOptions: true, + transform: (value: string) => value.toLowerCase(), + }), + booleanFlag("--help", "help", true, { repeatable: true }), + booleanFlag("-h", "help", true, { repeatable: true }), + ], + { + ignoreDoubleDash: false, + onUnhandledArg(arg: string) { + throw new Error(`unknown argument: ${arg}`); + }, + }, + ); + validateOutputName(options.outputName); + return options; +} + +function validateOutputName(value: string) { + if (!/^[A-Za-z0-9][A-Za-z0-9._-]*\.t(?:ar\.)?gz$/u.test(value)) { + throw new Error(`--output-name must be a tarball filename, not a path: ${value}`); + } +} + +function resolvePackedOpenClawTarballFilename(value: unknown) { + const filename = typeof value === "string" ? value.trim() : ""; + if ( + !/^openclaw-[A-Za-z0-9._-]+\.tgz$/u.test(filename) || + filename.includes("\0") || + filename !== path.basename(filename) || + filename !== path.win32.basename(filename) + ) { + throw new Error( + `npm pack reported unsafe OpenClaw tarball filename: ${JSON.stringify(filename)}`, + ); + } + return filename; +} + +export function validateOpenClawPackageSpec(spec: string) { + if (!OPENCLAW_PACKAGE_SPEC_RE.test(spec)) { + throw new Error( + `package_spec must be openclaw@alpha, openclaw@beta, openclaw@extended-stable, openclaw@latest, or an exact OpenClaw release version; got: ${spec}`, + ); + } +} + +export function resolveNpmPackageCandidatePackRunner( + packageSpec: string, + outputDir: string, + params: { + comSpec?: string; + env?: NodeJS.ProcessEnv; + execPath?: string; + existsSync?: (candidate: string) => boolean; + platform?: typeof process.platform; + } = {}, +): NpmPackageCandidatePackRunner { + validateOpenClawPackageSpec(packageSpec); + return resolveNpmRunner({ + comSpec: params.comSpec, + env: params.env, + execPath: params.execPath, + existsSync: params.existsSync, + npmArgs: ["pack", packageSpec, "--ignore-scripts", "--json", "--pack-destination", outputDir], + platform: params.platform, + }); +} + +function numericTimerValueMs(valueMs: unknown) { + const value = Number(valueMs); + return Number.isFinite(value) ? Math.floor(value) : undefined; +} + +function resolvePackageCandidateTimeoutMs( + valueMs: unknown, + fallbackMs: unknown = MAX_TIMER_TIMEOUT_MS, +) { + const value = numericTimerValueMs(valueMs) ?? numericTimerValueMs(fallbackMs); + return Math.min(Math.max(value ?? MAX_TIMER_TIMEOUT_MS, 1), MAX_TIMER_TIMEOUT_MS); +} + +function resolveOptionalTimerTimeoutMs(valueMs: unknown) { + if (valueMs === undefined) { + return undefined; + } + return resolvePackageCandidateTimeoutMs(valueMs, 1); +} + +function run(command: string, args: readonly string[], options: RunOptions = {}) { + return new Promise<string>((resolve, reject) => { + const resolvedTimeoutMs = resolveOptionalTimerTimeoutMs(options.timeoutMs); + const resolvedKillAfterMs = resolvePackageCandidateTimeoutMs( + options.killAfterMs, + COMMAND_TIMEOUT_KILL_AFTER_MS, + ); + const useProcessGroup = process.platform !== "win32"; + const spawnOptions: SpawnOptions = { + cwd: options.cwd ?? ROOT_DIR, + stdio: options.capture ? ["ignore", "pipe", "pipe"] : ["ignore", "inherit", "inherit"], + ...(options.env ? { env: options.env } : {}), + ...(options.shell !== undefined ? { shell: options.shell } : {}), + ...(options.windowsVerbatimArguments !== undefined + ? { windowsVerbatimArguments: options.windowsVerbatimArguments } + : {}), + detached: useProcessGroup, + }; + const child = spawn(command, args, spawnOptions); + let timedOut = false; + let killTimer: TimerHandle | undefined; + let forceKillAt: number | undefined; + const killChild: ChildKiller = (signal) => + terminateManagedChild(child, signal, { useProcessGroup }); + const terminateChild = () => { + killChild("SIGTERM"); + forceKillAt = Date.now() + resolvedKillAfterMs; + killTimer = setTimeout(() => { + killTimer = undefined; + forceKillAt = undefined; + killChild("SIGKILL"); + }, resolvedKillAfterMs); + }; + const timeout = + resolvedTimeoutMs === undefined + ? undefined + : setTimeout(() => { + timedOut = true; + terminateChild(); + }, resolvedTimeoutMs); + timeout?.unref?.(); + ACTIVE_CHILD_KILLERS.add(killChild); + let stdout = { text: "", truncatedChars: 0 }; + let stderr = { text: "", truncatedChars: 0 }; + if (options.capture) { + child.stdout?.setEncoding("utf8").on("data", (chunk: string) => { + stdout = appendBoundedTail(stdout, chunk, COMMAND_STDOUT_CAPTURE_MAX_CHARS); + }); + child.stderr?.setEncoding("utf8").on("data", (chunk: string) => { + stderr = appendBoundedTail(stderr, chunk, COMMAND_STDERR_CAPTURE_MAX_CHARS); + }); + } + child.on("error", (error: Error) => { + ACTIVE_CHILD_KILLERS.delete(killChild); + reject(toErrorObject(error, "Non-Error rejection")); + }); + child.on("close", (status: number | null, signal: ChildSignal) => { + if (timeout) { + clearTimeout(timeout); + } + if (killTimer && !timedOut) { + clearTimeout(killTimer); + forceKillAt = undefined; + } + ACTIVE_CHILD_KILLERS.delete(killChild); + if ( + forwardedSignalExitCode !== undefined && + ACTIVE_CHILD_KILLERS.size === 0 && + forwardedSignalForceKillTimer === undefined + ) { + process.exit(forwardedSignalExitCode); + } + if (forwardedSignalExitCode !== undefined) { + return; + } + if (timedOut) { + const timeoutError = new Error( + `${command} ${args.join(" ")} timed out after ${resolvedTimeoutMs}ms`, + ); + if (killTimer) { + void finishTimedOutProcessTree(child, { + forceKillAt, + killChild, + killTimer, + killAfterMs: resolvedKillAfterMs, + useProcessGroup, + }).then(() => reject(timeoutError), reject); + return; + } + reject(timeoutError); + return; + } + if (status === 0) { + if (stdout.truncatedChars > 0) { + reject( + new Error( + `${command} ${args.join(" ")} produced more than ${COMMAND_STDOUT_CAPTURE_MAX_CHARS} captured stdout chars`, + ), + ); + return; + } + resolve(stdout.text); + return; + } + const stderrText = formatCapturedCommandOutput(stderr).trim(); + const detail = stderrText ? `\n${stderrText}` : ""; + reject(new Error(`${command} ${args.join(" ")} failed with ${status ?? signal}${detail}`)); + }); + }); +} + +async function finishTimedOutProcessTree( + child: ProcessTreeChild, + { + forceKillAt, + killAfterMs, + killChild, + killTimer, + useProcessGroup, + }: { + forceKillAt: number | undefined; + killAfterMs: number; + killChild: ChildKiller; + killTimer: TimerHandle; + useProcessGroup: boolean; + }, +) { + const graceRemainingMs = + forceKillAt === undefined ? killAfterMs : Math.max(0, forceKillAt - Date.now()); + if (graceRemainingMs > 0) { + await waitForProcessTreeExit(child, graceRemainingMs, useProcessGroup); + } + clearTimeout(killTimer); + if (processTreeIsAlive(child, useProcessGroup)) { + killChild("SIGKILL"); + await waitForProcessTreeExit(child, killAfterMs, useProcessGroup); + } +} + +function childHasExited(child: ProcessTreeChild) { + return child.exitCode !== null || child.signalCode !== null; +} + +function processTreeIsAlive(child: ProcessTreeChild, useProcessGroup: boolean) { + if (!child || typeof child.pid !== "number") { + return false; + } + if (!useProcessGroup) { + return !childHasExited(child); + } + try { + process.kill(-child.pid, 0); + return true; + } catch (error) { + return errorCode(error) === "EPERM"; + } +} + +async function waitForProcessTreeExit( + child: ProcessTreeChild, + timeoutMs: number, + useProcessGroup: boolean, +) { + const deadlineAt = Date.now() + timeoutMs; + while (Date.now() < deadlineAt) { + if (!processTreeIsAlive(child, useProcessGroup)) { + return true; + } + await new Promise((resolvePoll) => { + setTimeout(resolvePoll, COMMAND_PROCESS_TREE_EXIT_POLL_MS); + }); + } + return !processTreeIsAlive(child, useProcessGroup); +} + +function formatCapturedCommandOutput(buffer: CommandOutputBuffer) { + if (buffer.truncatedChars === 0) { + return buffer.text; + } + return `[output truncated ${buffer.truncatedChars} chars; showing tail]\n${buffer.text}`; +} + +export const runCommandForTest = run; + +async function sha256(file: string) { + const hash = createHash("sha256"); + const handle = await fs.open(file, "r"); + try { + for await (const chunk of handle.createReadStream()) { + hash.update(chunk); + } + } finally { + await handle.close(); + } + return hash.digest("hex"); +} + +function assertSha256(value: string) { + if (!/^[a-f0-9]{64}$/iu.test(value)) { + throw new Error(`package_sha256 must be a lowercase or uppercase 64-character SHA-256 digest`); + } +} + +async function assertExpectedSha256(file: string, expected: string) { + if (!expected) { + return await sha256(file); + } + assertSha256(expected); + const actual = await sha256(file); + if (actual !== expected.toLowerCase()) { + throw new Error(`package SHA-256 mismatch: expected ${expected}, got ${actual}`); + } + return actual; +} + +export const assertExpectedSha256ForTest = assertExpectedSha256; + +async function findSingleTarball(dir: string, maxEntries = ARTIFACT_TARBALL_SCAN_MAX_ENTRIES) { + const root = path.resolve(ROOT_DIR, dir); + const pending = [root]; + const tarballs = []; + let scannedEntries = 0; + + while (pending.length > 0) { + const currentDir = pending.pop(); + if (!currentDir) { + continue; + } + const handle = await fs.opendir(currentDir); + for await (const entry of handle) { + scannedEntries += 1; + if (scannedEntries > maxEntries) { + throw new Error( + `source=artifact scan exceeded ${maxEntries} filesystem entries under ${dir}; provide a smaller artifact directory containing exactly one .tgz.`, + ); + } + + const absolute = path.join(currentDir, entry.name); + if (entry.isDirectory()) { + pending.push(absolute); + continue; + } + if (entry.isFile() && /\.t(?:ar\.)?gz$/u.test(entry.name)) { + tarballs.push(absolute); + if (tarballs.length > 1) { + const relativeTarballs = tarballs + .map((tarball) => path.relative(root, tarball)) + .toSorted((a, b) => a.localeCompare(b)); + throw new Error( + `source=artifact requires exactly one .tgz under ${dir}; found at least 2: ${relativeTarballs.join(", ")}`, + ); + } + } + } + } + + if (tarballs.length !== 1) { + throw new Error( + `source=artifact requires exactly one .tgz under ${dir}; found ${tarballs.length}: ${tarballs.join(", ")}`, + ); + } + return tarballs[0]!; +} + +export const findSingleTarballForTest = findSingleTarball; + +export async function readArtifactPackageCandidateMetadata(dir: string) { + const metadataPath = path.join(path.resolve(ROOT_DIR, dir), "package-candidate.json"); + let raw: string; + try { + raw = await fs.readFile(metadataPath, "utf8"); + } catch (error) { + if (errorCode(error) === "ENOENT") { + return {}; + } + throw error; + } + const parsed: unknown = JSON.parse(raw); + if (!isJsonRecord(parsed)) { + throw new Error(`artifact package-candidate.json must contain a JSON object`); + } + const packageSourceSha = + typeof parsed.packageSourceSha === "string" ? parsed.packageSourceSha.trim() : ""; + if (packageSourceSha && !/^[0-9a-f]{40}$/iu.test(packageSourceSha)) { + throw new Error( + "artifact package-candidate.json packageSourceSha must be a 40-character commit SHA", + ); + } + if (typeof parsed.packageSourceSha === "string") { + return packageSourceSha + ? { ...parsed, packageSourceSha: packageSourceSha.toLowerCase() } + : { ...parsed, packageSourceSha: "" }; + } + return parsed; +} + +async function revParseTrustedInputRef(ref: string) { + const candidates = [ref, `refs/remotes/origin/${ref}`, `refs/tags/${ref}`]; + for (const candidate of candidates) { + const resolved = await run("git", ["rev-parse", "--verify", `${candidate}^{commit}`], { + capture: true, + }).then( + (value) => value.trim(), + () => "", + ); + if (resolved) { + return resolved; + } + } + throw new Error(`package_ref does not resolve to a commit: ${ref}`); +} + +async function resolveTrustedRepoRef(ref: string) { + if (!ref || ref.trim() === "" || ref.startsWith("-")) { + throw new Error( + `package_ref must be a branch, tag, or full commit SHA; got: ${ref || "<empty>"}`, + ); + } + + await run("git", ["fetch", "--no-tags", "origin", "+refs/heads/*:refs/remotes/origin/*"]); + await run("git", ["fetch", "--tags", "origin", "+refs/tags/*:refs/tags/*"]); + + const selectedSha = await revParseTrustedInputRef(ref); + const isMainAncestor = await run("git", [ + "merge-base", + "--is-ancestor", + selectedSha, + "refs/remotes/origin/main", + ]).then( + () => true, + () => false, + ); + if (isMainAncestor) { + return { selectedSha, trustedReason: "main-ancestor" }; + } + + const releaseTags = (await run("git", ["tag", "--points-at", selectedSha], { capture: true })) + .split(/\r?\n/u) + .map((line) => line.trim()) + .filter(Boolean); + if (releaseTags.some((tag) => tag.startsWith("v"))) { + return { selectedSha, trustedReason: "release-tag" }; + } + + const containingBranches = ( + await run( + "git", + [ + "for-each-ref", + "--format=%(refname:short)", + "--contains", + selectedSha, + "refs/remotes/origin", + ], + { capture: true }, + ) + ) + .split(/\r?\n/u) + .map((line) => line.trim()) + .filter(Boolean); + if (containingBranches.some((branch) => branch.startsWith("origin/"))) { + return { selectedSha, trustedReason: "repository-branch-history" }; + } + + throw new Error( + `package_ref ${ref} resolved to ${selectedSha}, which is not reachable from an OpenClaw branch or release tag`, + ); +} + +async function preparePackageSourceWorktree(ref: string) { + const { selectedSha, trustedReason } = await resolveTrustedRepoRef(ref); + const sourceDir = path.join( + process.env.RUNNER_TEMP || os.tmpdir(), + `openclaw-package-source-${process.pid}`, + ); + await fs.rm(sourceDir, { recursive: true, force: true }); + await run("git", ["worktree", "add", "--detach", sourceDir, selectedSha]); + return { selectedSha, sourceDir, trustedReason }; +} + +async function cleanupPackageSourceWorktree( + sourceDir: string, + { + resolveError, + runImpl = run, + consoleError = console.error, + }: { + resolveError?: unknown; + runImpl?: RunCommand; + consoleError?: (message: string) => void; + } = {}, +) { + try { + await runImpl("git", ["worktree", "remove", "--force", sourceDir]); + } catch (cleanupError) { + if (!resolveError) { + throw cleanupError; + } + const message = cleanupError instanceof Error ? cleanupError.message : String(cleanupError); + consoleError( + `warning: failed to remove temporary package source worktree ${sourceDir}: ${message}`, + ); + } +} + +export const cleanupPackageSourceWorktreeForTest = cleanupPackageSourceWorktree; + +async function installPackageSourceDeps(sourceDir: string) { + await run( + "pnpm", + [ + "install", + "--frozen-lockfile", + "--config.ignore-scripts=false", + "--config.engine-strict=false", + "--config.enable-pre-post-scripts=true", + ], + { cwd: sourceDir }, + ); +} + +async function moveNewestPackedTarball(outputDir: string, packOutput: string, outputName: string) { + let filename = ""; + let parsed; + try { + parsed = JSON.parse(packOutput); + } catch {} + if (parsed !== undefined) { + const packedEntry = resolveNpmJsonEntries(parsed).find( + (entry): entry is Record<string, unknown> => + isJsonRecord(entry) && typeof entry.filename === "string", + ); + const packedFilename = + packedEntry && typeof packedEntry.filename === "string" ? packedEntry.filename : ""; + if (packedFilename) { + filename = resolvePackedOpenClawTarballFilename(packedFilename); + } + } + if (!filename) { + for (const line of packOutput.split(/\r?\n/u)) { + const trimmed = line.trim(); + if ( + trimmed.endsWith(".tgz") && + (trimmed.startsWith("openclaw-") || + trimmed.includes(":") || + trimmed.includes("/") || + trimmed.includes("\\")) + ) { + filename = resolvePackedOpenClawTarballFilename(trimmed); + } + } + } + if (!filename) { + const entries = await fs.readdir(outputDir); + filename = + entries + .filter((entry) => { + try { + return resolvePackedOpenClawTarballFilename(entry) === entry; + } catch { + return false; + } + }) + .toSorted((a, b) => a.localeCompare(b)) + .at(-1) ?? ""; + } + if (!filename) { + throw new Error(`npm pack produced no OpenClaw tarball in ${outputDir}`); + } + const packed = path.join(outputDir, filename); + const target = path.join(outputDir, outputName); + if (packed !== target) { + await fs.rm(target, { force: true }); + await fs.rename(packed, target); + } + return target; +} + +export const moveNewestPackedTarballForTest = moveNewestPackedTarball; + +async function cleanPackedOpenClawTarballs(outputDir: string) { + let entries: string[]; + try { + entries = await fs.readdir(outputDir); + } catch (error) { + if (errorCode(error) === "ENOENT") { + entries = []; + } else { + throw error; + } + } + await Promise.all( + entries + .filter((entry) => { + try { + return resolvePackedOpenClawTarballFilename(entry) === entry; + } catch { + return false; + } + }) + .map((entry) => fs.rm(path.join(outputDir, entry), { force: true })), + ); +} + +export const cleanPackedOpenClawTarballsForTest = cleanPackedOpenClawTarballs; + +function normalizeUrlHostname(hostname: string): string { + return hostname.replace(/^\[/u, "").replace(/\]$/u, "").replace(/\.+$/u, "").toLowerCase(); +} + +function parseIpv4(address: string): Ipv4Octets | null { + const parts = address.split("."); + if (parts.length !== 4) { + return null; + } + const octets = parts.map((part) => Number(part)) as Ipv4Octets; + if (octets.some((octet) => !Number.isInteger(octet) || octet < 0 || octet > 255)) { + return null; + } + return [octets[0], octets[1], octets[2], octets[3]]; +} + +function ipv4ToInt(octets: Ipv4Octets) { + return ((octets[0] << 24) >>> 0) + (octets[1] << 16) + (octets[2] << 8) + octets[3]; +} + +function ipv4InCidr(octets: Ipv4Octets, base: Ipv4Octets, bits: number) { + const mask = bits === 0 ? 0 : (0xffffffff << (32 - bits)) >>> 0; + return (ipv4ToInt(octets) & mask) === (ipv4ToInt(base) & mask); +} + +function isUnsafeIpv4(address: string | Ipv4Octets) { + const octets = Array.isArray(address) ? address : parseIpv4(address); + if (!octets) { + return true; + } + const blockedRanges: Array<[Ipv4Octets, number]> = [ + [[0, 0, 0, 0], 8], + [[10, 0, 0, 0], 8], + [[100, 64, 0, 0], 10], + [[127, 0, 0, 0], 8], + [[169, 254, 0, 0], 16], + [[172, 16, 0, 0], 12], + [[192, 0, 0, 0], 24], + [[192, 0, 2, 0], 24], + [[192, 168, 0, 0], 16], + [[198, 18, 0, 0], 15], + [[198, 51, 100, 0], 24], + [[203, 0, 113, 0], 24], + [[224, 0, 0, 0], 4], + [[240, 0, 0, 0], 4], + ]; + return blockedRanges.some(([base, bits]) => ipv4InCidr(octets, base, bits)); +} + +function ipv4FromHextets(high: number, low: number): Ipv4Octets { + return [(high >>> 8) & 0xff, high & 0xff, (low >>> 8) & 0xff, low & 0xff]; +} + +function ipv4OctetsToHextets(octets: Ipv4Octets): [string, string] { + return [((octets[0] << 8) | octets[1]).toString(16), ((octets[2] << 8) | octets[3]).toString(16)]; +} + +function parseIpv6Parts(address: string): number[] | null { + const normalized = address.toLowerCase().replace(/%[0-9a-z_.-]+$/u, ""); + const dottedIpv4 = normalized.match(/^(.*:)(\d{1,3}(?:\.\d{1,3}){3})$/u); + let canonical = normalized; + if (dottedIpv4) { + const dottedIpv4Octets = parseIpv4(dottedIpv4[2]!); + if (!dottedIpv4Octets) { + return null; + } + const [high, low] = ipv4OctetsToHextets(dottedIpv4Octets); + canonical = `${dottedIpv4[1]!}${high}:${low}`; + } + if (canonical.includes(":::") || canonical.split("::").length > 2) { + return null; + } + const [leftRaw = "", rightRaw = ""] = canonical.split("::"); + const parseParts = (value: string): number[] => { + if (!value) { + return []; + } + return value.split(":").map((part) => { + if (!/^[0-9a-f]{1,4}$/u.test(part)) { + return Number.NaN; + } + return Number.parseInt(part, 16); + }); + }; + const left = parseParts(leftRaw); + const right = parseParts(rightRaw); + if ([...left, ...right].some((part) => !Number.isInteger(part) || part < 0 || part > 0xffff)) { + return null; + } + const zeroCount = canonical.includes("::") ? 8 - left.length - right.length : 0; + if (zeroCount < 0 || (!canonical.includes("::") && left.length !== 8)) { + return null; + } + return [...left, ...Array.from({ length: zeroCount }, () => 0), ...right]; +} + +function extractUnsafeEmbeddedIpv4FromIpv6(address: string): Ipv4Octets | null { + const parts = parseIpv6Parts(address); + if (!parts || parts.length !== 8) { + return null; + } + const hextets = parts as [number, number, number, number, number, number, number, number]; + const candidates: Ipv4Octets[] = []; + if (hextets.slice(0, 5).every((part) => part === 0) && hextets[5] === 0xffff) { + candidates.push(ipv4FromHextets(hextets[6], hextets[7])); + } + if (hextets.slice(0, 6).every((part) => part === 0)) { + candidates.push(ipv4FromHextets(hextets[6], hextets[7])); + } + if ( + hextets[0] === 0x0064 && + hextets[1] === 0xff9b && + hextets.slice(2, 6).every((part) => part === 0) + ) { + candidates.push(ipv4FromHextets(hextets[6], hextets[7])); + } + if ( + hextets[0] === 0x0064 && + hextets[1] === 0xff9b && + hextets[2] === 0x0001 && + hextets.slice(3, 6).every((part) => part === 0) + ) { + candidates.push(ipv4FromHextets(hextets[6], hextets[7])); + } + if (hextets[0] === 0x2002) { + candidates.push(ipv4FromHextets(hextets[1], hextets[2])); + } + if (hextets[0] === 0x2001 && hextets[1] === 0x0000) { + candidates.push(ipv4FromHextets(hextets[6] ^ 0xffff, hextets[7] ^ 0xffff)); + } + if ((hextets[4] & 0xfcff) === 0 && hextets[5] === 0x5efe) { + candidates.push(ipv4FromHextets(hextets[6], hextets[7])); + } + return candidates.find((candidate) => isUnsafeIpv4(candidate)) ?? null; +} + +function isUnsafeIpv6(address: string) { + const normalized = address.toLowerCase(); + if (extractUnsafeEmbeddedIpv4FromIpv6(normalized)) { + return true; + } + return ( + normalized === "::" || + normalized === "::1" || + normalized.startsWith("fc") || + normalized.startsWith("fd") || + /^fe[89ab]/u.test(normalized) || + normalized.startsWith("ff") || + normalized.startsWith("64:ff9b:") || + normalized.startsWith("100:") || + normalized.startsWith("2001:2:") || + normalized.startsWith("2001:db8:") + ); +} + +function isUnsafeIpAddress(address: string) { + const normalized = normalizeUrlHostname(address); + const family = isIP(normalized); + if (family === 4) { + return isUnsafeIpv4(normalized); + } + if (family === 6) { + return isUnsafeIpv6(normalized); + } + return true; +} + +function isBlockedPackageHostname(hostname: string) { + const normalized = normalizeUrlHostname(hostname); + return ( + BLOCKED_PACKAGE_HOSTNAMES.has(normalized) || + normalized.endsWith(".localhost") || + normalized.endsWith(".local") || + normalized.endsWith(".internal") || + (isIP(normalized) !== 0 && isUnsafeIpAddress(normalized)) + ); +} + +function packageUrlPort(parsed: URL) { + return parsed.port ? Number(parsed.port) : 443; +} + +function toUniqueNormalizedHostList(value: unknown, field: string, sourceId: string) { + if (!Array.isArray(value) || value.length === 0) { + throw new Error(`trusted package source ${sourceId} must define non-empty ${field}`); + } + return [...new Set(value.map((entry) => normalizeUrlHostname(String(entry))).filter(Boolean))]; +} + +function toTrustedPorts(value: unknown, sourceId: string) { + const ports = value === undefined ? [443] : value; + if (!Array.isArray(ports) || ports.length === 0) { + throw new Error(`trusted package source ${sourceId} must define non-empty ports`); + } + const normalized = ports.map((port) => parseTrustedPort(port)); + if (normalized.some((port) => !Number.isInteger(port) || port < 1 || port > 65535)) { + throw new Error(`trusted package source ${sourceId} has invalid ports`); + } + return [...new Set(normalized)].toSorted((a, b) => a - b); +} + +function parseTrustedPort(value: unknown) { + if (typeof value === "number") { + return value; + } + if (typeof value === "string" && /^[0-9]+$/u.test(value)) { + return Number(value); + } + return Number.NaN; +} + +function pathnameMatchesTrustedPrefix(pathname: string, prefix: string) { + if (prefix === "/") { + return true; + } + const normalizedPrefix = prefix.endsWith("/") ? prefix : `${prefix}/`; + return pathname === prefix || pathname.startsWith(normalizedPrefix); +} + +function toPathPrefixes(value: unknown, sourceId: string) { + const prefixes = value === undefined ? ["/"] : value; + if (!Array.isArray(prefixes) || prefixes.length === 0) { + throw new Error(`trusted package source ${sourceId} must define non-empty pathPrefixes`); + } + return prefixes.map((prefix) => { + const text = String(prefix); + if (!text.startsWith("/")) { + throw new Error(`trusted package source ${sourceId} pathPrefixes must start with /`); + } + return text; + }); +} + +function normalizeTrustedPackageSource(id: string, raw: unknown): TrustedPackageSource { + if (!/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/u.test(id)) { + throw new Error(`Invalid trusted package source id: ${id}`); + } + if (!isJsonRecord(raw)) { + throw new Error(`trusted package source ${id} must be an object`); + } + const hosts = toUniqueNormalizedHostList(raw.hosts, "hosts", id); + const redirectHosts = raw.redirectHosts + ? toUniqueNormalizedHostList(raw.redirectHosts, "redirectHosts", id) + : hosts; + const rawAuth = raw.auth; + let auth: TrustedPackageSource["auth"]; + if (rawAuth !== undefined) { + if (!isJsonRecord(rawAuth) || rawAuth.type !== "bearer") { + throw new Error(`trusted package source ${id} auth must be {"type":"bearer"}`); + } + const authKeys = Object.keys(rawAuth); + if (authKeys.some((key) => key !== "type")) { + throw new Error(`trusted package source ${id} auth only supports type`); + } + auth = { type: "bearer" }; + } + return { + allowPrivateNetwork: raw.allowPrivateNetwork === true, + auth, + hosts, + id, + pathPrefixes: toPathPrefixes(raw.pathPrefixes, id), + ports: toTrustedPorts(raw.ports, id), + redirectHosts, + }; +} + +export async function loadTrustedPackageSource( + id: string, + policyPath: string = TRUSTED_PACKAGE_SOURCE_POLICY, +): Promise<TrustedPackageSource> { + if (!id) { + throw new Error("source=trusted-url requires --trusted-source-id"); + } + const absolutePolicyPath = path.resolve(ROOT_DIR, policyPath); + const sourceId = id; + let policy: unknown; + try { + policy = JSON.parse(await fs.readFile(absolutePolicyPath, "utf8")); + } catch (error) { + throw new Error(`Unable to read trusted package source policy: ${policyPath}`, { + cause: error, + }); + } + if (!isJsonRecord(policy) || policy.schemaVersion !== 1) { + throw new Error(`Trusted package source policy must use schemaVersion 1: ${policyPath}`); + } + const sources = policy.sources; + if (!isJsonRecord(sources)) { + throw new Error(`Trusted package source policy must define sources: ${policyPath}`); + } + if (!Object.hasOwn(sources, sourceId)) { + throw new Error(`Unknown trusted package source: ${sourceId}`); + } + return normalizeTrustedPackageSource(sourceId, sources[sourceId]); +} + +function validateTrustedPackageDownloadUrl( + parsed: URL, + trustedSource: TrustedPackageSource, + options: { isRedirect?: boolean } = {}, +) { + if (parsed.protocol !== "https:") { + throw new Error(`package_url must use https: ${parsed.toString()}`); + } + if (parsed.username || parsed.password) { + throw new Error(`package_url must not include credentials: ${parsed.origin}`); + } + const hostname = normalizeUrlHostname(parsed.hostname); + const allowedHosts = options.isRedirect ? trustedSource.redirectHosts : trustedSource.hosts; + if (!allowedHosts.includes(hostname)) { + throw new Error( + `package_url host ${parsed.hostname} is not allowed by trusted package source ${trustedSource.id}`, + ); + } + if (!trustedSource.ports.includes(packageUrlPort(parsed))) { + throw new Error( + `package_url port ${packageUrlPort(parsed)} is not allowed by trusted package source ${trustedSource.id}`, + ); + } + if ( + !trustedSource.pathPrefixes.some((prefix) => + pathnameMatchesTrustedPrefix(parsed.pathname, prefix), + ) + ) { + throw new Error( + `package_url path is not allowed by trusted package source ${trustedSource.id}`, + ); + } + if (!trustedSource.allowPrivateNetwork && isBlockedPackageHostname(parsed.hostname)) { + throw new Error( + `Blocked hostname or private/internal/special-use IP address: ${parsed.hostname}`, + ); + } +} + +function createTrustedPackageAuthHeaders( + trustedSource: TrustedPackageSource | undefined, + parsed: URL, + initialOrigin: string, +) { + if (!trustedSource?.auth) { + return undefined; + } + if (parsed.origin !== initialOrigin) { + return undefined; + } + const token = process.env[TRUSTED_PACKAGE_SOURCE_TOKEN_ENV]; + if (!token) { + throw new Error( + `trusted package source ${trustedSource.id} requires ${TRUSTED_PACKAGE_SOURCE_TOKEN_ENV}`, + ); + } + return { authorization: `Bearer ${token}` }; +} + +function validatePackageDownloadUrl(parsed: URL) { + if (parsed.protocol !== "https:") { + throw new Error(`package_url must use https: ${parsed.toString()}`); + } + if (parsed.username || parsed.password) { + throw new Error(`package_url must not include credentials: ${parsed.origin}`); + } + if (parsed.port && parsed.port !== "443") { + throw new Error(`package_url must use the default HTTPS port: ${parsed.origin}`); + } + if (isBlockedPackageHostname(parsed.hostname)) { + throw new Error( + `Blocked hostname or private/internal/special-use IP address: ${parsed.hostname}`, + ); + } +} + +async function defaultLookupHost(hostname: string) { + return await dnsLookup(hostname, { all: true, verbatim: true }); +} + +function normalizeLookupResults(results: unknown): LookupAddress[] { + const entries = Array.isArray(results) ? results : [results]; + return entries + .map((entry) => ({ + address: isJsonRecord(entry) && typeof entry.address === "string" ? entry.address : "", + family: Number(isJsonRecord(entry) ? (entry.family ?? 0) : 0), + })) + .filter((entry) => entry.address && (entry.family === 4 || entry.family === 6)); +} + +function createPinnedLookup(hostname: string, addresses: string[]): LookupFunction { + const normalizedHost = normalizeUrlHostname(hostname); + const records: LookupAddress[] = addresses.map((address) => ({ + address, + family: isIP(normalizeUrlHostname(address)), + })); + type LookupCallback = ( + error: (Error & { code?: string }) | null, + address: string | LookupAddress[], + family?: number, + ) => void; + type PinnedLookupOptions = { + all?: boolean; + family?: number; + hints?: number; + verbatim?: boolean; + }; + const lookup = ( + host: string, + options: PinnedLookupOptions | LookupCallback, + callback?: LookupCallback, + ): void => { + const cb = typeof options === "function" ? options : callback; + if (!cb) { + return; + } + if (normalizeUrlHostname(host) !== normalizedHost) { + if (typeof options === "function") { + dnsLookupCb(host, cb); + return; + } + dnsLookupCb(host, options, cb); + return; + } + const opts = typeof options === "object" && options !== null ? options : {}; + const filtered = opts.family + ? records.filter((record) => record.family === opts.family) + : records; + const usable = filtered.length > 0 ? filtered : records; + // Keep custom lookup delivery asynchronous so HTTPS owns immediate socket errors. + if (opts.all) { + process.nextTick(() => { + cb(null, usable); + }); + return; + } + const chosen = usable[0]!; + process.nextTick(() => { + cb(null, chosen.address, chosen.family); + }); + }; + return lookup as LookupFunction; +} + +async function resolvePackageDownloadAddresses( + parsed: URL, + lookupHost: PackageLookupHost, + trustedSource: TrustedPackageSource | undefined, +) { + const hostname = normalizeUrlHostname(parsed.hostname); + if (isIP(hostname)) { + if (!trustedSource?.allowPrivateNetwork && isUnsafeIpAddress(hostname)) { + throw new Error( + `Blocked: package_url resolves to private/internal/special-use IP address: ${hostname}`, + ); + } + return [hostname]; + } + const results = normalizeLookupResults(await lookupHost(hostname)); + if (results.length === 0) { + throw new Error(`Unable to resolve package_url hostname: ${parsed.hostname}`); + } + if (!trustedSource?.allowPrivateNetwork) { + const blocked = results.find((entry) => isUnsafeIpAddress(entry.address)); + if (blocked) { + throw new Error( + `Blocked: package_url resolves to private/internal/special-use IP address: ${blocked.address}`, + ); + } + } + return [...new Set(results.map((entry) => entry.address))]; +} + +function responseStatus(response: PackageDownloadResponse) { + return response.status; +} + +function responseOk(response: PackageDownloadResponse) { + const status = responseStatus(response); + return status >= 200 && status < 300; +} + +function responseHeader(response: PackageDownloadResponse, name: string) { + return response.headers?.get?.(name) ?? null; +} + +function createPackageDownloadTimeoutError(parsed: URL, timeoutMs: number) { + return Object.assign( + new Error(`package_url download timed out after ${timeoutMs}ms: ${parsed.toString()}`), + { + code: "ETIMEDOUT", + }, + ); +} + +async function closeResponseBody(body: PackageResponseBody | null) { + if (!body) { + return; + } + if (isWebResponseBody(body)) { + await body.cancel().catch(() => {}); + return; + } + body.destroy(); +} + +async function openFetchPackageDownloadResponse( + parsed: URL, + options: Required<Pick<PackageDownloadOptions, "fetchImpl">> & + Pick<PackageDownloadOptions, "headers"> & { timeoutMs: number }, +) { + const controller = new AbortController(); + const timeoutError = createPackageDownloadTimeoutError(parsed, options.timeoutMs); + let timeout!: TimerHandle; + const timeoutPromise = new Promise<never>((_resolve, reject) => { + timeout = setTimeout(() => { + controller.abort(timeoutError); + reject(timeoutError); + }, options.timeoutMs); + timeout.unref?.(); + }); + timeoutPromise.catch(() => {}); + const response = await Promise.resolve( + options.fetchImpl(parsed, { + headers: options.headers, + redirect: "manual", + signal: controller.signal, + }), + ).catch((error: unknown) => { + clearTimeout(timeout); + if (errorName(error) === "AbortError") { + throw Object.assign(timeoutError, { cause: error }); + } + throw error; + }); + return { + close: async () => closeResponseBody(response.body), + response, + timeout, + timeoutPromise, + timeoutMs: options.timeoutMs, + }; +} + +async function openHttpsPackageDownloadResponse( + parsed: URL, + options: Required<Pick<PackageDownloadOptions, "addresses">> & + Pick<PackageDownloadOptions, "headers"> & { timeoutMs: number }, +) { + const controller = new AbortController(); + const timeoutError = createPackageDownloadTimeoutError(parsed, options.timeoutMs); + let timeout!: TimerHandle; + const timeoutPromise = new Promise<never>((_resolve, reject) => { + timeout = setTimeout(() => { + controller.abort(timeoutError); + reject(timeoutError); + }, options.timeoutMs); + timeout.unref?.(); + }); + timeoutPromise.catch(() => {}); + const lookup = createPinnedLookup(parsed.hostname, options.addresses); + const response = await new Promise<PackageDownloadResponse>((resolve, reject) => { + const request = httpsRequest( + parsed, + { + headers: options.headers, + lookup, + signal: controller.signal, + }, + (message) => { + resolve({ + body: message, + headers: { + get(name) { + const value = message.headers[name.toLowerCase()]; + if (Array.isArray(value)) { + return value[0] ?? null; + } + return value ?? null; + }, + }, + status: message.statusCode ?? 0, + }); + }, + ); + request.on("error", reject); + request.end(); + }).catch((error: unknown) => { + clearTimeout(timeout); + if (errorName(error) === "AbortError" || errorCode(error) === "ABORT_ERR") { + throw Object.assign(timeoutError, { cause: error }); + } + throw error; + }); + return { + close: async () => closeResponseBody(response.body), + response, + timeout, + timeoutPromise, + timeoutMs: options.timeoutMs, + }; +} + +async function openPackageDownloadResponse(url: string, options: PackageDownloadOptions) { + const lookupHost = options.lookupHost ?? defaultLookupHost; + const timeoutMs = resolvePackageCandidateTimeoutMs( + options.timeoutMs, + PACKAGE_URL_DOWNLOAD_TIMEOUT_MS, + ); + const maxRedirects = options.maxRedirects ?? PACKAGE_URL_MAX_REDIRECTS; + const trustedSource = options.trustedSource; + let parsed = new URL(url); + const initialOrigin = parsed.origin; + for (let redirectCount = 0; redirectCount <= maxRedirects; redirectCount += 1) { + if (trustedSource) { + validateTrustedPackageDownloadUrl(parsed, trustedSource, { isRedirect: redirectCount > 0 }); + } else { + validatePackageDownloadUrl(parsed); + } + const addresses = await resolvePackageDownloadAddresses(parsed, lookupHost, trustedSource); + const headers = createTrustedPackageAuthHeaders(trustedSource, parsed, initialOrigin); + const opened = options.fetchImpl + ? await openFetchPackageDownloadResponse(parsed, { + fetchImpl: options.fetchImpl, + headers, + timeoutMs, + }) + : await openHttpsPackageDownloadResponse(parsed, { + addresses, + headers, + timeoutMs, + }); + const status = responseStatus(opened.response); + if ([301, 302, 303, 307, 308].includes(status)) { + clearTimeout(opened.timeout); + await opened.close(); + const location = responseHeader(opened.response, "location"); + if (!location) { + throw new Error(`package_url redirect missing Location header: HTTP ${status}`); + } + parsed = new URL(location, parsed); + continue; + } + return opened; + } + throw new Error(`package_url exceeded ${maxRedirects} redirects: ${url}`); +} + +async function* limitWebResponseBody( + body: WebResponseBody, + maxBytes: number, + timeoutPromise: Promise<never> | undefined, +): AsyncGenerator<string | Uint8Array> { + let downloaded = 0; + const reader = body.getReader(); + let timedOut = false; + let timeoutFailure: unknown; + const timeoutRead = timeoutPromise?.catch((error: unknown) => { + timedOut = true; + timeoutFailure = error; + void reader.cancel().catch(() => {}); + throw error; + }); + try { + for (;;) { + const next = reader.read(); + const { done, value } = timeoutRead ? await Promise.race([next, timeoutRead]) : await next; + if (timedOut) { + throw toErrorObject(timeoutFailure, "package_url download timed out"); + } + if (done) { + return; + } + const size = typeof value === "string" ? Buffer.byteLength(value) : value.byteLength; + downloaded += size; + if (downloaded > maxBytes) { + await reader.cancel().catch(() => {}); + throw new Error(`package_url exceeds maximum download size of ${maxBytes} bytes`); + } + yield value; + } + } finally { + if (!timedOut) { + reader.releaseLock(); + } + } +} + +async function* limitResponseBody( + body: PackageResponseBody, + maxBytes: number, + timeoutPromise: Promise<never> | undefined, +): AsyncGenerator<string | Uint8Array> { + if (isWebResponseBody(body)) { + yield* limitWebResponseBody(body, maxBytes, timeoutPromise); + return; + } + let downloaded = 0; + for await (const chunk of body as AsyncIterable<string | Uint8Array>) { + const size = typeof chunk === "string" ? Buffer.byteLength(chunk) : chunk.byteLength; + downloaded += size; + if (downloaded > maxBytes) { + throw new Error(`package_url exceeds maximum download size of ${maxBytes} bytes`); + } + yield chunk; + } +} + +export async function downloadUrl( + urlText: string, + targetPath: string, + typedOptions: PackageDownloadOptions = {}, +) { + const maxBytes = typedOptions.maxBytes ?? PACKAGE_URL_MAX_BYTES; + const { close, response, timeout, timeoutMs, timeoutPromise } = await openPackageDownloadResponse( + urlText, + typedOptions, + ); + const tempTarget = `${targetPath}.tmp`; + let waitForOutputClose: (() => Promise<void>) | undefined; + try { + if (!responseOk(response) || !response.body) { + throw new Error(`failed to download package_url: HTTP ${responseStatus(response)}`); + } + const rawContentLength = responseHeader(response, "content-length"); + const contentLength = + rawContentLength && /^\d+$/u.test(rawContentLength) ? Number(rawContentLength) : undefined; + if ( + contentLength !== undefined && + (!Number.isSafeInteger(contentLength) || contentLength > maxBytes) + ) { + throw new Error(`package_url exceeds maximum download size of ${maxBytes} bytes`); + } + await fs.rm(tempTarget, { force: true }); + const output = createWriteStream(tempTarget); + waitForOutputClose = async () => { + if (!output.closed) { + await once(output, "close").catch(() => {}); + } + }; + await pipeline(limitResponseBody(response.body, maxBytes, timeoutPromise), output); + await fs.rename(tempTarget, targetPath); + } catch (error) { + if (errorCode(error) === "ETIMEDOUT") { + throw error; + } + if (errorName(error) === "AbortError") { + throw new Error(`package_url download timed out after ${timeoutMs}ms: ${urlText}`, { + cause: error, + }); + } + throw error; + } finally { + clearTimeout(timeout); + await close(); + await waitForOutputClose?.(); + await fs.rm(tempTarget, { force: true }); + } +} + +async function readPackageJson(tarball: string) { + const raw = await run("tar", ["-xOf", tarball, "package/package.json"], { capture: true }); + const pkg: unknown = JSON.parse(raw); + return { + name: isJsonRecord(pkg) && typeof pkg.name === "string" ? pkg.name : "", + version: isJsonRecord(pkg) && typeof pkg.version === "string" ? pkg.version : "", + }; +} + +export async function readPackageBuildSourceSha(tarball: string) { + const raw = await run("tar", ["-xOf", tarball, "package/dist/build-info.json"], { + capture: true, + }).then( + (value) => value, + () => "", + ); + if (!raw.trim()) { + return ""; + } + const buildInfo: unknown = JSON.parse(raw); + const commit = + isJsonRecord(buildInfo) && typeof buildInfo.commit === "string" ? buildInfo.commit.trim() : ""; + return /^[0-9a-f]{40}$/iu.test(commit) ? commit.toLowerCase() : ""; +} + +async function appendGithubOutputs(file: string, outputs: Record<string, unknown>) { + if (!file) { + return; + } + const body = Object.entries(outputs) + .map(([key, value]) => `${key}=${String(value).replace(/\n/gu, " ")}`) + .join("\n"); + await fs.appendFile(file, `${body}\n`); +} + +async function resolveCandidate(options: PackageCandidateOptions) { + const outputDir = path.resolve(ROOT_DIR, options.outputDir); + const target = path.join(outputDir, options.outputName || DEFAULT_OUTPUT_NAME); + await fs.mkdir(outputDir, { recursive: true }); + await fs.rm(target, { force: true }); + let packageRef = ""; + let packageSourceSha = ""; + let packageTrustedReason = ""; + let packageTrustedSourceId = ""; + let packageWorktreeDir = ""; + let packageBuildSourceSha: string | undefined; + let pluginRegistrySource: Awaited<ReturnType<typeof preparePackageSourceWorktree>> | undefined; + let artifactMetadata: ArtifactMetadata = {}; + let pluginRegistryIdentity: + | { candidateVersion: string; manifestSha256: string; sourceSha: string } + | undefined; + let resolveError: unknown; + + try { + if (options.source === "ref") { + packageRef = options.packageRef || "main"; + const packageSource = await preparePackageSourceWorktree(packageRef); + packageWorktreeDir = packageSource.sourceDir; + if (options.pluginRegistryOutputDir) { + pluginRegistrySource = packageSource; + } + packageSourceSha = packageSource.selectedSha; + packageTrustedReason = packageSource.trustedReason; + validatePackageSourceDir(packageSource.sourceDir, { allowUnreleasedChangelog: true }); + await installPackageSourceDeps(packageSource.sourceDir); + await run("node", [ + "scripts/package-openclaw-for-docker.mjs", + "--allow-unreleased-changelog", + "--source-dir", + packageSource.sourceDir, + "--output-dir", + outputDir, + "--output-name", + options.outputName || DEFAULT_OUTPUT_NAME, + ]); + } else if (options.source === "npm") { + const npmPackRunner = resolveNpmPackageCandidatePackRunner(options.packageSpec, outputDir, { + env: process.env, + }); + await cleanPackedOpenClawTarballs(outputDir); + const packOutput = await run(npmPackRunner.command, npmPackRunner.args, { + capture: true, + env: npmPackRunner.env, + shell: npmPackRunner.shell, + windowsVerbatimArguments: npmPackRunner.windowsVerbatimArguments, + }); + await moveNewestPackedTarball( + outputDir, + packOutput, + options.outputName || DEFAULT_OUTPUT_NAME, + ); + } else if (options.source === "url" || options.source === "trusted-url") { + if (!options.packageUrl) { + throw new Error(`${options.source} requires --package-url`); + } + if (!options.packageSha256) { + throw new Error(`${options.source} requires --package-sha256`); + } + if (options.source === "trusted-url") { + const trustedSource = await loadTrustedPackageSource( + options.trustedSourceId, + options.trustedSourcePolicy, + ); + await downloadUrl(options.packageUrl, target, { trustedSource }); + packageTrustedReason = `trusted-url-policy:${trustedSource.id}`; + packageTrustedSourceId = trustedSource.id; + } else { + if (options.trustedSourceId) { + throw new Error("--trusted-source-id is only allowed with source=trusted-url"); + } + await downloadUrl(options.packageUrl, target); + } + } else if (options.source === "artifact") { + if (!options.artifactDir) { + throw new Error("source=artifact requires --artifact-dir"); + } + artifactMetadata = await readArtifactPackageCandidateMetadata(options.artifactDir); + packageRef = + typeof artifactMetadata.packageRef === "string" ? artifactMetadata.packageRef : ""; + packageSourceSha = + typeof artifactMetadata.packageSourceSha === "string" + ? artifactMetadata.packageSourceSha + : ""; + packageTrustedReason = + typeof artifactMetadata.packageTrustedReason === "string" + ? artifactMetadata.packageTrustedReason + : ""; + const input = await findSingleTarball(options.artifactDir); + await fs.copyFile(input, target); + packageBuildSourceSha = await readPackageBuildSourceSha(target); + if (packageSourceSha && packageBuildSourceSha && packageSourceSha !== packageBuildSourceSha) { + throw new Error( + `artifact packageSourceSha ${packageSourceSha} does not match package build-info commit ${packageBuildSourceSha}`, + ); + } + if (!packageSourceSha && packageBuildSourceSha) { + packageSourceSha = packageBuildSourceSha; + packageTrustedReason = "package-build-info"; + } + if (options.pluginRegistryOutputDir) { + if (!packageBuildSourceSha) { + throw new Error( + "source=artifact requires a valid package build-info commit for prerelease plugin registry creation", + ); + } + packageTrustedReason ||= "package-build-info"; + } + } else { + throw new Error( + `source must be one of: ref, npm, url, trusted-url, artifact. Got: ${options.source}`, + ); + } + if (options.pluginRegistryOutputDir && !pluginRegistrySource) { + if (options.source !== "npm" && options.source !== "artifact") { + throw new Error( + "--plugin-registry-output-dir is only supported with source=ref, source=npm, or source=artifact", + ); + } + if (options.source === "npm") { + packageRef = options.packageRef; + } + pluginRegistrySource = await preparePackageSourceWorktree( + options.source === "npm" ? packageRef : packageSourceSha, + ); + packageWorktreeDir = pluginRegistrySource.sourceDir; + await installPackageSourceDeps(pluginRegistrySource.sourceDir); + } + if (options.pluginRegistryOutputDir && pluginRegistrySource) { + const requiredPackages = JSON.parse(options.requiredPluginPackagesJson) as string[]; + const rootPackage = JSON.parse( + await fs.readFile(path.join(pluginRegistrySource.sourceDir, "package.json"), "utf8"), + ) as { version: string }; + const registry = createPrepublishPluginRegistryArtifact({ + repoRoot: pluginRegistrySource.sourceDir, + outputDir: path.resolve(ROOT_DIR, options.pluginRegistryOutputDir), + sourceSha: pluginRegistrySource.selectedSha, + candidateVersion: rootPackage.version, + requiredPackages, + }); + pluginRegistryIdentity = { + candidateVersion: rootPackage.version, + manifestSha256: registry.manifestSha256, + sourceSha: pluginRegistrySource.selectedSha, + }; + } + } catch (error) { + resolveError = error; + throw error; + } finally { + if (packageWorktreeDir) { + await cleanupPackageSourceWorktree(packageWorktreeDir, { resolveError }); + } + } + + const artifactSha256 = typeof artifactMetadata.sha256 === "string" ? artifactMetadata.sha256 : ""; + const digest = await assertExpectedSha256(target, options.packageSha256 || artifactSha256); + console.error(`Checking OpenClaw package tarball: ${target}`); + const checkStartedAt = Date.now(); + await run("node", ["scripts/check-openclaw-package-tarball.mjs", target], { + timeoutMs: 5 * 60 * 1000, + }); + console.error( + `OpenClaw package tarball check finished in ${Math.round((Date.now() - checkStartedAt) / 1000)}s`, + ); + const pkg = await readPackageJson(target); + if (!packageSourceSha) { + packageSourceSha = packageBuildSourceSha ?? (await readPackageBuildSourceSha(target)); + if (packageSourceSha && !packageTrustedReason) { + packageTrustedReason = "package-build-info"; + } + } + if ( + pluginRegistryIdentity && + (pluginRegistryIdentity.candidateVersion !== pkg.version || + pluginRegistryIdentity.sourceSha !== packageSourceSha) + ) { + throw new Error( + "prepublish plugin registry source SHA/version differs from the package candidate", + ); + } + const metadata = { + name: pkg.name, + packageRef, + packageSpec: options.packageSpec || "", + packageSourceSha, + packageTrustedReason, + pluginRegistryManifestSha256: pluginRegistryIdentity?.manifestSha256 ?? "", + trustedSourceId: packageTrustedSourceId, + sha256: digest, + source: options.source, + tarball: path.relative(ROOT_DIR, target), + version: pkg.version, + }; + + if (pkg.name !== "openclaw") { + throw new Error(`package candidate must be named "openclaw"; got: ${pkg.name || "<missing>"}`); + } + if (!pkg.version) { + throw new Error("package candidate package.json has no version"); + } + + if (options.metadata) { + await fs.mkdir(path.dirname(path.resolve(ROOT_DIR, options.metadata)), { recursive: true }); + await fs.writeFile( + path.resolve(ROOT_DIR, options.metadata), + `${JSON.stringify(metadata, null, 2)}\n`, + ); + } + await appendGithubOutputs(options.githubOutput, { + package_name: pkg.name, + package_source_sha: packageSourceSha, + package_version: pkg.version, + plugin_registry_manifest_sha256: pluginRegistryIdentity?.manifestSha256 ?? "", + sha256: digest, + tarball: metadata.tarball, + }); + return metadata; +} + +export async function main(argv: string[] = process.argv.slice(2)) { + const options = parseArgs(argv); + if (options.help) { + console.log(usage()); + return; + } + if (!options.outputDir) { + throw new Error("--output-dir is required"); + } + const metadata = await resolveCandidate(options); + console.log(JSON.stringify(metadata, null, 2)); +} + +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + await main().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : String(error)); + console.error(usage()); + process.exit(1); + }); +} + +function errorCode(value: unknown) { + return isPropertyContainer(value) ? value.code : undefined; +} + +function errorName(value: unknown) { + return isPropertyContainer(value) ? value.name : undefined; +} + +function isPropertyContainer(value: unknown): value is { code?: unknown; name?: unknown } { + return (typeof value === "object" && value !== null) || typeof value === "function"; +} + +function isWebResponseBody(body: PackageResponseBody): body is WebResponseBody { + return "getReader" in body && typeof body.getReader === "function"; +} diff --git a/scripts/resolve-upgrade-survivor-baselines.mts b/scripts/resolve-upgrade-survivor-baselines.mts new file mode 100644 index 0000000000000000000000000000000000000000..b6bad05c1194d20b03d646ef2af46ee3aadc9f1c --- /dev/null +++ b/scripts/resolve-upgrade-survivor-baselines.mts @@ -0,0 +1,308 @@ +// Resolves Docker upgrade-survivor baseline specs from requested tokens and +// live release history JSON captured by release workflows. +import { readFileSync, writeFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { normalizeUpgradeSurvivorBaselineSpec } from "./lib/docker-e2e-plan.mts"; +import { resolveNpmJsonEntries } from "./lib/npm-json-output.mts"; +import { compareReleaseVersions, parseReleaseVersion } from "./lib/release-version.mjs"; +import { OLDEST_SUPPORTED_UPGRADE_SURVIVOR_BASELINE } from "./lib/upgrade-survivor-policy.mjs"; + +type ReleaseRecord = Partial<Record<"isPrerelease" | "publishedAt" | "tagName", unknown>>; + +function scalarText(value: unknown): string { + return typeof value === "string" || typeof value === "number" ? String(value) : ""; +} + +export function parseArgs(argv: string[]) { + const args = new Map<string, string>(); + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === undefined || !arg.startsWith("--")) { + throw new Error(`unexpected argument: ${arg}`); + } + const key = arg.slice(2); + const value = argv[index + 1]; + if (value === undefined || value.startsWith("-")) { + throw new Error(`missing value for --${key}`); + } + args.set(key, value); + index += 1; + } + return args; +} + +function splitSpecs(raw: unknown) { + return scalarText(raw) + .split(/[,\s]+/u) + .map((token) => token.trim()) + .filter(Boolean); +} + +function dedupeSpecs(specs: string[]) { + const normalized = specs + .map(normalizeUpgradeSurvivorBaselineSpec) + .filter((spec) => spec !== undefined); + return [...new Set(normalized)]; +} + +function omitUnpublishedCandidateBaseline(args: Map<string, string>, specs: string[]) { + if (args.get("candidate-published") !== "false") { + return specs; + } + const candidateVersion = args.get("candidate-version"); + if (!candidateVersion) { + return specs; + } + const candidateSpec = normalizeUpgradeSurvivorBaselineSpec(candidateVersion); + if (!candidateSpec) { + throw new Error(`invalid candidate version: ${candidateVersion}`); + } + return specs.filter((spec) => normalizeUpgradeSurvivorBaselineSpec(spec) !== candidateSpec); +} + +function parsePositiveInteger(value: unknown, label: string) { + const text = scalarText(value).trim(); + if (!/^[1-9]\d*$/u.test(text)) { + throw new Error(`${label} must be a positive integer`); + } + const parsed = Number(text); + if (!Number.isSafeInteger(parsed)) { + throw new Error(`${label} must be a positive integer`); + } + return parsed; +} + +function readPublishedVersions(file: string | undefined) { + if (!file) { + return undefined; + } + const parsed: unknown = JSON.parse(readFileSync(file, "utf8")); + if (!Array.isArray(parsed)) { + throw new Error(`npm versions list must be a JSON array: ${file}`); + } + return new Set(parsed.filter((version) => typeof version === "string")); +} + +function stableVersionFromTag(tagName: unknown) { + const version = typeof tagName === "string" ? tagName.replace(/^v/u, "") : ""; + return parseStableVersion(version) ? version : undefined; +} + +function parseStableVersion(version: unknown) { + const parsed = parseReleaseVersion(typeof version === "string" ? version : ""); + return parsed?.channel === "stable" ? parsed : undefined; +} + +function compareStableVersions(left: string, right: string) { + if (!parseStableVersion(left) || !parseStableVersion(right)) { + throw new Error(`cannot compare release versions: ${left} ${right}`); + } + const comparison = compareReleaseVersions(left, right); + if (comparison === null) { + throw new Error(`cannot compare release versions: ${left} ${right}`); + } + return comparison; +} + +function npmPublishedVersion(version: string | undefined, publishedVersions?: Set<string>) { + if (!version || !publishedVersions) { + return version; + } + if (publishedVersions.has(version)) { + return version; + } + const baseVersion = version.replace(/-[0-9]+$/u, ""); + return publishedVersions.has(baseVersion) ? baseVersion : undefined; +} + +function isReleaseRecord(value: unknown): value is ReleaseRecord { + return typeof value === "object" && value !== null; +} + +function readStableReleases(file: string, publishedVersions?: Set<string>) { + const ansiEscape = new RegExp(`${String.fromCharCode(27)}\\[[0-?]*[ -/]*[@-~]`, "g"); + const raw = readFileSync(file, "utf8").replace(ansiEscape, ""); + const parsed: unknown = JSON.parse(raw); + if (!Array.isArray(parsed)) { + throw new Error(`release list must be a JSON array: ${file}`); + } + return parsed + .filter(isReleaseRecord) + .filter((release) => !release.isPrerelease) + .flatMap((release) => { + const publishedAt = typeof release.publishedAt === "string" ? release.publishedAt : undefined; + const version = npmPublishedVersion(stableVersionFromTag(release.tagName), publishedVersions); + return publishedAt && version ? [{ publishedAt, version }] : []; + }) + .toSorted((a, b) => b.publishedAt.localeCompare(a.publishedAt)); +} + +/** + * Expands the release-history token into recent stable plus pinned historical baselines. + */ +function resolveReleaseHistory(args: Map<string, string>) { + const releasesJson = args.get("releases-json"); + if (!releasesJson) { + throw new Error("--releases-json is required when requested baselines include release-history"); + } + const historyCount = parsePositiveInteger(args.get("history-count") ?? "6", "--history-count"); + const includeVersion = args.get("include-version") ?? "2026.4.23"; + const preDate = args.get("pre-date") ?? "2026-03-15T00:00:00Z"; + const publishedVersions = readPublishedVersions(args.get("npm-versions-json")); + const releases = readStableReleases(releasesJson, publishedVersions); + const versions = omitUnpublishedCandidateBaseline( + args, + releases.map((release) => release.version), + ).slice(0, historyCount); + const exact = releases.find((release) => release.version === includeVersion); + if (exact) { + versions.push(exact.version); + } + const preDateRelease = releases.find( + (release) => new Date(release.publishedAt).getTime() < new Date(preDate).getTime(), + ); + if (preDateRelease) { + versions.push(preDateRelease.version); + } + return dedupeSpecs(versions); +} + +/** + * Resolves the last N stable release versions from release metadata. + */ +function resolveLastStable(args: Map<string, string>, count: number) { + const releasesJson = args.get("releases-json"); + if (!releasesJson) { + throw new Error("--releases-json is required when requested baselines include last-stable-*"); + } + if (!Number.isInteger(count) || count < 1) { + throw new Error(`invalid last-stable baseline count: ${count}`); + } + const publishedVersions = readPublishedVersions(args.get("npm-versions-json")); + const releases = readStableReleases(releasesJson, publishedVersions); + return dedupeSpecs( + omitUnpublishedCandidateBaseline( + args, + releases.map((release) => release.version), + ).slice(0, count), + ); +} + +/** + * Resolves all stable release versions at or after the requested minimum. + */ +function resolveAllSince(args: Map<string, string>, minimumVersion: string) { + const releasesJson = args.get("releases-json"); + if (!releasesJson) { + throw new Error("--releases-json is required when requested baselines include all-since-*"); + } + const publishedVersions = readPublishedVersions(args.get("npm-versions-json")); + const releases = readStableReleases(releasesJson, publishedVersions); + return omitUnpublishedCandidateBaseline( + args, + dedupeSpecs( + releases + .map((release) => release.version) + .filter((version) => compareStableVersions(version, minimumVersion) >= 0), + ), + ); +} + +function resolveSupportedLines(args: Map<string, string>) { + const tagsFile = args.get("npm-dist-tags-json"); + const versions = readPublishedVersions(args.get("npm-versions-json")); + if (!tagsFile || !versions) { + throw new Error("supported-lines requires --npm-dist-tags-json and --npm-versions-json"); + } + const tagResults = resolveNpmJsonEntries(JSON.parse(readFileSync(tagsFile, "utf8"))); + const tags = tagResults.length === 1 ? tagResults[0] : undefined; + if (typeof tags !== "object" || tags === null || Array.isArray(tags)) { + throw new Error("npm dist-tags must be a JSON object"); + } + const latest = "latest" in tags ? tags.latest : undefined; + if (typeof latest !== "string" || !parseStableVersion(latest) || !versions.has(latest)) { + throw new Error("npm latest must name a published stable version"); + } + const previous = [...versions] + .filter((version) => parseStableVersion(version) && compareStableVersions(version, latest) < 0) + .toSorted((left, right) => compareStableVersions(right, left))[0]; + if (!previous) { + throw new Error(`no previous stable npm version before ${latest}`); + } + if (!versions.has(OLDEST_SUPPORTED_UPGRADE_SURVIVOR_BASELINE)) { + throw new Error( + `oldest supported baseline is not published: ${OLDEST_SUPPORTED_UPGRADE_SURVIVOR_BASELINE}`, + ); + } + const extended = "extended-stable" in tags ? tags["extended-stable"] : undefined; + if ( + extended !== undefined && + (typeof extended !== "string" || !parseStableVersion(extended) || !versions.has(extended)) + ) { + throw new Error("npm extended-stable must name a published stable version when present"); + } + return omitUnpublishedCandidateBaseline( + args, + dedupeSpecs([ + latest, + previous, + ...(typeof extended === "string" ? [extended] : []), + OLDEST_SUPPORTED_UPGRADE_SURVIVOR_BASELINE, + ]), + ); +} + +/** + * Expands requested baseline tokens into normalized package/version specs. + */ +export function resolveBaselines(args: Map<string, string>) { + const requested = args.get("requested") ?? ""; + const fallback = args.get("fallback") ?? "openclaw@latest"; + const requestedTokens = splitSpecs(requested); + if (requestedTokens.length === 0) { + return dedupeSpecs([fallback]); + } + const resolved: string[] = []; + for (const token of requestedTokens) { + if (token === "supported-lines") { + resolved.push(...resolveSupportedLines(args)); + } else if (token === "release-history") { + resolved.push(...resolveReleaseHistory(args)); + } else if (token.startsWith("last-stable-")) { + const count = parsePositiveInteger( + token.slice("last-stable-".length), + "last-stable baseline count", + ); + resolved.push(...resolveLastStable(args, count)); + } else if (token.startsWith("all-since-")) { + const minimumVersion = token.slice("all-since-".length); + if (!parseStableVersion(minimumVersion)) { + throw new Error(`invalid all-since baseline token: ${token}`); + } + resolved.push(...resolveAllSince(args, minimumVersion)); + } else { + resolved.push(token); + } + } + return dedupeSpecs(resolved); +} + +const isMain = process.argv[1] ? fileURLToPath(import.meta.url) === process.argv[1] : false; + +if (isMain) { + const args = parseArgs(process.argv.slice(2)); + const baselines = resolveBaselines(args).join(" "); + process.stdout.write(`${baselines}\n`); + + const githubOutput = args.get("github-output"); + if (githubOutput) { + const requestedTokens = splitSpecs(args.get("requested")); + const baselineScope = + requestedTokens.length > 0 && requestedTokens.every((token) => token === "supported-lines") + ? "legacy-operator-state" + : "all-scenarios"; + writeFileSync(githubOutput, `baselines=${baselines}\nbaseline_scope=${baselineScope}\n`, { + flag: "a", + }); + } +} diff --git a/scripts/root-dependency-ownership-audit.mts b/scripts/root-dependency-ownership-audit.mts new file mode 100644 index 0000000000000000000000000000000000000000..7a1a865b1db7ebe1b46f63be2ee64987add24fac --- /dev/null +++ b/scripts/root-dependency-ownership-audit.mts @@ -0,0 +1,473 @@ +#!/usr/bin/env node + +// Audits root package runtime dependencies against source imports and bundled +// plugin ownership so extension-owned deps can move out of root. +import fs from "node:fs"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { isRecord } from "@openclaw/normalization-core/record-coerce"; +import { collectExcludedPackagedExtensionDirs } from "./lib/packaged-extension-dirs.mts"; +import { packageNameFromSpecifier } from "./lib/plugin-package-dependencies.mts"; + +const DEFAULT_SCAN_ROOTS = ["src", "extensions", "packages", "ui", "scripts", "test"]; +const SCANNED_EXTENSIONS = new Set([".cjs", ".cts", ".js", ".jsx", ".mjs", ".mts", ".ts", ".tsx"]); +const IMPORT_PATTERNS = [ + /\bfrom\s*["']([^"']+)["']/g, + /\bimport\s*\(\s*["']([^"']+)["']\s*\)/g, + /\brequire\s*\(\s*["']([^"']+)["']\s*\)/g, + /\b(?:require|[_$A-Za-z][\w$]*require[\w$]*)\.resolve\s*\(\s*["']([^"']+)["']\s*\)/gi, +]; +const STRING_CONSTANT_PATTERN = /\b(?:const|let|var)\s+([_$A-Za-z][\w$]*)\s*=\s*["']([^"']+)["']/g; +const DYNAMIC_CONSTANT_IMPORT_PATTERNS = [ + /\bimport\s*\(\s*([_$A-Za-z][\w$]*)\s*\)/g, + /\brequire\s*\(\s*([_$A-Za-z][\w$]*)\s*\)/g, + /\b(?:require|[_$A-Za-z][\w$]*require[\w$]*)\.resolve\s*\(\s*([_$A-Za-z][\w$]*)\s*\)/gi, +]; +const ROOT_OWNED_EXTENSION_RUNTIME_DEPENDENCIES = new Map([ + [ + "@homebridge/ciao", + "keep at root; the Bonjour runtime is shipped with packaged startup surfaces even though the bundled plugin also declares it", + ], + [ + "playwright-core", + "keep at root; the internal browser runtime is shipped with core even though downloadable browser-adjacent plugins also declare it", + ], +]); + +type JsonObject = Record<string, unknown>; +type OwnershipClassificationInput = { + depName?: string; + internalizedBundledRuntimeOwners?: string[]; + sections: Iterable<string>; +}; +type WorkingAuditRecord = { + declaredInExtensions: string[]; + depName: string; + files: Set<string>; + internalizedBundledRuntimeOwners: string[]; + sections: Set<string>; + spec: unknown; +}; +type OwnershipCheckRecord = { + category: string; + declaredInExtensions: string[]; + depName: string; + recommendation: string; + sampleFiles: string[]; +}; + +function readJson(filePath: string): JsonObject { + const value: unknown = JSON.parse(fs.readFileSync(filePath, "utf8")); + if (!isRecord(value)) { + throw new Error(`${filePath} must contain a JSON object`); + } + return value; +} + +function isScannableSourceFile(fileName: string) { + return SCANNED_EXTENSIONS.has(path.extname(fileName)); +} + +function shouldSkipDir(dirName: string) { + return dirName === "dist" || dirName === "node_modules" || dirName === ".git"; +} + +function walkFiles(rootDir: string): string[] { + if (!fs.existsSync(rootDir)) { + return []; + } + const files: string[] = []; + const queue = [rootDir]; + while (queue.length > 0) { + const current = queue.shift(); + if (!current) { + continue; + } + for (const entry of fs.readdirSync(current, { withFileTypes: true })) { + const fullPath = path.join(current, entry.name); + if (entry.isDirectory()) { + if (shouldSkipDir(entry.name)) { + continue; + } + queue.push(fullPath); + continue; + } + if (entry.isFile() && isScannableSourceFile(entry.name)) { + files.push(fullPath); + } + } + } + return files.toSorted((left, right) => left.localeCompare(right)); +} + +function normalizeRelativePath(filePath: string, repoRoot: string) { + return path.relative(repoRoot, filePath).replaceAll(path.sep, "/"); +} + +function sectionFor(relativePath: string) { + const [section = "other"] = relativePath.split("/"); + return section; +} + +/** + * Collects static and simple constant-backed package specifiers from source text. + */ +export function collectModuleSpecifiers(source: string): Set<string> { + const sourceText = source; + const specifiers = new Set<string>(); + for (const pattern of IMPORT_PATTERNS) { + for (const match of sourceText.matchAll(pattern)) { + if (match[1]) { + specifiers.add(match[1]); + } + } + } + const stringConstants = new Map<string, string>(); + for (const match of sourceText.matchAll(STRING_CONSTANT_PATTERN)) { + if (match[1] && match[2]) { + stringConstants.set(match[1], match[2]); + } + } + for (const pattern of DYNAMIC_CONSTANT_IMPORT_PATTERNS) { + for (const match of sourceText.matchAll(pattern)) { + const specifier = match[1] ? stringConstants.get(match[1]) : undefined; + if (specifier) { + specifiers.add(specifier); + } + } + } + return specifiers; +} + +function collectExtensionDependencyDeclarations(repoRoot: string) { + const declarations = new Map<string, string[]>(); + const extensionsRoot = path.join(repoRoot, "extensions"); + if (!fs.existsSync(extensionsRoot)) { + return declarations; + } + + for (const entry of fs.readdirSync(extensionsRoot, { withFileTypes: true })) { + if (!entry.isDirectory()) { + continue; + } + const packageJsonPath = path.join(extensionsRoot, entry.name, "package.json"); + if (!fs.existsSync(packageJsonPath)) { + continue; + } + const packageJson = readJson(packageJsonPath); + for (const section of [ + "dependencies", + "optionalDependencies", + "devDependencies", + "peerDependencies", + ]) { + const sectionRecord = packageJson[section]; + for (const depName of Object.keys(isRecord(sectionRecord) ? sectionRecord : {})) { + const existing = declarations.get(depName) ?? []; + existing.push(`${entry.name}:${section}`); + declarations.set(depName, existing); + } + } + } + + for (const values of declarations.values()) { + values.sort((left, right) => left.localeCompare(right)); + } + + return declarations; +} + +function collectInternalizedBundledExtensionRuntimeDependencies( + repoRoot: string, + rootPackageJson: JsonObject, +) { + const dependencies = new Map<string, string[]>(); + const extensionsRoot = path.join(repoRoot, "extensions"); + if (!fs.existsSync(extensionsRoot)) { + return dependencies; + } + + const excluded = collectExcludedPackagedExtensionDirs(rootPackageJson); + for (const entry of fs.readdirSync(extensionsRoot, { withFileTypes: true })) { + if (!entry.isDirectory() || excluded.has(entry.name)) { + continue; + } + const packageJsonPath = path.join(extensionsRoot, entry.name, "package.json"); + const manifestPath = path.join(extensionsRoot, entry.name, "openclaw.plugin.json"); + if (!fs.existsSync(packageJsonPath) || !fs.existsSync(manifestPath)) { + continue; + } + const packageJson = readJson(packageJsonPath); + for (const section of ["dependencies", "optionalDependencies"]) { + const sectionRecord = packageJson[section]; + for (const depName of Object.keys(isRecord(sectionRecord) ? sectionRecord : {})) { + const existing = dependencies.get(depName) ?? []; + existing.push(`${entry.name}:${section}`); + dependencies.set(depName, existing); + } + } + } + + for (const values of dependencies.values()) { + values.sort((left, right) => left.localeCompare(right)); + } + + return dependencies; +} + +function sectionSetContainsCore(sectionSet: Set<string>) { + return sectionSet.has("src") || sectionSet.has("packages") || sectionSet.has("ui"); +} + +function sectionSetIsSubsetOf(sectionSet: Set<string>, allowed: Set<string>) { + for (const value of sectionSet) { + if (!allowed.has(value)) { + return false; + } + } + return sectionSet.size > 0; +} + +/** + * Classifies whether a root dependency is core-owned, shared, or extension-local. + */ +export function classifyRootDependencyOwnership(record: OwnershipClassificationInput): { + category: string; + recommendation: string; +} { + const sections = new Set(record.sections); + + if (sections.size === 0) { + return { + category: "unreferenced", + recommendation: "investigate removal; no direct source imports found in scanned files", + }; + } + + if (sectionSetIsSubsetOf(sections, new Set(["scripts", "test"]))) { + return { + category: "script_or_test_only", + recommendation: "consider moving from dependencies to devDependencies", + }; + } + + if (sectionSetContainsCore(sections)) { + if (sections.has("extensions")) { + return { + category: "shared_core_and_extension", + recommendation: + "keep at root until shared code is split or extension/core boundary changes", + }; + } + return { + category: "core_runtime", + recommendation: "keep at root", + }; + } + + const rootOwnedExtensionRuntime = record.depName + ? ROOT_OWNED_EXTENSION_RUNTIME_DEPENDENCIES.get(record.depName) + : undefined; + const internalizedOwners = record.internalizedBundledRuntimeOwners ?? []; + if ( + rootOwnedExtensionRuntime && + sectionSetIsSubsetOf(sections, new Set(["extensions", "test"])) + ) { + return { + category: "root_owned_extension_runtime", + recommendation: rootOwnedExtensionRuntime, + }; + } + + if ( + internalizedOwners.length > 0 && + sectionSetIsSubsetOf(sections, new Set(["extensions", "test"])) + ) { + return { + category: "root_owned_extension_runtime", + recommendation: `keep at root while bundled plugin runtime dependencies are internalized; owners: ${internalizedOwners.join(", ")}`, + }; + } + + if (sectionSetIsSubsetOf(sections, new Set(["extensions", "test"]))) { + return { + category: "extension_only_localizable", + recommendation: + "remove from root package.json and rely on owning extension manifests plus doctor --fix", + }; + } + + return { + category: "mixed_noncore", + recommendation: "inspect manually; usage spans non-core surfaces", + }; +} + +/** + * Builds dependency ownership records from root package.json and scanned imports. + */ +export function collectRootDependencyOwnershipAudit( + params: { repoRoot?: string; scanRoots?: string[] } = {}, +) { + const repoRoot = path.resolve(params.repoRoot ?? process.cwd()); + const rootPackageJson = readJson(path.join(repoRoot, "package.json")); + const dependencies = isRecord(rootPackageJson.dependencies) ? rootPackageJson.dependencies : {}; + const optionalDependencies = isRecord(rootPackageJson.optionalDependencies) + ? rootPackageJson.optionalDependencies + : {}; + const rootDependencies = { + ...dependencies, + ...optionalDependencies, + }; + const records = new Map<string, WorkingAuditRecord>( + Object.keys(rootDependencies).map((depName) => [ + depName, + { + depName, + sections: new Set<string>(), + files: new Set<string>(), + declaredInExtensions: [] as string[], + internalizedBundledRuntimeOwners: [] as string[], + spec: rootDependencies[depName], + }, + ]), + ); + + const scanRoots = params.scanRoots ?? DEFAULT_SCAN_ROOTS; + for (const scanRoot of scanRoots) { + for (const filePath of walkFiles(path.join(repoRoot, scanRoot))) { + const relativePath = normalizeRelativePath(filePath, repoRoot); + const source = fs.readFileSync(filePath, "utf8"); + for (const specifier of collectModuleSpecifiers(source)) { + const depName = packageNameFromSpecifier(specifier); + if (!depName) { + continue; + } + const record = records.get(depName); + if (!record) { + continue; + } + record.sections.add(sectionFor(relativePath)); + record.files.add(relativePath); + } + } + } + + const extensionDeclarations = collectExtensionDependencyDeclarations(repoRoot); + for (const [depName, declarations] of extensionDeclarations) { + const record = records.get(depName); + if (record) { + record.declaredInExtensions = declarations; + } + } + + const internalizedBundledRuntimeDependencies = + collectInternalizedBundledExtensionRuntimeDependencies(repoRoot, rootPackageJson); + for (const [depName, owners] of internalizedBundledRuntimeDependencies) { + const record = records.get(depName); + if (record) { + record.internalizedBundledRuntimeOwners = owners; + } + } + + return [...records.values()] + .map((record) => { + const classification = classifyRootDependencyOwnership({ + ...record, + sections: [...record.sections].toSorted((left, right) => left.localeCompare(right)), + }); + return { + depName: record.depName, + spec: record.spec, + sections: [...record.sections].toSorted((left, right) => left.localeCompare(right)), + fileCount: record.files.size, + sampleFiles: [...record.files].slice(0, 5), + declaredInExtensions: record.declaredInExtensions, + internalizedBundledRuntimeOwners: record.internalizedBundledRuntimeOwners, + category: classification.category, + recommendation: classification.recommendation, + }; + }) + .toSorted((left, right) => left.depName.localeCompare(right.depName)); +} + +type AuditRecord = ReturnType<typeof collectRootDependencyOwnershipAudit>[number]; + +/** + * Returns actionable errors for dependencies that should not remain root-owned. + */ +export function collectRootDependencyOwnershipCheckErrors( + records: OwnershipCheckRecord[], +): string[] { + return records + .filter((record) => record.category === "extension_only_localizable") + .map((record) => { + const declaredInExtensions = + record.declaredInExtensions.length > 0 + ? `; extension declarations: ${record.declaredInExtensions.join(", ")}` + : ""; + const sampleFiles = + record.sampleFiles.length > 0 ? `; sample imports: ${record.sampleFiles.join(", ")}` : ""; + return ( + `root dependency '${record.depName}' is extension-owned (${record.recommendation})` + + `${declaredInExtensions}${sampleFiles}` + ); + }); +} + +function printTextReport(records: AuditRecord[]) { + const grouped = new Map<string, AuditRecord[]>(); + for (const record of records) { + const existing = grouped.get(record.category) ?? []; + existing.push(record); + grouped.set(record.category, existing); + } + + for (const category of [...grouped.keys()].toSorted((left, right) => left.localeCompare(right))) { + console.log(`\n## ${category}`); + for (const record of grouped.get(category) ?? []) { + const details = [`sections=${record.sections.join(",") || "-"}`, `files=${record.fileCount}`]; + const spec = + typeof record.spec === "string" + ? record.spec + : (JSON.stringify(record.spec) ?? "<undefined>"); + if (record.declaredInExtensions.length > 0) { + details.push(`extensions=${record.declaredInExtensions.join(",")}`); + } + if (record.internalizedBundledRuntimeOwners.length > 0) { + details.push(`internalized=${record.internalizedBundledRuntimeOwners.join(",")}`); + } + console.log(`- ${record.depName}@${spec} :: ${details.join(" | ")}`); + console.log(` ${record.recommendation}`); + } + } +} + +function main(argv: string[] = process.argv.slice(2)) { + const asJson = argv.includes("--json"); + const check = argv.includes("--check"); + const records = collectRootDependencyOwnershipAudit(); + if (check) { + const errors = collectRootDependencyOwnershipCheckErrors(records); + if (errors.length > 0) { + for (const error of errors) { + console.error(`[root-dependency-ownership] ${error}`); + } + process.exitCode = 1; + return; + } + if (!asJson) { + console.error("[root-dependency-ownership] ok"); + return; + } + } + if (asJson) { + console.log(JSON.stringify(records, null, 2)); + return; + } + printTextReport(records); +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { + main(); +} diff --git a/scripts/run-additional-boundary-checks.mts b/scripts/run-additional-boundary-checks.mts new file mode 100644 index 0000000000000000000000000000000000000000..0ae79a900ff3954b2dede8f32a22dc6ce30a154c --- /dev/null +++ b/scripts/run-additional-boundary-checks.mts @@ -0,0 +1,662 @@ +#!/usr/bin/env node +// Runs additional architecture and boundary checks with sharding, concurrency, +// timeout handling, and grouped CI output. +import { spawn, type ChildProcess } from "node:child_process"; +import { performance } from "node:perf_hooks"; +import pMap from "p-map"; +import prettyMilliseconds from "pretty-ms"; +import { + MAX_TIMER_TIMEOUT_MS, + resolveTimerTimeoutMs, +} from "../packages/normalization-core/src/number-coercion.ts"; +import { isDirectRunUrl } from "./lib/direct-run.mjs"; +import { + inspectManagedProcessGroup, + terminateManagedChild, + waitForManagedProcessGroupExit, +} from "./lib/managed-child-process.mts"; + +const DEFAULT_CHECK_TIMEOUT_MS = 10 * 60 * 1000; +const DEFAULT_OUTPUT_MAX_BYTES = 512 * 1024; +// Boundary checks are disposable subprocesses; bound descendant cleanup after timeout. +const TIMEOUT_KILL_GRACE_MS = 250; +const POST_FORCE_KILL_WAIT_MS = 250; + +type ProcessSignal = `SIG${string}`; +type TimerHandle = ReturnType<typeof setTimeout>; +type BoundaryCheck = { args: string[]; command: string; label: string }; + +type BoundaryShard = { count: number; index: number; label: string }; +type OutputWriter = { write(chunk: string): boolean }; +type BoundaryCheckResult = { + check: BoundaryCheck; + code: number; + durationMs: number; + output: string; + signal: ProcessSignal | null; + timedOut: boolean; +}; +type CheckExecutionOptions = { + checkTimeoutMs?: number; + cwd: string; + env: NodeJS.ProcessEnv; + outputMaxBytes?: number; +}; +type RunSingleCheckOptions = CheckExecutionOptions & { activeChildren?: Set<ChildProcess> }; +type RunChecksOptions = Partial<CheckExecutionOptions> & { + concurrency?: number; + output?: OutputWriter; +}; + +/** Ordered list of supplemental boundary checks used by CI sharding. */ +// prompt:snapshots:check is intentionally absent: it regenerates snapshots by +// running real embedded-agent turns (~2min) and owns a dedicated CI lane +// (check-prompt-snapshots) so no boundary shard carries that wall clock. +export const BOUNDARY_CHECKS = ( + [ + ["plugin-extension-boundary", "pnpm", ["run", "lint:plugins:no-extension-imports"]], + ["lint:docker-e2e", "pnpm", ["run", "lint:docker-e2e"]], + ["lint:tmp:no-random-messaging", "pnpm", ["run", "lint:tmp:no-random-messaging"]], + [ + "lint:tmp:channel-agnostic-boundaries", + "pnpm", + ["run", "lint:tmp:channel-agnostic-boundaries"], + ], + ["lint:tmp:tsgo-core-boundary", "pnpm", ["run", "lint:tmp:tsgo-core-boundary"]], + ["lint:tmp:no-raw-channel-fetch", "pnpm", ["run", "lint:tmp:no-raw-channel-fetch"]], + ["lint:tmp:no-raw-http2-imports", "pnpm", ["run", "lint:tmp:no-raw-http2-imports"]], + ["lint:agent:ingress-owner", "pnpm", ["run", "lint:agent:ingress-owner"]], + // This full-root pass runs all four focused rules, including the narrower + // HTTP/window.open guards and both public assertion aliases. + ["lint:no-chained-type-assertions", "pnpm", ["run", "lint:no-chained-type-assertions"]], + [ + "lint:plugins:no-monolithic-plugin-sdk-entry-imports", + "pnpm", + ["run", "lint:plugins:no-monolithic-plugin-sdk-entry-imports"], + ], + [ + "lint:plugins:no-extension-src-imports", + "pnpm", + ["run", "lint:plugins:no-extension-src-imports"], + ], + [ + "lint:plugins:no-extension-test-core-imports", + "pnpm", + ["run", "lint:plugins:no-extension-test-core-imports"], + ], + [ + "lint:plugins:plugin-sdk-subpaths-exported", + "pnpm", + ["run", "lint:plugins:plugin-sdk-subpaths-exported"], + ], + ["deps:root-ownership:check", "pnpm", ["deps:root-ownership:check"]], + ["web-fetch-provider-boundary", "pnpm", ["run", "lint:web-fetch-provider-boundaries"]], + [ + "extension-plugin-sdk-boundaries", + "node", + ["--import", "./scripts/tsx.mjs", "scripts/check-extension-plugin-sdk-boundary.mts", "--all"], + ], + [ + "lint:extensions:telegram-grammy-types", + "pnpm", + ["run", "lint:extensions:telegram-grammy-types"], + ], + ["native-state-schema-version", "node", ["scripts/check-native-state-schema-version.mjs"]], + ] satisfies Array<[label: string, command: string, args: string[]]> +).map(([label, command, args]) => ({ label, command, args })); + +/** + * Resolves the configured boundary-check concurrency. + */ +export function resolveConcurrency(value: unknown, fallback = 4, label = "concurrency") { + return resolvePositiveInteger(value, fallback, label); +} + +function scalarText(value: unknown): string | undefined { + return typeof value === "string" || + typeof value === "number" || + typeof value === "boolean" || + typeof value === "bigint" + ? String(value) + : undefined; +} + +function displayValue(value: unknown): string { + return scalarText(value) ?? JSON.stringify(value) ?? "<unserializable>"; +} + +/** + * Parses positive integer CLI/env options with a fallback. + */ +export function resolvePositiveInteger(value: unknown, fallback: number, label = "value") { + if (value === undefined || value === null || value === "") { + return fallback; + } + const text = scalarText(value)?.trim(); + if (text === undefined || !/^\d+$/u.test(text)) { + throw new Error(`${label} must be a positive integer; got: ${displayValue(value)}`); + } + const parsed = Number(text); + if (!Number.isSafeInteger(parsed) || parsed < 1) { + throw new Error(`${label} must be a positive integer; got: ${displayValue(value)}`); + } + return parsed; +} + +/** + * Parses one N/TOTAL shard selector into zero-based index form. + */ +export function parseShardSpec(value: unknown): BoundaryShard | null { + if (!value) { + return null; + } + const match = scalarText(value)?.match(/^(\d+)\/(\d+)$/u); + if (!match) { + throw new Error(`Invalid shard spec '${displayValue(value)}' (expected N/TOTAL)`); + } + const index = Number(match[1]); + const count = Number(match[2]); + if ( + !Number.isSafeInteger(index) || + !Number.isSafeInteger(count) || + index < 1 || + count < 1 || + index > count + ) { + throw new Error(`Invalid shard spec '${displayValue(value)}' (expected 1 <= N <= TOTAL)`); + } + return { count, index: index - 1, label: `${index}/${count}` }; +} + +/** + * Parses a comma-separated list of N/TOTAL shard selectors. + */ +export function parseShardSelection(value: unknown) { + if (!value) { + return null; + } + const text = scalarText(value); + if (text === undefined) { + throw new Error(`Invalid shard selection '${displayValue(value)}'`); + } + return text + .split(",") + .map((part) => part.trim()) + .filter(Boolean) + .map((part) => { + const shard = parseShardSpec(part); + if (!shard) { + throw new Error(`Invalid shard spec '${displayValue(value)}'`); + } + return shard; + }); +} + +/** + * Selects checks whose ordinal belongs to the requested shard set. + */ +export function selectChecksForShard( + checks: BoundaryCheck[], + shardSpec: string | BoundaryShard | BoundaryShard[] | null, + coreTestBoundaryOwner: "additional" | "test-types" = "additional", +) { + const shards = + typeof shardSpec === "string" + ? parseShardSelection(shardSpec) + : Array.isArray(shardSpec) + ? shardSpec + : shardSpec + ? [shardSpec] + : null; + // Transfer only this obligation, after partitioning so other checks keep their owner. + return checks.filter( + (check, index) => + (!shards?.length || shards.some((shard) => index % shard.count === shard.index)) && + (coreTestBoundaryOwner !== "test-types" || check.label !== "lint:tmp:tsgo-core-boundary"), + ); +} + +/** + * Formats a check command for CI group output. + */ +export function formatCommand({ command, args }: Pick<BoundaryCheck, "args" | "command">) { + return [command, ...args].join(" "); +} + +function decodeUtf8Tail(buffer: Buffer) { + let start = 0; + while (start < buffer.length && (buffer[start]! & 0b1100_0000) === 0b1000_0000) { + start += 1; + } + // Appends are complete JS strings; only a byte slice's leading boundary can be partial. + return buffer.subarray(start).toString("utf8"); +} + +/** + * Keeps only the tail of noisy check output so failure logs stay bounded. + */ +export function createBoundedOutputBuffer(maxBytes = DEFAULT_OUTPUT_MAX_BYTES) { + const limit = Math.max(1, maxBytes); + const chunks: string[] = []; + let bytes = 0; + let truncated = false; + + const append = (value: unknown) => { + const text = String(value); + const textBytes = Buffer.byteLength(text); + if (textBytes >= limit) { + const buffer = Buffer.from(text); + const tail = decodeUtf8Tail(buffer.subarray(buffer.length - limit)); + chunks.splice(0, chunks.length, tail); + bytes = Buffer.byteLength(tail); + truncated = true; + return; + } + + chunks.push(text); + bytes += textBytes; + while (bytes > limit && chunks.length > 0) { + const first = chunks[0]!; + const firstBytes = Buffer.byteLength(first); + const overflow = bytes - limit; + if (firstBytes <= overflow) { + chunks.shift(); + bytes -= firstBytes; + truncated = true; + continue; + } + + const buffer = Buffer.from(first); + const tail = decodeUtf8Tail(buffer.subarray(overflow)); + chunks[0] = tail; + bytes = chunks.reduce((total, chunk) => total + Buffer.byteLength(chunk), 0); + truncated = true; + } + }; + + return { + append, + read() { + const output = chunks.join(""); + return truncated ? `[output truncated to last ${limit} bytes]\n${output}` : output; + }, + }; +} + +function terminateChild(child: ChildProcess, signal: ProcessSignal) { + terminateManagedChild(child, signal as NodeJS.Signals, { + onChildSignalError(error) { + throw error; + }, + useWindowsTaskkill: false, + }); +} + +function processGroupAlive(child: ChildProcess) { + return inspectManagedProcessGroup(child, { errorPolicy: "alive-on-eperm" }) === "live"; +} + +function waitForProcessGroupExit(child: ChildProcess, timeoutMs: number) { + return waitForManagedProcessGroupExit(child, timeoutMs, { errorPolicy: "alive-on-eperm" }); +} + +async function finishTerminatedProcessTree( + child: ChildProcess, + timeoutKillGraceMs = TIMEOUT_KILL_GRACE_MS, +) { + if (processGroupAlive(child)) { + await waitForProcessGroupExit(child, timeoutKillGraceMs); + } + if (processGroupAlive(child)) { + terminateChild(child, "SIGKILL"); + await waitForProcessGroupExit(child, POST_FORCE_KILL_WAIT_MS); + } +} + +function terminateActiveChildren(activeChildren: Iterable<ChildProcess>, signal: ProcessSignal) { + for (const child of activeChildren) { + terminateChild(child, signal); + } +} + +function installActiveChildCleanup(activeChildren: Set<ChildProcess>) { + let active = true; + let shutdownChildren: ChildProcess[] = []; + let shutdownPromise: Promise<void> | null = null; + let shutdownForceKillTimer: TimerHandle | null = null; + let resolveShutdownForceKill: (() => void) | null = null; + const removeHandlers = () => { + for (const [signal, handler] of signalHandlers) { + process.off(signal, handler); + } + process.off("exit", exitHandler); + }; + const forceKillShutdownChildren = () => { + if (shutdownForceKillTimer) { + clearTimeout(shutdownForceKillTimer); + shutdownForceKillTimer = null; + } + terminateActiveChildren(shutdownChildren, "SIGKILL"); + resolveShutdownForceKill?.(); + }; + const cleanup = ( + signal: ProcessSignal, + { waitForExit = false }: { waitForExit?: boolean } = {}, + ) => { + if (!active) { + return shutdownPromise ?? Promise.resolve(); + } + active = false; + shutdownChildren = [...activeChildren]; + terminateActiveChildren(shutdownChildren, signal); + if (!waitForExit) { + return Promise.resolve(); + } + shutdownPromise = new Promise<void>((resolveForceKill) => { + resolveShutdownForceKill = resolveForceKill; + // Keep this timer ref'ed: once the leader exits, group liveness can look + // gone while descendants are still running and still need the force kill. + shutdownForceKillTimer = setTimeout(forceKillShutdownChildren, TIMEOUT_KILL_GRACE_MS); + }) + .then(() => + Promise.all( + shutdownChildren.map((child) => waitForProcessGroupExit(child, POST_FORCE_KILL_WAIT_MS)), + ), + ) + .then(() => undefined); + return shutdownPromise; + }; + const signalHandlers = new Map<ProcessSignal, () => void>(); + const signals: ProcessSignal[] = + process.platform === "win32" ? ["SIGINT", "SIGTERM"] : ["SIGINT", "SIGTERM", "SIGHUP"]; + for (const signal of signals) { + const handler = () => { + if (shutdownPromise) { + forceKillShutdownChildren(); + return; + } + void cleanup(signal, { waitForExit: true }).finally(() => { + removeHandlers(); + process.kill(process.pid, signal as NodeJS.Signals); + }); + }; + signalHandlers.set(signal, handler); + process.on(signal, handler); + } + const exitHandler = () => { + void cleanup("SIGTERM"); + }; + process.once("exit", exitHandler); + + return () => { + if (shutdownPromise) { + return; + } + active = false; + removeHandlers(); + }; +} + +/** + * Runs one boundary check with timeout and process-group termination. + */ +export function runSingleCheck( + check: BoundaryCheck, + { + activeChildren, + checkTimeoutMs = DEFAULT_CHECK_TIMEOUT_MS, + cwd, + env, + outputMaxBytes = DEFAULT_OUTPUT_MAX_BYTES, + }: RunSingleCheckOptions, +) { + return new Promise<BoundaryCheckResult>((resolve) => { + const resolvedCheckTimeoutMs = resolveTimerTimeoutMs(checkTimeoutMs, MAX_TIMER_TIMEOUT_MS); + const startedAt = performance.now(); + const child = spawn(check.command, check.args, { + cwd, + env, + detached: process.platform !== "win32", + shell: false, + stdio: ["ignore", "pipe", "pipe"], + }); + activeChildren?.add(child); + const output = createBoundedOutputBuffer(outputMaxBytes); + let settled = false; + let timedOut = false; + let forceKillTimer: TimerHandle | null = null; + const finish = (code: number | null, signal: ProcessSignal | null) => { + if (settled) { + return; + } + settled = true; + clearTimeout(timeout); + if (forceKillTimer) { + clearTimeout(forceKillTimer); + } + activeChildren?.delete(child); + resolve({ + check, + code: timedOut ? 1 : (code ?? 1), + durationMs: Math.round(performance.now() - startedAt), + signal, + timedOut, + output: output.read(), + }); + }; + const finishAfterTimeoutTeardown = async ( + code: number | null, + signal: ProcessSignal | null, + ) => { + await finishTerminatedProcessTree(child, TIMEOUT_KILL_GRACE_MS); + finish(code, signal); + }; + const timeout = setTimeout(() => { + timedOut = true; + output.append( + `\n[boundary-check] ${check.label} timed out after ${formatDuration(resolvedCheckTimeoutMs)}; terminating process group\n`, + ); + terminateChild(child, "SIGTERM"); + forceKillTimer = setTimeout(() => { + output.append( + `[boundary-check] ${check.label} still running after ${formatDuration(TIMEOUT_KILL_GRACE_MS)}; sending SIGKILL\n`, + ); + terminateChild(child, "SIGKILL"); + }, TIMEOUT_KILL_GRACE_MS); + forceKillTimer.unref?.(); + }, resolvedCheckTimeoutMs); + timeout.unref?.(); + + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk) => output.append(chunk)); + child.stderr.on("data", (chunk) => output.append(chunk)); + child.on("error", (error) => { + output.append(`${error.stack ?? error.message}\n`); + finish(1, null); + }); + child.on("close", (code, signal) => { + if (timedOut) { + void finishAfterTimeoutTeardown(code, signal); + return; + } + finish(code, signal); + }); + }); +} + +function formatDuration(ms: number) { + if (!Number.isFinite(ms)) { + return ""; + } + const roundedMs = ms < 1000 ? Math.round(ms) : Math.round(ms / 100) * 100; + return prettyMilliseconds(Math.max(0, roundedMs), { + unitCount: 1, + }); +} + +function writeGroupedResult(result: BoundaryCheckResult, output: OutputWriter) { + const success = result.code === 0; + output.write(`::group::${result.check.label}\n`); + output.write(`$ ${formatCommand(result.check)}\n`); + if (result.output) { + output.write(result.output.endsWith("\n") ? result.output : `${result.output}\n`); + } + if (success) { + output.write(`[ok] ${result.check.label} in ${formatDuration(result.durationMs)}\n`); + } else { + const suffix = result.timedOut + ? " (timeout)" + : result.signal + ? ` (signal ${result.signal})` + : ` (exit ${result.code})`; + output.write( + `::error title=${result.check.label} failed::${result.check.label} failed${suffix} after ${formatDuration(result.durationMs)}\n`, + ); + } + output.write("::endgroup::\n"); +} + +function writeTimingSummary(results: BoundaryCheckResult[], output: OutputWriter) { + output.write("Additional boundary check timings:\n"); + for (const result of [...results].toSorted((left, right) => right.durationMs - left.durationMs)) { + output.write( + `${result.check.label.padEnd(48)} ${formatDuration(result.durationMs).padStart(8)}\n`, + ); + } +} + +/** + * Runs boundary checks with bounded concurrency and returns the failure count. + */ +export async function runChecks( + checks: BoundaryCheck[] = BOUNDARY_CHECKS, + { + checkTimeoutMs = DEFAULT_CHECK_TIMEOUT_MS, + concurrency = 4, + cwd = process.cwd(), + env = process.env, + output = process.stdout, + outputMaxBytes = DEFAULT_OUTPUT_MAX_BYTES, + }: RunChecksOptions = {}, +) { + const activeChildren = new Set<ChildProcess>(); + const removeActiveChildCleanup = installActiveChildCleanup(activeChildren); + let results: BoundaryCheckResult[]; + + try { + results = await pMap( + checks, + (check) => + runSingleCheck(check, { + activeChildren, + checkTimeoutMs, + cwd, + env, + outputMaxBytes, + }), + { concurrency, stopOnError: true }, + ); + } finally { + removeActiveChildCleanup(); + } + + let failures = 0; + for (const result of results) { + writeGroupedResult(result, output); + if (result.code !== 0) { + failures += 1; + } + } + writeTimingSummary(results, output); + return failures; +} + +function usage() { + return `Usage: node --import tsx scripts/run-additional-boundary-checks.mts [--shard <N/TOTAL>[,<N/TOTAL>]] + +Runs supplemental architecture and boundary checks with bounded concurrency. + +Options: + --shard <spec> Run only checks selected by one or more N/TOTAL shard specs + --core-test-boundary-owner=test-types The required type job owns the core graph boundary + -h, --help Show this help +`; +} + +export function parseCliArgs(args: string[], env: NodeJS.ProcessEnv = process.env) { + let shardSpec = env.OPENCLAW_ADDITIONAL_BOUNDARY_SHARD ?? ""; + let help = false; + let coreTestBoundaryOwner: "additional" | "test-types" = "additional"; + for (let index = 0; index < args.length; index += 1) { + const arg = args[index]!; + if (arg === "--core-test-boundary-owner=test-types") { + coreTestBoundaryOwner = "test-types"; + continue; + } + if (arg === "-h" || arg === "--help") { + help = true; + continue; + } + if (arg === "--shard") { + const value = args[index + 1]; + if (!value || value.startsWith("-")) { + throw new Error("--shard requires a value"); + } + shardSpec = value; + index += 1; + continue; + } + if (arg.startsWith("--shard=")) { + const value = arg.slice("--shard=".length); + if (!value) { + throw new Error("--shard requires a value"); + } + shardSpec = value; + continue; + } + throw new Error(`Unknown argument: ${arg}`); + } + return { help, shardSpec, coreTestBoundaryOwner }; +} + +if (isDirectRunUrl(process.argv[1], import.meta.url)) { + try { + const cliArgs = parseCliArgs(process.argv.slice(2), process.env); + if (cliArgs.help) { + process.stdout.write(usage()); + process.exitCode = 0; + } else { + const concurrencyRaw = + process.env.OPENCLAW_ADDITIONAL_BOUNDARY_CONCURRENCY ?? + process.env.OPENCLAW_EXTENSION_BOUNDARY_CONCURRENCY; + const concurrencyLabel = + process.env.OPENCLAW_ADDITIONAL_BOUNDARY_CONCURRENCY === undefined + ? "OPENCLAW_EXTENSION_BOUNDARY_CONCURRENCY" + : "OPENCLAW_ADDITIONAL_BOUNDARY_CONCURRENCY"; + const concurrency = resolveConcurrency(concurrencyRaw, 4, concurrencyLabel); + const checkTimeoutMs = resolvePositiveInteger( + process.env.OPENCLAW_ADDITIONAL_BOUNDARY_TIMEOUT_MS, + DEFAULT_CHECK_TIMEOUT_MS, + "OPENCLAW_ADDITIONAL_BOUNDARY_TIMEOUT_MS", + ); + const outputMaxBytes = resolvePositiveInteger( + process.env.OPENCLAW_ADDITIONAL_BOUNDARY_OUTPUT_MAX_BYTES, + DEFAULT_OUTPUT_MAX_BYTES, + "OPENCLAW_ADDITIONAL_BOUNDARY_OUTPUT_MAX_BYTES", + ); + const shards = parseShardSelection(cliArgs.shardSpec); + const checks = selectChecksForShard(BOUNDARY_CHECKS, shards, cliArgs.coreTestBoundaryOwner); + if (shards) { + process.stdout.write( + `Running ${checks.length}/${BOUNDARY_CHECKS.length} additional boundary checks (shard ${shards.map((shard) => shard.label).join(",")})\n`, + ); + } + const failures = await runChecks(checks, { checkTimeoutMs, concurrency, outputMaxBytes }); + process.exitCode = failures === 0 ? 0 : 1; + } + } catch (error) { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n\n${usage()}`); + process.exitCode = 1; + } +} diff --git a/scripts/run-node-package-bin.mts b/scripts/run-node-package-bin.mts new file mode 100644 index 0000000000000000000000000000000000000000..56a112c9ea0b9e402ab36398a407f1b45f723cb8 --- /dev/null +++ b/scripts/run-node-package-bin.mts @@ -0,0 +1,31 @@ +import { readFileSync } from "node:fs"; +import { createRequire } from "node:module"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; + +export function resolveNodePackageBin( + tool: string, + requireFromPackage: ReturnType<typeof createRequire>, +): string { + // Hoisting can leave executable local shims pointing at removed packages. + // Resolve the declared bin from the caller's actual dependency scope instead. + const manifestPath = requireFromPackage.resolve(`${tool}/package.json`); + const { bin } = JSON.parse(readFileSync(manifestPath, "utf8")) as { + bin: Record<string, string>; + }; + const entry = bin[tool]; + if (!entry) { + throw new Error(`${tool} does not declare its executable`); + } + return path.resolve(path.dirname(manifestPath), entry); +} + +if (import.meta.main) { + const tool = process.argv[2]; + if (!tool) { + throw new Error("Usage: node scripts/run-node-package-bin.mts <package> [...args]"); + } + const entry = resolveNodePackageBin(tool, createRequire(path.resolve("package.json"))); + process.argv.splice(1, 2, entry); + await import(pathToFileURL(entry).href); +} diff --git a/scripts/run-node-watch-paths.mts b/scripts/run-node-watch-paths.mts new file mode 100644 index 0000000000000000000000000000000000000000..2e05df01dad492d6530550ca3157be5b183cc9cd --- /dev/null +++ b/scripts/run-node-watch-paths.mts @@ -0,0 +1,134 @@ +// Defines source/config paths that pnpm dev watches for rebuilds and restarts. +import path from "node:path"; +import { + BUNDLED_PLUGIN_PATH_PREFIX, + BUNDLED_PLUGIN_ROOT_DIR, +} from "./lib/bundled-plugin-paths.mjs"; +import { listGeneratedExtensionAssetSources } from "./lib/static-extension-assets.mts"; + +const RUN_NODE_PACKAGE_SOURCE_ROOTS = [ + // Root runtime code imports these package sources through tsconfig aliases, + // while pnpm dev/watch still runs the root dist entrypoint. Treat them like + // src/ so edits restart the same process that consumes them. + "packages/ai/src", + "packages/gateway-client/src", + "packages/gateway-protocol/src", + "packages/markdown-core/src", + "packages/llm-core/src", + "packages/media-core/src", + "packages/media-generation-core/src", + "packages/media-understanding-common/src", + "packages/normalization-core/src", + "packages/retry/src", + "packages/acp-core/src", + "packages/terminal-core/src", + "packages/net-policy/src", +]; + +/** Source roots whose changes require the root dev build pipeline. */ +export const runNodeSourceRoots = [ + "src", + ...RUN_NODE_PACKAGE_SOURCE_ROOTS, + BUNDLED_PLUGIN_ROOT_DIR, +]; +/** Root config files whose changes invalidate the dev build. */ +export const runNodeConfigFiles = ["tsconfig.json", "package.json", "tsdown.config.ts"]; +/** Combined watch list used by the run-node wrapper. */ +export const runNodeWatchedPaths = [...runNodeSourceRoots, ...runNodeConfigFiles]; +/** Plugin metadata files that require a runtime restart even without source edits. */ +export const extensionRestartMetadataFiles = new Set(["openclaw.plugin.json", "package.json"]); + +const ignoredRunNodeRepoPathPatterns = [ + /^extensions\/[^/]+\/src\/host\/.+\/\.bundle\.hash$/u, + /^extensions\/[^/]+\/src\/host\/.+\/[^/]+\.bundle\.js$/u, +]; +const extensionSourceFilePattern = /\.(?:[cm]?[jt]sx?)$/; + +/** Canonicalizes native paths without treating POSIX filename backslashes as separators. */ +export const normalizeRunNodePath = (filePath: unknown): string => + (typeof filePath === "string" ? filePath : "").replaceAll(path.sep, "/").replace(/^\.\/+/, ""); + +const isIgnoredSourcePath = (relativePath: string): boolean => + relativePath.endsWith(".test.ts") || + relativePath.endsWith(".test.tsx") || + relativePath.endsWith("test-helpers.ts"); + +const isBuildRelevantSourcePath = (relativePath: string): boolean => + extensionSourceFilePattern.test(relativePath) && !isIgnoredSourcePath(relativePath); + +const isRestartRelevantExtensionPath = (relativePath: string): boolean => { + if (extensionRestartMetadataFiles.has(path.posix.basename(relativePath))) { + return true; + } + return isBuildRelevantSourcePath(relativePath); +}; + +const isRelevantRunNodePath = ( + repoPath: unknown, + isRelevantBundledPluginPath: (relativePath: string) => boolean, + generatedPluginAssetPaths: ReadonlySet<string>, +): boolean => { + const normalizedPath = normalizeRunNodePath(repoPath); + if ( + generatedPluginAssetPaths.has(normalizedPath) || + ignoredRunNodeRepoPathPatterns.some((pattern) => pattern.test(normalizedPath)) + ) { + return false; + } + if (runNodeConfigFiles.includes(normalizedPath)) { + return true; + } + if (normalizedPath.startsWith("src/")) { + return !isIgnoredSourcePath(normalizedPath.slice("src/".length)); + } + for (const sourceRoot of RUN_NODE_PACKAGE_SOURCE_ROOTS) { + if (normalizedPath.startsWith(`${sourceRoot}/`)) { + return !isIgnoredSourcePath(normalizedPath.slice(sourceRoot.length + 1)); + } + } + if (normalizedPath.startsWith(BUNDLED_PLUGIN_PATH_PREFIX)) { + return isRelevantBundledPluginPath(normalizedPath.slice(BUNDLED_PLUGIN_PATH_PREFIX.length)); + } + return false; +}; + +/** Creates a path classifier whose generated-output metadata can be refreshed. */ +export function createRunNodePathClassifier(params: { rootDir?: string } = {}) { + const rootDir = params.rootDir ?? process.cwd(); + let generatedPluginAssetPaths = new Set<string>(); + + return { + refreshGeneratedPluginAssetPaths() { + generatedPluginAssetPaths = new Set(listGeneratedExtensionAssetSources({ rootDir })); + }, + isBuildRelevantRunNodePath(repoPath: unknown) { + return isRelevantRunNodePath(repoPath, isBuildRelevantSourcePath, generatedPluginAssetPaths); + }, + isRestartRelevantRunNodePath(repoPath: unknown) { + return isRelevantRunNodePath( + repoPath, + isRestartRelevantExtensionPath, + generatedPluginAssetPaths, + ); + }, + }; +} + +type RunNodePathClassifier = ReturnType<typeof createRunNodePathClassifier>; + +let defaultRunNodePathClassifier: RunNodePathClassifier | undefined; +const getDefaultRunNodePathClassifier = (): RunNodePathClassifier => { + if (!defaultRunNodePathClassifier) { + defaultRunNodePathClassifier = createRunNodePathClassifier(); + defaultRunNodePathClassifier.refreshGeneratedPluginAssetPaths(); + } + return defaultRunNodePathClassifier; +}; + +/** Returns true when a repo path should trigger a dev rebuild. */ +export const isBuildRelevantRunNodePath = (repoPath: unknown): boolean => + getDefaultRunNodePathClassifier().isBuildRelevantRunNodePath(repoPath); + +/** Returns true when a repo path should restart the running dev process. */ +export const isRestartRelevantRunNodePath = (repoPath: unknown): boolean => + getDefaultRunNodePathClassifier().isRestartRelevantRunNodePath(repoPath); diff --git a/scripts/run-node.mjs b/scripts/run-node.mjs new file mode 100644 index 0000000000000000000000000000000000000000..b238631ca7f9c38b139d138af9b04fa4c6a49403 --- /dev/null +++ b/scripts/run-node.mjs @@ -0,0 +1,7 @@ +#!/usr/bin/env node +import { runTsxCliShim } from "./lib/tsx-cli-shim.mjs"; + +await runTsxCliShim(import.meta.url, { + implementation: "./run-node.mts", + terminationOwner: "implementation", +}); diff --git a/scripts/run-openclaw-podman.sh b/scripts/run-openclaw-podman.sh new file mode 100644 index 0000000000000000000000000000000000000000..804c0191851cc8988ea72335458a9e28d38952b2 --- /dev/null +++ b/scripts/run-openclaw-podman.sh @@ -0,0 +1,423 @@ +#!/usr/bin/env bash +# Bash 5.3+ can deadlock writing heredoc pipes on macOS before the reader starts. +if [[ ${OSTYPE:-} == darwin* && $BASH != /bin/bash ]] && ((BASH_VERSINFO[0] > 5 || (BASH_VERSINFO[0] == 5 && BASH_VERSINFO[1] >= 3))); then + exec /bin/bash "$0" "$@" +fi +# Rootless OpenClaw in Podman: run after one-time setup. +# +# One-time setup (from repo root): ./scripts/podman/setup.sh +# Then: +# ./scripts/run-openclaw-podman.sh launch # Start gateway +# ./scripts/run-openclaw-podman.sh launch setup # Onboarding wizard +# +# Manage the running container from the host CLI: +# openclaw --container openclaw dashboard --no-open +# openclaw --container openclaw channels login +# +# Legacy: "setup-host" delegates to the Podman setup script + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=scripts/lib/host-timeout.sh +source "$SCRIPT_DIR/lib/host-timeout.sh" +# shellcheck source=scripts/podman/common.sh +source "$SCRIPT_DIR/podman/common.sh" +PLATFORM_NAME="$(uname -s 2>/dev/null || echo unknown)" + +run_podman_detached() { + openclaw_host_timeout_cmd "$PODMAN_RUN_TIMEOUT" podman run "$@" +} + +ensure_safe_existing_regular_file() { + local label="$1" + local file="$2" + validate_absolute_path "$label" "$file" + [[ -e "$file" ]] || fail "Missing $label: $file" + [[ ! -L "$file" ]] || fail "Unsafe $label: symlinks are not allowed ($file)" + [[ -f "$file" ]] || fail "Unsafe $label: expected a regular file ($file)" +} + +ensure_private_existing_regular_file_owned_by_user() { + local label="$1" + local file="$2" + local uid="" + local mode="" + ensure_safe_existing_regular_file "$label" "$file" + uid="$(stat_uid "$file")" + [[ "$uid" == "$(id -u)" ]] || fail "Unsafe $label: not owned by current user ($file)" + mode="$(stat_mode "$file")" + (( (8#$mode & 0077) == 0 )) || fail "Unsafe $label: expected owner-only permissions ($file)" +} + +load_podman_env_file() { + local file="$1" + local line="" + local key="" + local value="" + local trimmed="" + local dir="" + ensure_private_existing_regular_file_owned_by_user "Podman env file" "$file" + dir="$(dirname "$file")" + ensure_private_existing_dir_owned_by_user "Podman env directory" "$dir" + exec 9<"$file" || fail "Unable to open Podman env file: $file" + while IFS= read -r line <&9 || [[ -n "$line" ]]; do + trimmed="${line#"${line%%[![:space:]]*}"}" + [[ -z "$trimmed" || "${trimmed:0:1}" == "#" ]] && continue + [[ "$line" == *"="* ]] || continue + key="${line%%=*}" + value="${line#*=}" + key="${key#"${key%%[![:space:]]*}"}" + key="${key%"${key##*[![:space:]]}"}" + [[ "$key" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]] || continue + case "$key" in + OPENCLAW_GATEWAY_TOKEN|OPENCLAW_PODMAN_CONTAINER|OPENCLAW_PODMAN_IMAGE|OPENCLAW_IMAGE|OPENCLAW_PODMAN_PULL|OPENCLAW_PODMAN_RUN_TIMEOUT|OPENCLAW_PODMAN_GATEWAY_HOST_PORT|OPENCLAW_GATEWAY_PORT|OPENCLAW_PODMAN_BRIDGE_HOST_PORT|OPENCLAW_BRIDGE_PORT|OPENCLAW_GATEWAY_BIND|OPENCLAW_PODMAN_USERNS|OPENCLAW_BIND_MOUNT_OPTIONS|OPENCLAW_PODMAN_PUBLISH_HOST) + ;; + *) + continue + ;; + esac + if [[ "$value" =~ ^\".*\"$ || "$value" =~ ^\'.*\'$ ]]; then + value="${value:1:${#value}-2}" + fi + printf -v "$key" '%s' "$value" + export "$key" + done + exec 9<&- +} + +EFFECTIVE_USER="$(id -un)" +EFFECTIVE_HOME="${HOME:-}" +if [[ -z "$EFFECTIVE_HOME" ]]; then + EFFECTIVE_HOME="$(resolve_user_home "$EFFECTIVE_USER")" +fi +if [[ "$(id -u)" -eq 0 ]]; then + fail "Run run-openclaw-podman.sh as your normal user so Podman stays rootless." +fi + +# Legacy: setup-host -> run the Podman setup script +if [[ "${1:-}" == "setup-host" ]]; then + shift + REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + SETUP_PODMAN="$REPO_ROOT/scripts/podman/setup.sh" + if [[ -f "$SETUP_PODMAN" ]]; then + exec "$SETUP_PODMAN" "$@" + fi + echo "Podman setup script not found. Run from repo root: ./scripts/podman/setup.sh" >&2 + exit 1 +fi + +if [[ "${1:-}" == "launch" ]]; then + shift +fi + +if [[ -z "${EFFECTIVE_HOME:-}" ]]; then + EFFECTIVE_HOME="/tmp" +fi +validate_absolute_path "effective home" "$EFFECTIVE_HOME" + +CONFIG_DIR="${OPENCLAW_CONFIG_DIR:-$EFFECTIVE_HOME/.openclaw}" +ENV_FILE="${OPENCLAW_PODMAN_ENV:-$CONFIG_DIR/.env}" +# Bootstrap `.env` may set runtime/container options, but it must not +# relocate the config/workspace/env paths mid-run. Those path overrides are +# only honored from the parent process environment before bootstrap. +if [[ -f "$ENV_FILE" ]]; then + load_podman_env_file "$ENV_FILE" +fi + +CONFIG_DIR="${OPENCLAW_CONFIG_DIR:-$EFFECTIVE_HOME/.openclaw}" +ENV_FILE="${OPENCLAW_PODMAN_ENV:-$CONFIG_DIR/.env}" +WORKSPACE_DIR="${OPENCLAW_WORKSPACE_DIR:-$CONFIG_DIR/workspace}" +CONTAINER_NAME="${OPENCLAW_PODMAN_CONTAINER:-openclaw}" +OPENCLAW_IMAGE="${OPENCLAW_PODMAN_IMAGE:-${OPENCLAW_IMAGE:-openclaw:local}}" +PODMAN_PULL="${OPENCLAW_PODMAN_PULL:-never}" +PODMAN_RUN_TIMEOUT="${OPENCLAW_PODMAN_RUN_TIMEOUT:-600s}" +HOST_GATEWAY_PORT="${OPENCLAW_PODMAN_GATEWAY_HOST_PORT:-${OPENCLAW_GATEWAY_PORT:-18789}}" +HOST_BRIDGE_PORT="${OPENCLAW_PODMAN_BRIDGE_HOST_PORT:-${OPENCLAW_BRIDGE_PORT:-18790}}" +PUBLISH_HOST="${OPENCLAW_PODMAN_PUBLISH_HOST:-127.0.0.1}" +validate_mount_source_path "config directory" "$CONFIG_DIR" +validate_mount_source_path "workspace directory" "$WORKSPACE_DIR" +validate_absolute_path "env file path" "$ENV_FILE" +validate_single_line_value "container name" "$CONTAINER_NAME" +validate_single_line_value "image name" "$OPENCLAW_IMAGE" +validate_single_line_value "publish host" "$PUBLISH_HOST" +validate_port "gateway host port" "$HOST_GATEWAY_PORT" +validate_port "bridge host port" "$HOST_BRIDGE_PORT" + +cd "$EFFECTIVE_HOME" 2>/dev/null || cd /tmp 2>/dev/null || true + +RUN_SETUP=false +if [[ "${1:-}" == "setup" || "${1:-}" == "onboard" ]]; then + RUN_SETUP=true + shift +fi + +mkdir -p "$CONFIG_DIR" "$WORKSPACE_DIR" +mkdir -p "$CONFIG_DIR/canvas" "$CONFIG_DIR/cron" +chmod 700 "$CONFIG_DIR" "$WORKSPACE_DIR" +ensure_private_existing_dir_owned_by_user "config directory" "$CONFIG_DIR" +ensure_private_existing_dir_owned_by_user "workspace directory" "$WORKSPACE_DIR" + +resolve_config_gateway_bind() { + local config_dir="$1" + if ! command -v openclaw >/dev/null 2>&1; then + return 0 + fi + OPENCLAW_CONTAINER="" OPENCLAW_CONFIG_DIR="$config_dir" \ + openclaw config get gateway.bind 2>/dev/null || true +} + +# For published container ports, the gateway must listen on the container +# interface, so the Podman launcher defaults to lan. Respect an explicit +# OPENCLAW_GATEWAY_BIND first, then gateway.bind in local config. +CONFIG_GATEWAY_BIND="$(resolve_config_gateway_bind "$CONFIG_DIR")" +GATEWAY_BIND="${OPENCLAW_GATEWAY_BIND:-${CONFIG_GATEWAY_BIND:-lan}}" + +create_token_env_file() { + local file="$1" + local token="$2" + local dir="" + local tmp="" + dir="$(dirname "$file")" + ensure_private_existing_dir_owned_by_user "token env directory" "$dir" + tmp="$(mktemp "$dir/.token.env.XXXXXX")" + chmod 600 "$tmp" + printf 'OPENCLAW_GATEWAY_TOKEN=%s\n' "$token" >"$tmp" + printf '%s' "$tmp" +} + +sync_local_control_ui_origins_via_cli() { + local file="$1" + local port="$2" + local config_dir="" + local allowed_json="" + local merged_json="" + config_dir="$(dirname "$file")" + if ! command -v openclaw >/dev/null 2>&1; then + echo "Warning: openclaw not found; unable to sync gateway.controlUi.allowedOrigins in $file." >&2 + return 0 + fi + if ! command -v python3 >/dev/null 2>&1; then + OPENCLAW_CONTAINER="" OPENCLAW_CONFIG_DIR="$config_dir" \ + openclaw config set gateway.controlUi.allowedOrigins \ + "[\"http://127.0.0.1:${port}\",\"http://localhost:${port}\"]" \ + --strict-json >/dev/null + return 0 + fi + allowed_json="$( + OPENCLAW_CONTAINER="" OPENCLAW_CONFIG_DIR="$config_dir" \ + openclaw config get gateway.controlUi.allowedOrigins --json 2>/dev/null || true + )" + merged_json="$(python3 - "$port" "$allowed_json" <<'PY' +import json +import sys + +port = sys.argv[1] +raw = sys.argv[2] if len(sys.argv) > 2 else "" +desired = [ + f"http://127.0.0.1:{port}", + f"http://localhost:{port}", +] +allowed = [] +if raw: + try: + parsed = json.loads(raw) + if isinstance(parsed, list): + allowed = parsed + except json.JSONDecodeError: + allowed = [] +cleaned = [] +seen = set() +for origin in allowed + desired: + if not isinstance(origin, str): + continue + normalized = origin.strip() + if not normalized or normalized in seen: + continue + cleaned.append(normalized) + seen.add(normalized) +print(json.dumps(cleaned)) +PY + )" + OPENCLAW_CONTAINER="" OPENCLAW_CONFIG_DIR="$config_dir" \ + openclaw config set gateway.controlUi.allowedOrigins "$merged_json" --strict-json >/dev/null +} + +sync_local_control_ui_origins() { + local file="$1" + local port="$2" + local dir="" + local tmp="" + ensure_safe_write_file_path "config file" "$file" + if ! command -v python3 >/dev/null 2>&1; then + echo "Warning: python3 not found; unable to sync gateway.controlUi.allowedOrigins in $file." >&2 + return 0 + fi + dir="$(dirname "$file")" + ensure_private_existing_dir_owned_by_user "config file directory" "$dir" + tmp="$(mktemp "$dir/.config.tmp.XXXXXX")" + if ! python3 - "$file" "$port" "$tmp" <<'PY' +import json +import sys + +path = sys.argv[1] +port = sys.argv[2] +tmp = sys.argv[3] +try: + with open(path, "r", encoding="utf-8") as fh: + data = json.load(fh) +except json.JSONDecodeError as exc: + print( + f"Warning: unable to sync gateway.controlUi.allowedOrigins in {path}: existing config is not strict JSON ({exc}). Leaving file unchanged.", + file=sys.stderr, + ) + raise SystemExit(1) +if not isinstance(data, dict): + raise SystemExit(f"{path}: expected top-level object") +gateway = data.setdefault("gateway", {}) +if not isinstance(gateway, dict): + raise SystemExit(f"{path}: expected gateway object") +gateway.setdefault("mode", "local") +control_ui = gateway.setdefault("controlUi", {}) +if not isinstance(control_ui, dict): + raise SystemExit(f"{path}: expected gateway.controlUi object") +allowed = control_ui.get("allowedOrigins") +desired = [ + f"http://127.0.0.1:{port}", + f"http://localhost:{port}", +] +if not isinstance(allowed, list): + allowed = [] +cleaned = [] +seen = set() +for origin in allowed: + if not isinstance(origin, str): + continue + normalized = origin.strip() + if not normalized or normalized in seen: + continue + cleaned.append(normalized) + seen.add(normalized) +for origin in desired: + if origin not in seen: + cleaned.append(origin) + seen.add(origin) +control_ui["allowedOrigins"] = cleaned +with open(tmp, "w", encoding="utf-8") as fh: + json.dump(data, fh, indent=2) + fh.write("\n") +PY + then + rm -f "$tmp" + sync_local_control_ui_origins_via_cli "$file" "$port" + return 0 + fi + [[ -s "$tmp" ]] || { + rm -f "$tmp" + return 0 + } + chmod 600 "$tmp" 2>/dev/null || true + mv -f "$tmp" "$file" +} + +TOKEN_ENV_FILE="" +cleanup_token_env_file() { + if [[ -n "$TOKEN_ENV_FILE" && -f "$TOKEN_ENV_FILE" ]]; then + rm -f "$TOKEN_ENV_FILE" + fi +} +trap cleanup_token_env_file EXIT + +if [[ -z "${OPENCLAW_GATEWAY_TOKEN:-}" ]]; then + export OPENCLAW_GATEWAY_TOKEN="$(generate_token_hex_32)" + mkdir -p "$(dirname "$ENV_FILE")" + ensure_safe_existing_dir "env file directory" "$(dirname "$ENV_FILE")" + upsert_env_var "$ENV_FILE" "OPENCLAW_GATEWAY_TOKEN" "$OPENCLAW_GATEWAY_TOKEN" + echo "Generated OPENCLAW_GATEWAY_TOKEN and wrote it to $ENV_FILE." >&2 +fi + +CONFIG_JSON="$CONFIG_DIR/openclaw.json" +if [[ ! -f "$CONFIG_JSON" ]]; then + ( + umask 077 + write_file_atomically "$CONFIG_JSON" 600 <<'JSON' +{ "gateway": { "mode": "local" } } +JSON + ) + echo "Created $CONFIG_JSON (minimal gateway.mode=local)." >&2 +fi +sync_local_control_ui_origins "$CONFIG_JSON" "$HOST_GATEWAY_PORT" + +PODMAN_USERNS="${OPENCLAW_PODMAN_USERNS:-keep-id}" +USERNS_ARGS=() +RUN_USER_ARGS=() +case "$PODMAN_USERNS" in + ""|auto) ;; + keep-id) USERNS_ARGS=(--userns=keep-id) ;; + host) USERNS_ARGS=(--userns=host) ;; + *) + echo "Unsupported OPENCLAW_PODMAN_USERNS=$PODMAN_USERNS (expected: keep-id, auto, host)." >&2 + exit 2 + ;; +esac + +RUN_UID="$(id -u)" +RUN_GID="$(id -g)" +if [[ "$PODMAN_USERNS" == "keep-id" ]]; then + RUN_USER_ARGS=(--user "${RUN_UID}:${RUN_GID}") +else + echo "Starting container without --user (OPENCLAW_PODMAN_USERNS=$PODMAN_USERNS), mounts may require ownership fixes." >&2 +fi + +SELINUX_MOUNT_OPTS="" +if [[ -z "${OPENCLAW_BIND_MOUNT_OPTIONS:-}" ]]; then + if [[ "$(uname -s 2>/dev/null)" == "Linux" ]] && command -v getenforce >/dev/null 2>&1; then + _selinux_mode="$(getenforce 2>/dev/null || true)" + if [[ "$_selinux_mode" == "Enforcing" || "$_selinux_mode" == "Permissive" ]]; then + SELINUX_MOUNT_OPTS=",Z" + fi + fi +else + SELINUX_MOUNT_OPTS="${OPENCLAW_BIND_MOUNT_OPTIONS#:}" + [[ -n "$SELINUX_MOUNT_OPTS" ]] && SELINUX_MOUNT_OPTS=",$SELINUX_MOUNT_OPTS" +fi + +if [[ "$RUN_SETUP" == true ]]; then + TOKEN_ENV_FILE="$(create_token_env_file "$ENV_FILE" "$OPENCLAW_GATEWAY_TOKEN")" + podman run --pull="$PODMAN_PULL" --rm -it \ + --init \ + ${USERNS_ARGS[@]+"${USERNS_ARGS[@]}"} ${RUN_USER_ARGS[@]+"${RUN_USER_ARGS[@]}"} \ + -e HOME=/home/node -e TERM=xterm-256color -e BROWSER=echo \ + -e NPM_CONFIG_CACHE=/home/node/.openclaw/.npm \ + -e OPENCLAW_NO_RESPAWN=1 \ + --env-file "$TOKEN_ENV_FILE" \ + -v "$CONFIG_DIR:/home/node/.openclaw:rw${SELINUX_MOUNT_OPTS}" \ + -v "$WORKSPACE_DIR:/home/node/.openclaw/workspace:rw${SELINUX_MOUNT_OPTS}" \ + "$OPENCLAW_IMAGE" \ + node dist/index.js onboard "$@" + exit 0 +fi + +TOKEN_ENV_FILE="$(create_token_env_file "$ENV_FILE" "$OPENCLAW_GATEWAY_TOKEN")" +run_podman_detached --pull="$PODMAN_PULL" -d --replace \ + --name "$CONTAINER_NAME" \ + --init \ + ${USERNS_ARGS[@]+"${USERNS_ARGS[@]}"} ${RUN_USER_ARGS[@]+"${RUN_USER_ARGS[@]}"} \ + -e HOME=/home/node -e TERM=xterm-256color \ + -e NPM_CONFIG_CACHE=/home/node/.openclaw/.npm \ + -e OPENCLAW_NO_RESPAWN=1 \ + --env-file "$TOKEN_ENV_FILE" \ + -v "$CONFIG_DIR:/home/node/.openclaw:rw${SELINUX_MOUNT_OPTS}" \ + -v "$WORKSPACE_DIR:/home/node/.openclaw/workspace:rw${SELINUX_MOUNT_OPTS}" \ + -p "${PUBLISH_HOST}:${HOST_GATEWAY_PORT}:18789" \ + -p "${PUBLISH_HOST}:${HOST_BRIDGE_PORT}:18790" \ + "$OPENCLAW_IMAGE" \ + node dist/index.js gateway --bind "$GATEWAY_BIND" --port 18789 >/dev/null + +echo "Container $CONTAINER_NAME started: http://127.0.0.1:${HOST_GATEWAY_PORT}/" +echo "podman exec -it $CONTAINER_NAME openclaw dashboard --no-open" +echo "podman exec -it $CONTAINER_NAME openclaw devices approve --latest # if pairing required" +echo "podman logs -f $CONTAINER_NAME" +if [[ "$PLATFORM_NAME" == "Linux" ]]; then + echo "For auto-start/restarts, use: ./scripts/podman/setup.sh --quadlet (Quadlet + systemd user service)." +fi diff --git a/scripts/run-oxlint-shards.mts b/scripts/run-oxlint-shards.mts new file mode 100644 index 0000000000000000000000000000000000000000..50a8bc61c91b54f477bff900e009c75554a0d29e --- /dev/null +++ b/scripts/run-oxlint-shards.mts @@ -0,0 +1,881 @@ +// Splits oxlint into resource-aware shards with heartbeat and timeout handling. +import { spawn, type ChildProcess } from "node:child_process"; +import fs, { type Dirent } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { + distArtifactEntryArgs, + withDistArtifactOwnership, +} from "./lib/dist-artifact-ownership.mts"; +import { runWithFailedTrailer } from "./lib/failed-trailer.mts"; +import { + CI_PARALLEL_MIN_MEMORY_BYTES, + isConstrainedCiCheckHost, + resolveLocalCheckEnv, +} from "./lib/local-check-runtime.mts"; +import { + inspectManagedProcessGroup, + runManagedCommand, + terminateManagedChild, + waitForManagedProcessGroupExit, +} from "./lib/managed-child-process.mts"; +import { shouldPrepareExtensionPackageBoundaryArtifacts } from "./run-oxlint.mts"; + +const DEFAULT_EXTENSION_CHUNK_SIZE = 8; +const DEFAULT_CONSTRAINED_CORE_STRIPES = 5; +const DEFAULT_SHARD_HEARTBEAT_MS = 30_000; +const DEFAULT_SHARD_TIMEOUT_MS = 15 * 60_000; +const DEFAULT_SHARD_KILL_GRACE_MS = 5_000; +const POST_FORCE_KILL_WAIT_MS = 1_000; +const DEFAULT_SPLIT_CORE_SHARD_CONCURRENCY = 4; +const FAST_LOCAL_CHECK_MIN_CPUS = 12; +const FAST_LOCAL_CHECK_MIN_MEMORY_BYTES = 48 * 1024 ** 3; +const EXTENSION_TS_CONFIG = "extensions/tsconfig.json"; +const EXTENSIONS_DIR = "extensions"; +const OXLINT_SOURCE_FILE_PATTERN = /\.[cm]?[jt]sx?$/; +const PARENT_TERMINATION_SIGNALS = ["SIGINT", "SIGTERM"] satisfies NodeJS.Signals[]; + +type OxlintShard = { name: string; args: string[] }; +type ShardStripe = { index: number; total: number }; +type HostResources = { logicalCpuCount: number; totalMemoryBytes: number }; +type ReadDirectoryEntries = (target: string, options: { withFileTypes: true }) => Dirent[]; +type DirectoryOptions = { cwd?: string; readDir?: ReadDirectoryEntries }; +type DirectoryLookup = Required<DirectoryOptions>; +type ShardOptions = DirectoryOptions & { env?: NodeJS.ProcessEnv }; +type PlatformOptions = { env?: NodeJS.ProcessEnv; platform?: NodeJS.Platform }; +type PlatformShardOptions = ShardOptions & + ResourceOptions & { splitCore?: boolean; splitExtensions?: boolean }; +type ResourceOptions = PlatformOptions & { hostResources?: HostResources }; +type RunnerOptions = { + env: NodeJS.ProcessEnv; + extraArgs: string[]; + runner: string; +}; +type ShardRunnerOptions = RunnerOptions & { shard: OxlintShard }; +type ShardBatchOptions = RunnerOptions & { concurrency: number; entries: OxlintShard[] }; +type ActiveShardChild = { child: ChildProcess; killGraceMs: number }; + +const ACTIVE_SHARD_CHILDREN = new Set<ActiveShardChild>(); +let parentTerminationSignal: (typeof PARENT_TERMINATION_SIGNALS)[number] | null = null; +let parentTerminationForceKill: ReturnType<typeof setTimeout> | null = null; +const parentSignalHandlers = new Map<NodeJS.Signals, () => void>(); + +const CORE_SHARD = { + name: "core", + args: ["--tsconfig", "config/tsconfig/oxlint.core.json", "src", "ui", "packages"], +}; +const CORE_TS_CONFIG = "config/tsconfig/oxlint.core.json"; +const CORE_SPLIT_TARGETS = ["ui", "packages"]; +const EXTENSIONS_SHARD = { + name: "extensions", + args: ["--tsconfig", EXTENSION_TS_CONFIG, EXTENSIONS_DIR], +}; +const SCRIPTS_SHARD = { + name: "scripts", + args: ["--tsconfig", "config/tsconfig/oxlint.scripts.json", "scripts"], +}; + +/** + * Builds the platform-specific oxlint shard list. + */ +export function createOxlintShards({ + cwd = process.cwd(), + env = process.env, + platform = process.platform, + hostResources = resolveHostResources(), + readDir = fs.readdirSync, + splitCore = false, + splitExtensions = false, +}: PlatformShardOptions = {}) { + const constrainedSerial = + hostResources.totalMemoryBytes < CI_PARALLEL_MIN_MEMORY_BYTES && + shouldRunOxlintShardsSerial({ env, platform, hostResources }); + const coreGroups = + splitCore || constrainedSerial ? createCoreOxlintShards({ cwd, readDir }) : [CORE_SHARD]; + // Bound semantic checker caches without rebuilding the full type graph for every directory. + const coreShards = + constrainedSerial && !splitCore + ? Array.from({ length: DEFAULT_CONSTRAINED_CORE_STRIPES }, (_, index) => + selectCoreOxlintStripe(coreGroups, { + index: index + 1, + total: DEFAULT_CONSTRAINED_CORE_STRIPES, + }), + ).flat() + : coreGroups; + // Unsplit plugin lint can exceed small-host RAM even with a single lint thread. + // Chunk serial runs; explicit stripes use independently bounded Programs that stay serial. + const chunkExtensions = splitExtensions || platform === "win32" || constrainedSerial; + const extensionShards = chunkExtensions + ? createExtensionOxlintShards({ cwd, env, platform, readDir }) + : [EXTENSIONS_SHARD]; + + return [...coreShards, ...extensionShards, SCRIPTS_SHARD]; +} + +/** + * Splits core oxlint targets into smaller source/package/UI shards. + */ +function createCoreOxlintShards({ + cwd = process.cwd(), + readDir = fs.readdirSync, +}: DirectoryOptions = {}) { + const sourceShards = listSourceRootTargetGroups({ cwd, readDir }).map((targets) => ({ + name: targets.length === 1 ? `core:${targets.join("").replaceAll("/", ":")}` : "core:src:root", + args: ["--tsconfig", CORE_TS_CONFIG, ...targets], + })); + const sourceEntries = sourceShards.length > 0 ? sourceShards : [createCoreShard("src")]; + + return [...sourceEntries, ...CORE_SPLIT_TARGETS.map((target) => createCoreShard(target))]; +} + +function createCoreShard(target: string) { + return { + name: `core:${target}`, + args: ["--tsconfig", CORE_TS_CONFIG, target], + }; +} + +/** + * Chunks plugin lint targets for Windows and memory-constrained serial runs. + */ +export function createExtensionOxlintShards({ + cwd = process.cwd(), + env = process.env, + platform = process.platform, + readDir = fs.readdirSync, +}: ShardOptions & PlatformOptions = {}) { + const entries = listExtensionEntries({ cwd, readDir }); + if (entries.dirs.length === 0 && entries.rootFiles.length === 0) { + return [EXTENSIONS_SHARD]; + } + + const chunkSize = + platform === "win32" ? resolveWindowsExtensionChunkSize(env) : DEFAULT_EXTENSION_CHUNK_SIZE; + const shards: OxlintShard[] = []; + + if (entries.rootFiles.length > 0) { + shards.push({ + name: "extensions:root", + args: ["--tsconfig", EXTENSION_TS_CONFIG, ...entries.rootFiles], + }); + } + + for (let index = 0; index < entries.dirs.length; index += chunkSize) { + const chunk = entries.dirs.slice(index, index + chunkSize); + const chunkNumber = String(index / chunkSize + 1).padStart(2, "0"); + shards.push({ + name: `extensions:${chunkNumber}`, + args: ["--tsconfig", EXTENSION_TS_CONFIG, ...chunk], + }); + } + return shards; +} + +/** + * Reads the Windows extension shard chunk size. + */ +export function resolveWindowsExtensionChunkSize(env: NodeJS.ProcessEnv = process.env) { + return resolvePositiveEnvIntWithFallback( + env, + "OPENCLAW_OXLINT_WINDOWS_EXTENSION_CHUNK_SIZE", + DEFAULT_EXTENSION_CHUNK_SIZE, + ); +} + +/** + * Chooses serial shard execution for constrained hosts or Windows. + */ +export function shouldRunOxlintShardsSerial({ + env = process.env, + platform = process.platform, + hostResources, +}: ResourceOptions = {}) { + const explicitMode = env.OPENCLAW_OXLINT_SHARDS_SERIAL?.trim(); + if (explicitMode === "1") { + return true; + } + if (platform === "win32") { + return true; + } + if (explicitMode === "0") { + return false; + } + const localCheckMode = env.OPENCLAW_LOCAL_CHECK_MODE?.trim().toLowerCase(); + if (!isRemoteChangedGateEnv(env)) { + if (localCheckMode === "full" || localCheckMode === "fast") { + return false; + } + if (localCheckMode === "throttled" || localCheckMode === "low-memory") { + return true; + } + } + const resources = resolveHostResources(hostResources); + if (env.CI === "true" || env.GITHUB_ACTIONS === "true") { + return isConstrainedCiCheckHost(resources); + } + return ( + resources.totalMemoryBytes < FAST_LOCAL_CHECK_MIN_MEMORY_BYTES || + resources.logicalCpuCount < FAST_LOCAL_CHECK_MIN_CPUS + ); +} + +function isRemoteChangedGateEnv(env: NodeJS.ProcessEnv) { + return ( + env.OPENCLAW_CHECK_CHANGED_REMOTE_CHILD === "1" || env.OPENCLAW_CHANGED_LANES_RAW_SYNC === "1" + ); +} + +function readDirectoryEntries(readDir: ReadDirectoryEntries, target: string) { + try { + return readDir(target, { withFileTypes: true }); + } catch { + return []; + } +} + +function listExtensionEntries({ cwd, readDir }: DirectoryLookup) { + const entries = readDirectoryEntries(readDir, path.join(cwd, EXTENSIONS_DIR)); + + const dirs = entries + .filter((entry) => entry.isDirectory()) + .map((entry) => `${EXTENSIONS_DIR}/${entry.name}`) + .toSorted((left, right) => left.localeCompare(right)); + const rootFiles = entries + .filter((entry) => entry.isFile() && OXLINT_SOURCE_FILE_PATTERN.test(entry.name)) + .map((entry) => `${EXTENSIONS_DIR}/${entry.name}`) + .toSorted((left, right) => left.localeCompare(right)); + + return { + dirs, + rootFiles, + }; +} + +function listSourceRootTargetGroups({ cwd, readDir }: DirectoryLookup) { + const entries = readDirectoryEntries(readDir, path.join(cwd, "src")); + + const dirs = entries + .filter((entry) => entry.isDirectory()) + .map((entry) => `src/${entry.name}`) + .toSorted((left, right) => left.localeCompare(right)); + const rootFiles = entries + .filter((entry) => entry.isFile() && OXLINT_SOURCE_FILE_PATTERN.test(entry.name)) + .map((entry) => `src/${entry.name}`) + .toSorted((left, right) => left.localeCompare(right)); + + return [...dirs.map((target) => [target]), ...(rootFiles.length > 0 ? [rootFiles] : [])]; +} + +/** + * Runs selected oxlint shards and returns process-style success/failure. + */ +export async function main( + extraArgs: string[] = process.argv.slice(2), + runtimeEnv: NodeJS.ProcessEnv = process.env, +) { + const runner = path.resolve("scripts", "run-oxlint.mts"); + const shardArgs = parseShardRunnerArgs(extraArgs); + const env = resolveLocalCheckEnv(runtimeEnv); + const hostResources = resolveHostResources(); + const splitExtensions = shardArgs.extensionStripe !== undefined; + const shards = createOxlintShards({ + cwd: process.cwd(), + env, + platform: process.platform, + hostResources, + splitCore: shardArgs.splitCore, + splitExtensions, + }); + const selectedShards = selectExtensionOxlintStripe( + selectCoreOxlintStripe(filterOxlintShards(shards, shardArgs.only), shardArgs.coreStripe), + shardArgs.extensionStripe, + ); + + const needsArtifacts = shouldPrepareExtensionPackageBoundaryArtifactsForShards( + selectedShards, + shardArgs.oxlintArgs, + ); + const run = async () => { + if (needsArtifacts) { + const code = await runManagedCommand({ + bin: process.execPath, + args: distArtifactEntryArgs( + path.resolve("scripts/prepare-extension-package-boundary-artifacts.mts"), + ["--mode=package-boundary"], + ), + env, + requireProcessTreeExit: process.platform !== "win32", + }); + if (code !== 0) { + return code; + } + } + const shardConcurrency = resolveOxlintShardConcurrency({ + env, + platform: process.platform, + hostResources, + splitCore: shardArgs.splitCore, + splitExtensions, + }); + // stderr: stdout may carry machine-readable oxlint output for callers. + console.error( + `[oxlint] shard concurrency ${Math.max(1, Math.min(shardConcurrency, selectedShards.length))} ` + + `(cpus=${hostResources.logicalCpuCount}, memGB=${Math.round(hostResources.totalMemoryBytes / 1024 ** 3)})`, + ); + const results = await runShards({ + concurrency: Math.max(1, Math.min(shardConcurrency, selectedShards.length)), + entries: selectedShards, + env, + extraArgs: shardArgs.oxlintArgs, + runner, + }); + return results.find((status) => status !== 0) ?? 0; + }; + return needsArtifacts ? await withDistArtifactOwnership(process.cwd(), run) : await run(); +} + +if (import.meta.main) { + // Imported batches leave final reporting to their outer pipeline, after ownership settles. + await runWithFailedTrailer("oxlint", async () => { + process.exitCode = await main(); + }); +} + +function resolveHostResources(hostResources?: HostResources) { + if (hostResources) { + return hostResources; + } + + return { + totalMemoryBytes: os.totalmem(), + logicalCpuCount: + typeof os.availableParallelism === "function" ? os.availableParallelism() : os.cpus().length, + }; +} + +/** + * Parses shard-runner flags separately from forwarded oxlint args. + */ +export function parseShardRunnerArgs(args: string[]) { + const only = new Set<string>(); + const oxlintArgs: string[] = []; + let coreStripe: ShardStripe | undefined; + let extensionStripe: ShardStripe | undefined; + let splitCore = false; + + for (let index = 0; index < args.length; index += 1) { + const arg = args[index]; + if (arg === undefined) { + break; + } + if (arg === "--split-core") { + splitCore = true; + continue; + } + if (arg === "--core-stripe") { + coreStripe = parseShardStripe(args[index + 1], "--core-stripe"); + index += 1; + continue; + } + if (arg.startsWith("--core-stripe=")) { + coreStripe = parseShardStripe(arg.slice("--core-stripe=".length), "--core-stripe"); + continue; + } + if (arg === "--extension-stripe") { + extensionStripe = parseShardStripe(args[index + 1], "--extension-stripe"); + index += 1; + continue; + } + if (arg.startsWith("--extension-stripe=")) { + extensionStripe = parseShardStripe( + arg.slice("--extension-stripe=".length), + "--extension-stripe", + ); + continue; + } + if (arg === "--only") { + only.add(requireShardSelector(args[index + 1])); + index += 1; + continue; + } + if (arg.startsWith("--only=")) { + only.add(requireShardSelector(arg.slice("--only=".length))); + continue; + } + oxlintArgs.push(arg); + } + + if (coreStripe && !splitCore) { + throw new Error("--core-stripe requires --split-core"); + } + return { coreStripe, extensionStripe, only, oxlintArgs, splitCore }; +} + +function parseShardStripe(value: string | undefined, flag: string): ShardStripe { + const match = /^(\d+)\/(\d+)$/u.exec(value ?? ""); + const index = Number(match?.[1]); + const total = Number(match?.[2]); + if ( + !Number.isSafeInteger(index) || + !Number.isSafeInteger(total) || + index < 1 || + total < 1 || + index > total + ) { + throw new Error(`${flag} requires INDEX/TOTAL with 1 <= INDEX <= TOTAL; got: ${value}`); + } + return { index, total }; +} + +/** + * Filters shards by optional shard names and rejects unknown selectors. + */ +export function filterOxlintShards<T extends { name: string }>(shards: T[], only: Set<string>) { + if (only.size === 0) { + return shards; + } + + const selectors = [...only]; + const unknownSelectors = selectors.filter( + (selector) => !shards.some((shard) => matchesShardSelector(shard, selector)), + ); + if (unknownSelectors.length > 0) { + throw new Error( + `Unknown oxlint shard selector${unknownSelectors.length === 1 ? "" : "s"}: ${unknownSelectors.join(", ")}`, + ); + } + + return shards.filter((shard) => + selectors.some((selector) => matchesShardSelector(shard, selector)), + ); +} + +/** Aggregate one deterministic, disjoint stripe into a single core Program. */ +export function selectCoreOxlintStripe(shards: OxlintShard[], stripe: ShardStripe | undefined) { + if (!stripe) { + return shards; + } + if (shards.length === 0 || shards.some((shard) => !shard.name.startsWith("core:"))) { + throw new Error("--core-stripe requires a non-empty core-only shard selection"); + } + const targets = shards + .filter((_, index) => index % stripe.total === stripe.index - 1) + .flatMap((shard) => shard.args.slice(2)); + if (targets.length === 0) { + return []; + } + return [ + { + name: `core:stripe:${stripe.index}`, + args: ["--tsconfig", CORE_TS_CONFIG, ...targets], + }, + ]; +} + +/** Select one deterministic, disjoint stripe of independently bounded extension Programs. */ +export function selectExtensionOxlintStripe( + shards: OxlintShard[], + stripe: ShardStripe | undefined, +) { + if (!stripe) { + return shards; + } + if (shards.length === 0) { + return []; + } + if (shards.some((shard) => !shard.name.startsWith("extensions:"))) { + throw new Error("--extension-stripe requires an extension-only shard selection"); + } + return shards.filter((_, index) => index % stripe.total === stripe.index - 1); +} + +export function shouldPrepareExtensionPackageBoundaryArtifactsForShards( + shards: readonly OxlintShard[], + extraArgs: readonly string[] = [], +) { + return shards.some((shard) => + shouldPrepareExtensionPackageBoundaryArtifacts([...shard.args, ...extraArgs]), + ); +} + +function requireShardSelector(value: string | undefined) { + if (!value || value.startsWith("-")) { + throw new Error("--only requires a shard name"); + } + return value; +} + +function matchesShardSelector(shard: { name: string }, selector: string) { + return selector === shard.name || selector === shard.name.split(":")[0]; +} + +/** + * Resolves shard concurrency from env, platform, and host resources. + */ +export function resolveOxlintShardConcurrency({ + env = process.env, + platform = process.platform, + hostResources, + splitCore = false, + splitExtensions = false, +}: ResourceOptions & { splitCore?: boolean; splitExtensions?: boolean } = {}) { + if (splitExtensions || shouldRunOxlintShardsSerial({ env, platform, hostResources })) { + return 1; + } + + const explicitConcurrency = resolvePositiveEnvInt(env, "OPENCLAW_OXLINT_SHARD_CONCURRENCY"); + if (explicitConcurrency !== null) { + return explicitConcurrency; + } + + if (!splitCore) { + return Number.MAX_SAFE_INTEGER; + } + + const resources = resolveHostResources(hostResources); + return Math.max( + 1, + Math.min(DEFAULT_SPLIT_CORE_SHARD_CONCURRENCY, Math.floor(resources.logicalCpuCount / 4)), + ); +} + +async function runShards({ concurrency, entries, env, extraArgs, runner }: ShardBatchOptions) { + // Dependency-less worktrees establish their primary-checkout toolchain link + // before this lazy import, avoiding a top-level package-resolution failure. + const { default: pMap } = await import("p-map"); + const results = await pMap( + entries, + async (shard) => { + if (isParentTerminationRequested()) { + return undefined; + } + return await runShard({ env, extraArgs, runner, shard }); + }, + { concurrency, stopOnError: false }, + ); + return results.filter((status) => status !== undefined); +} + +/** + * Runs one oxlint shard with bounded output, heartbeat, and forced cleanup. + */ +export async function runShard({ env, extraArgs, runner, shard }: ShardRunnerOptions) { + console.error(`[oxlint:${shard.name}] starting`); + const startedAt = Date.now(); + const heartbeatMs = resolveShardHeartbeatMs(env); + const timeoutMs = resolveShardTimeoutMs(env); + const killGraceMs = resolveShardKillGraceMs(env); + // The batch owns reporting and artifacts. Raw children must propagate cleanup + // errors to the private artifact entry without catching or reporting them here. + const args = + runner === path.resolve("scripts", "run-oxlint.mts") + ? shouldPrepareExtensionPackageBoundaryArtifactsForShards([shard], extraArgs) + ? distArtifactEntryArgs(runner, [...shard.args, ...extraArgs]) + : [ + "--import", + new URL("./tsx.mjs", import.meta.url).href, + runner, + ...shard.args, + ...extraArgs, + ] + : [runner, ...shard.args, ...extraArgs]; + const child = spawn(process.execPath, args, { + stdio: ["inherit", "pipe", "pipe"], + detached: process.platform !== "win32", + env: { + ...env, + OPENCLAW_OXLINT_SKIP_PREPARE: "1", + }, + }); + child.stdout.pipe(process.stdout, { end: false }); + child.stderr.pipe(process.stderr, { end: false }); + const unregisterShardChild = registerShardChild({ child, killGraceMs }); + + return await new Promise<number>((resolve, reject) => { + let finished = false; + let timedOut = false; + let forceKill: ReturnType<typeof setTimeout> | null = null; + let forceKillAt: number | null = null; + const heartbeat = + heartbeatMs > 0 + ? setInterval(() => { + const elapsedSeconds = Math.round((Date.now() - startedAt) / 1000); + console.error(`[oxlint:${shard.name}] still running after ${elapsedSeconds}s`); + }, heartbeatMs) + : null; + heartbeat?.unref(); + const timeout = + timeoutMs > 0 + ? setTimeout(() => { + timedOut = true; + const elapsedSeconds = Math.round((Date.now() - startedAt) / 1000); + console.error( + `[oxlint:${shard.name}] timed out after ${elapsedSeconds}s; terminating shard`, + ); + signalChildProcess(child, "SIGTERM"); + if (killGraceMs > 0) { + forceKillAt = Date.now() + killGraceMs; + forceKill = setTimeout(() => { + console.error(`[oxlint:${shard.name}] did not exit cleanly; killing shard`); + signalChildProcess(child, "SIGKILL"); + }, killGraceMs); + forceKill.unref(); + } else { + signalChildProcess(child, "SIGKILL"); + } + }, timeoutMs) + : null; + timeout?.unref(); + const finish = (status: number, error?: Error) => { + if (finished) { + return; + } + finished = true; + if (heartbeat) { + clearInterval(heartbeat); + } + if (timeout) { + clearTimeout(timeout); + } + if (forceKill) { + clearTimeout(forceKill); + } + forceKillAt = null; + unregisterShardChild(); + console.error( + `[oxlint:${shard.name}] ${status === 0 ? "passed" : `failed (exit ${status})`}`, + ); + if (error) { + reject(error); + } else { + resolve(status); + } + }; + const finishAfterForcedTeardown = async (status: number) => { + const graceRemainingMs = + forceKillAt === null ? killGraceMs : Math.max(0, forceKillAt - Date.now()); + if (graceRemainingMs > 0) { + await waitForChildProcessGroupExit(child, graceRemainingMs); + } + const requiresForceKill = isChildProcessGroupAlive(child); + if (requiresForceKill) { + signalChildProcess(child, "SIGKILL"); + } + await waitForChildProcessGroupExit(child, POST_FORCE_KILL_WAIT_MS); + if (isChildProcessGroupAlive(child)) { + finish( + 1, + Object.assign(new Error("oxlint shard process group did not exit"), { + code: "EPROCESSGROUP_CLEANUP_FAILED", + processTreeState: "live", + }), + ); + } else { + finish(requiresForceKill ? status || 1 : status); + } + }; + child.once("error", (error) => { + console.error(error); + finish(1); + }); + child.once("close", (status) => { + const exitStatus = parentTerminationSignal + ? getSignalExitCode(parentTerminationSignal) + : timedOut + ? 124 + : (status ?? 1); + if (isChildProcessGroupAlive(child)) { + void finishAfterForcedTeardown(exitStatus); + return; + } + finish(exitStatus); + }); + }); +} + +/** + * Reads the shard heartbeat interval. + */ +export function resolveShardHeartbeatMs(env: NodeJS.ProcessEnv) { + return resolveNonNegativeEnvInt( + env, + "OPENCLAW_OXLINT_SHARD_HEARTBEAT_MS", + DEFAULT_SHARD_HEARTBEAT_MS, + ); +} + +/** + * Reads the per-shard timeout. + */ +export function resolveShardTimeoutMs(env: NodeJS.ProcessEnv) { + return resolveNonNegativeEnvInt( + env, + "OPENCLAW_OXLINT_SHARD_TIMEOUT_MS", + DEFAULT_SHARD_TIMEOUT_MS, + ); +} + +/** + * Reads the graceful shutdown window before SIGKILL. + */ +export function resolveShardKillGraceMs(env: NodeJS.ProcessEnv) { + return resolveNonNegativeEnvInt( + env, + "OPENCLAW_OXLINT_SHARD_KILL_GRACE_MS", + DEFAULT_SHARD_KILL_GRACE_MS, + ); +} + +function resolveNonNegativeEnvInt(env: NodeJS.ProcessEnv, key: string, defaultValue: number) { + const rawValue = env[key]; + if (rawValue === undefined || rawValue === "") { + return defaultValue; + } + + const text = rawValue.trim(); + if (!/^\d+$/u.test(text)) { + throw new Error(`${key} must be a non-negative integer; got: ${rawValue}`); + } + const parsedValue = Number(text); + if (!Number.isSafeInteger(parsedValue)) { + throw new Error(`${key} must be a non-negative integer; got: ${rawValue}`); + } + return parsedValue; +} + +function resolvePositiveEnvInt(env: NodeJS.ProcessEnv, key: string) { + const rawValue = env[key]; + if (rawValue === undefined || rawValue === "") { + return null; + } + + return parsePositiveEnvInt(rawValue, key); +} + +function resolvePositiveEnvIntWithFallback( + env: NodeJS.ProcessEnv, + key: string, + defaultValue: number, +) { + const rawValue = env[key]; + if (rawValue === undefined || rawValue === "") { + return defaultValue; + } + + return parsePositiveEnvInt(rawValue, key); +} + +function parsePositiveEnvInt(rawValue: string, key: string) { + const text = rawValue.trim(); + if (!/^\d+$/u.test(text)) { + throw new Error(`${key} must be a positive integer; got: ${rawValue}`); + } + const parsedValue = Number(text); + if (!Number.isSafeInteger(parsedValue) || parsedValue <= 0) { + throw new Error(`${key} must be a positive integer; got: ${rawValue}`); + } + return parsedValue; +} + +function signalChildProcess(child: ChildProcess, signal: NodeJS.Signals) { + if (!child.pid) { + return; + } + + const reportSignalError = (error: unknown) => { + if (!isNodeErrorCode(error, "ESRCH")) { + console.error(error); + } + }; + terminateManagedChild(child, signal, { + onChildSignalError: reportSignalError, + onProcessGroupSignalError: reportSignalError, + processGroupFallback: "never", + useWindowsTaskkill: false, + }); +} + +function isChildProcessGroupAlive(child: ChildProcess) { + return inspectManagedProcessGroup(child, { errorPolicy: "alive-on-eperm" }) === "live"; +} + +function waitForChildProcessGroupExit(child: ChildProcess, timeoutMs: number) { + return waitForManagedProcessGroupExit(child, timeoutMs, { errorPolicy: "alive-on-eperm" }); +} + +function registerShardChild(entry: ActiveShardChild) { + installParentSignalForwarding(); + ACTIVE_SHARD_CHILDREN.add(entry); + return () => { + ACTIVE_SHARD_CHILDREN.delete(entry); + if (ACTIVE_SHARD_CHILDREN.size === 0 && parentTerminationForceKill) { + clearTimeout(parentTerminationForceKill); + parentTerminationForceKill = null; + } + if (ACTIVE_SHARD_CHILDREN.size === 0) { + for (const [signal, handler] of parentSignalHandlers) { + process.off(signal, handler); + } + parentSignalHandlers.clear(); + process.off("exit", onParentExit); + } + }; +} + +function installParentSignalForwarding() { + if (parentSignalHandlers.size > 0) { + return; + } + for (const signal of PARENT_TERMINATION_SIGNALS) { + const handler = () => { + parentTerminationSignal = signal; + process.exitCode = getSignalExitCode(signal); + signalActiveShardChildren(signal); + scheduleParentTerminationForceKill(); + }; + parentSignalHandlers.set(signal, handler); + process.on(signal, handler); + } + process.once("exit", onParentExit); +} + +function onParentExit() { + signalActiveShardChildren("SIGTERM"); +} + +function isParentTerminationRequested() { + return parentTerminationSignal !== null; +} + +function signalActiveShardChildren(signal: NodeJS.Signals) { + for (const entry of ACTIVE_SHARD_CHILDREN) { + signalChildProcess(entry.child, signal); + } +} + +function scheduleParentTerminationForceKill() { + if (parentTerminationForceKill) { + return; + } + const killGraceMs = Math.max( + 0, + ...Array.from(ACTIVE_SHARD_CHILDREN, (entry) => entry.killGraceMs), + ); + if (killGraceMs === 0) { + signalActiveShardChildren("SIGKILL"); + return; + } + parentTerminationForceKill = setTimeout(() => { + parentTerminationForceKill = null; + signalActiveShardChildren("SIGKILL"); + }, killGraceMs); + parentTerminationForceKill.unref(); +} + +function getSignalExitCode(signal: NodeJS.Signals) { + return signal === "SIGINT" ? 130 : 143; +} + +function isNodeErrorCode(error: unknown, code: string) { + return Boolean(error && typeof error === "object" && "code" in error && error.code === code); +} diff --git a/scripts/run-oxlint.mts b/scripts/run-oxlint.mts new file mode 100644 index 0000000000000000000000000000000000000000..9897971b9bfd01f853aafa53d50605e70b43b572 --- /dev/null +++ b/scripts/run-oxlint.mts @@ -0,0 +1,307 @@ +// Runs oxlint with local resource policy, sparse-checkout filtering, and +// plugin package-boundary artifact preparation when needed. +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { isDirectRunUrl } from "./lib/direct-run.mjs"; +import { + distArtifactEntryArgs, + withDistArtifactOwnership, +} from "./lib/dist-artifact-ownership.mts"; +import { + applyLocalOxlintPolicy, + resolveLocalCheckEnv, + resolveRepoToolBinPath, +} from "./lib/local-check-runtime.mts"; +import { createManagedCommandInvocation, runManagedCommand } from "./lib/managed-child-process.mts"; +import { resolvePathEnvKey } from "./windows-cmd-helpers.mjs"; + +const PREPARE_EXTENSION_BOUNDARY_ARGS = distArtifactEntryArgs( + path.resolve("scripts", "prepare-extension-package-boundary-artifacts.mts"), + ["--mode=package-boundary"], +); +const OXLINT_PREPARE_SKIP_FLAGS = new Set([ + "--help", + "-h", + "--version", + "-V", + "--print-config", + "--rules", + "--init", + "--lsp", +]); +const OXLINT_VALUE_FLAGS = new Set([ + "--config", + "--deny", + "--env", + "--format", + "--globals", + "--ignore-path", + "--max-warnings", + "--output-file", + "--plugin", + "--rules", + "--tsconfig", + "--warn", +]); +const OXLINT_BOUNDARY_FREE_TS_CONFIGS = new Set([ + "config/tsconfig/oxlint.core.json", + "config/tsconfig/oxlint.scripts.json", + "test/tsconfig/tsconfig.test.root.json", +]); +const OPENCLAW_FOCUSED_CONFIG_FLAG = "--openclaw-focused-config"; + +/** + * Returns whether oxlint args need package-boundary declaration artifacts first. + */ +export function shouldPrepareExtensionPackageBoundaryArtifacts(args: string[]) { + if (args.some((arg) => OXLINT_PREPARE_SKIP_FLAGS.has(arg))) { + return false; + } + + const tsconfigs = args.flatMap((arg, index) => { + if (arg === "--tsconfig") { + const value = args[index + 1]; + return value === undefined ? [] : [value]; + } + return arg.startsWith("--tsconfig=") ? [arg.slice("--tsconfig=".length)] : []; + }); + // Core, script, and root-test lint resolve sources through the root tsconfig; + // generated plugin package declarations are only an extension-lint input. + return ( + tsconfigs.length === 0 || + tsconfigs.some((tsconfig) => !OXLINT_BOUNDARY_FREE_TS_CONFIGS.has(tsconfig)) + ); +} + +/** + * Drops tracked-but-missing sparse-checkout targets so narrow sparse checks can pass. + */ +export function filterSparseMissingOxlintTargets( + args: string[], + { + cwd = process.cwd(), + fileExists = fs.existsSync, + isSparseCheckoutEnabled = getSparseCheckoutEnabled, + isTrackedPath = hasTrackedPath, + }: Partial<{ + cwd?: string; + fileExists?: (target: string) => boolean; + isSparseCheckoutEnabled?: (params: { cwd: string }) => boolean; + isTrackedPath?: (params: { cwd: string; target: string }) => boolean; + }> = {}, +) { + if (!isSparseCheckoutEnabled({ cwd })) { + return { + args, + hadExplicitTargets: false, + remainingExplicitTargets: 0, + skippedTargets: [], + skippedConfigs: [], + }; + } + + const filteredArgs = []; + const skippedTargets = []; + const skippedConfigs = []; + let hadExplicitTargets = false; + let remainingExplicitTargets = 0; + let consumeNextValue = false; + + for (let index = 0; index < args.length; index += 1) { + const arg = args[index]; + if (arg === undefined) { + continue; + } + if (consumeNextValue) { + filteredArgs.push(arg); + consumeNextValue = false; + continue; + } + + if (arg === "--") { + filteredArgs.push(arg); + continue; + } + + if (arg.startsWith("--")) { + if (arg === "--tsconfig") { + const value = args[index + 1]; + if (value !== undefined) { + index += 1; + if (!fileExists(path.resolve(cwd, value)) && isTrackedPath({ cwd, target: value })) { + skippedConfigs.push(value); + continue; + } + filteredArgs.push(arg, value); + continue; + } + } + if (arg.startsWith("--tsconfig=")) { + const value = arg.slice("--tsconfig=".length); + if ( + value && + !fileExists(path.resolve(cwd, value)) && + isTrackedPath({ cwd, target: value }) + ) { + skippedConfigs.push(value); + continue; + } + } + filteredArgs.push(arg); + if (!arg.includes("=") && OXLINT_VALUE_FLAGS.has(arg)) { + consumeNextValue = true; + } + continue; + } + + if (arg.startsWith("-")) { + filteredArgs.push(arg); + continue; + } + + hadExplicitTargets = true; + const absoluteTarget = path.resolve(cwd, arg); + if (!fileExists(absoluteTarget) && isTrackedPath({ cwd, target: arg })) { + skippedTargets.push(arg); + continue; + } + + remainingExplicitTargets += 1; + filteredArgs.push(arg); + } + + return { + args: filteredArgs, + hadExplicitTargets, + remainingExplicitTargets, + skippedTargets, + skippedConfigs, + }; +} + +function getSparseCheckoutEnabled({ cwd }: { cwd: string }) { + const git = createManagedCommandInvocation({ + args: ["config", "--get", "--bool", "core.sparseCheckout"], + bin: "git", + }); + const result = spawnSync(git.command, git.args, { + cwd, + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + shell: git.shell, + windowsVerbatimArguments: git.windowsVerbatimArguments, + }); + + return result.status === 0 && result.stdout.trim() === "true"; +} + +function hasTrackedPath({ cwd, target }: { cwd: string; target: string }) { + const git = createManagedCommandInvocation({ + args: ["ls-files", "--", target], + bin: "git", + }); + const result = spawnSync(git.command, git.args, { + cwd, + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + shell: git.shell, + windowsVerbatimArguments: git.windowsVerbatimArguments, + }); + + return result.status === 0 && result.stdout.trim().length > 0; +} + +function resolveOxlintToolchainEnv( + oxlintPath: string, + env: NodeJS.ProcessEnv, + platform: NodeJS.Platform = process.platform, +) { + const pathKey = platform === "win32" ? resolvePathEnvKey(env) : "PATH"; + const delimiter = platform === "win32" ? ";" : path.delimiter; + const currentPath = env[pathKey]?.trim(); + return { + ...env, + // Type-aware oxlint resolves its optional tsgolint peer through PATH, so + // keep the selected checkout's toolchain together in dependency-less worktrees. + [pathKey]: [path.dirname(oxlintPath), currentPath].filter(Boolean).join(delimiter), + }; +} + +async function prepareExtensionPackageBoundaryArtifacts(env: NodeJS.ProcessEnv) { + const status = await runManagedCommand({ + bin: process.execPath, + args: PREPARE_EXTENSION_BOUNDARY_ARGS, + env, + requireProcessTreeExit: process.platform !== "win32", + }); + + if (status !== 0) { + throw new Error(`prepare-extension-package-boundary-artifacts failed with exit code ${status}`); + } +} + +/** + * Applies wrapper policy and runs oxlint with the final argument list. + */ +async function runOxlint( + argv: string[] = process.argv.slice(2), + runtimeEnv: NodeJS.ProcessEnv = process.env, +): Promise<number> { + const focusedConfig = argv.includes(OPENCLAW_FOCUSED_CONFIG_FLAG); + const oxlintArgs = argv.filter((arg) => arg !== OPENCLAW_FOCUSED_CONFIG_FLAG); + const localEnv = resolveLocalCheckEnv(runtimeEnv); + // Focused configs are syntax-only guards; keep wrapper process handling + // without the broad type-aware policy or package artifact preparation. + const { args: policyArgs, env } = focusedConfig + ? { args: oxlintArgs, env: localEnv } + : applyLocalOxlintPolicy(oxlintArgs, localEnv, { + logicalCpuCount: os.availableParallelism(), + totalMemoryBytes: os.totalmem(), + }); + const sparseTargets = filterSparseMissingOxlintTargets(policyArgs); + const finalArgs = sparseTargets.args; + const oxlintPath = resolveRepoToolBinPath("oxlint"); + const needsArtifactPreparation = + !focusedConfig && + env.OPENCLAW_OXLINT_SKIP_PREPARE !== "1" && + shouldPrepareExtensionPackageBoundaryArtifacts(finalArgs); + if (sparseTargets.skippedTargets.length > 0) { + console.error( + `[oxlint] sparse checkout is missing tracked target(s); skipping ${sparseTargets.skippedTargets.join(", ")}`, + ); + } + if (sparseTargets.skippedConfigs.length > 0) { + console.error( + `[oxlint] sparse checkout is missing tracked config(s); skipping oxlint: ${sparseTargets.skippedConfigs.join(", ")}`, + ); + return 0; + } + if (sparseTargets.hadExplicitTargets && sparseTargets.remainingExplicitTargets === 0) { + console.error("[oxlint] no present sparse-checkout targets remain; skipping oxlint."); + return 0; + } + + if (needsArtifactPreparation) { + // Declaration compilation owns its Go policy; lint limits belong to the oxlint child. + await prepareExtensionPackageBoundaryArtifacts(localEnv); + } + return await runManagedCommand({ + bin: oxlintPath, + args: finalArgs, + env: resolveOxlintToolchainEnv(oxlintPath, env), + requireProcessTreeExit: process.platform !== "win32", + }); +} + +if (isDirectRunUrl(process.argv[1], import.meta.url)) { + const argv = process.argv.slice(2); + // Skip-prepare callers still consume shared declarations. Source-only lint + // remains independent; sharded lint inherits its parent's owner. + process.exitCode = + !argv.includes(OPENCLAW_FOCUSED_CONFIG_FLAG) && + shouldPrepareExtensionPackageBoundaryArtifacts(argv) + ? await withDistArtifactOwnership(process.cwd(), () => runOxlint(argv)) + : await runOxlint(argv); +} diff --git a/scripts/run-tsgo-core-test-shards.mjs b/scripts/run-tsgo-core-test-shards.mjs new file mode 100644 index 0000000000000000000000000000000000000000..a98c7bd7d183e1abc5e58e2ee1003cf9103e0cd1 --- /dev/null +++ b/scripts/run-tsgo-core-test-shards.mjs @@ -0,0 +1,8 @@ +#!/usr/bin/env node + +import { runTsxCliShim } from "./lib/tsx-cli-shim.mjs"; + +await runTsxCliShim(import.meta.url, { + implementation: "./run-tsgo-core-test-shards.mts", + failureTool: "tsgo:core:test", +}); diff --git a/scripts/run-tsgo.mts b/scripts/run-tsgo.mts new file mode 100644 index 0000000000000000000000000000000000000000..23168107a7831061e9f0a4b4eb69a7b482b9ffae --- /dev/null +++ b/scripts/run-tsgo.mts @@ -0,0 +1,124 @@ +// Runs tsgo through local resource policy and sparse-checkout guards. +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { readFlagValue } from "./lib/arg-utils.mts"; +import { isDirectRunUrl } from "./lib/direct-run.mjs"; +import { + applyLocalTsgoPolicy, + resolveLocalCheckEnv, + resolveRepoToolBinPath, +} from "./lib/local-check-runtime.mts"; +import { runManagedCommand } from "./lib/managed-child-process.mts"; +import { readPositiveEnvInt } from "./lib/numeric-options.mjs"; +import { findRepoRoot } from "./lib/repo-root.mjs"; +import { + getSparseTsgoGuardError, + shouldSkipSparseTsgoGuardError, +} from "./lib/tsgo-sparse-guard.mts"; + +// Declared locally, as sibling scripts do, rather than imported from packages/: +// a static import there resolves before the sparse-checkout guard can report a +// missing project, turning a clean skip into ERR_MODULE_NOT_FOUND. Mirrors +// normalization-core's MAX_TIMER_TIMEOUT_MS. +const MAX_TIMER_TIMEOUT_MS = 2_147_000_000; + +export function resolveTsgoTimeoutMs(env: NodeJS.ProcessEnv): number | undefined { + if (!env.OPENCLAW_TSGO_TIMEOUT_MS?.trim()) { + return undefined; + } + return Math.min( + readPositiveEnvInt("OPENCLAW_TSGO_TIMEOUT_MS", env, MAX_TIMER_TIMEOUT_MS), + MAX_TIMER_TIMEOUT_MS, + ); +} + +/** Prepare one compiler invocation; the caller owns its process group and deadline. */ +export function prepareTsgoCommand( + args: string[], + baseEnv: NodeJS.ProcessEnv = process.env, + cwd = process.cwd(), +) { + const hostResources = { + logicalCpuCount: + typeof os.availableParallelism === "function" ? os.availableParallelism() : os.cpus().length, + totalMemoryBytes: os.totalmem(), + }; + const { args: finalArgs, env } = applyLocalTsgoPolicy( + args, + resolveLocalCheckEnv(baseEnv), + hostResources, + ); + + const sparseGuardError = getSparseTsgoGuardError(finalArgs, { cwd }); + if (sparseGuardError) { + if (shouldSkipSparseTsgoGuardError(env)) { + console.error(sparseGuardError); + console.error("[tsgo] skipping sparse-missing project because OPENCLAW_TSGO_SPARSE_SKIP=1"); + return null; + } + throw new Error(sparseGuardError); + } + + // Subdirectories share checkout ownership, but another checkout's install never does. + const tsgoPath = resolveRepoToolBinPath("tsgo", { cwd: findRepoRoot(cwd) ?? cwd }); + let timeoutMs: number | undefined; + try { + timeoutMs = resolveTsgoTimeoutMs(env); + } catch { + throw new Error( + `[tsgo] OPENCLAW_TSGO_TIMEOUT_MS must be plain decimal digits with no leading zero, sign, exponent, or decimal point, between 1 and ${Number.MAX_SAFE_INTEGER}; got ${env.OPENCLAW_TSGO_TIMEOUT_MS}. Unset it to disable the watchdog.`, + ); + } + return { + args: finalArgs, + bin: tsgoPath, + cwd, + env, + shell: process.platform === "win32", + timeoutMs, + }; +} + +async function main(): Promise<void> { + let command: ReturnType<typeof prepareTsgoCommand>; + try { + command = prepareTsgoCommand(process.argv.slice(2)); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + return; + } + if (!command) { + return; + } + try { + // Preflight must refuse or skip before installed bootstrap dependencies load. + // Output mutation and the compiler still remain inside their checkout owner. + const { withDistArtifactOwnership } = await import("./lib/dist-artifact-ownership.mts"); + await withDistArtifactOwnership(command.cwd, async () => { + const tsBuildInfoFile = readFlagValue(command.args, "--tsBuildInfoFile"); + if (tsBuildInfoFile) { + fs.mkdirSync(path.dirname(path.resolve(command.cwd, tsBuildInfoFile)), { recursive: true }); + } + // Managed cleanup forwards SIGTERM before bounded SIGKILL escalation, then + // joins the compiler group and output before reporting a timeout. + process.exitCode = await runManagedCommand({ + ...command, + requireProcessTreeExit: process.platform !== "win32", + }); + }); + } catch (error) { + if ((error as { code?: string } | undefined)?.code !== "ETIMEDOUT") { + throw error; + } + console.error( + `[tsgo] no completion after ${command.timeoutMs}ms; killed the tsgo process tree. Raise OPENCLAW_TSGO_TIMEOUT_MS for intentionally longer builds, or unset it to disable the watchdog.`, + ); + process.exitCode = 1; + } +} + +if (isDirectRunUrl(process.argv[1], import.meta.url)) { + await main(); +} diff --git a/scripts/run-vitest.mts b/scripts/run-vitest.mts new file mode 100644 index 0000000000000000000000000000000000000000..ead43b4b2a4dac87f36d7891559a88157db19f2e --- /dev/null +++ b/scripts/run-vitest.mts @@ -0,0 +1,1055 @@ +// Runs Vitest through repo project selection, local scheduling policy, output +// watchdogs, and process-group cleanup. +import fs from "node:fs"; +import { constants as osConstants } from "node:os"; +import path from "node:path"; +import { assertTestHomeSelection } from "../test/test-home-policy.mts"; +import { agentVitestProjectOwners } from "../test/vitest/vitest.agents-paths.mjs"; +import { toolingIsolatedTestFiles } from "../test/vitest/vitest.tooling-isolated-paths.mjs"; +import { + isPluginControlUiPath, + isUiBrowserTestFile, + isUiTestTarget, + uiTimingTestFiles, +} from "../test/vitest/vitest.ui-paths.mjs"; +import { boundaryTestFiles } from "../test/vitest/vitest.unit-paths.mjs"; +import { parsePermissiveBooleanToken } from "./lib/arg-utils.mts"; +import { resolveExtensionTestConfig } from "./lib/extension-test-plan.mts"; +import { createGatewayServerTestTargetChunks } from "./lib/gateway-server-test-plan.mts"; +import { resolveRepoRoot } from "./lib/repo-root.mjs"; +import { + prepareE2eVitestRuntime, + resolveVitestCliEntry, + prepareVitestRuntime, +} from "./lib/vitest-build-prerequisites.mts"; +import { + hasNonRunVitestSubcommand, + collectVitestFileFilters, + resolveBooleanModeFlag, + resolveExplicitVitestMode, + vitestOptionConsumesNextArg, +} from "./lib/vitest-cli-mode.mts"; +import { parseVitestExecutionArgs } from "./lib/vitest-cli.mts"; +import { resolveVitestHomeSelection } from "./lib/vitest-home-selection.mts"; +import { + resolveVitestProcessEnv, + resolveVitestNodeArgs, + resolveRunVitestSpawnEnv, + resolveVitestNoOutputTimeoutMs, + resolveVitestNoOutputHeartbeatMs, + resolveVitestCompileCacheSafeEnv, + resolveVitestConfigArg, + normalizeVitestConfigPath, + matchesVitestConfigPath, +} from "./lib/vitest-process-env.mts"; +import { + spawnOwnedVitestProcess, + runVitestCli, + type exitVitestBySignal, +} from "./lib/vitest-process.mts"; +import { resolveVitestRuntimeCliSelections } from "./lib/vitest-runtime-selection.mts"; +import { + createVitestUnhandledErrorDetector, + stripVitestAnsi, + writeVitestUnhandledErrorSummary, +} from "./lib/vitest-unhandled-errors.mts"; +import { createVitestWorkerRun, type VitestWorkerRun } from "./lib/vitest-worker-run.mts"; +import { createPnpmRunnerSpawnSpec, type PnpmRunnerParams } from "./pnpm-runner.mts"; +import { + forwardSignalToVitestProcessGroup, + installVitestProcessGroupCleanup, + shouldUseDetachedVitestProcessGroup, + terminateVitestProcessGroupForTimeout, +} from "./vitest-process-group.mts"; + +type VitestPathFs = Pick<typeof fs, "existsSync" | "statSync">; +type WatchdogStream = { + on(event: string, listener: (...args: unknown[]) => void): unknown; + off(event: string, listener: (...args: unknown[]) => void): unknown; +}; +type NodeSignal = keyof typeof osConstants.signals; +type VitestOutputStream = { + setEncoding(encoding: "utf8"): unknown; + on(event: "data", listener: (chunk: string) => void): unknown; + on(event: "end", listener: () => void): unknown; +}; +type VitestOutputTarget = { + write(chunk: string): unknown; +}; + +const SUPPRESSED_VITEST_STDERR_PATTERNS = ["[PLUGIN_TIMINGS]"]; +const UI_VITEST_CONFIG = "test/vitest/vitest.ui.config.ts"; +const UI_BROWSER_VITEST_CONFIG = "test/vitest/vitest.ui-browser.config.ts"; +const TOOLING_DOCKER_VITEST_CONFIG = "test/vitest/vitest.tooling-docker.config.ts"; +const TOOLING_VITEST_CONFIG = "test/vitest/vitest.tooling.config.ts"; +const GATEWAY_SERVER_VITEST_CONFIG = "test/vitest/vitest.gateway-server.config.ts"; +const E2E_VITEST_CONFIG = "test/vitest/vitest.e2e.config.ts"; +const E2E_TEST_PROCESS_COUNT = 4; +export const TOOLING_EXCLUDED_TESTS = new Set([ + ...boundaryTestFiles, + "test/scripts/docker-build-helper.test.ts", + ...toolingIsolatedTestFiles, +]); +const EXPLICIT_FILE_TARGET_RE = /\.(?:[cm]?[jt]sx?)$/u; +const EXPLICIT_TEST_FILE_RE = /\.(?:test|e2e|live)\.(?:[cm]?[jt]sx?)$/u; +const GLOB_PATTERN_CHARS_RE = /[*?[\]{}]/u; +const UNBOUNDED_CONFIG_ONLY_OPTIONS = [ + "--changed", + "--coverage", + "--dir", + "--mergeReports", + "--outputFile", + "--project", + "--root", + "--shard", +]; + +function isErrorWithCode(error: unknown, code: string): error is NodeJS.ErrnoException { + return error instanceof Error && "code" in error && error.code === code; +} + +function isNodeSignal(signal: string): signal is NodeSignal { + return Object.hasOwn(osConstants.signals, signal); +} + +function normalizeNodeSignal(signal: string | null): NodeSignal | null { + if (!signal) { + return null; + } + const unknownSignalMessage = `child process exited with unknown signal: ${signal}`; + if (!isNodeSignal(signal)) { + throw new Error(unknownSignalMessage); + } + return signal; +} + +function hasVitestOption(argv: string[], option: string): boolean { + for (const arg of argv) { + if (arg === "--") { + return false; + } + if (arg === option || arg.startsWith(option + "=") || arg.startsWith(option + ".")) { + return true; + } + } + return false; +} + +function insertVitestTargets(argv: string[], targets: string[]): string[] { + const separatorIndex = argv.indexOf("--"); + const insertionIndex = separatorIndex < 0 ? argv.length : separatorIndex; + return [...argv.slice(0, insertionIndex), ...targets, ...argv.slice(insertionIndex)]; +} + +/** + * Bounds shared-worker lifetime with fresh processes so broad non-isolated + * runs do not exhaust the worker heap. + */ +export function resolveBoundedVitestInvocations( + argv: string[], + options: { + cwd?: string; + env?: NodeJS.ProcessEnv; + gatewayServerTargetChunks?: string[][]; + } = {}, +): string[][] { + const config = resolveVitestConfigArg(argv); + const normalizedConfig = config === null ? "" : normalizeVitestConfigPath(config); + const env = options.env ?? process.env; + const cwd = options.cwd ?? process.cwd(); + const mode = resolveExplicitVitestMode(argv); + const isE2E = matchesVitestConfigPath(normalizedConfig, E2E_VITEST_CONFIG); + if ( + (!isE2E && !matchesVitestConfigPath(normalizedConfig, GATEWAY_SERVER_VITEST_CONFIG)) || + mode === "watch" || + hasExplicitDisabledRunFlag(argv) || + (mode !== "run" && parsePermissiveBooleanToken(env.CI) !== true) || + hasNonRunVitestSubcommand(argv) || + hasAlternateVitestRootArg(argv) || + collectExplicitProjectRouterTargetArgs(argv, cwd).length > 0 || + [...UNBOUNDED_CONFIG_ONLY_OPTIONS, "--bail"].some((option) => hasVitestOption(argv, option)) + ) { + return [argv]; + } + if (isE2E) { + // Let Vitest own include/exclude and shard membership. Filtered and report-producing + // calls retain one process so small selections and aggregate outputs keep their semantics. + if ( + collectExplicitFileTargetArgs(argv, (arg) => arg !== "run").length > 0 || + argv.includes("--") || + [ + "--exclude", + "--testNamePattern", + "-t", + "--tagsFilter", + "--sequence.sequencer", + "--reporter", + "--reporters", + "--listTags", + "--clearCache", + "--standalone", + "--help", + "-h", + "--version", + "-v", + ].some((option) => hasVitestOption(argv, option)) + ) { + return [argv]; + } + return Array.from({ length: E2E_TEST_PROCESS_COUNT }, (_, index) => [ + ...argv, + `--shard=${index + 1}/${E2E_TEST_PROCESS_COUNT}`, + ]); + } + const chunks = options.gatewayServerTargetChunks ?? createGatewayServerTestTargetChunks(cwd); + return chunks.length > 1 ? chunks.map((targets) => insertVitestTargets(argv, targets)) : [argv]; +} + +/** + * Builds spawn options for the primary Vitest child process. + */ +export function resolveVitestSpawnParams( + env: NodeJS.ProcessEnv = process.env, + platform: NodeJS.Platform = process.platform, +): PnpmRunnerParams { + return { + env: resolveVitestProcessEnv(resolveVitestCompileCacheSafeEnv(env)), + detached: shouldUseDetachedVitestProcessGroup(platform), + stdio: ["inherit", "pipe", "pipe"], + }; +} + +/** + * Filters known noisy Vitest stderr lines after stripping ANSI escapes. + */ +export function shouldSuppressVitestStderrLine(line: string): boolean { + const normalizedLine = stripVitestAnsi(line); + return SUPPRESSED_VITEST_STDERR_PATTERNS.some((pattern) => normalizedLine.includes(pattern)); +} + +/** + * Detects pnpm exec node invocations so the wrapper can spawn Node directly. + */ +export function resolveDirectNodeVitestArgs(pnpmArgs: string[]): string[] | null { + return pnpmArgs[0] === "exec" && pnpmArgs[1] === "node" ? pnpmArgs.slice(2) : null; +} + +function hasExplicitVitestConfigArg(argv: string[]): boolean { + // Delegation consumes separators; keep config-bearing argv direct so its tail + // cannot become an active override in the project runner. + return argv.some( + (arg) => + arg === "--config" || arg === "-c" || arg.startsWith("--config=") || arg.startsWith("-c="), + ); +} + +function isPathLikeExplicitFileArg(arg: string): boolean { + return ( + path.isAbsolute(arg) || arg.startsWith("./") || arg.startsWith("../") || /[/\\]/u.test(arg) + ); +} + +function isExplicitFileTargetArg(arg: string): boolean { + if (!EXPLICIT_FILE_TARGET_RE.test(arg) || GLOB_PATTERN_CHARS_RE.test(arg)) { + return false; + } + return isPathLikeExplicitFileArg(arg); +} + +function isExplicitTestFileArg(arg: string): boolean { + return EXPLICIT_TEST_FILE_RE.test(arg) && isExplicitFileTargetArg(arg); +} + +function isDelegableBroadProjectRouterTarget(arg: string, cwd: string): boolean { + const relative = toRepoRelativeArg(arg, cwd).replace(/\/+$/u, ""); + return ( + relative === "test/scripts" || + relative === "test/scripts/*.test.ts" || + relative === "test/scripts/**/*.test.ts" + ); +} + +function isPathAtOrUnder(value: string, root: string): boolean { + return value === root || value.startsWith(`${root}/`); +} + +function isOwnedAgentDirectoryTarget(arg: string, cwd: string, fsImpl: VitestPathFs): boolean { + const relative = toRepoRelativeArg(arg, cwd).replace(/\/+$/u, ""); + return ( + isPathAtOrUnder(relative, agentVitestProjectOwners.all.root) && + isExplicitDirectoryTargetArg(arg, cwd, fsImpl) + ); +} + +function isOwnedExtensionRootTarget(arg: string, cwd: string, fsImpl: VitestPathFs): boolean { + const relative = toRepoRelativeArg(arg, cwd).replace(/\/+$/u, ""); + const [root, extensionId, ...remainder] = relative.split("/"); + if ( + root !== "extensions" || + !extensionId || + remainder.length > 0 || + !isExplicitDirectoryTargetArg(arg, cwd, fsImpl) + ) { + return false; + } + // Extension roots delegate so the bounded planner owns process lifetime (#124413). + // Raw Vitest would run the workspace config as one process. + return resolveExtensionTestConfig(relative).length > 0; +} + +function isExplicitProjectRouterTargetArg( + arg: string, + cwd = process.cwd(), + fsImpl: VitestPathFs = fs, +): boolean { + if (!isPathLikeExplicitFileArg(arg)) { + return false; + } + if (GLOB_PATTERN_CHARS_RE.test(arg)) { + return isDelegableBroadProjectRouterTarget(arg, cwd); + } + if (isExplicitFileTargetArg(arg)) { + return true; + } + const filePath = path.isAbsolute(arg) ? arg : path.resolve(cwd, arg); + return fsImpl.existsSync(filePath) + ? isDelegableBroadProjectRouterTarget(arg, cwd) || + isOwnedAgentDirectoryTarget(arg, cwd, fsImpl) || + isPluginControlUiPath(toRepoRelativeArg(arg, cwd)) || + isOwnedExtensionRootTarget(arg, cwd, fsImpl) + : path.extname(arg) === "" && + /^(?:src|test|extensions|ui|packages|apps)\//u.test(toRepoRelativeArg(arg, cwd)); +} + +function collectExplicitFileTargetArgs( + argv: string[], + predicate: (arg: string) => boolean = isExplicitFileTargetArg, +): string[] { + return collectVitestFileFilters(argv).filter(predicate); +} + +function collectExplicitProjectRouterTargetArgs( + argv: string[], + cwd = process.cwd(), + fsImpl: VitestPathFs = fs, +): string[] { + return collectExplicitFileTargetArgs(argv, (arg) => + isExplicitProjectRouterTargetArg(arg, cwd, fsImpl), + ); +} + +function isExplicitDirectoryTargetArg( + arg: string, + cwd = process.cwd(), + fsImpl: VitestPathFs = fs, +): boolean { + if (!isPathLikeExplicitFileArg(arg) || GLOB_PATTERN_CHARS_RE.test(arg)) { + return false; + } + const targetPath = path.isAbsolute(arg) ? arg : path.resolve(cwd, arg); + try { + return fsImpl.statSync(targetPath).isDirectory(); + } catch { + return false; + } +} + +function collectExplicitDirectoryTargetArgs( + argv: string[], + cwd = process.cwd(), + fsImpl: VitestPathFs = fs, +): string[] { + return collectExplicitFileTargetArgs(argv, (arg) => + isExplicitDirectoryTargetArg(arg, cwd, fsImpl), + ); +} + +function collectExplicitTestFileArgs(argv: string[]): string[] { + return collectExplicitFileTargetArgs(argv, isExplicitTestFileArg); +} + +/** + * Forces explicit test-file targets to fail when Vitest finds no matching tests. + */ +export async function resolveExplicitTestFileNoPassArgs(argv: string[]): Promise<string[]> { + if (collectExplicitTestFileArgs(argv).length === 0) { + return argv; + } + const policyArgs: string[] = []; + const spellings = new Set(["passWithNoTests"]); + for (const [index, arg] of argv.entries()) { + if (arg === "--") { + break; + } + const spelling = /^(?:--(?!-)(?:no-)?|-+no-)([^=.]+)/u.exec(arg)?.[1]; + if ( + !spelling || + spelling.replace(/([a-z])-([a-z])/gu, (_, a: string, b: string) => a + b.toUpperCase()) !== + "passWithNoTests" + ) { + continue; + } + policyArgs.push(arg); + spellings.add(spelling); + const value = argv[index + 1]; + if (arg.replace(/[=]$/u, "") === `--${spelling}` && (value === "true" || value === "false")) { + policyArgs.push(value); + } + } + if (policyArgs.length) { + const { parseCLI } = await import("vitest/node"); + // Validate only the scalar policy can overwrite; the child owns help/version + // and unrelated errors. parseCLI mutates its input, so always give it a copy. + parseCLI(["vitest", ...policyArgs], { allowUnknownOptions: true }); + } + // CAC camelcases after parsing. Negate every raw spelling so a later alias + // cannot overwrite strict policy, without erasing caller scalar errors. + return insertVitestTargets( + argv, + Array.from(spellings, (name) => `--no-${name}`), + ); +} + +function hasAlternateVitestRootArg(argv: string[]): boolean { + return argv.some( + (arg) => + arg === "--root" || + arg === "-r" || + arg === "--dir" || + arg.startsWith("--root=") || + arg.startsWith("--dir="), + ); +} + +function hasExplicitVitestProjectArg(argv: string[]): boolean { + return argv.some((arg) => arg === "--project" || arg.startsWith("--project=")); +} + +function hasExplicitDisabledRunFlag(argv: string[]): boolean { + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === undefined) { + break; + } + if (arg === "--") { + break; + } + const runFlag = resolveBooleanModeFlag(argv, index, "run"); + if (!runFlag) { + if (vitestOptionConsumesNextArg(arg, argv[index + 1])) { + index += 1; + } + continue; + } + if (runFlag.consumedNext) { + index += 1; + } + if (!runFlag.value) { + return true; + } + } + return false; +} + +function resolveDelegatedVitestArgs(argv: string[]): string[] { + const positionalArgs: string[] = []; + const optionArgs: string[] = []; + let canRemoveRunSubcommand = true; + let passthrough = false; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === undefined) { + break; + } + if (arg === "--") { + passthrough = true; + canRemoveRunSubcommand = false; + continue; + } + if (passthrough) { + optionArgs.push(arg); + continue; + } + if (vitestOptionConsumesNextArg(arg, argv[index + 1])) { + optionArgs.push(arg); + const optionValue = argv[index + 1]; + if (optionValue !== undefined) { + optionArgs.push(optionValue); + index += 1; + } + continue; + } + if (arg.startsWith("-")) { + optionArgs.push(arg); + continue; + } + if (canRemoveRunSubcommand && arg === "run") { + canRemoveRunSubcommand = false; + continue; + } + canRemoveRunSubcommand = false; + positionalArgs.push(arg); + } + return optionArgs.length > 0 ? [...positionalArgs, "--", ...optionArgs] : positionalArgs; +} + +/** + * Delegates explicit path runs to the repo test-projects runner. + */ +export function resolveTestProjectsDelegationArgs( + argv: string[], + cwd = process.cwd(), +): string[] | null { + if ( + hasExplicitVitestConfigArg(argv) || + hasAlternateVitestRootArg(argv) || + hasExplicitVitestProjectArg(argv) || + resolveExplicitVitestMode(argv) === "watch" || + hasNonRunVitestSubcommand(argv) || + hasExplicitDisabledRunFlag(argv) || + collectExplicitProjectRouterTargetArgs(argv, cwd).length === 0 + ) { + return null; + } + return resolveDelegatedVitestArgs(argv); +} + +/** + * Lists explicit test file targets missing from the current checkout. + */ +export function resolveMissingExplicitTestFiles( + argv: string[], + cwd = process.cwd(), + fsImpl: { existsSync(filePath: string): boolean } = fs, +): string[] { + if (hasExplicitVitestConfigArg(argv) || hasAlternateVitestRootArg(argv)) { + return []; + } + return collectExplicitFileTargetArgs(argv) + .filter((arg) => { + const filePath = path.isAbsolute(arg) ? arg : path.resolve(cwd, arg); + return !fsImpl.existsSync(filePath); + }) + .map((arg) => toRepoRelativeArg(arg, cwd)); +} + +function toRepoRelativeArg(arg: string, cwd: string): string { + const normalized = path.isAbsolute(arg) ? path.relative(cwd, arg) : arg; + return normalized.replaceAll(path.sep, "/").replace(/^\.\//u, ""); +} + +function withImplicitVitestConfig(argv: string[], config: string): string[] { + if (argv[0] === "run") { + return ["run", "--config", config, ...argv.slice(1)]; + } + return ["--config", config, ...argv]; +} + +function isToolingTestTarget(target: string): boolean { + return ( + target.startsWith("test/") && target.endsWith(".test.ts") && !TOOLING_EXCLUDED_TESTS.has(target) + ); +} + +function isToolingDockerTestTarget(target: string): boolean { + return target === "test/scripts/docker-build-helper.test.ts"; +} + +/** + * Resolves config defaults and explicit-file handling for wrapper-inferred runs. + */ +export function resolveImplicitVitestArgs(argv: string[], cwd = process.cwd()): string[] { + if (hasExplicitVitestConfigArg(argv)) { + return argv; + } + const separatorIndex = argv.indexOf("--"); + const optionArgs = separatorIndex < 0 ? argv : argv.slice(0, separatorIndex); + const hasExplicitIsolation = optionArgs.some( + (arg) => arg === "--isolate" || arg === "--no-isolate" || arg.startsWith("--isolate="), + ); + if (!hasExplicitIsolation && collectExplicitDirectoryTargetArgs(argv, cwd).length > 1) { + // Mixed directory selectors can activate overlapping Vitest projects. + // Isolate their module caches so one project's mocks cannot poison another. + const resolved = [...argv]; + resolved.splice(separatorIndex < 0 ? resolved.length : separatorIndex, 0, "--isolate"); + return resolved; + } + if (collectExplicitDirectoryTargetArgs(argv, cwd).length > 0) { + return argv; + } + const testTargets = collectVitestFileFilters(argv) + .filter((arg) => !arg.startsWith("-") && arg.endsWith(".test.ts")) + .map((arg) => toRepoRelativeArg(arg, cwd)); + if (testTargets.length > 0 && testTargets.every(isToolingDockerTestTarget)) { + return withImplicitVitestConfig(argv, TOOLING_DOCKER_VITEST_CONFIG); + } + if (testTargets.length > 0 && testTargets.every(isToolingTestTarget)) { + return withImplicitVitestConfig(argv, TOOLING_VITEST_CONFIG); + } + // Glob selectors can span both browser owners; the root project matrix partitions them. + if (testTargets.some((target) => /[*?[\]{}]|[@+!]\(/u.test(target))) { + return argv; + } + if (testTargets.length > 0 && testTargets.every(isUiBrowserTestFile)) { + return withImplicitVitestConfig(argv, UI_BROWSER_VITEST_CONFIG); + } + if ( + testTargets.length > 0 && + testTargets.every((target) => isUiTestTarget(target) && !isUiBrowserTestFile(target)) + ) { + // Mixed timing/ordinary UI selection needs the root matrix to preserve groups. + if (testTargets.some((target) => uiTimingTestFiles.includes(target))) { + return argv; + } + return withImplicitVitestConfig(argv, UI_VITEST_CONFIG); + } + return argv; +} + +/** + * Installs the no-output watchdog for long-running Vitest children. + */ +export function installVitestNoOutputWatchdog(params: { + streams?: Array<WatchdogStream | null>; + timeoutMs: number | null; + heartbeatMs?: number | null; + forceKillAfterMs?: number; + log?: (message: string) => void; + onTimeout?: () => void; + onForceKill?: () => void; + setTimeoutFn?: typeof setTimeout; + clearTimeoutFn?: typeof clearTimeout; +}): () => void { + const timeoutMs = params.timeoutMs; + if (!timeoutMs || timeoutMs <= 0) { + return () => {}; + } + + const setTimeoutFn = params.setTimeoutFn ?? setTimeout; + const clearTimeoutFn = params.clearTimeoutFn ?? clearTimeout; + const forceKillAfterMs = params.forceKillAfterMs ?? 5_000; + const heartbeatMs = + params.heartbeatMs && params.heartbeatMs > 0 && params.heartbeatMs < timeoutMs + ? params.heartbeatMs + : null; + const streams = + params.streams?.filter((stream): stream is WatchdogStream => stream !== null) ?? []; + + let active = true; + let silenceTimer: ReturnType<typeof setTimeout> | null = null; + let forceKillTimer: ReturnType<typeof setTimeout> | null = null; + let heartbeatTimer: ReturnType<typeof setTimeout> | null = null; + let silentForMs = 0; + let timedOut = false; + + const clearHeartbeatTimer = () => { + if (heartbeatTimer !== null) { + clearTimeoutFn(heartbeatTimer); + heartbeatTimer = null; + } + }; + + const clearForceKillTimer = () => { + if (forceKillTimer !== null) { + clearTimeoutFn(forceKillTimer); + forceKillTimer = null; + } + }; + + const clearSilenceTimer = () => { + if (silenceTimer !== null) { + clearTimeoutFn(silenceTimer); + silenceTimer = null; + } + }; + + const scheduleHeartbeatTimer = () => { + if (!active || heartbeatMs === null) { + return; + } + clearHeartbeatTimer(); + heartbeatTimer = setTimeoutFn(() => { + if (!active) { + return; + } + silentForMs += heartbeatMs; + params.log?.(`[vitest] still running with no output for ${silentForMs}ms.`); + if (silentForMs + heartbeatMs < timeoutMs) { + scheduleHeartbeatTimer(); + } + }, heartbeatMs); + }; + + const resetSilenceTimer = () => { + if (!active) { + return; + } + clearSilenceTimer(); + silentForMs = 0; + scheduleHeartbeatTimer(); + silenceTimer = setTimeoutFn(() => { + if (!active) { + return; + } + clearHeartbeatTimer(); + timedOut = true; + params.log?.( + `[vitest] no output for ${timeoutMs}ms; terminating stalled Vitest process group.`, + ); + if (forceKillAfterMs > 0) { + clearForceKillTimer(); + forceKillTimer = setTimeoutFn(() => { + if (!active) { + return; + } + params.log?.( + `[vitest] process group still alive after ${forceKillAfterMs}ms; sending SIGKILL.`, + ); + params.onForceKill?.(); + }, forceKillAfterMs); + } + params.onTimeout?.(); + }, timeoutMs); + }; + + const handleActivity = () => { + if (timedOut) { + return; + } + clearForceKillTimer(); + resetSilenceTimer(); + }; + + const listeners = streams.map((stream) => { + const handler = () => { + handleActivity(); + }; + stream.on("data", handler); + return { stream, handler }; + }); + + resetSilenceTimer(); + + return () => { + if (!active) { + return; + } + active = false; + clearSilenceTimer(); + clearForceKillTimer(); + clearHeartbeatTimer(); + for (const { stream, handler } of listeners) { + stream.off("data", handler); + } + }; +} + +/** + * Forwards child output while optionally suppressing complete stderr lines. + */ +function forwardVitestOutput( + stream: VitestOutputStream | null, + target: VitestOutputTarget, + shouldSuppressLine: (line: string) => boolean = () => false, + observeLine: (line: string) => void = () => {}, +): Promise<void> { + if (!stream) { + return Promise.resolve(); + } + + let buffered = ""; + const forwardLine = (line: string) => { + if (shouldSuppressLine(line)) { + return; + } + observeLine(line); + target.write(line); + }; + stream.setEncoding("utf8"); + stream.on("data", (chunk) => { + buffered += chunk; + while (true) { + const newlineIndex = buffered.indexOf("\n"); + if (newlineIndex === -1) { + break; + } + const line = buffered.slice(0, newlineIndex + 1); + buffered = buffered.slice(newlineIndex + 1); + forwardLine(line); + } + }); + return new Promise((resolve) => { + stream.on("end", () => { + if (buffered.length > 0) { + forwardLine(buffered); + } + resolve(); + }); + }); +} + +/** + * Joins watched Vitest processes and keeps expired deadlines failed after cooperative exits. + */ +export function spawnWatchedVitestProcess({ + pnpmArgs, + spawnParams, + env, + onNoOutputTimeout, + workerRun, + homeMode = resolveVitestHomeSelection(pnpmArgs, { cwd: spawnParams.cwd, env }), +}: { + pnpmArgs: string[]; + spawnParams: PnpmRunnerParams; + env: NodeJS.ProcessEnv; + onNoOutputTimeout?: () => void; + workerRun?: VitestWorkerRun; + homeMode?: Parameters<typeof spawnOwnedVitestProcess>[0]["homeMode"]; +}) { + if (homeMode !== "tooling") { + assertTestHomeSelection(env, homeMode); + } + let timeoutCompletion: Promise<boolean> | null = null; + const directNodeArgs = resolveDirectNodeVitestArgs(pnpmArgs); + if (workerRun && directNodeArgs) { + // Preserve Node flags while giving the same owned child its private generation. + const cliIndex = directNodeArgs.findIndex((arg) => path.basename(arg) === "vitest.mjs"); + if (cliIndex < 0) { + throw new Error("Compiled subprocess owner requires a native Vitest CLI spec"); + } + directNodeArgs.splice( + cliIndex, + 0, + path.join(resolveRepoRoot(import.meta.url), "scripts/lib/vitest-worker-bootstrap.mts"), + workerRun.descriptor.directory, + ); + } + const childSpawnParams: PnpmRunnerParams = workerRun + ? { + ...spawnParams, + stdio: [ + ...(Array.isArray(spawnParams.stdio) + ? spawnParams.stdio + : [ + spawnParams.stdio ?? "pipe", + spawnParams.stdio ?? "pipe", + spawnParams.stdio ?? "pipe", + ]), + "ipc", + ], + } + : spawnParams; + const { child, completion: childCompletion } = spawnOwnedVitestProcess({ + ...(directNodeArgs + ? { command: process.execPath, args: directNodeArgs, options: childSpawnParams } + : createPnpmRunnerSpawnSpec({ pnpmArgs, ...childSpawnParams })), + homeMode, + }); + const childCleanup = installVitestProcessGroupCleanup({ + child, + forceSignal: "SIGKILL", + forceSignalDelayMs: 100, + }); + const teardownNoOutputWatchdog = installVitestNoOutputWatchdog({ + streams: [child.stdout, child.stderr], + timeoutMs: resolveVitestNoOutputTimeoutMs(env), + heartbeatMs: resolveVitestNoOutputHeartbeatMs(env), + log: (message) => { + console.error(message); + }, + onTimeout: () => { + const termination = terminateVitestProcessGroupForTimeout({ + child, + kill: process.kill.bind(process), + log: (message) => { + console.error(message); + }, + onTimeout: onNoOutputTimeout, + }); + timeoutCompletion = termination.diagnostics.then(() => true); + }, + onForceKill: () => { + forwardSignalToVitestProcessGroup({ + child, + signal: "SIGKILL", + kill: process.kill.bind(process), + }); + }, + }); + const unhandledErrors = createVitestUnhandledErrorDetector(); + const forwardedOutput = Promise.all([ + forwardVitestOutput(child.stdout, process.stdout, undefined, unhandledErrors.observe), + forwardVitestOutput( + child.stderr, + process.stderr, + shouldSuppressVitestStderrLine, + unhandledErrors.observe, + ), + ]); + + const teardown = () => { + childCleanup.teardown(); + teardownNoOutputWatchdog(); + }; + const completion = Promise.all([childCompletion, forwardedOutput]) + .then(async ([{ code: childCode, signal, groupJoined }]) => { + const code = (await timeoutCompletion) && childCode === 0 ? 1 : childCode; + const result = unhandledErrors.finish(); + if (result) { + writeVitestUnhandledErrorSummary(result, env); + } + return { code, signal: normalizeNodeSignal(signal), groupJoined }; + }) + .finally(teardown); + + return { + child, + completion: workerRun ? workerRun.borrow(child, completion) : completion, + getForwardedSignal: childCleanup.getForwardedSignal, + teardown, + }; +} + +export async function runVitest( + exitBySignal: typeof exitVitestBySignal, + argv: string[] = process.argv.slice(2), + env: NodeJS.ProcessEnv = process.env, +): Promise<void> { + if (argv.length === 0) { + console.error("usage: node scripts/run-vitest.mjs <vitest args...>"); + process.exitCode = 1; + return; + } + + const missingTestFiles = resolveMissingExplicitTestFiles(argv); + if (missingTestFiles.length > 0) { + console.error( + [ + "[vitest] explicit test/source file(s) not found:", + ...missingTestFiles.map((file) => ` - ${file}`), + ].join("\n"), + ); + process.exitCode = 1; + return; + } + + const delegatedArgs = resolveTestProjectsDelegationArgs(argv); + if (delegatedArgs) { + const { runTestProjects } = await import("./test-projects-run.mts"); + return runTestProjects(exitBySignal, delegatedArgs, env); + } + + const vitestArgs = resolveImplicitVitestArgs(argv); + assertTestHomeSelection(env, resolveVitestHomeSelection(vitestArgs, { env })); + const repoRoot = resolveRepoRoot(import.meta.url); + let vitestCliEntry; + try { + // Resolution owns hydrated-module linking and missing-dependency guidance; + // it must finish before the execution parser imports Vitest. + vitestCliEntry = resolveVitestCliEntry({ baseDir: repoRoot }); + } catch (error) { + if (isErrorWithCode(error, "OPENCLAW_MISSING_VITEST")) { + console.error(error.message); + process.exitCode = 1; + return; + } + throw error; + } + const { parseCLI } = await import("vitest/node"); + const execution = parseVitestExecutionArgs(vitestArgs, parseCLI); + const invocations = execution + ? resolveBoundedVitestInvocations(vitestArgs, { env }) + : [vitestArgs]; + const config = resolveVitestConfigArg(vitestArgs); + const relativeConfig = config ? toRepoRelativeArg(path.resolve(config), repoRoot) : ""; + const invocationEnv = + invocations.length > 1 && relativeConfig === E2E_VITEST_CONFIG + ? { ...env, ...(await prepareE2eVitestRuntime(env)) } + : env; + // Canonical configs have known project scopes. Custom roots/projects keep + // their own setup; never infer their runtime selection from a config name. + if ( + execution && + config && + !hasAlternateVitestRootArg(vitestArgs) && + !hasExplicitVitestProjectArg(vitestArgs) && + !hasNonRunVitestSubcommand(vitestArgs) && + !hasExplicitDisabledRunFlag(vitestArgs) + ) { + const code = await prepareVitestRuntime( + invocations.flatMap((cliArgs) => + resolveVitestRuntimeCliSelections(relativeConfig, cliArgs, invocationEnv), + ), + invocationEnv, + ); + if (code !== 0) { + process.exitCode = code; + return; + } + } + const sourceMode = + !execution || execution.options.watch || resolveExplicitVitestMode(vitestArgs) === "watch"; + const workers = sourceMode + ? undefined + : createVitestWorkerRun(resolveVitestProcessEnv(invocationEnv)); + let interrupted: NodeJS.Signals | undefined; + const onSignal = (signal: NodeJS.Signals) => { + interrupted ??= signal; + }; + // The invocation outlives child-scoped handlers when admission or final + // verification is still reading. Retain signal ownership through disposal. + process.on("SIGINT", onSignal); + process.on("SIGTERM", onSignal); + try { + let failedExitCode = 0; + for (const [index, invocation] of invocations.entries()) { + const guardedVitestArgs = await resolveExplicitTestFileNoPassArgs(invocation); + const spawnEnv = resolveRunVitestSpawnEnv(invocationEnv, guardedVitestArgs); + if (invocations.length > 1) { + console.error("[vitest] bounded process " + (index + 1) + "/" + invocations.length); + } + const handle = spawnWatchedVitestProcess({ + workerRun: workers, + pnpmArgs: [ + "exec", + "node", + ...resolveVitestNodeArgs(invocationEnv), + vitestCliEntry, + ...guardedVitestArgs, + ], + spawnParams: resolveVitestSpawnParams(spawnEnv), + env: spawnEnv, + }); + const { code, signal } = await handle.completion; + interrupted ??= handle.getForwardedSignal() ?? signal ?? undefined; + const exitCode = code ?? 1; + // Ordinary test failures must not hide later files; interruptions stop + // admission and are re-raised only after the invocation has been disposed. + if (interrupted || (exitCode !== 0 && exitCode !== 1)) { + process.exitCode = exitCode; + return; + } + failedExitCode ||= exitCode; + } + process.exitCode = failedExitCode; + } finally { + try { + await workers?.dispose().catch((error: unknown) => { + process.exitCode ||= 1; + console.error(error); + }); + } finally { + process.off("SIGINT", onSignal); + process.off("SIGTERM", onSignal); + if (interrupted) { + await exitBySignal(interrupted); + } + } + } +} + +if (import.meta.main) { + // The project owner imports our spawn helpers; top-level await would deadlock + // its dynamic import when this module is also the native entrypoint. + void runVitestCli("vitest", runVitest); +} diff --git a/scripts/sandbox-browser-entrypoint.sh b/scripts/sandbox-browser-entrypoint.sh new file mode 100644 index 0000000000000000000000000000000000000000..ed8c60057dc331779eddf58c5454c18ee9d38fbc --- /dev/null +++ b/scripts/sandbox-browser-entrypoint.sh @@ -0,0 +1,333 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +export DBUS_SESSION_BUS_ADDRESS=/dev/null + +export DISPLAY=:1 +export HOME=/tmp/openclaw-home +export XDG_CONFIG_HOME="${HOME}/.config" +export XDG_CACHE_HOME="${HOME}/.cache" + +CDP_PORT="${OPENCLAW_BROWSER_CDP_PORT:-9222}" +CDP_SOURCE_RANGE="${OPENCLAW_BROWSER_CDP_SOURCE_RANGE:-}" +CDP_AUTH_TOKEN="${OPENCLAW_BROWSER_CDP_AUTH_TOKEN:-}" +VNC_PORT="${OPENCLAW_BROWSER_VNC_PORT:-5900}" +NOVNC_PORT="${OPENCLAW_BROWSER_NOVNC_PORT:-6080}" +ENABLE_NOVNC="${OPENCLAW_BROWSER_ENABLE_NOVNC:-1}" +HEADLESS="${OPENCLAW_BROWSER_HEADLESS:-0}" +ALLOW_NO_SANDBOX="${OPENCLAW_BROWSER_NO_SANDBOX:-0}" +NOVNC_PASSWORD="${OPENCLAW_BROWSER_NOVNC_PASSWORD:-}" + +DISABLE_GRAPHICS_FLAGS="${OPENCLAW_BROWSER_DISABLE_GRAPHICS_FLAGS:-1}" +DISABLE_EXTENSIONS="${OPENCLAW_BROWSER_DISABLE_EXTENSIONS:-1}" +RENDERER_PROCESS_LIMIT="${OPENCLAW_BROWSER_RENDERER_PROCESS_LIMIT:-2}" +AUTO_START_TIMEOUT_MS="${OPENCLAW_BROWSER_AUTO_START_TIMEOUT_MS:-12000}" + +validate_uint() { + local name="$1" + local value="$2" + local min="${3:-0}" + local max="${4:-4294967295}" + + if ! [[ "$value" =~ ^[0-9]+$ ]]; then + echo "[sandbox] ERROR: $name must be an integer, got: ${value}" >&2 + exit 1 + fi + if (( value < min || value > max )); then + echo "[sandbox] ERROR: $name out of range (${min}..${max}), got: ${value}" >&2 + exit 1 + fi +} + +validate_uint "CDP_PORT" "$CDP_PORT" 1 65535 +validate_uint "VNC_PORT" "$VNC_PORT" 1 65535 +validate_uint "NOVNC_PORT" "$NOVNC_PORT" 1 65535 +validate_uint "AUTO_START_TIMEOUT_MS" "$AUTO_START_TIMEOUT_MS" 1 2147483647 +if [[ -n "$RENDERER_PROCESS_LIMIT" ]]; then + validate_uint "RENDERER_PROCESS_LIMIT" "$RENDERER_PROCESS_LIMIT" 0 2147483647 +fi + +cleanup() { + local code="${1:-1}" + trap - EXIT INT TERM + + local pids=() + local pid + + for pid in "${WEBSOCKIFY_PID:-}" "${X11VNC_PID:-}" "${CDP_RELAY_PID:-}" "${CHROME_PID:-}" "${XVFB_PID:-}"; do + if [[ -n "${pid:-}" ]]; then + pids+=("$pid") + fi + done + + if ((${#pids[@]} > 0)); then + kill -TERM "${pids[@]}" 2>/dev/null || true + + for _ in {1..10}; do + local alive=0 + for pid in "${pids[@]}"; do + if kill -0 "$pid" 2>/dev/null; then + alive=1 + break + fi + done + if [[ "$alive" == "0" ]]; then + break + fi + sleep 0.2 + done + + kill -KILL "${pids[@]}" 2>/dev/null || true + wait 2>/dev/null || true + fi + + exit "$code" +} + +trap 'cleanup "$?"' EXIT +trap 'cleanup 130' INT +trap 'cleanup 143' TERM + +mkdir -p "${HOME}" "${HOME}/.chrome" "${XDG_CONFIG_HOME}" "${XDG_CACHE_HOME}" + +Xvfb :1 -screen 0 1280x800x24 -ac -nolisten tcp & +XVFB_PID=$! +echo "[sandbox] Xvfb started (PID: ${XVFB_PID})" + +if [[ "${CDP_PORT}" -ge 65535 ]]; then + CHROME_CDP_PORT="$((CDP_PORT - 1))" +else + CHROME_CDP_PORT="$((CDP_PORT + 1))" +fi + +CHROME_ARGS=( + "--remote-debugging-address=127.0.0.1" + "--remote-debugging-port=${CHROME_CDP_PORT}" + "--user-data-dir=${HOME}/.chrome" + "--no-first-run" + "--no-default-browser-check" + "--disable-dev-shm-usage" + "--disable-background-networking" + "--disable-breakpad" + "--disable-crash-reporter" + "--no-zygote" + "--metrics-recording-only" + "--password-store=basic" + "--use-mock-keychain" +) + +if [[ "${HEADLESS}" == "1" ]]; then + CHROME_ARGS+=("--headless=new") +fi + +if [[ "${ALLOW_NO_SANDBOX}" == "1" ]]; then + CHROME_ARGS+=("--no-sandbox" "--disable-setuid-sandbox") +fi + +DISABLE_GRAPHICS_FLAGS_LOWER="${DISABLE_GRAPHICS_FLAGS,,}" +if [[ "${DISABLE_GRAPHICS_FLAGS_LOWER}" =~ ^(1|true|yes|on)$ ]]; then + CHROME_ARGS+=( + "--disable-3d-apis" + "--disable-gpu" + "--disable-software-rasterizer" + ) +fi + +DISABLE_EXTENSIONS_LOWER="${DISABLE_EXTENSIONS,,}" +if [[ "${DISABLE_EXTENSIONS_LOWER}" =~ ^(1|true|yes|on)$ ]]; then + CHROME_ARGS+=("--disable-extensions") +fi + +if [[ "${RENDERER_PROCESS_LIMIT}" =~ ^[0-9]+$ && "${RENDERER_PROCESS_LIMIT}" -gt 0 ]]; then + CHROME_ARGS+=("--renderer-process-limit=${RENDERER_PROCESS_LIMIT}") +fi + +echo "[sandbox] Starting Chromium..." +chromium "${CHROME_ARGS[@]}" about:blank & +CHROME_PID=$! +echo "[sandbox] Chromium started (PID: ${CHROME_PID})" + +start_ms=$(date +%s%3N) +deadline_ms=$(( start_ms + AUTO_START_TIMEOUT_MS )) +CDP_READY=0 +probe_url="http://127.0.0.1:${CHROME_CDP_PORT}/json/version" + +echo "[sandbox] Waiting up to ${AUTO_START_TIMEOUT_MS}ms for CDP on port ${CHROME_CDP_PORT}..." + +while (( $(date +%s%3N) < deadline_ms )); do + if ! kill -0 "${CHROME_PID}" 2>/dev/null; then + echo "[sandbox] ERROR: Chromium exited before CDP became ready." + exit 1 + fi + + if curl -fsS --max-time 0.5 "${probe_url}" >/dev/null; then + CDP_READY=1 + break + fi + + sleep 0.2 +done + +if [[ "${CDP_READY}" == "0" ]]; then + echo "[sandbox] ERROR: CDP failed to start within ${AUTO_START_TIMEOUT_MS}ms." + exit 1 +fi + +echo "[sandbox] CDP ready. Starting relay..." + +if [[ -z "${CDP_AUTH_TOKEN}" ]]; then + echo "[sandbox-browser] WARNING: CDP auth token unset; CDP relay will not start." >&2 +else + OPENCLAW_BROWSER_CHROME_CDP_PORT="${CHROME_CDP_PORT}" python3 - <<'PY' & +import base64 +import hmac +import ipaddress +import os +import select +import socket +import socketserver +import sys +import time + +LISTEN_PORT = int(os.environ["OPENCLAW_BROWSER_CDP_PORT"]) +UPSTREAM_PORT = int(os.environ["OPENCLAW_BROWSER_CHROME_CDP_PORT"]) +AUTH_TOKEN = os.environ["OPENCLAW_BROWSER_CDP_AUTH_TOKEN"] +SOURCE_RANGE = os.environ.get("OPENCLAW_BROWSER_CDP_SOURCE_RANGE", "").strip() +MAX_HEADER_BYTES = 65536 +HEADER_READ_TIMEOUT_SECONDS = 5.0 + +try: + SOURCE_NETWORK = ipaddress.ip_network(SOURCE_RANGE, strict=False) if SOURCE_RANGE else None +except ValueError: + print(f"[sandbox-browser] ERROR: invalid CDP source range: {SOURCE_RANGE}", file=sys.stderr) + raise SystemExit(1) + +EXPECTED_BASIC = "Basic " + base64.b64encode(f"openclaw:{AUTH_TOKEN}".encode()).decode() +EXPECTED_BEARER = "Bearer " + AUTH_TOKEN + + +def source_allowed(host): + if SOURCE_NETWORK is None: + return True + try: + return ipaddress.ip_address(host) in SOURCE_NETWORK + except ValueError: + return False + + +def has_auth(header_bytes): + try: + text = header_bytes.decode("iso-8859-1") + except UnicodeDecodeError: + return False + for line in text.split("\r\n")[1:]: + name, sep, value = line.partition(":") + if sep and name.strip().lower() == "authorization": + auth = value.strip() + basic_ok = hmac.compare_digest(auth, EXPECTED_BASIC) + bearer_ok = hmac.compare_digest(auth, EXPECTED_BEARER) + return basic_ok or bearer_ok + return False + + +def read_headers(conn, deadline): + data = b"" + while b"\r\n\r\n" not in data: + remaining = deadline - time.monotonic() + if remaining <= 0: + return b"" + conn.settimeout(remaining) + try: + chunk = conn.recv(4096) + except socket.timeout: + return b"" + if not chunk: + return b"" + data += chunk + if len(data) > MAX_HEADER_BYTES: + return b"" + return data + + +def relay(left, right): + sockets = [left, right] + try: + while sockets: + readable, _, _ = select.select(sockets, [], []) + for src in readable: + dst = right if src is left else left + data = src.recv(65536) + if not data: + return + dst.sendall(data) + finally: + for sock in (left, right): + try: + sock.shutdown(socket.SHUT_RDWR) + except OSError: + pass + try: + sock.close() + except OSError: + pass + + +class Handler(socketserver.BaseRequestHandler): + def handle(self): + client_host = self.client_address[0] + if not source_allowed(client_host): + return + header_deadline = time.monotonic() + HEADER_READ_TIMEOUT_SECONDS + header_bytes = read_headers(self.request, header_deadline) + if not header_bytes: + return + if not has_auth(header_bytes): + self.request.sendall( + b"HTTP/1.1 401 Unauthorized\r\n" + b'WWW-Authenticate: Basic realm="OpenClaw CDP"\r\n' + b"Connection: close\r\n" + b"Content-Length: 0\r\n\r\n" + ) + return + upstream = socket.create_connection(("127.0.0.1", UPSTREAM_PORT), timeout=5) + upstream.settimeout(None) + self.request.settimeout(None) + upstream.sendall(header_bytes) + relay(self.request, upstream) + + +class Server(socketserver.ThreadingTCPServer): + allow_reuse_address = True + daemon_threads = True + + +with Server(("0.0.0.0", LISTEN_PORT), Handler) as server: + print("[sandbox] CDP relay started", flush=True) + server.serve_forever() +PY + CDP_RELAY_PID=$! + echo "[sandbox] CDP relay started (PID: ${CDP_RELAY_PID})" +fi + +if [[ "${ENABLE_NOVNC}" == "1" && "${HEADLESS}" != "1" ]]; then + if [[ -z "${NOVNC_PASSWORD}" ]]; then + NOVNC_PASSWORD="$(< /proc/sys/kernel/random/uuid)" + NOVNC_PASSWORD="${NOVNC_PASSWORD//-/}" + NOVNC_PASSWORD="${NOVNC_PASSWORD:0:8}" + fi + + mkdir -p "${HOME}/.vnc" + x11vnc -storepasswd "${NOVNC_PASSWORD}" "${HOME}/.vnc/passwd" >/dev/null + chmod 600 "${HOME}/.vnc/passwd" + + x11vnc -display :1 -rfbport "${VNC_PORT}" -shared -forever -rfbauth "${HOME}/.vnc/passwd" -localhost & + X11VNC_PID=$! + echo "[sandbox] x11vnc started (PID: ${X11VNC_PID})" + + websockify --web /usr/share/novnc/ "${NOVNC_PORT}" "localhost:${VNC_PORT}" & + WEBSOCKIFY_PID=$! + echo "[sandbox] websockify started (PID: ${WEBSOCKIFY_PID})" +fi + +echo "[sandbox] Container running. Monitoring all sub-processes..." +wait -n diff --git a/scripts/sandbox-browser-setup.sh b/scripts/sandbox-browser-setup.sh new file mode 100644 index 0000000000000000000000000000000000000000..aff6e950a59bf7df71ef719ba7b6796a25fd4c83 --- /dev/null +++ b/scripts/sandbox-browser-setup.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# Bash 5.3+ can deadlock writing heredoc pipes on macOS before the reader starts. +if [[ ${OSTYPE:-} == darwin* && $BASH != /bin/bash ]] && ((BASH_VERSINFO[0] > 5 || (BASH_VERSINFO[0] == 5 && BASH_VERSINFO[1] >= 3))); then + exec /bin/bash "$0" "$@" +fi +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +source "$ROOT_DIR/scripts/lib/docker-build.sh" + +IMAGE_NAME="openclaw-sandbox-browser:bookworm-slim" + +docker_build_exec -t "${IMAGE_NAME}" -f "$ROOT_DIR/scripts/docker/sandbox/Dockerfile.browser" "$ROOT_DIR" +echo "Built ${IMAGE_NAME}" diff --git a/scripts/stage-cua-driver-macos.sh b/scripts/stage-cua-driver-macos.sh new file mode 100644 index 0000000000000000000000000000000000000000..697ff0c122eb94b96da3f7a88a9c843956afecdf --- /dev/null +++ b/scripts/stage-cua-driver-macos.sh @@ -0,0 +1,60 @@ +#!/bin/bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "$0")/.." && pwd)" +ARTIFACT_MANIFEST="$ROOT_DIR/extensions/cua-computer/package.json" +VERSION="$(node -e 'const manifest = require(process.argv[1]); process.stdout.write(manifest.dependencies["@trycua/cua-driver"]);' "$ARTIFACT_MANIFEST")" +TAG="cua-driver-rs-v${VERSION}" +ASSET="cua-driver-rs-${VERSION}-darwin-universal-binary.tar.gz" +EXPECTED_SHA256="$(node -e 'const manifest = require(process.argv[1]); process.stdout.write(manifest.cuaDriverArtifacts["darwin-universal-binary"].archiveSha256);' "$ARTIFACT_MANIFEST")" +DOWNLOAD_URL="https://github.com/trycua/cua/releases/download/${TAG}/${ASSET}" +CACHE_DIR="$ROOT_DIR/apps/macos/.build/cua-driver/${TAG}" +ARCHIVE="$CACHE_DIR/$ASSET" +DESTINATION="${1:-}" + +if [[ -z "$DESTINATION" || "$DESTINATION" == -* ]]; then + echo "Usage: scripts/stage-cua-driver-macos.sh <destination>" >&2 + exit 2 +fi + +verify_archive() { + [[ -f "$ARCHIVE" ]] || return 1 + local actual + actual="$(shasum -a 256 "$ARCHIVE" | awk '{print $1}')" + [[ "$actual" == "$EXPECTED_SHA256" ]] +} + +mkdir -p "$CACHE_DIR" +if ! verify_archive; then + rm -f "$ARCHIVE" + partial="$ARCHIVE.partial.$$" + trap 'rm -f "$partial"' EXIT + curl --fail --location --retry 3 --retry-delay 2 --output "$partial" "$DOWNLOAD_URL" + actual="$(shasum -a 256 "$partial" | awk '{print $1}')" + if [[ "$actual" != "$EXPECTED_SHA256" ]]; then + echo "ERROR: CUA driver archive sha256 mismatch: got $actual" >&2 + exit 1 + fi + mv "$partial" "$ARCHIVE" + trap - EXIT +fi + +extract_dir="$(mktemp -d "${TMPDIR:-/tmp}/openclaw-cua-driver.XXXXXX")" +trap 'rm -rf "$extract_dir"' EXIT +tar -xzf "$ARCHIVE" -C "$extract_dir" cua-driver +source_binary="$extract_dir/cua-driver" +if [[ ! -f "$source_binary" || -L "$source_binary" ]]; then + echo "ERROR: CUA driver archive did not contain a regular cua-driver executable" >&2 + exit 1 +fi + +archs="$(/usr/bin/lipo -archs "$source_binary")" +if [[ " $archs " != *" arm64 "* || " $archs " != *" x86_64 "* ]]; then + echo "ERROR: CUA driver is not universal (architectures: $archs)" >&2 + exit 1 +fi + +mkdir -p "$(dirname "$DESTINATION")" +cp "$source_binary" "$DESTINATION" +chmod 0755 "$DESTINATION" +echo "Staged cua-driver $VERSION at $DESTINATION" diff --git a/scripts/stage-mac-node-worker.sh b/scripts/stage-mac-node-worker.sh new file mode 100644 index 0000000000000000000000000000000000000000..aef91b85c1b1b149af8e0f63ef20edfadfbb1fb3 --- /dev/null +++ b/scripts/stage-mac-node-worker.sh @@ -0,0 +1,76 @@ +#!/bin/bash +set -euo pipefail + +# Called after the canonical source build, before the app is signed. The +# complete npm artifact owns dependency selection; this is not a dist closure. +ROOT_DIR="$(cd "$(dirname "$0")/.." && pwd)" +DESTINATION="$1" +shift +[[ "$#" -gt 0 ]] || { echo "ERROR: No worker architectures requested" >&2; exit 1; } +case "${OPENCLAW_MAC_SIGNING_VARIANT:-standard}" in + standard|elevation-host) ;; + *) echo "ERROR: Unknown Mac signing variant" >&2; exit 1 ;; +esac +# Scratch follows the caller's temp volume; only installed payloads need to +# share the destination volume so publishing remains a rename, not a copy. +SCRATCH="$(mktemp -d "${TMPDIR:-/tmp}/openclaw-mac-worker.XXXXXX")" +trap 'rm -rf "$SCRATCH"' EXIT +mkdir -p "$(dirname "$DESTINATION")" +STAGE="$(cd "$(dirname "$DESTINATION")" && mktemp -d "$PWD/.openclaw-mac-worker.XXXXXX")" +trap 'rm -rf "$STAGE" "$SCRATCH"' EXIT +mkdir -p "$SCRATCH/home" "$SCRATCH/package" + +# Lifecycle hooks must never see operator config, credentials, or state; +# installer main discovers launchd by UID even with a new HOME. +# Use the build's pinned pnpm packer, not the host npm's expanding file globs. +TARBALL="$(env -i HOME="$SCRATCH/home" PATH="$PATH" TMPDIR="$SCRATCH" \ + node "$ROOT_DIR/scripts/package-openclaw-for-docker.mjs" \ + --skip-build --pnpm-pack --allow-unreleased-changelog --output-dir "$SCRATCH/package" \ + --output-name openclaw.tgz)" +[[ -f "$TARBALL" ]] || { echo "ERROR: Canonical worker package missing" >&2; exit 1; } + +for arch in "$@"; do + case "$arch" in + arm64) node_arch=arm64 ;; + x86_64) node_arch=x64 ;; + *) echo "ERROR: Unsupported Mac worker architecture: $arch" >&2; exit 1 ;; + esac + mkdir -p "$SCRATCH/$arch/home" "$SCRATCH/$arch/tmp" + env -i HOME="$SCRATCH/$arch/home" PATH="/usr/bin:/bin:/usr/sbin:/sbin" \ + TMPDIR="$SCRATCH/$arch/tmp" OPENCLAW_INSTALL_CLI_SH_NO_RUN=1 \ + OPENCLAW_NODE_VERSION="${OPENCLAW_NODE_VERSION:-}" \ + /bin/bash -c ' + set -euo pipefail + source "$1/scripts/install-cli.sh" + PREFIX="$2/prefix" + OPENCLAW_VERSION="$3" + install_node darwin "$4" + export PATH="$(node_dir)/bin:$PATH" + "$(node_bin)" -e '\''if (process.arch !== process.argv[1]) process.exit(1)'\'' "$4" || { + echo "ERROR: Cannot execute requested Node architecture $4; x64 on ARM requires Rosetta" >&2 + exit 1 + } + install_openclaw + mv "$(node_dir)" "$2/installed" + ' bash "$ROOT_DIR" "$STAGE/$arch" "$TARBALL" "$node_arch" + # Unused Intel prebuilds can trigger macOS compatibility warnings even when + # the app and its selected worker are native Apple silicon. + env -i HOME="$SCRATCH/$arch/home" PATH="/usr/bin:/bin:/usr/sbin:/sbin" \ + TMPDIR="$SCRATCH/$arch/tmp" /usr/bin/python3 -B "$ROOT_DIR/scripts/materialize-mac-node-worker.py" \ + "$STAGE/$arch/installed" "$STAGE/$arch/runtime" "$STAGE/$arch" "$arch" + # Validate after moving out of its install prefix: absolute wrappers/symlinks + # cannot accidentally make a non-relocatable payload pass the proof. + env -i HOME="$SCRATCH/$arch/home" PATH="/usr/bin:/bin:/usr/sbin:/sbin" \ + TMPDIR="$SCRATCH/$arch/tmp" \ + "$STAGE/$arch/runtime/bin/node" "$ROOT_DIR/scripts/verify-mac-node-worker.mjs" \ + "$STAGE/$arch/runtime" "$ROOT_DIR/dist/build-info.json" +done + +mkdir -p "$DESTINATION" +# The packager owns this private parent; reject every occupant before any move. +for arch in "$@"; do + [[ ! -e "$DESTINATION/$arch" && ! -L "$DESTINATION/$arch" ]] || { echo "ERROR: Worker destination exists: $DESTINATION/$arch" >&2; exit 1; } +done +for arch in "$@"; do + mv "$STAGE/$arch/runtime" "$DESTINATION/$arch" +done diff --git a/scripts/sync-labels.ts b/scripts/sync-labels.ts new file mode 100644 index 0000000000000000000000000000000000000000..10e294a77c9c6eb99bfb75ff81417a47fefd4854 --- /dev/null +++ b/scripts/sync-labels.ts @@ -0,0 +1,107 @@ +// Sync Labels script supports OpenClaw repository automation. +import { execFileSync } from "node:child_process"; +import { readFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { parse } from "yaml"; +import { resolveGitHubRepoFromOrigin } from "./lib/github-repo.ts"; + +const SYNC_LABELS_TIMEOUT_MS = 120_000; + +type RepoLabel = { + name: string; + color?: string; + description?: string; +}; + +const COLOR_BY_PREFIX = new Map<string, string>([ + ["channel", "0969DA"], + ["app", "6E7781"], + ["extensions", "6E7781"], + ["plugin", "6E7781"], + ["docs", "0A3069"], + ["cli", "0A3069"], + ["gateway", "57606A"], + ["commands", "0A3069"], + ["scripts", "57606A"], + ["docker", "D6E3DA"], + ["size", "8C959F"], +]); + +const EXTRA_LABEL_METADATA = new Map< + string, + { + color: string; + description?: string; + } +>([ + [ + "beta-blocker", + { + color: "D93F0B", + description: "Plugin beta-release blocker pending stable cutoff triage", + }, + ], +]); + +const configPath = resolve(".github/labeler.yml"); +const EXTRA_LABELS = [ + "size: XS", + "size: S", + "size: M", + "size: L", + "size: XL", + "beta-blocker", +] as const; +const labelerConfig = parse(readFileSync(configPath, "utf8")) as Record<string, unknown>; +const labelNames = [...new Set([...Object.keys(labelerConfig), ...EXTRA_LABELS])]; + +const repo = resolveGitHubRepoFromOrigin(); +const existing = fetchExistingLabels(repo); + +const missing = labelNames.filter((label) => !existing.has(label)); +if (!missing.length) { + console.log("All labeler labels already exist."); + process.exit(0); +} + +for (const label of missing) { + const metadata = resolveLabelMetadata(label); + const args = [ + "api", + "-X", + "POST", + `repos/${repo}/labels`, + "-f", + `name=${label}`, + "-f", + `color=${metadata.color}`, + ]; + if (metadata.description) { + args.push("-f", `description=${metadata.description}`); + } + execFileSync("gh", args, { + stdio: "inherit", + timeout: SYNC_LABELS_TIMEOUT_MS, + killSignal: "SIGKILL", + }); + console.log(`Created label: ${label}`); +} + +function resolveLabelMetadata(label: string): { color: string; description?: string } { + const extraMetadata = EXTRA_LABEL_METADATA.get(label); + if (extraMetadata) { + return extraMetadata; + } + const prefix = label.includes(":") ? label.slice(0, label.indexOf(":")).trim() : label.trim(); + return { color: COLOR_BY_PREFIX.get(prefix) ?? "ededed" }; +} + +function fetchExistingLabels(repoLocal: string): Map<string, RepoLabel> { + const raw = execFileSync("gh", ["api", `repos/${repoLocal}/labels?per_page=100`, "--paginate"], { + encoding: "utf8", + timeout: SYNC_LABELS_TIMEOUT_MS, + killSignal: "SIGKILL", + }); + const labels = JSON.parse(raw) as RepoLabel[]; + return new Map(labels.map((label) => [label.name, label])); +} diff --git a/scripts/sync-plugin-versions.ts b/scripts/sync-plugin-versions.ts new file mode 100644 index 0000000000000000000000000000000000000000..4beeca333d38ba3c42b067448853c4317c7d5292 --- /dev/null +++ b/scripts/sync-plugin-versions.ts @@ -0,0 +1,212 @@ +// Sync Plugin Versions script supports OpenClaw repository automation. +import { existsSync, readdirSync, readFileSync, writeFileSync } from "node:fs"; +import { join, resolve } from "node:path"; + +type PackageJson = { + name?: string; + version?: string; + devDependencies?: Record<string, string>; + peerDependencies?: Record<string, string>; + openclaw?: { + install?: { + minHostVersion?: string; + }; + compat?: { + pluginApi?: string; + }; + build?: { + openclawVersion?: string; + }; + }; +}; + +type SyncPluginVersionsOptions = { + write?: boolean; +}; + +const OPENCLAW_VERSION_RANGE_RE = /^>=\d{4}\.\d{1,2}\.\d{1,2}(?:[-.][^"\s]+)?$/u; +const VERSION_ALIGNED_PACKAGE_DIRS = [ + "packages/ai", + "packages/gateway-client", + "packages/gateway-protocol", +] as const; + +function syncOpenClawDependencyRange( + deps: Record<string, string> | undefined, + targetVersion: string, +): boolean { + const current = deps?.openclaw; + if (!current || current === "workspace:*" || !OPENCLAW_VERSION_RANGE_RE.test(current)) { + return false; + } + const next = `>=${targetVersion}`; + if (current === next) { + return false; + } + deps.openclaw = next; + return true; +} + +function syncPluginApiVersion(pkg: PackageJson, targetVersion: string): boolean { + const compat = pkg.openclaw?.compat; + const current = compat?.pluginApi; + if (!current || !OPENCLAW_VERSION_RANGE_RE.test(current)) { + return false; + } + const next = `>=${targetVersion}`; + if (current === next) { + return false; + } + compat.pluginApi = next; + return true; +} + +function syncBuildOpenClawVersion(pkg: PackageJson, targetVersion: string): boolean { + const build = pkg.openclaw?.build; + const current = build?.openclawVersion; + if (!current) { + return false; + } + if (current === targetVersion) { + return false; + } + build.openclawVersion = targetVersion; + return true; +} + +function changelogVersionForPackageVersion(version: string): string { + return version.replace(/-beta\.\d+$/u, ""); +} + +function ensureChangelogEntry(changelogPath: string, version: string, write: boolean): boolean { + if (!existsSync(changelogPath)) { + return false; + } + const content = readFileSync(changelogPath, "utf8"); + if (content.includes(`## ${version}`)) { + return false; + } + const entry = `## ${version}\n\n### Changes\n- Version alignment with core OpenClaw release numbers.\n\n`; + if (content.startsWith("# Changelog\n\n")) { + const next = content.replace("# Changelog\n\n", `# Changelog\n\n${entry}`); + if (write) { + writeFileSync(changelogPath, next); + } + return true; + } + const next = `# Changelog\n\n${entry}${content.trimStart()}`; + if (write) { + writeFileSync(changelogPath, `${next}\n`); + } + return true; +} + +export function syncPluginVersions( + rootDir = resolve("."), + options: SyncPluginVersionsOptions = {}, +) { + const write = options.write ?? true; + const rootPackagePath = join(rootDir, "package.json"); + const rootPackage = JSON.parse(readFileSync(rootPackagePath, "utf8")) as PackageJson; + const targetVersion = rootPackage.version; + if (!targetVersion) { + throw new Error("Root package.json missing version."); + } + + const extensionsDir = join(rootDir, "extensions"); + const dirs = readdirSync(extensionsDir, { withFileTypes: true }).filter((entry) => + entry.isDirectory(), + ); + + const updated: string[] = []; + const changelogged: string[] = []; + const skipped: string[] = []; + + for (const packageDir of VERSION_ALIGNED_PACKAGE_DIRS) { + const packagePath = join(rootDir, packageDir, "package.json"); + if (!existsSync(packagePath)) { + continue; + } + const pkg = JSON.parse(readFileSync(packagePath, "utf8")) as PackageJson; + if (!pkg.name || pkg.version === targetVersion) { + continue; + } + pkg.version = targetVersion; + if (write) { + writeFileSync(packagePath, `${JSON.stringify(pkg, null, 2)}\n`); + } + updated.push(pkg.name); + } + + for (const dir of dirs) { + const packagePath = join(extensionsDir, dir.name, "package.json"); + let pkg: PackageJson; + try { + pkg = JSON.parse(readFileSync(packagePath, "utf8")) as PackageJson; + } catch { + continue; + } + + if (!pkg.name) { + skipped.push(dir.name); + continue; + } + + const changelogPath = join(extensionsDir, dir.name, "CHANGELOG.md"); + const changelogVersion = changelogVersionForPackageVersion(targetVersion); + if (ensureChangelogEntry(changelogPath, changelogVersion, write)) { + changelogged.push(pkg.name); + } + + const versionChanged = pkg.version !== targetVersion; + const devDependencyChanged = syncOpenClawDependencyRange(pkg.devDependencies, targetVersion); + const peerDependencyChanged = syncOpenClawDependencyRange(pkg.peerDependencies, targetVersion); + // minHostVersion is a compatibility floor, not release alignment metadata. + // Keep it stable unless the owning plugin intentionally raises it. + const pluginApiChanged = syncPluginApiVersion(pkg, targetVersion); + const buildOpenClawVersionChanged = syncBuildOpenClawVersion(pkg, targetVersion); + const packageChanged = + versionChanged || + devDependencyChanged || + peerDependencyChanged || + pluginApiChanged || + buildOpenClawVersionChanged; + if (!packageChanged) { + skipped.push(pkg.name); + continue; + } + + if (versionChanged) { + pkg.version = targetVersion; + } + if (write) { + writeFileSync(packagePath, `${JSON.stringify(pkg, null, 2)}\n`); + } + updated.push(pkg.name); + } + + return { + targetVersion, + updated, + changelogged, + skipped, + }; +} + +if (import.meta.main) { + const check = process.argv.includes("--check"); + const summary = syncPluginVersions(resolve("."), { write: !check }); + console.log( + `Synced plugin versions to ${summary.targetVersion}. Updated: ${summary.updated.length}. Changelogged: ${summary.changelogged.length}. Skipped: ${summary.skipped.length}.`, + ); + if (check && (summary.updated.length > 0 || summary.changelogged.length > 0)) { + for (const packageName of summary.updated) { + console.error(` update required: ${packageName}`); + } + for (const packageName of summary.changelogged) { + console.error(` changelog entry required: ${packageName}`); + } + console.error("Run `pnpm plugins:sync` and commit the plugin version alignment."); + process.exit(1); + } +} diff --git a/scripts/termux-auth-widget.sh b/scripts/termux-auth-widget.sh new file mode 100644 index 0000000000000000000000000000000000000000..f8ddca7d9f0a10c8819515cb0e6411eb96308106 --- /dev/null +++ b/scripts/termux-auth-widget.sh @@ -0,0 +1,81 @@ +#!/data/data/com.termux/files/usr/bin/bash +# OpenClaw Auth Widget for Termux +# Place in ~/.shortcuts/ for Termux:Widget +# +# This widget checks auth status and helps with re-auth if needed. +# It's designed for quick one-tap checking from phone home screen. + +# Server hostname (via Tailscale or SSH config) +SERVER="${OPENCLAW_SERVER:-openclaw-host}" + +# Check auth status +termux-toast "Checking OpenClaw auth..." + +STATUS=$(ssh "$SERVER" '$HOME/openclaw/scripts/claude-auth-status.sh simple' 2>&1) +EXIT_CODE=$? + +case "$STATUS" in + OK) + # Get remaining time + DETAILS=$(ssh "$SERVER" '$HOME/openclaw/scripts/claude-auth-status.sh json' 2>&1) + HOURS=$(echo "$DETAILS" | jq -r '.claude_code.status' | grep -oP '\d+(?=h)' || echo "?") + + termux-vibrate -d 50 + termux-toast "Auth OK (${HOURS}h left)" + ;; + + CLAUDE_EXPIRING|OPENCLAW_EXPIRING) + termux-vibrate -d 100 + + # Ask if user wants to re-auth now + CHOICE=$(termux-dialog radio -t "Auth Expiring Soon" -v "Re-auth now,Check later,Dismiss") + SELECTED=$(echo "$CHOICE" | jq -r '.text // "Dismiss"') + + case "$SELECTED" in + "Re-auth now") + termux-toast "Opening auth page..." + termux-open-url "https://console.anthropic.com/settings/api-keys" + + # Show instructions + termux-dialog confirm -t "Re-auth Instructions" -i "1. Create/copy API key from browser +2. Return here and tap OK +3. SSH to server and paste key" + + # Open terminal to server + am start -n com.termux/com.termux.app.TermuxActivity -a android.intent.action.MAIN + termux-toast "Run: ssh $SERVER '$HOME/openclaw/scripts/mobile-reauth.sh'" + ;; + *) + termux-toast "Reminder: Auth expires soon" + ;; + esac + ;; + + CLAUDE_EXPIRED|OPENCLAW_EXPIRED) + termux-vibrate -d 300 + + CHOICE=$(termux-dialog radio -t "Auth Expired!" -v "Re-auth now,Dismiss") + SELECTED=$(echo "$CHOICE" | jq -r '.text // "Dismiss"') + + case "$SELECTED" in + "Re-auth now") + termux-toast "Opening auth page..." + termux-open-url "https://console.anthropic.com/settings/api-keys" + + termux-dialog confirm -t "Re-auth Steps" -i "1. Create/copy API key from browser +2. Return here and tap OK to SSH" + + am start -n com.termux/com.termux.app.TermuxActivity -a android.intent.action.MAIN + termux-toast "Run: ssh $SERVER '$HOME/openclaw/scripts/mobile-reauth.sh'" + ;; + *) + termux-toast "Warning: OpenClaw won't work until re-auth" + ;; + esac + ;; + + *) + termux-vibrate -d 200 + termux-toast "Error: $STATUS" + ;; +esac diff --git a/scripts/termux-sync-widget.sh b/scripts/termux-sync-widget.sh new file mode 100644 index 0000000000000000000000000000000000000000..5c038b1f5f1dfdaa2ef757155f0f38c373c3e7ac --- /dev/null +++ b/scripts/termux-sync-widget.sh @@ -0,0 +1,25 @@ +#!/data/data/com.termux/files/usr/bin/bash +# OpenClaw OAuth Sync Widget +# Syncs Claude Code tokens to OpenClaw over SSH +# Place in ~/.shortcuts/ on phone for Termux:Widget + +termux-toast "Syncing OpenClaw auth..." + +# Run sync on the configured OpenClaw host. +SERVER="${OPENCLAW_SERVER:-openclaw-host}" +RESULT=$(ssh "$SERVER" '$HOME/openclaw/scripts/sync-claude-code-auth.sh' 2>&1) +EXIT_CODE=$? + +if [ $EXIT_CODE -eq 0 ]; then + # Extract expiry time from output + EXPIRY=$(echo "$RESULT" | grep "Token expires:" | cut -d: -f2-) + + termux-vibrate -d 100 + termux-toast "OpenClaw synced! Expires:${EXPIRY}" + + # Optional: restart openclaw service + ssh "$SERVER" 'systemctl --user restart openclaw' 2>/dev/null +else + termux-vibrate -d 300 + termux-toast "Sync failed: ${RESULT}" +fi diff --git a/scripts/test-built-plugin-singleton.mts b/scripts/test-built-plugin-singleton.mts new file mode 100644 index 0000000000000000000000000000000000000000..b5a3ede1a6ea9cd2db5913ab7e7fc55582ac32c2 --- /dev/null +++ b/scripts/test-built-plugin-singleton.mts @@ -0,0 +1,491 @@ +// Smoke-tests the built plugin loader singleton and bundled plugin runtime overlay. +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { withDistArtifactOwnership } from "./lib/dist-artifact-ownership.mts"; +import { resolveRepoRoot } from "./lib/repo-root.mjs"; +import { installProcessWarningFilter } from "./process-warning-filter.mts"; +import { prepareBundledPluginRuntime } from "./stage-bundled-plugin-runtime.mts"; + +installProcessWarningFilter(); + +const repoRoot = resolveRepoRoot(import.meta.url); +async function runBuiltPluginSingletonSmoke(signal: AbortSignal) { + signal.throwIfAborted(); + const smokeEntryPath = path.join(repoRoot, "dist", "plugins", "build-smoke-entry.js"); + assert.ok(fs.existsSync(smokeEntryPath), `missing build output: ${smokeEntryPath}`); + + const { + buildPluginRuntimeLoadOptions, + clearPluginCommands, + getPluginCommandSpecs, + getPluginModuleLoaderStats, + loadOpenClawPlugins, + matchPluginCommand, + resolvePluginRuntimeLoadContext, + } = await import(pathToFileURL(smokeEntryPath).href); + signal.throwIfAborted(); + + assert.equal(typeof loadOpenClawPlugins, "function", "built loader export missing"); + assert.equal(typeof clearPluginCommands, "function", "clearPluginCommands missing"); + assert.equal(typeof getPluginCommandSpecs, "function", "getPluginCommandSpecs missing"); + assert.equal(typeof getPluginModuleLoaderStats, "function", "plugin loader stats missing"); + assert.equal(typeof matchPluginCommand, "function", "matchPluginCommand missing"); + + const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-build-smoke-")); + const pluginId = "build-smoke-plugin"; + const distPluginDir = path.join(repoRoot, "dist", "extensions", pluginId); + const runtimePluginDir = path.join(repoRoot, "dist-runtime", "extensions", pluginId); + + function cleanup() { + clearPluginCommands(); + fs.rmSync(distPluginDir, { recursive: true, force: true }); + fs.rmSync(runtimePluginDir, { recursive: true, force: true }); + fs.rmSync(tempRoot, { recursive: true, force: true }); + } + + process.on("exit", cleanup); + + try { + fs.mkdirSync(distPluginDir, { recursive: true }); + fs.writeFileSync( + path.join(distPluginDir, "package.json"), + JSON.stringify( + { + name: "@openclaw/build-smoke-plugin", + type: "module", + openclaw: { + extensions: ["./index.js"], + }, + }, + null, + 2, + ), + "utf8", + ); + fs.writeFileSync( + path.join(distPluginDir, "openclaw.plugin.json"), + JSON.stringify( + { + id: pluginId, + configSchema: { + type: "object", + additionalProperties: false, + properties: {}, + }, + }, + null, + 2, + ), + "utf8", + ); + fs.writeFileSync( + path.join(distPluginDir, "index.js"), + [ + "import { emptyPluginConfigSchema } from 'openclaw/plugin-sdk/plugin-entry';", + "", + "export default {", + ` id: ${JSON.stringify(pluginId)},`, + " configSchema: emptyPluginConfigSchema(),", + " register(api) {", + " api.registerCommand({", + " name: 'pair',", + " description: 'Pair a device',", + " acceptsArgs: true,", + " nativeNames: { telegram: 'pair', discord: 'pair' },", + " async handler({ args }) {", + " return { text: `paired:${args ?? ''}` };", + " },", + " });", + " },", + "};", + "", + ].join("\n"), + "utf8", + ); + + const runtime = prepareBundledPluginRuntime({ repoRoot }); + try { + await runtime.publish(() => signal.throwIfAborted()); + } catch (error) { + try { + await runtime.cleanup(); + } catch (cleanupError) { + throw new AggregateError([error, cleanupError], "Smoke runtime setup and cleanup failed.", { + cause: cleanupError, + }); + } + throw error; + } + await runtime.cleanup(); + signal.throwIfAborted(); + + const runtimeEntryPath = path.join(runtimePluginDir, "index.js"); + assert.ok(fs.existsSync(runtimeEntryPath), "runtime overlay entry missing"); + const smsRuntimeEntryPath = path.join( + repoRoot, + "dist-runtime", + "extensions", + "sms", + "index.js", + ); + assert.ok(fs.existsSync(smsRuntimeEntryPath), "compiled SMS runtime entry missing"); + assert.ok( + fs.existsSync( + path.join(repoRoot, "dist-runtime", "extensions", "mxc", "mxc-spawn-launcher.mjs"), + ), + "compiled MXC runtime asset missing", + ); + assert.equal( + fs.existsSync(path.join(repoRoot, "dist-runtime", "plugins", "commands.js")), + false, + "dist-runtime must not stage a duplicate commands module", + ); + + clearPluginCommands(); + + const smsStatsBefore = getPluginModuleLoaderStats(); + // Prepared runtimes carry this context into late, plugin-scoped loads. Prove that the load-options + // projection retains the built-artifact choice instead of reopening source transformation. + const smsRegistry = loadOpenClawPlugins( + buildPluginRuntimeLoadOptions( + resolvePluginRuntimeLoadContext({ + config: { + plugins: { + enabled: true, + allow: ["sms"], + entries: { sms: { enabled: true } }, + }, + }, + env: { + ...process.env, + OPENCLAW_BUNDLED_PLUGINS_DIR: path.join(repoRoot, "extensions"), + }, + workspaceDir: tempRoot, + }), + { cache: false, onlyPluginIds: ["sms"] }, + ), + ); + const smsRecord = smsRegistry.plugins.find((entry: { id: string }) => entry.id === "sms"); + assert.ok(smsRecord, "SMS plugin missing from registry"); + assert.equal(smsRecord.status, "loaded", smsRecord.error ?? "SMS plugin failed to load"); + const smsStatsAfter = getPluginModuleLoaderStats(); + assert.ok( + smsStatsAfter.nativeHits > smsStatsBefore.nativeHits, + "compiled SMS runtime did not use native loading", + ); + for (const counter of [ + "nativeMisses", + "sourceTransformForced", + "sourceTransformFallbacks", + ] as const) { + assert.equal( + smsStatsAfter[counter], + smsStatsBefore[counter], + `compiled SMS runtime changed ${counter}`, + ); + } + assert.equal( + smsStatsAfter.topSourceTransformTargets.some(({ target }: { target: string }) => + target.replaceAll("\\", "/").includes("/extensions/sms/"), + ), + false, + "compiled SMS runtime reached the source transformer", + ); + + // Exercise the real built load owner with omitted preferences, as provider and + // tool callers do. Source-owned metadata must not force source execution. + const { resolvePluginDiscoveryProvidersRuntime } = await import( + pathToFileURL(path.join(repoRoot, "dist", "plugins", "provider-discovery.runtime.js")).href + ); + signal.throwIfAborted(); + const { createPluginMetadataSnapshotFixture } = + await import("../src/plugins/plugin-metadata.test-support.js"); + signal.throwIfAborted(); + const { withPluginRuntimeGenerationScope } = + await import("../src/plugins/runtime/generation-scope.js"); + signal.throwIfAborted(); + const { setPluginRuntimeLoadContext } = await import("../src/plugins/runtime/load-context.js"); + signal.throwIfAborted(); + const artifactPluginId = "build-artifact-selection"; + const artifactSourceRoot = path.join(tempRoot, "extensions", artifactPluginId); + const artifactBuiltRoot = path.join(tempRoot, "dist", "extensions", artifactPluginId); + fs.mkdirSync(artifactSourceRoot, { recursive: true }); + fs.mkdirSync(artifactBuiltRoot, { recursive: true }); + fs.mkdirSync(path.join(artifactSourceRoot, "dist")); + const artifactEntry = (label: string) => `export default { + id: ${JSON.stringify(artifactPluginId)}, + register(api) { + api.registerProvider({ id: ${JSON.stringify(artifactPluginId)}, label: ${JSON.stringify(label)}, auth: [] }); + api.registerTool({ name: "artifact_probe", label: "Artifact", description: ${JSON.stringify(label)}, + parameters: { type: "object", properties: {} }, + async execute() { return { content: [{ type: "text", text: ${JSON.stringify(label)} }] }; } + }); + } +};\n`; + const artifactSource = path.join(artifactSourceRoot, "index.ts"); + fs.writeFileSync(artifactSource, artifactEntry("source")); + fs.writeFileSync(path.join(artifactBuiltRoot, "index.js"), artifactEntry("compiled")); + fs.writeFileSync( + path.join(artifactSourceRoot, "dist", "index.js"), + artifactEntry("package-local"), + ); + fs.writeFileSync( + path.join(artifactSourceRoot, "package.json"), + JSON.stringify({ type: "module" }), + ); + for (const [rootDir, extension, label] of [ + [artifactSourceRoot, ".ts", "source"], + [artifactBuiltRoot, ".js", "compiled"], + ] as const) { + fs.writeFileSync( + path.join(rootDir, `provider-discovery${extension}`), + `export default { + id: ${JSON.stringify(artifactPluginId)}, label: ${JSON.stringify(label)}, auth: [], + catalog: { run: async () => ({ providers: {} }) } + };\n`, + ); + } + fs.writeFileSync( + path.join(artifactBuiltRoot, "package.json"), + JSON.stringify({ + type: "module", + openclaw: { extensions: ["./index.js"] }, + }), + ); + const artifactConfig = { + plugins: { allow: [artifactPluginId], entries: { [artifactPluginId]: { enabled: true } } }, + }; + const artifactManifest = { + id: artifactPluginId, + rootDir: artifactSourceRoot, + source: artifactSource, + origin: "bundled" as const, + channels: [], + providers: [artifactPluginId], + cliBackends: [], + skills: [], + hooks: [], + contracts: { tools: ["artifact_probe"] }, + manifestPath: path.join(artifactSourceRoot, "openclaw.plugin.json"), + providerDiscoverySource: path.join(artifactSourceRoot, "provider-discovery.ts"), + configSchema: { type: "object", properties: {}, additionalProperties: false }, + packageManifest: { extensions: ["./index.ts"], build: { bundledDist: false } }, + }; + const artifactManifestRegistry = { + plugins: [artifactManifest], + diagnostics: [], + }; + const artifactMetadataSnapshot = createPluginMetadataSnapshotFixture(artifactManifestRegistry); + for (const toolDiscovery of [false, true]) { + for (const preferBuiltPluginArtifacts of [undefined, false]) { + const values = { + config: artifactConfig, + env: { ...process.env, OPENCLAW_STATE_DIR: path.join(tempRoot, "artifact-state") }, + manifestRegistry: artifactManifestRegistry, + installRecords: {}, + preferBuiltPluginArtifacts, + workspaceDir: tempRoot, + }; + const options = toolDiscovery + ? buildPluginRuntimeLoadOptions(resolvePluginRuntimeLoadContext(values)) + : values; + const selected = loadOpenClawPlugins({ + ...options, + cache: false, + activate: false, + toolDiscovery, + onlyPluginIds: [artifactPluginId], + }); + const label = preferBuiltPluginArtifacts === false ? "source" : "compiled"; + assert.equal(selected.plugins[0]?.status, "loaded", selected.plugins[0]?.error); + assert.equal(selected.providers[0]?.provider.label, label); + const tool = selected.tools[0]?.factory({ config: artifactConfig }); + assert.equal(tool?.description, label); + if (!toolDiscovery) { + setPluginRuntimeLoadContext(selected, resolvePluginRuntimeLoadContext(values)); + const providers = withPluginRuntimeGenerationScope( + { metadataSnapshot: artifactMetadataSnapshot, pluginRegistry: selected }, + () => + resolvePluginDiscoveryProvidersRuntime({ + config: artifactConfig, + pluginMetadataSnapshot: artifactMetadataSnapshot, + onlyPluginIds: [artifactPluginId], + discoveryEntriesOnly: true, + }), + ); + assert.equal( + providers[0]?.label, + label, + "provider discovery chose a different artifact policy", + ); + } + } + } + + // Implicit built-host policy leaves installed source alone; explicit true can + // use its package-local output. Neither cache call order may contaminate the other. + for (const [orderIndex, preferences] of [ + [undefined, true], + [true, undefined], + ].entries()) { + const options = { + config: artifactConfig, + env: { ...process.env, OPENCLAW_STATE_DIR: path.join(tempRoot, "artifact-state") }, + workspaceDir: path.join(tempRoot, `cache-order-${orderIndex}`), + manifestRegistry: { + ...artifactManifestRegistry, + plugins: [{ ...artifactManifest, origin: "global" }], + }, + installRecords: {}, + onlyPluginIds: [artifactPluginId], + cache: true, + activate: false, + }; + const registries = preferences.map((preferBuiltPluginArtifacts) => { + const selected = loadOpenClawPlugins({ ...options, preferBuiltPluginArtifacts }); + const label = preferBuiltPluginArtifacts ? "package-local" : "source"; + assert.equal(selected.plugins[0]?.status, "loaded", selected.plugins[0]?.error); + assert.equal( + selected.providers[0]?.provider.label, + label, + `cache call order ${orderIndex}`, + ); + assert.equal(selected.tools[0]?.factory({ config: artifactConfig })?.description, label); + return selected; + }); + for (const [index, preferBuiltPluginArtifacts] of preferences.entries()) { + assert.equal( + loadOpenClawPlugins({ ...options, preferBuiltPluginArtifacts }), + registries[index], + "repeated selection did not reuse its own cached registry", + ); + } + } + + clearPluginCommands(); + + const registry = loadOpenClawPlugins({ + cache: false, + workspaceDir: tempRoot, + env: { + ...process.env, + OPENCLAW_BUNDLED_PLUGINS_DIR: path.join(repoRoot, "dist-runtime", "extensions"), + }, + config: { + plugins: { + enabled: true, + allow: [pluginId], + entries: { + [pluginId]: { enabled: true }, + }, + }, + }, + }); + + const record = registry.plugins.find((entry: { id: string }) => entry.id === pluginId); + assert.ok(record, "smoke plugin missing from registry"); + assert.equal(record.status, "loaded", record.error ?? "smoke plugin failed to load"); + + assert.deepEqual( + getPluginCommandSpecs().filter((command: { name: string }) => command.name === "pair"), + [{ name: "pair", description: "Pair a device", acceptsArgs: true }], + ); + + const match = matchPluginCommand("/pair now"); + assert.ok(match, "canonical built command registry did not receive the command"); + assert.equal(match.args, "now"); + const result = await match.command.handler({ args: match.args }); + signal.throwIfAborted(); + assert.deepEqual(result, { text: "paired:now" }); + + // Keep these imports after the cold native checks so they cannot prewarm the loader. + const { buildBundleMcpToolsFromCatalog } = await import( + pathToFileURL(path.join(repoRoot, "dist", "agents", "agent-bundle-mcp-materialize.js")).href + ); + signal.throwIfAborted(); + const { getPluginToolMeta } = await import( + pathToFileURL(path.join(repoRoot, "dist", "plugins", "tool-metadata.js")).href + ); + signal.throwIfAborted(); + const { getPluginToolMeta: getSdkPluginToolMeta } = await import( + pathToFileURL(path.join(repoRoot, "dist", "plugin-sdk", "agent-harness-runtime.js")).href + ); + signal.throwIfAborted(); + const [mcpTool] = buildBundleMcpToolsFromCatalog({ + catalog: { + version: 1, + generatedAt: 0, + servers: { + "build-smoke-mcp": { + serverName: "build-smoke-mcp", + safeServerName: "build-smoke-mcp", + launchSummary: "build smoke inventory fixture", + toolCount: 1, + }, + }, + tools: [ + { + serverName: "build-smoke-mcp", + safeServerName: "build-smoke-mcp", + toolName: "lookup", + inputSchema: { type: "object", properties: {} }, + fallbackDescription: "Look up a build smoke inventory item", + }, + ], + }, + }); + assert.ok(mcpTool, "compiled MCP materializer did not produce a tool"); + const mcpMetadata = getPluginToolMeta(mcpTool); + assert.ok(mcpMetadata, "canonical built metadata owner did not receive MCP tool metadata"); + assert.equal(mcpMetadata.pluginId, "bundle-mcp"); + assert.equal(mcpMetadata.mcp?.serverName, "build-smoke-mcp"); + assert.equal(mcpMetadata.mcp?.safeServerName, "build-smoke-mcp"); + assert.equal(mcpMetadata.mcp?.toolName, "lookup"); + assert.equal(mcpMetadata.mcp?.operation, "tool"); + assert.strictEqual( + getSdkPluginToolMeta(mcpTool), + mcpMetadata, + "public agent-harness-runtime SDK did not read the canonical MCP metadata record", + ); + } finally { + process.off("exit", cleanup); + cleanup(); + } + signal.throwIfAborted(); + const remaining = prepareBundledPluginRuntime({ repoRoot }); + const changed = remaining.changed; + await remaining.cleanup(); + assert.equal(changed, false, "singleton smoke left noncanonical runtime artifacts"); + process.stdout.write("[build-smoke] built plugin singleton smoke passed\n"); +} + +const controller = new AbortController(); +let exitCode = 0; +const cancel = (code: number) => { + if (!controller.signal.aborted) { + exitCode = code; + controller.abort(new Error("Built plugin singleton smoke canceled.")); + } +}; +const onSigint = () => cancel(130); +const onSigterm = () => cancel(143); +try { + await withDistArtifactOwnership(repoRoot, async () => { + process.on("SIGINT", onSigint); + process.on("SIGTERM", onSigterm); + // Imports cannot be interrupted; keep ownership until each pending read joins. + await runBuiltPluginSingletonSmoke(controller.signal); + }); + controller.signal.throwIfAborted(); +} catch (error) { + if (!controller.signal.aborted || error !== controller.signal.reason) { + throw error; + } + process.exitCode = exitCode; +} finally { + process.off("SIGINT", onSigint); + process.off("SIGTERM", onSigterm); +} diff --git a/scripts/test-docker-all.mjs b/scripts/test-docker-all.mjs new file mode 100644 index 0000000000000000000000000000000000000000..a4a2a1ed3a6097559491127abcdd093f40f4fafc --- /dev/null +++ b/scripts/test-docker-all.mjs @@ -0,0 +1,5 @@ +import { runTsxCliShim } from "./lib/tsx-cli-shim.mjs"; + +await runTsxCliShim(import.meta.url, { + implementation: "./test-docker-all.mts", +}); diff --git a/scripts/test-extension-batch.mts b/scripts/test-extension-batch.mts new file mode 100644 index 0000000000000000000000000000000000000000..da9e1c5936558a6bf5cc80c81125190f65791729 --- /dev/null +++ b/scripts/test-extension-batch.mts @@ -0,0 +1,507 @@ +#!/usr/bin/env node + +// Runs grouped Vitest plans for one or more bundled plugins. +import path from "node:path"; +import pMap from "p-map"; +import { waitForever } from "../src/cli/wait.ts"; +import { assertTestHomeSelection, combineTestHomeSelections } from "../test/test-home-policy.mts"; +import { + DATABASE_WORKER_WATCH_OWNER_ENV_KEY, + DATABASE_WORKER_WATCH_TESTS_ENV_KEY, +} from "../test/vitest/vitest.database-worker-core-paths.mjs"; +import { databaseWorkerExtensionTestFiles } from "../test/vitest/vitest.extension-database-workers-paths.mjs"; +import { collectVitestExcludePatterns } from "../test/vitest/vitest.pattern-file.ts"; +import { resolveVitestFsModuleCacheRoot } from "../test/vitest/vitest.performance-config.ts"; +import { + createExtensionTestProcessTargetChunks, + listExtensionTestFilesForRoots, + resolveExtensionBatchPlan, + shouldSplitExtensionTestProcesses, + splitExtensionTestProcessTargets, +} from "./lib/extension-test-plan.mts"; +import type { ExtensionBatchPlan, ExtensionTestPlanGroup } from "./lib/extension-test-plan.mts"; +import { + normalizeRelativePath, + relativizeExtensionVitestArgs, + relativizeExtensionVitestPath, +} from "./lib/extension-vitest-paths.mts"; +import { parsePositiveInt } from "./lib/numeric-options.mjs"; +import { isDirectScriptRun, runVitestBatch } from "./lib/vitest-batch-runner.mts"; +import type { VitestBatchRunParams } from "./lib/vitest-batch-runner.mts"; +import { prepareVitestRuntime } from "./lib/vitest-build-prerequisites.mts"; +import { resolveVitestHomeSelection } from "./lib/vitest-home-selection.mts"; +import { createVitestReportOwner, type VitestReportOutcome } from "./lib/vitest-report-owner.mts"; +import { resolveVitestRuntimeCliSelections } from "./lib/vitest-runtime-selection.mts"; + +const DATABASE_WORKER_CONFIG = "test/vitest/vitest.extension-database-workers.config.ts"; +const FS_MODULE_CACHE_PATH_ENV_KEY = "OPENCLAW_VITEST_FS_MODULE_CACHE_PATH"; +const PARALLEL_ENV_KEY = "OPENCLAW_EXTENSION_BATCH_PARALLEL"; +const ALLOW_NO_TESTS_FLAG = "--allow-no-tests"; +const ALLOW_EMPTY_AFTER_EXCLUDE_FLAG = "--allow-empty-after-exclude"; + +function printUsage() { + console.error( + `Usage: pnpm test:extensions:batch <extension[,extension...]> [${ALLOW_NO_TESTS_FLAG}] [${ALLOW_EMPTY_AFTER_EXCLUDE_FLAG}] [vitest args...]`, + ); + console.error( + ` node --import tsx scripts/test-extension-batch.mts <extension[,extension...]> [${ALLOW_NO_TESTS_FLAG}] [${ALLOW_EMPTY_AFTER_EXCLUDE_FLAG}] [vitest args...]`, + ); +} + +/** + * Parses comma-separated plugin ids and separates Vitest passthrough args. + */ +export function parseExtensionIds(rawArgs: string[]) { + const normalizedArgs = rawArgs[0] === "--" ? rawArgs.slice(1) : rawArgs; + const separatorIndex = normalizedArgs.indexOf("--"); + const args = separatorIndex >= 0 ? normalizedArgs.slice(0, separatorIndex) : [...normalizedArgs]; + const separatorPassthroughArgs = + separatorIndex >= 0 ? normalizedArgs.slice(separatorIndex + 1) : []; + const extensionIds: string[] = []; + + while (args[0] && !args[0].startsWith("-")) { + const extensionArg = args.shift(); + if (!extensionArg) { + break; + } + extensionIds.push( + ...extensionArg + .split(",") + .map((value) => value.trim()) + .filter(Boolean), + ); + } + + return { + extensionIds, + passthroughArgs: separatorIndex >= 0 ? [...args, ...separatorPassthroughArgs] : args, + }; +} + +/** + * Resolves bounded parallelism for extension test config groups. + */ +export function resolveExtensionBatchParallelism(groupCount: number, env = process.env) { + const raw = env[PARALLEL_ENV_KEY]?.trim(); + const override = raw ? parsePositiveInt(raw, PARALLEL_ENV_KEY) : 1; + return Math.min(Math.max(1, override), Math.max(1, groupCount)); +} + +function sanitizeCacheSegment(value: string) { + return ( + value + .replace(/[^a-zA-Z0-9._-]+/gu, "-") + .replace(/^-+|-+$/gu, "") + .slice(0, 180) || "default" + ); +} + +function createGroupEnv({ + baseEnv, + group, + groupIndex, + useDedicatedCache, +}: { + baseEnv: NodeJS.ProcessEnv; + group: ExtensionTestPlanGroup; + groupIndex: number; + useDedicatedCache: boolean; +}) { + if (!useDedicatedCache || baseEnv[FS_MODULE_CACHE_PATH_ENV_KEY]?.trim()) { + return baseEnv; + } + + return { + ...baseEnv, + [FS_MODULE_CACHE_PATH_ENV_KEY]: path.join( + resolveVitestFsModuleCacheRoot(), + "extension-batch", + sanitizeCacheSegment(`${groupIndex}-${group.config}`), + ), + }; +} + +function orderPlanGroups(planGroups: ExtensionTestPlanGroup[], parallelism: number) { + if (parallelism <= 1) { + return planGroups; + } + return [...planGroups].toSorted((left, right) => { + if (left.estimatedCost !== right.estimatedCost) { + return right.estimatedCost - left.estimatedCost; + } + if (left.testFileCount !== right.testFileCount) { + return right.testFileCount - left.testFileCount; + } + return left.config.localeCompare(right.config); + }); +} + +function isExactExcludePath(inputPath: string) { + return !/[*!?[\]{}]/u.test(inputPath); +} + +function addExactExcludePath(excludePaths: Set<string>, value: string) { + const normalized = normalizeRelativePath(value); + excludePaths.add(normalized); + if (!normalized.startsWith("extensions/")) { + excludePaths.add(`extensions/${normalized}`); + } +} + +/** + * Collects exact --exclude paths so empty groups can be reported accurately. + */ +export function parseExactVitestExcludePaths(vitestArgs: string[]) { + const excludePaths = new Set<string>(); + for (const value of collectVitestExcludePatterns(vitestArgs)) { + if (isExactExcludePath(value)) { + addExactExcludePath(excludePaths, value); + } + } + return excludePaths; +} + +function resolveGroupTargets(group: ExtensionTestPlanGroup, exactExcludePaths: Set<string>) { + if (exactExcludePaths.size === 0) { + return group.roots; + } + + const testFiles = listExtensionTestFilesForRoots(group.roots); + if (!testFiles) { + return group.roots; + } + + return testFiles.filter( + (file) => + !exactExcludePaths.has(file) && + (group.config === DATABASE_WORKER_CONFIG || !databaseWorkerExtensionTestFiles.includes(file)), + ); +} + +function preparePlanGroup( + group: ExtensionTestPlanGroup, + groupIndex: number, + env: NodeJS.ProcessEnv, + vitestArgs: string[], + exactExcludePaths: Set<string>, + useDedicatedCache: boolean, +) { + const targets = resolveGroupTargets(group, exactExcludePaths); + const targetChunks = + targets.length === 0 + ? [] + : exactExcludePaths.size > 0 + ? shouldSplitExtensionTestProcesses(group.config, vitestArgs) + ? splitExtensionTestProcessTargets(group.config, targets) + : [targets] + : createExtensionTestProcessTargetChunks(group.config, group.roots, vitestArgs); + return { + group, + invocations: targetChunks.map<VitestBatchRunParams & { env: NodeJS.ProcessEnv }>((chunk) => ({ + args: relativizeExtensionVitestArgs(vitestArgs), + config: group.config, + env: createGroupEnv({ baseEnv: env, group, groupIndex, useDedicatedCache }), + targets: chunk.map((target) => relativizeExtensionVitestPath(target)), + })), + }; +} + +function combineSinglePluginGroups( + preparedGroups: ReturnType<typeof preparePlanGroup>[], + vitestArgs: string[], + env: NodeJS.ProcessEnv, + homeMode: VitestBatchRunParams["homeMode"], +): ReturnType<typeof preparePlanGroup>[] { + const worker = preparedGroups.find(({ group }) => group.config === DATABASE_WORKER_CONFIG); + const owner = preparedGroups.find(({ group }) => group.config !== DATABASE_WORKER_CONFIG); + if ( + preparedGroups.length !== 2 || + !worker || + !owner || + shouldSplitExtensionTestProcesses(DATABASE_WORKER_CONFIG, vitestArgs) + ) { + return preparedGroups; + } + const targets = [ + ...new Set( + preparedGroups.flatMap(({ invocations }) => + invocations.flatMap((invocation) => invocation.targets), + ), + ), + ]; + const config = "test/vitest/vitest.database-worker-watch.config.ts"; + return [ + { + group: { ...owner.group, config }, + invocations: + targets.length === 0 + ? [] + : [ + { + config, + args: relativizeExtensionVitestArgs(vitestArgs), + targets: targets.filter( + (target) => !targets.some((root) => target.startsWith(`${root}/`)), + ), + env: { + ...env, + [DATABASE_WORKER_WATCH_OWNER_ENV_KEY]: owner.group.config, + [DATABASE_WORKER_WATCH_TESTS_ENV_KEY]: JSON.stringify(worker.group.roots), + }, + homeMode, + }, + ], + }, + ]; +} + +async function runPlanGroup( + { group, invocations }: ReturnType<typeof preparePlanGroup>, + runGroup: (params: ReturnType<typeof preparePlanGroup>["invocations"][number]) => Promise<number>, + allowEmptyAfterExclude: boolean, + isCancelled: () => boolean, +) { + if (invocations.length === 0) { + console.error(`[test-extension-batch] ${group.config}: no test files remain after excludes`); + return allowEmptyAfterExclude ? 0 : 1; + } + let finalExitCode = 0; + for (const [index, invocation] of invocations.entries()) { + if (isCancelled()) { + break; + } + console.log( + `[test-extension-batch] ${group.config}: ${group.extensionIds.join(", ")} (${invocation.targets.length} targets${invocations.length > 1 ? `, chunk ${index + 1}/${invocations.length}` : ""})`, + ); + const exitCode = await runGroup(invocation); + if (exitCode !== 0 && finalExitCode === 0) { + finalExitCode = exitCode; + } + } + return finalExitCode; +} + +/** + * Runs a resolved extension batch plan, optionally in parallel config groups. + */ +export async function runExtensionBatchPlan( + batchPlan: ExtensionBatchPlan, + params: { + allowEmptyAfterExclude?: boolean; + expandExactExcludes?: boolean; + env?: NodeJS.ProcessEnv; + runGroup?: (params: VitestBatchRunParams) => Promise<number>; + vitestArgs?: string[]; + } = {}, +) { + const env = params.env ?? process.env; + const vitestArgs = params.vitestArgs ?? []; + // Single-plugin CLI historically leaves exact exclusions to Vitest. + const exactExcludePaths = + params.expandExactExcludes === false + ? new Set<string>() + : parseExactVitestExcludePaths(vitestArgs); + const runGroup = params.runGroup ?? runVitestBatch; + const parallelism = resolveExtensionBatchParallelism(batchPlan.planGroups.length, env); + const orderedGroups = orderPlanGroups(batchPlan.planGroups, parallelism); + const useDedicatedCache = parallelism > 1; + const allowEmptyAfterExclude = params.allowEmptyAfterExclude ?? false; + + if (parallelism > 1) { + console.log(`[test-extension-batch] Running up to ${parallelism} config groups in parallel`); + } + + const leafGroups = orderedGroups.map((group, index) => + preparePlanGroup(group, index, env, vitestArgs, exactExcludePaths, useDedicatedCache), + ); + const leafInvocations = leafGroups.flatMap((group) => group.invocations); + const cwd = path.resolve(import.meta.dirname, ".."); + const homeMode = combineTestHomeSelections( + leafInvocations.map(({ config, args, targets, env: invocationEnv }) => + resolveVitestHomeSelection(["--config", config, ...args, ...targets], { + cwd, + env: invocationEnv, + }), + ), + ); + // Admit the whole selection before report or runtime preparation can import code. + assertTestHomeSelection(env, homeMode); + const preparedGroups = + batchPlan.extensionCount === 1 + ? combineSinglePluginGroups(leafGroups, vitestArgs, env, homeMode) + : leafGroups; + const invocations = preparedGroups.flatMap((group) => group.invocations); + const reports = await createVitestReportOwner( + invocations.map((invocation) => ({ + config: invocation.config, + args: ["run", "--config", invocation.config, ...invocation.args, ...invocation.targets], + })), + cwd, + ); + const termination: { signal: NodeJS.Signals | null } = { signal: null }; + const onSignal = (value: NodeJS.Signals) => { + termination.signal ??= value; + }; + if (reports) { + process.on("SIGTERM", onSignal); + process.on("SIGINT", onSignal); + } + let reportFailure: string | undefined; + let exitCode = 0; + const started: Promise<unknown>[] = []; + const runInvocation = async ( + invocation: ReturnType<typeof preparePlanGroup>["invocations"][number], + ) => { + const attempt = reports?.attempt(invocations.indexOf(invocation), invocation.args); + if (!attempt) { + return runGroup(invocation); + } + try { + let outcome: VitestReportOutcome | undefined; + const code = await runGroup({ + ...invocation, + args: attempt.args, + onComplete(value) { + outcome = value; + termination.signal ??= value.signal; + }, + }); + attempt.complete(outcome ?? { code, signal: null }); + return code; + } catch (error) { + attempt.fail(error); + throw error; + } + }; + try { + // No reader may start while a shared generation is being replaced. Select + // from the exact emitted chunks, including existing exact-exclude expansion. + const preparationCode = await prepareVitestRuntime( + leafInvocations.flatMap(({ config, args, targets, env: invocationEnv }) => + resolveVitestRuntimeCliSelections(config, [...args, ...targets], invocationEnv), + ), + env, + ); + if (preparationCode !== 0) { + exitCode = preparationCode; + reportFailure = "Runtime preparation failed; invocations unstarted"; + return exitCode; + } + try { + await pMap( + preparedGroups, + async (group) => { + if ((exitCode !== 0 && batchPlan.extensionCount > 1) || termination.signal) { + return; + } + const running = runPlanGroup( + group, + runInvocation, + allowEmptyAfterExclude, + () => termination.signal !== null, + ).then((groupExitCode) => { + if (groupExitCode !== 0 && exitCode === 0) { + exitCode = groupExitCode; + } + }); + started.push(running); + await running; + }, + { concurrency: parallelism, stopOnError: true }, + ); + } finally { + await Promise.allSettled(started); + } + } catch (error) { + reportFailure = String(error); + throw error; + } finally { + if (reports) { + try { + const reportCode = await reports.finish( + async (mergeArgs) => { + const args = mergeArgs.slice(1); + const configIndex = args.indexOf("--config"); + const config = args.splice(configIndex, 2)[1]!; + let outcome: VitestReportOutcome | undefined; + const code = await runVitestBatch({ + config, + args, + targets: [], + env, + homeMode, + onComplete(value) { + outcome = value; + termination.signal ??= value.signal; + }, + }); + return outcome ?? { code, signal: null }; + }, + termination.signal ? `Cancelled by ${termination.signal}` : reportFailure, + ); + exitCode ||= reportCode; + } finally { + process.off("SIGTERM", onSignal); + process.off("SIGINT", onSignal); + if (termination.signal) { + process.kill(process.pid, termination.signal); + // Keep the loop alive for dependency signal handlers to finish cleanup + // and re-raise; a numeric return can win the race with signal delivery. + await waitForever(); + } + } + } + } + return exitCode; +} + +async function run() { + const rawArgs = process.argv.slice(2); + if (rawArgs.includes("--help") || rawArgs.includes("-h")) { + printUsage(); + return; + } + + const allowNoTests = rawArgs.includes(ALLOW_NO_TESTS_FLAG); + const allowEmptyAfterExclude = rawArgs.includes(ALLOW_EMPTY_AFTER_EXCLUDE_FLAG); + const controlArgs = new Set([ALLOW_NO_TESTS_FLAG, ALLOW_EMPTY_AFTER_EXCLUDE_FLAG]); + const args = rawArgs.filter((arg) => !controlArgs.has(arg)); + const { extensionIds, passthroughArgs: vitestArgs } = parseExtensionIds(args); + if (extensionIds.length === 0) { + printUsage(); + process.exit(1); + } + + const batchPlan = resolveExtensionBatchPlan({ cwd: process.cwd(), extensionIds }); + const noTestExtensionIds = batchPlan.noTestExtensionIds ?? []; + if (noTestExtensionIds.length > 0 && !allowNoTests) { + console.error( + `[test-extension-batch] No tests found for requested extension(s): ${noTestExtensionIds.join(", ")}`, + ); + process.exit(1); + } + if (!batchPlan.hasTests) { + console.error("[test-extension-batch] No tests found for the requested extensions."); + if (!allowNoTests) { + process.exit(1); + } + return; + } + + console.log( + `[test-extension-batch] Running ${batchPlan.testFileCount} test files across ${batchPlan.extensionCount} extensions`, + ); + + const exitCode = await runExtensionBatchPlan(batchPlan, { + allowEmptyAfterExclude, + env: process.env, + vitestArgs, + }); + if (exitCode !== 0) { + process.exit(exitCode); + } +} + +if (isDirectScriptRun(import.meta.url)) { + await run(); +} diff --git a/scripts/test-freebsd-service-inspect.mjs b/scripts/test-freebsd-service-inspect.mjs new file mode 100644 index 0000000000000000000000000000000000000000..17e923b2ac7118f1dfe706cb9dcb05623f060b22 --- /dev/null +++ b/scripts/test-freebsd-service-inspect.mjs @@ -0,0 +1,578 @@ +#!/usr/bin/env node +// Native proof runs in a disposable FreeBSD VM; all cases use a private chroot. +import assert from "node:assert/strict"; +import { execFileSync, spawn, spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +if (process.platform !== "freebsd" || process.geteuid() !== 0) { + process.stderr.write("Run this native fixture as root in a disposable FreeBSD VM.\n"); + process.stderr.write("[test-freebsd-service-inspect] FAILED (exit 1)\n"); + process.exit(1); +} + +const work = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-rc-inspect-")); +// The inner root must be traversable after chroot drops to nobody. Its outer +// directory stays private, especially while testing a writable helper file. +fs.chmodSync(work, 0o700); +const root = path.join(work, "root"); +fs.mkdirSync(root, { mode: 0o755 }); +const helper = "/usr/local/libexec/openclaw-service-inspect.mjs"; +const discoveryHelper = "/usr/local/libexec/lib/freebsd-service-discovery.mjs"; +const escapedCase = "deadline closes captures held by an escaped daemon"; +const escapedOnly = process.argv[2] === "--escaped-pipe-only"; +assert.ok(process.argv.length === 2 || (escapedOnly && process.argv.length === 3)); +const escapedScript = `/pipe-holder-${path.basename(work)}.mjs`; +let escapedLaunched = false; +let mounted = false; +let noexecMounted = false; +let passed = 0; +let signalFixtureActive = false; +const fixturePath = (filename) => path.join(root, filename); +const write = (filename, contents, mode = 0o644) => { + fs.mkdirSync(path.dirname(fixturePath(filename)), { recursive: true }); + fs.writeFileSync(fixturePath(filename), contents, { mode }); + fs.chmodSync(fixturePath(filename), mode); +}; + +function nodeArguments(args, user) { + return [ + ...(user ? ["-u", user] : []), + root, + "/usr/bin/env", + "-i", + "HOME=/", + "PATH=/sbin:/bin:/usr/sbin:/usr/bin", + "LC_ALL=C", + "/usr/local/bin/node", + ...args, + ]; +} + +function runNode(args, { user, env = {} } = {}) { + return spawnSync("/usr/sbin/chroot", nodeArguments(args, user), { + encoding: "utf8", + timeout: 20_000, + // The fixture deadline must stop its own child even if it handles SIGTERM. + killSignal: "SIGKILL", + env: { PATH: "/sbin:/bin:/usr/sbin:/usr/bin", ...env }, + }); +} + +function diagnostic(result) { + return JSON.stringify({ + status: result.status, + signal: result.signal, + error: result.error?.code, + stderr: String(result.stderr ?? "") + .replaceAll("private-fixture-value", "[redacted]") + .slice(-4096), + }); +} + +function run({ args = [], user, env = {} } = {}) { + const result = runNode([helper, ...args], { user, env }); + assert.ifError(result.error); + assert.equal(result.signal, null); + assert.ok(result.stdout.length < 64 * 1024); + assert.ok(!result.stdout.includes("private-fixture-value")); + assert.ok(!result.stderr.includes("private-fixture-value")); + assert.ok(result.stdout.trim(), `Inspector returned no JSON: ${diagnostic(result)}`); + const output = JSON.parse(result.stdout); + assert.equal(output.authority, "diagnostic-only"); + assert.equal(output.service, "openclaw"); + assert.equal(output.schema, 1); + if (output.status === "unknown") { + assert.equal(result.status, 1); + assert.ok(result.stderr.endsWith("[freebsd-service-inspect] FAILED (exit 1)\n")); + } else { + assert.equal(result.status, 0); + assert.equal(result.stderr, ""); + } + return output; +} + +async function check(name, test) { + if (escapedOnly && name !== escapedCase) { + return; + } + process.stdout.write(`CASE ${name}\n`); + await test(); + assert.equal(fs.existsSync(fixturePath("/service-executed")), false); + passed++; + process.stdout.write(`PASS ${name}\n`); +} + +async function waitFor(predicate) { + const deadline = performance.now() + 5_000; + do { + if (predicate()) { + return true; + } + await new Promise((resolve) => { + setTimeout(resolve, 25); + }); + } while (performance.now() < deadline); + return false; +} + +async function checkSignalCancellation(signal, caller) { + const holder = `/signal-holder-${path.basename(work)}.mjs`; + const pidPath = fixturePath("/signal-holder.pid"); + fs.rmSync(pidPath, { force: true }); + write( + holder, + 'import fs from "node:fs"; fs.writeFileSync("/signal-holder.pid", String(process.pid)); setTimeout(() => {}, 60_000);\n', + ); + write("/etc/rc.conf", `exec /usr/local/bin/node ${holder}\n`); + signalFixtureActive = true; + const child = spawn("/usr/sbin/chroot", nodeArguments([caller]), { + env: { PATH: "/sbin:/bin:/usr/sbin:/usr/bin" }, + stdio: ["ignore", "pipe", "pipe"], + }); + let closed = false; + let spawnError; + let outputBytes = 0; + child.once("error", (error) => { + spawnError = error; + }); + child.once("close", () => { + closed = true; + }); + child.stdout.on("data", (chunk) => { + outputBytes += chunk.length; + }); + child.stderr.on("data", (chunk) => { + outputBytes += chunk.length; + }); + let pid; + const queryIsLive = () => { + const result = spawnSync("/bin/ps", ["-ww", "-p", pid, "-o", "stat=", "-o", "command="], { + encoding: "utf8", + timeout: 2_000, + killSignal: "SIGKILL", + maxBuffer: 4096, + }); + assert.ifError(result.error); + assert.equal(result.signal, null); + assert.ok(result.status === 0 || result.status === 1); + assert.equal(result.stderr, ""); + const state = result.stdout.trim(); + if (!state || state.startsWith("Z")) { + return false; + } + assert.equal(state.replace(/^\S+\s+/, ""), `/usr/local/bin/node ${holder}`); + return true; + }; + try { + assert.ok( + await waitFor( + () => + closed || + (fs.existsSync(pidPath) && /^[1-9][0-9]*$/.test(fs.readFileSync(pidPath, "utf8").trim())), + ), + "signal query did not start", + ); + assert.ifError(spawnError); + pid = fs.readFileSync(pidPath, "utf8").trim(); + assert.match(pid, /^[1-9][0-9]*$/); + assert.ok(queryIsLive(), "signal query exited before interruption"); + assert.equal(child.kill(signal), true); + assert.ok(await waitFor(() => closed), "caller did not settle after signal"); + assert.equal(outputBytes, 0, "caller continued to publish a query result"); + assert.equal(fs.existsSync(fixturePath("/query-continued")), false); + if (caller === helper) { + assert.equal(child.exitCode, null); + assert.equal(child.signalCode, signal); + } else { + assert.equal(child.exitCode, 73); + assert.equal(child.signalCode, null); + assert.equal(fs.readFileSync(fixturePath("/host-signal"), "utf8"), signal); + } + assert.ok(await waitFor(() => !queryIsLive()), "owned configuration query remains live"); + } finally { + // Stop only this child and the recorded unique fixture, never an argv census. + // Retain the chroot if either identity or completion cannot be established. + if (!closed) { + child.kill("SIGKILL"); + } + assert.ok(await waitFor(() => closed), "signal caller cleanup did not finish"); + if (!pid) { + pid = fs.readFileSync(pidPath, "utf8").trim(); + assert.match(pid, /^[1-9][0-9]*$/); + } + if (queryIsLive()) { + process.kill(Number(pid), "SIGTERM"); + } + assert.ok(await waitFor(() => !queryIsLive()), "signal query cleanup did not finish"); + signalFixtureActive = false; + write("/etc/rc.conf", ""); + } +} + +try { + // Real base utilities and loader, not fake rc.subr or production test hooks. + for (const directory of [ + "/bin", + "/sbin", + "/lib", + "/libexec", + "/usr/bin", + "/usr/sbin", + "/usr/lib", + "/usr/libexec", + "/usr/local/lib", + ]) { + fs.mkdirSync(path.dirname(fixturePath(directory)), { recursive: true }); + // Preserve native modes and ownership, but not immutable flags on disposable copies. + execFileSync("/bin/cp", ["-RpN", directory, fixturePath(directory)]); + } + fs.mkdirSync(fixturePath("/usr/local/bin"), { recursive: true }); + fs.copyFileSync(process.execPath, fixturePath("/usr/local/bin/node")); + fs.chmodSync(fixturePath("/usr/local/bin/node"), 0o755); + write( + helper, + fs.readFileSync(fileURLToPath(new URL("./freebsd-service-inspect.mjs", import.meta.url))), + ); + const discoverySource = fs.readFileSync( + fileURLToPath(new URL("./lib/freebsd-service-discovery.mjs", import.meta.url)), + ); + write(discoveryHelper, discoverySource); + for (const filename of ["/etc/rc.subr", "/etc/defaults/rc.conf", "/etc/passwd", "/etc/group"]) { + write(filename, fs.readFileSync(filename)); + } + // Packaged Node dependencies live outside rtld's built-in /lib and /usr/lib paths. + fs.mkdirSync(fixturePath("/var/run"), { recursive: true }); + execFileSync("/usr/sbin/chroot", [root, "/sbin/ldconfig", "/lib", "/usr/lib", "/usr/local/lib"], { + timeout: 20_000, + }); + fs.mkdirSync(fixturePath("/etc/rc.d"), { recursive: true }); + fs.mkdirSync(fixturePath("/dev")); + execFileSync("/sbin/mount", ["-t", "devfs", "devfs", fixturePath("/dev")]); + mounted = true; + write("/etc/rc.conf", ""); + + const node = runNode(["--version"]); + assert.equal(node.status, 0, `Chroot Node failed: ${diagnostic(node)}`); + assert.match(node.stdout.trim(), /^v\d+\.\d+\.\d+$/); + process.stdout.write(`chroot-node=${node.stdout.trim()}\n`); + + await check("native default absence", () => assert.equal(run().status, "absent")); + write( + "/preload.mjs", + 'import fs from "node:fs"; fs.writeFileSync("/preloaded", "unexpected");\n', + ); + await check("documented invocation excludes Node preloads", () => { + assert.equal(run({ env: { NODE_OPTIONS: "--import /preload.mjs" } }).status, "absent"); + assert.equal(fs.existsSync(fixturePath("/preloaded")), false); + }); + await check("non-root refusal", () => + assert.equal(run({ user: "nobody" }).reason, "root-required"), + ); + await check("fixed operation rejects arguments", () => + assert.equal(run({ args: ["start"] }).reason, "no-arguments-accepted"), + ); + + const definition = "#!/bin/sh\ntouch /service-executed\n"; + write("/custom/rc.d/openclaw", definition, 0o755); + write("/second/rc.d/openclaw", definition, 0o755); + write("/etc/rc.conf", ". /etc/root-only.conf\n"); + write("/etc/root-only.conf", 'local_startup="/custom/rc.d /second/rc.d"\n', 0o600); + await check("root-only include and shadowed definitions", () => { + const result = run(); + assert.equal(result.status, "present"); + assert.equal(result.selected, "/custom/rc.d/openclaw"); + assert.deepEqual( + result.definitions.map((entry) => entry.path), + ["/custom/rc.d/openclaw", "/second/rc.d/openclaw"], + ); + }); + await check("inherited environment cannot skip native configuration", () => { + assert.equal( + run({ env: { _rc_conf_loaded: "true", local_startup: "/absent", NODE_OPTIONS: "" } }) + .selected, + "/custom/rc.d/openclaw", + ); + }); + fs.chmodSync(fixturePath("/custom/rc.d/openclaw"), 0o644); + await check("non-executable definition remains present", () => { + const result = run(); + assert.equal(result.definitions[0].executable, false); + assert.equal(result.selected, "/second/rc.d/openclaw"); + }); + fs.chmodSync(fixturePath("/second/rc.d/openclaw"), 0o644); + await check("no executable definition is not absence", () => { + assert.equal(run().status, "present"); + assert.equal(run().selected, null); + }); + write("/etc/rc.conf.d/openclaw", 'local_startup="/absent"\n'); + await check("per-service overrides do not redirect global discovery", () => + assert.equal(run().status, "present"), + ); + write("/etc/rc.conf", 'rc_conf_files="/etc/second.conf"\n'); + write("/etc/second.conf", 'local_startup="/custom/rc.d"\n'); + await check("native rc_conf_files chaining", () => + assert.equal(run().definitions[0].path, "/custom/rc.d/openclaw"), + ); + write("/etc/rc.conf", 'local_startup="/cus*/rc.d"\n'); + await check("native startup glob expansion", () => + assert.equal(run().definitions[0].path, "/custom/rc.d/openclaw"), + ); + fs.mkdirSync(fixturePath("/noexec")); + execFileSync("/sbin/mount", ["-t", "tmpfs", "-o", "noexec", "tmpfs", fixturePath("/noexec")]); + noexecMounted = true; + write("/noexec/rc.d/openclaw", definition, 0o755); + write("/etc/rc.conf", 'local_startup="/noexec/rc.d"\n'); + await check("executable selection matches native test on a noexec mount", () => { + const native = spawnSync("/usr/sbin/chroot", [ + root, + "/bin/test", + "-x", + "/noexec/rc.d/openclaw", + ]); + assert.ok(native.status === 0 || native.status === 1); + assert.equal(run().definitions[0].executable, native.status === 0); + }); + write("/etc/rc.conf", '[ "$HOME" = / ] && local_startup="/custom/rc.d"\n'); + await check("reported context matches native service discovery", () => { + const result = run(); + const native = execFileSync( + "/usr/sbin/chroot", + [ + root, + "/usr/bin/env", + "-i", + "HOME=/", + "PATH=/sbin:/bin:/usr/sbin:/usr/bin", + "LC_ALL=C", + "/usr/sbin/service", + "-v", + "-l", + ], + { encoding: "utf8" }, + ); + assert.deepEqual( + result.directories, + [...native.matchAll(/^From (.*):$/gm)].map((match) => match[1]), + ); + assert.equal(result.context.cwd, "/"); + assert.equal(result.context.env.HOME, "/"); + }); + fs.chmodSync(fixturePath("/custom/rc.d"), 0o777); + await check("writable startup directory is unknown", () => + assert.equal(run().reason, "unsafe-path-ownership"), + ); + fs.chmodSync(fixturePath("/custom/rc.d"), 0o755); + fs.renameSync(fixturePath("/custom/rc.d/openclaw"), fixturePath("/custom/rc.d/target")); + fs.symlinkSync("target", fixturePath("/custom/rc.d/openclaw")); + await check("symlink definition is unknown", () => + assert.equal(run().reason, "symbolic-link-needs-owner-inspection"), + ); + fs.unlinkSync(fixturePath("/custom/rc.d/openclaw")); + fs.renameSync(fixturePath("/custom/rc.d/target"), fixturePath("/custom/rc.d/openclaw")); + for (const redirect of ["", " >&2"]) { + write("/etc/rc.conf", `echo private-fixture-value${redirect}\n`); + await check(`native output is rejected and redacted ${redirect || "stdout"}`, () => + assert.equal(run().reason, "native-configuration-output-invalid"), + ); + } + const bytePath = Buffer.concat([Buffer.from("/nonutf8-"), Buffer.from([255])]); + fs.mkdirSync(Buffer.concat([Buffer.from(root), bytePath])); + fs.writeFileSync( + Buffer.concat([Buffer.from(root), bytePath, Buffer.from("/openclaw")]), + definition, + { mode: 0o755 }, + ); + write( + "/etc/rc.conf", + Buffer.concat([Buffer.from('local_startup="'), bytePath, Buffer.from('"\n')]), + ); + await check("invalid path bytes cannot become absence", () => + assert.equal(run().reason, "native-configuration-output-invalid"), + ); + write("/etc/rc.conf", "while :; do :; done\n"); + await check("native configuration deadline", () => + assert.equal(run().reason, "native-configuration-failed"), + ); + write("/etc/rc.conf", "/bin/sleep 60 &\necho $! > /child.pid\nwait\n"); + await check("deadline settles configuration children", () => { + assert.equal(run().reason, "native-configuration-failed"); + const pid = fs.readFileSync(fixturePath("/child.pid"), "utf8").trim(); + assert.match(pid, /^[1-9][0-9]*$/); + let alive = true; + for (let attempt = 0; attempt < 20; attempt++) { + const status = spawnSync("/bin/ps", ["-p", pid, "-o", "stat="], { encoding: "utf8" }); + if (status.status !== 0 || !status.stdout.trim() || status.stdout.trim().startsWith("Z")) { + alive = false; + break; + } + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 50); + } + assert.equal(alive, false, "configuration child remains live"); + }); + write( + escapedScript, + 'import fs from "node:fs"; fs.writeFileSync("/pipe-holder.pid", String(process.pid)); setTimeout(() => {}, 60_000);\n', + ); + write("/etc/rc.conf", `/usr/sbin/daemon /usr/local/bin/node ${escapedScript}\n`); + await check(escapedCase, () => { + escapedLaunched = true; + const started = performance.now(); + assert.equal(run().reason, "native-configuration-failed"); + process.stdout.write(`escaped-pipe elapsed-ms=${Math.round(performance.now() - started)}\n`); + const pid = fs.readFileSync(fixturePath("/pipe-holder.pid"), "utf8").trim(); + assert.match(pid, /^[1-9][0-9]*$/); + assert.equal( + execFileSync("/bin/ps", ["-ww", "-p", pid, "-o", "command="], { encoding: "utf8" }).trim(), + `/usr/local/bin/node ${escapedScript}`, + ); + }); + write("/etc/rc.conf", ""); + await check("shared observer honors a shorter caller deadline", () => { + write("/etc/rc.conf", "sleep 60\n"); + const started = performance.now(); + const result = runNode([ + "--input-type=module", + "--eval", + `import { discoverFreeBsdService } from ${JSON.stringify(`file://${discoveryHelper}`)}; +console.log(JSON.stringify(await discoverFreeBsdService({ + timeoutMs: 100, + registerExitCleanup(cleanup) { + process.once("exit", cleanup); + return () => process.off("exit", cleanup); + }, +})));`, + ]); + assert.ifError(result.error); + assert.equal(result.status, 0, diagnostic(result)); + assert.equal(JSON.parse(result.stdout).reason, "native-configuration-failed"); + assert.ok(performance.now() - started < 5_000, "short caller deadline was ignored"); + write("/etc/rc.conf", ""); + }); + write( + "/signal-caller.mjs", + ` +import assert from "node:assert/strict"; +import fs from "node:fs"; +import { discoverFreeBsdService } from ${JSON.stringify(`file://${discoveryHelper}`)}; +const cleanups = new Set(); +process.once("exit", () => { for (const cleanup of cleanups) { cleanup(); } }); +for (const signal of ["SIGINT", "SIGTERM"]) { + process.once(signal, () => { + fs.writeFileSync("/host-signal", signal); + process.exit(73); + }); +} +const handlers = [process.listeners("SIGINT"), process.listeners("SIGTERM")]; +const pending = discoverFreeBsdService({ + registerExitCleanup(cleanup) { + cleanups.add(cleanup); + return () => cleanups.delete(cleanup); + }, +}); +assert.deepEqual([process.listeners("SIGINT"), process.listeners("SIGTERM")], handlers); +await pending; +fs.writeFileSync("/query-continued", "unexpected"); +`, + ); + for (const signal of ["SIGINT", "SIGTERM"]) { + await check(`standalone ${signal} preserves termination and cleans up its query`, () => + checkSignalCancellation(signal, helper), + ); + await check(`shared observer leaves ${signal} policy with its caller`, () => + checkSignalCancellation(signal, "/signal-caller.mjs"), + ); + } + await check("writable shared helper is rejected before import", () => { + write( + discoveryHelper, + 'import fs from "node:fs"; fs.writeFileSync("/shared-helper-executed", "bad");\n', + 0o666, + ); + assert.equal(run().reason, "unsafe-path-ownership"); + assert.equal(fs.existsSync(fixturePath("/shared-helper-executed")), false); + write(discoveryHelper, discoverySource); + }); + await check("writable shared helper ancestor is rejected before import", () => { + const directory = fixturePath(path.dirname(discoveryHelper)); + fs.chmodSync(directory, 0o777); + assert.equal(run().reason, "unsafe-path-ownership"); + fs.chmodSync(directory, 0o755); + }); + fs.chmodSync(fixturePath(helper), 0o666); + await check("writable helper is rejected", () => + assert.equal(run().reason, "unsafe-path-ownership"), + ); + process.stdout.write(`${passed} native cases passed\n`); +} catch (error) { + process.stderr.write(`${error.stack}\n`); + process.exitCode = 1; +} finally { + let cleanupStage = "signal-query"; + try { + assert.equal(signalFixtureActive, false, "signal fixture cleanup is uncertain"); + cleanupStage = "escaped-pipe-holder"; + if (escapedLaunched) { + // This fixture owns the recorded PID and its unique script, not an argv census. + const pid = fs.readFileSync(fixturePath("/pipe-holder.pid"), "utf8").trim(); + assert.match(pid, /^[1-9][0-9]*$/); + assert.equal( + execFileSync("/bin/ps", ["-ww", "-p", pid, "-o", "command="], { encoding: "utf8" }).trim(), + `/usr/local/bin/node ${escapedScript}`, + ); + process.kill(Number(pid), "SIGTERM"); + let alive = true; + for (let attempt = 0; attempt < 100; attempt++) { + const status = spawnSync("/bin/ps", ["-p", pid, "-o", "stat="], { encoding: "utf8" }); + assert.ifError(status.error); + assert.equal(status.signal, null); + assert.ok(status.status === 0 || status.status === 1); + assert.equal(status.stderr, ""); + if (!status.stdout.trim() || status.stdout.trim().startsWith("Z")) { + alive = false; + break; + } + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 50); + } + assert.equal(alive, false, "escaped pipe holder remains live"); + process.stdout.write("escaped-pipe-holder cleanup=complete\n"); + } + cleanupStage = "unmount-noexec"; + if (noexecMounted) { + execFileSync("/sbin/umount", [fixturePath("/noexec")]); + } + cleanupStage = "unmount-devfs"; + if (mounted) { + execFileSync("/sbin/umount", [fixturePath("/dev")]); + } + const flags = spawnSync( + "/usr/bin/find", + [root, "-flags", "+schg,sappnd,uchg,uappnd", "-print"], + { encoding: "utf8", timeout: 20_000, maxBuffer: 4096 }, + ); + process.stdout.write( + `fixture-immutable-flags=${diagnostic(flags)} paths=${JSON.stringify((flags.stdout ?? "").slice(-4096))}\n`, + ); + if (flags.error || flags.status !== 0) { + process.exitCode = 1; + } + cleanupStage = "remove-fixture"; + fs.rmSync(work, { recursive: true, force: true }); + process.stdout.write("fixture-cleanup=complete\n"); + } catch (error) { + process.stderr.write( + `Fixture cleanup failed at ${cleanupStage}; retained ${work}: ${JSON.stringify({ + code: error.code, + syscall: error.syscall, + path: error.path, + stderr: String(error.stderr ?? "").slice(-4096), + })}\n`, + ); + process.exitCode = 1; + } +} +if (process.exitCode) { + process.stderr.write("[test-freebsd-service-inspect] FAILED (exit 1)\n"); +} diff --git a/scripts/test-group-report.mts b/scripts/test-group-report.mts new file mode 100644 index 0000000000000000000000000000000000000000..220f20770460f969bec976ef807d23e2c144460b --- /dev/null +++ b/scripts/test-group-report.mts @@ -0,0 +1,1181 @@ +// Builds grouped Vitest duration reports or compares two grouped reports. +import { spawn } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { isRecord } from "@openclaw/normalization-core/record-coerce"; +import pMap from "p-map"; +import { + booleanFlag, + parseFlagArgs, + requireOptionArgument, + stringFlag, + stringListFlag, + type FlagSpec, +} from "./lib/arg-utils.mts"; +import { coerceErrorMessage } from "./lib/error-format.mts"; +import { + inspectManagedProcessGroup, + terminateManagedChild, + waitForManagedProcessGroupExit, +} from "./lib/managed-child-process.mts"; +import { parsePositiveInt } from "./lib/numeric-options.mjs"; +import { + buildGroupedTestComparison, + buildGroupedTestReport, + formatBytesAsMb, + normalizeConfigLabel, + renderGroupedTestComparison, + renderGroupedTestReport, +} from "./lib/test-group-report.mts"; +import { resolveVitestNodeArgs } from "./lib/vitest-process-env.mts"; +import { formatMs } from "./lib/vitest-report-cli-utils.mts"; +import { + applyParallelVitestCachePaths, + buildFullSuiteVitestRunPlans, +} from "./test-projects.test-support.mts"; + +const DEFAULT_OUTPUT = ".artifacts/test-perf/group-report.json"; +const DEFAULT_COMPARE_OUTPUT = ".artifacts/test-perf/group-report-compare.json"; +const DEFAULT_RUN_TIMEOUT_MS = 30 * 60 * 1000; +const DEFAULT_TIMEOUT_KILL_GRACE_MS = 10_000; +const DEFAULT_SPAWN_LOG_MAX_BYTES = 1024 * 1024 * 256; +const DEFAULT_SPAWN_OUTPUT_MAX_BYTES = 1024 * 1024 * 64; +const DEFAULT_SPAWN_OUTPUT_TAIL_BYTES = 1024 * 256; + +type ProcessSignal = `SIG${string}`; +type TimerHandle = ReturnType<typeof setTimeout>; +export type TestGroupReportArgs = { + allowFailures: boolean; + compare: { after: string; before: string } | null; + concurrency: number | null; + configs: string[]; + fullSuite: boolean; + groupBy: string; + help?: boolean; + killGraceMs: number; + limit: number; + maxTestMs: number | null; + output: string | null; + reports: string[]; + rss: boolean; + timeoutMs: number; + topFiles: number; + vitestArgs: string[]; +}; + +export type TestGroupRunPlan = { + config: string; + forwardedArgs: string[]; + label: string; +}; + +export type TestGroupRunSpec = TestGroupRunPlan & { + env: NodeJS.ProcessEnv; + vitestArgs: string[]; +}; + +export type TestGroupRun = Awaited<ReturnType<typeof runVitestJsonReport>>; + +type ReportInputEntry = Pick<TestGroupRun, "config" | "reportPath"> & { + run?: TestGroupRun | null; +}; + +type SpawnTextOptions = { + cwd?: string; + env?: NodeJS.ProcessEnv; + killGraceMs?: number; + logPath?: string | null; + timeoutMs?: number; + maxBufferBytes?: number; + maxLogBytes?: number; + outputTailBytes?: number; +}; + +type SpawnTextResult = { + output: string; + status: number; + signal: ProcessSignal | null; + timedOut: boolean; +}; + +type GroupedComparisonInput = Parameters<typeof buildGroupedTestComparison>[0]["before"]; + +type RunVitestParams = TestGroupRunSpec & + Pick<TestGroupReportArgs, "killGraceMs" | "rss" | "timeoutMs"> & { + logPath: string; + reportPath: string; + }; + +function usage() { + return [ + "Usage: node --import tsx scripts/test-group-report.mts [options] [-- <vitest args>]", + "", + "Build a grouped Vitest duration report from one or more JSON reports.", + "", + "Options:", + " --config <path> Vitest config to run (repeatable)", + " --compare <before> <after>", + " Compare two grouped report JSON files", + " --report <path> Existing Vitest JSON report to read (repeatable)", + " --full-suite Run every full-suite leaf Vitest config serially", + " --group-by <mode> area | folder | top (default: area)", + " --output <path> JSON report path (default: .artifacts/test-perf/group-report.json)", + " --limit <count> Number of groups/configs to print (default: 25)", + " --top-files <count> Number of files to print (default: 25)", + " --max-test-ms <ms> Fail when any individual test exceeds this duration", + " --timeout-ms <ms> Per-config wall-clock timeout (default: 1800000)", + " --kill-grace-ms <ms> Grace after timeout before SIGKILL (default: 10000)", + " --concurrency <count> Run this many config reports at once (default: 2 for", + " repeated explicit configs, 1 for full-suite)", + " --allow-failures Write a report even when a Vitest run exits non-zero", + " --no-rss Skip max RSS measurement", + " --help Show this help", + "", + "Examples:", + " pnpm test:perf:groups --config test/vitest/vitest.unit-fast.config.ts", + " pnpm test:perf:groups --full-suite --allow-failures", + " pnpm test:perf:groups:compare .artifacts/test-perf/baseline-before.json .artifacts/test-perf/after-first-fix.json", + ].join("\n"); +} + +type PositiveIntArgKey = + | "concurrency" + | "killGraceMs" + | "limit" + | "maxTestMs" + | "timeoutMs" + | "topFiles"; + +const splitStringFlagOptions = { allowInline: false, rejectShortOptions: true } as const; + +function positiveIntFlag(flag: string, key: PositiveIntArgKey): FlagSpec<TestGroupReportArgs> { + return { + consume(argv, index) { + if (argv[index] !== flag) { + return null; + } + const value = parsePositiveInt(requireOptionArgument(argv, index, flag), flag); + return { + flag, + nextIndex: index + 1, + apply(args) { + args[key] = value; + }, + }; + }, + }; +} + +const compareFlag: FlagSpec<TestGroupReportArgs> = { + consume(argv, index) { + if (argv[index] !== "--compare") { + return null; + } + const before = requireOptionArgument(argv, index, "--compare"); + const after = requireOptionArgument(argv, index + 1, "--compare"); + return { + flag: "--compare", + nextIndex: index + 2, + apply(args) { + args.compare = { before, after }; + }, + }; + }, +}; + +/** + * Parses report, compare, and Vitest-run options for grouped test reports. + */ +export function parseTestGroupReportArgs(argv: string[]) { + const args: TestGroupReportArgs = { + allowFailures: false, + compare: null, + concurrency: null, + configs: [], + fullSuite: false, + groupBy: "area", + limit: 25, + killGraceMs: DEFAULT_TIMEOUT_KILL_GRACE_MS, + maxTestMs: null, + output: null, + reports: [], + rss: process.platform !== "win32", + timeoutMs: DEFAULT_RUN_TIMEOUT_MS, + topFiles: 25, + vitestArgs: [], + }; + const separatorIndex = argv.indexOf("--"); + const cliArgs = separatorIndex === -1 ? argv : argv.slice(0, separatorIndex); + args.vitestArgs = separatorIndex === -1 ? [] : argv.slice(separatorIndex + 1); + + parseFlagArgs( + cliArgs, + args, + [ + booleanFlag("--help", "help", true, { repeatable: true }), + booleanFlag("--allow-failures", "allowFailures", true, { repeatable: true }), + booleanFlag("--full-suite", "fullSuite", true, { repeatable: true }), + booleanFlag("--no-rss", "rss", false, { repeatable: true }), + stringListFlag("--config", "configs", splitStringFlagOptions), + compareFlag, + stringListFlag("--report", "reports", splitStringFlagOptions), + stringFlag("--group-by", "groupBy", splitStringFlagOptions), + stringFlag("--output", "output", splitStringFlagOptions), + positiveIntFlag("--limit", "limit"), + positiveIntFlag("--max-test-ms", "maxTestMs"), + positiveIntFlag("--timeout-ms", "timeoutMs"), + positiveIntFlag("--kill-grace-ms", "killGraceMs"), + positiveIntFlag("--concurrency", "concurrency"), + positiveIntFlag("--top-files", "topFiles"), + ], + { + onUnhandledArg(arg) { + throw new Error(`Unknown option: ${arg}`); + }, + }, + ); + + if (!["area", "folder", "top"].includes(args.groupBy)) { + throw new Error(`Unsupported --group-by value: ${args.groupBy}`); + } + if (args.compare && (!args.compare.before || !args.compare.after)) { + throw new Error("--compare requires before and after report paths"); + } + if ( + args.compare && + (args.configs.length > 0 || + args.fullSuite || + args.reports.length > 0 || + args.vitestArgs.length > 0) + ) { + throw new Error("--compare cannot be combined with test run or report input options"); + } + + return args; +} + +function sanitizePathSegment(value: string) { + return ( + value + .replace(/[^A-Za-z0-9._-]+/gu, "-") + .replace(/^-+|-+$/gu, "") + .slice(0, 180) || "report" + ); +} + +function resolveTimeArgs(command: [string, ...string[]]) { + if (process.platform === "darwin") { + return { command: "/usr/bin/time", args: ["-l", ...command] }; + } + if (process.platform === "linux") { + return { command: "/usr/bin/time", args: ["-v", ...command] }; + } + return { command: command[0], args: command.slice(1) }; +} + +function parseMaxRssBytes(output: string) { + const macMatch = output.match(/(\d+)\s+maximum resident set size/u); + if (macMatch) { + return Number.parseInt(macMatch[1] ?? "", 10); + } + const linuxMatch = output.match(/Maximum resident set size \(kbytes\):\s*(\d+)/u); + if (linuxMatch) { + return Number.parseInt(linuxMatch[1] ?? "", 10) * 1024; + } + return null; +} + +/** + * Runs a command, captures text output, and terminates timed-out process groups. + */ +export function spawnText(command: string, args: readonly string[], options: SpawnTextOptions) { + const maxBuffer = options.maxBufferBytes ?? DEFAULT_SPAWN_OUTPUT_MAX_BYTES; + const maxLogBytes = options.maxLogBytes ?? DEFAULT_SPAWN_LOG_MAX_BYTES; + const tailBytes = options.outputTailBytes ?? DEFAULT_SPAWN_OUTPUT_TAIL_BYTES; + const timeoutMs = options.timeoutMs ?? DEFAULT_RUN_TIMEOUT_MS; + const killGraceMs = options.killGraceMs ?? DEFAULT_TIMEOUT_KILL_GRACE_MS; + const useProcessGroup = process.platform !== "win32"; + const logPath = options.logPath ?? null; + return new Promise<SpawnTextResult>((resolve) => { + let logFd: number | null = null; + if (logPath) { + fs.mkdirSync(path.dirname(logPath), { recursive: true }); + logFd = fs.openSync(logPath, "w"); + } + const child = spawn(command, args, { + cwd: options.cwd, + detached: useProcessGroup, + env: options.env, + stdio: ["ignore", "pipe", "pipe"], + }); + let outputBytes = 0; + let outputTail: Buffer = Buffer.alloc(0); + let stderrTail: Buffer = Buffer.alloc(0); + let streamedLogBytes = 0; + let outputExceeded = false; + let timedOut = false; + let settled = false; + let killTimer: TimerHandle | null = null; + let killGraceDeadline: number | null = null; + let killGraceMessage: string | null = null; + let childClosedResult: SpawnTextResult | null = null; + let waitingForKillGrace = false; + const signalChild = (signal: ProcessSignal) => + terminateManagedChild(child, signal as NodeJS.Signals, { + onChildSignalError(error) { + throw error; + }, + onProcessGroupSignalError(error) { + appendDiagnostic( + `[test-group-report] failed to send ${signal} to process group: ${coerceErrorMessage(error)}\n`, + ); + }, + taskkillTimeoutMs: null, + }); + const parentSignalHandlers: { signal: ProcessSignal; handler: () => void }[] = []; + const cleanupParentSignalHandlers = () => { + for (const { signal, handler } of parentSignalHandlers) { + process.off(signal, handler); + } + parentSignalHandlers.length = 0; + }; + const relayParentSignal = (signal: ProcessSignal) => { + const handler = () => { + signalChild(signal); + signalChild("SIGKILL"); + cleanupParentSignalHandlers(); + process.kill(process.pid, signal as NodeJS.Signals); + }; + parentSignalHandlers.push({ signal, handler }); + process.once(signal, handler); + }; + if (useProcessGroup) { + relayParentSignal("SIGINT"); + relayParentSignal("SIGTERM"); + relayParentSignal("SIGHUP"); + } else if (process.platform === "win32") { + relayParentSignal("SIGINT"); + relayParentSignal("SIGTERM"); + } + const processGroupIsAlive = () => + inspectManagedProcessGroup(child, { errorPolicy: "alive-on-eperm" }) === "live"; + const finishAfterProcessGroupCleanup = async (result: SpawnTextResult) => { + const graceRemainingMs = + killGraceDeadline === null ? killGraceMs : Math.max(0, killGraceDeadline - Date.now()); + if (graceRemainingMs > 0) { + await waitForManagedProcessGroupExit(child, graceRemainingMs, { + errorPolicy: "alive-on-eperm", + }); + } + if (settled) { + return; + } + if (killTimer) { + clearTimeout(killTimer); + killTimer = null; + } + waitingForKillGrace = false; + killGraceDeadline = null; + if (processGroupIsAlive()) { + appendDiagnostic(killGraceMessage ?? ""); + signalChild("SIGKILL"); + } + killGraceMessage = null; + childClosedResult = null; + finish(result); + }; + const scheduleKill = (message: string) => { + if (waitingForKillGrace) { + return; + } + waitingForKillGrace = true; + killGraceDeadline = Date.now() + killGraceMs; + killGraceMessage = message; + killTimer = setTimeout(() => { + waitingForKillGrace = false; + killTimer = null; + killGraceDeadline = null; + appendDiagnostic(killGraceMessage ?? message); + killGraceMessage = null; + signalChild("SIGKILL"); + if (childClosedResult) { + finish(childClosedResult); + } + }, killGraceMs); + killTimer.unref?.(); + }; + const timeoutTimer = setTimeout(() => { + timedOut = true; + appendDiagnostic(`\n[test-group-report] command timed out after ${String(timeoutMs)}ms\n`); + signalChild("SIGTERM"); + scheduleKill( + `[test-group-report] command did not exit after ${String(killGraceMs)}ms grace; sending SIGKILL\n`, + ); + }, timeoutMs); + timeoutTimer.unref?.(); + const finish = (result: SpawnTextResult) => { + if (settled) { + return; + } + settled = true; + clearTimeout(timeoutTimer); + cleanupParentSignalHandlers(); + if (killTimer) { + clearTimeout(killTimer); + } + if (logFd !== null) { + fs.closeSync(logFd); + logFd = null; + } + resolve(result); + }; + function appendTail(chunk: Buffer, target = "output") { + if (tailBytes < 1) { + return; + } + const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(String(chunk), "utf8"); + const currentTail = target === "stderr" ? stderrTail : outputTail; + if (buffer.byteLength >= tailBytes) { + if (target === "stderr") { + stderrTail = buffer.subarray(buffer.byteLength - tailBytes); + } else { + outputTail = buffer.subarray(buffer.byteLength - tailBytes); + } + return; + } + let nextTail = Buffer.concat([currentTail, buffer]); + if (nextTail.byteLength > tailBytes) { + nextTail = nextTail.subarray(nextTail.byteLength - tailBytes); + } + if (target === "stderr") { + stderrTail = nextTail; + } else { + outputTail = nextTail; + } + } + function appendDiagnostic(message: string) { + const buffer = Buffer.from(message, "utf8"); + if (logFd !== null) { + fs.writeSync(logFd, buffer); + appendTail(buffer); + return; + } + appendTail(buffer); + } + const appendOutput = (chunk: Buffer, streamName: string) => { + if (logFd !== null) { + if (outputExceeded) { + return; + } + const remainingLogBytes = maxLogBytes - streamedLogBytes; + const chunkToWrite = + chunk.byteLength > remainingLogBytes ? chunk.subarray(0, remainingLogBytes) : chunk; + if (chunkToWrite.byteLength > 0) { + fs.writeSync(logFd, chunkToWrite); + streamedLogBytes += chunkToWrite.byteLength; + appendTail(chunkToWrite); + if (streamName === "stderr") { + appendTail(chunkToWrite, "stderr"); + } + } + if (chunk.byteLength > remainingLogBytes) { + outputExceeded = true; + appendDiagnostic( + `\n[test-group-report] output log exceeded ${String(maxLogBytes)} bytes\n`, + ); + signalChild("SIGTERM"); + scheduleKill( + "[test-group-report] command did not exit after output log limit; sending SIGKILL\n", + ); + } + return; + } + if (outputExceeded) { + return; + } + + const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(String(chunk), "utf8"); + outputBytes += buffer.byteLength; + appendTail(buffer); + if (streamName === "stderr") { + appendTail(buffer, "stderr"); + } + if (outputBytes > maxBuffer) { + outputExceeded = true; + appendDiagnostic(`\n[test-group-report] output exceeded ${String(maxBuffer)} bytes\n`); + signalChild("SIGTERM"); + scheduleKill( + "[test-group-report] command did not exit after output limit; sending SIGKILL\n", + ); + } + }; + function streamedOutput() { + const tail = outputTail.toString("utf8"); + const stderr = stderrTail.toString("utf8"); + if (!stderr || tail.includes(stderr)) { + return tail; + } + return `${tail}\n${stderr}`; + } + child.stdout?.on("data", (chunk) => appendOutput(chunk, "stdout")); + child.stderr?.on("data", (chunk) => appendOutput(chunk, "stderr")); + child.on("error", (error) => { + appendDiagnostic(`${String(error)}\n`); + }); + child.on("close", (code, signal) => { + const result = { + status: outputExceeded || timedOut ? 1 : (code ?? 1), + signal, + output: streamedOutput(), + timedOut, + }; + if (waitingForKillGrace && processGroupIsAlive()) { + killTimer?.ref?.(); + childClosedResult = result; + void finishAfterProcessGroupCleanup(result); + return; + } + finish(result); + }); + }); +} + +async function runVitestJsonReport(params: RunVitestParams) { + fs.mkdirSync(path.dirname(params.reportPath), { recursive: true }); + fs.mkdirSync(path.dirname(params.logPath), { recursive: true }); + const command: [string, ...string[]] = [ + process.execPath, + "scripts/run-vitest.mjs", + "run", + "--config", + params.config, + "--reporter=json", + "--outputFile", + params.reportPath, + ...params.forwardedArgs, + ...params.vitestArgs, + ]; + const startedAt = process.hrtime.bigint(); + const spawnCommand = params.rss + ? resolveTimeArgs(command) + : { command: command[0], args: command.slice(1) }; + const result = await spawnText(spawnCommand.command, spawnCommand.args, { + cwd: process.cwd(), + env: { + ...process.env, + ...params.env, + // The JSON reporter can stay silent for the entire config. The profiler + // owns the wall-clock timeout and process-group cleanup for this child. + OPENCLAW_VITEST_NO_OUTPUT_TIMEOUT_MS: "0", + NODE_OPTIONS: [ + (params.env?.NODE_OPTIONS ?? process.env.NODE_OPTIONS)?.trim(), + ...resolveVitestNodeArgs({ ...process.env, ...params.env }).filter( + (arg) => arg !== "--no-maglev", + ), + ] + .filter(Boolean) + .join(" "), + }, + killGraceMs: params.killGraceMs, + logPath: params.logPath, + timeoutMs: params.timeoutMs, + }); + const elapsedMs = Number.parseFloat(String(process.hrtime.bigint() - startedAt)) / 1_000_000; + const output = result.output; + return { + config: params.config, + elapsedMs, + label: params.label, + logPath: params.logPath, + maxRssBytes: params.rss ? parseMaxRssBytes(output) : null, + reportPath: params.reportPath, + status: result.status, + }; +} + +function readReportInput(entry: ReportInputEntry) { + const report = JSON.parse(fs.readFileSync(entry.reportPath, "utf8")) as unknown; + if (!isRecord(report) || !Array.isArray(report.testResults)) { + throw new Error("missing testResults array"); + } + if (report.testResults.length === 0) { + throw new Error("empty testResults array"); + } + return { + config: entry.config, + report, + reportPath: entry.reportPath, + run: entry.run ?? null, + }; +} + +function readReportInputs(entries: ReportInputEntry[]) { + const invalid: Array<{ entry: ReportInputEntry; reason: string }> = []; + const missing: ReportInputEntry[] = []; + const reports: ReturnType<typeof readReportInput>[] = []; + for (const entry of entries) { + if (!fs.existsSync(entry.reportPath)) { + missing.push(entry); + continue; + } + try { + reports.push(readReportInput(entry)); + } catch (error) { + invalid.push({ + entry, + reason: coerceErrorMessage(error), + }); + } + } + return { invalid, missing, reports }; +} + +function readGroupedReport(reportPath: fs.PathOrFileDescriptor) { + const report = JSON.parse(fs.readFileSync(reportPath, "utf8")) as unknown; + validateGroupedReport(report, reportPath); + return report; +} + +function isFiniteNumber(value: unknown) { + return typeof value === "number" && Number.isFinite(value); +} + +function displayReportPath(reportPath: fs.PathOrFileDescriptor) { + return typeof reportPath === "string" || typeof reportPath === "number" + ? String(reportPath) + : reportPath.toString(); +} + +function validateCounter( + counter: unknown, + reportPath: fs.PathOrFileDescriptor, + fieldName: string, + index: number | null = null, +) { + const label = index === null ? fieldName : `${fieldName}[${index}]`; + const displayPath = displayReportPath(reportPath); + if (!isRecord(counter)) { + throw new Error( + `[test-group-report] invalid grouped report ${displayPath}: ${label} must be an object`, + ); + } + for (const key of ["durationMs", "fileCount", "testCount"]) { + if (!isFiniteNumber(counter[key])) { + throw new Error( + `[test-group-report] invalid grouped report ${displayPath}: ${label}.${key} must be a finite number`, + ); + } + } +} + +function validateCounterRows( + report: Record<string, unknown>, + reportPath: fs.PathOrFileDescriptor, + fieldName: string, +) { + const displayPath = displayReportPath(reportPath); + const rows = report[fieldName]; + if (!Array.isArray(rows)) { + throw new Error( + `[test-group-report] invalid grouped report ${displayPath}: ${fieldName} must be an array`, + ); + } + rows.forEach((row, index) => { + validateCounter(row, reportPath, fieldName, index); + if (!isRecord(row) || typeof row.key !== "string" || !row.key) { + throw new Error( + `[test-group-report] invalid grouped report ${displayPath}: ${fieldName}[${index}].key must be a non-empty string`, + ); + } + }); + return rows; +} + +function validateTopFileRows(report: Record<string, unknown>, reportPath: fs.PathOrFileDescriptor) { + const displayPath = displayReportPath(reportPath); + if (!Array.isArray(report.topFiles)) { + throw new Error( + `[test-group-report] invalid grouped report ${displayPath}: topFiles must be an array`, + ); + } + report.topFiles.forEach((row, index) => { + if (!isRecord(row)) { + throw new Error( + `[test-group-report] invalid grouped report ${displayPath}: topFiles[${index}] must be an object`, + ); + } + for (const key of ["config", "file", "group"]) { + if (typeof row[key] !== "string" || !row[key]) { + throw new Error( + `[test-group-report] invalid grouped report ${displayPath}: topFiles[${index}].${key} must be a non-empty string`, + ); + } + } + for (const key of ["durationMs", "testCount"]) { + if (!isFiniteNumber(row[key])) { + throw new Error( + `[test-group-report] invalid grouped report ${displayPath}: topFiles[${index}].${key} must be a finite number`, + ); + } + } + }); + return report.topFiles; +} + +function validateRunRows(report: Record<string, unknown>, reportPath: fs.PathOrFileDescriptor) { + const displayPath = displayReportPath(reportPath); + if (!Array.isArray(report.runs)) { + throw new Error( + `[test-group-report] invalid grouped report ${displayPath}: runs must be an array`, + ); + } + report.runs.forEach((row, index) => { + if (!isRecord(row)) { + throw new Error( + `[test-group-report] invalid grouped report ${displayPath}: runs[${index}] must be an object`, + ); + } + if (typeof row.config !== "string" && typeof row.label !== "string") { + throw new Error( + `[test-group-report] invalid grouped report ${displayPath}: runs[${index}] must include config or label`, + ); + } + if (!isFiniteNumber(row.elapsedMs) || !isFiniteNumber(row.status)) { + throw new Error( + `[test-group-report] invalid grouped report ${displayPath}: runs[${index}] must include finite elapsedMs and status`, + ); + } + if ( + row.maxRssBytes !== null && + row.maxRssBytes !== undefined && + !isFiniteNumber(row.maxRssBytes) + ) { + throw new Error( + `[test-group-report] invalid grouped report ${displayPath}: runs[${index}].maxRssBytes must be finite when present`, + ); + } + }); + return report.runs; +} + +function validateGroupedReport( + report: unknown, + reportPath: fs.PathOrFileDescriptor, +): asserts report is Record<string, unknown> & GroupedComparisonInput { + const displayPath = displayReportPath(reportPath); + if (!isRecord(report)) { + throw new Error( + `[test-group-report] invalid grouped report ${displayPath}: report must be an object`, + ); + } + if (report.command !== "test-group-report") { + throw new Error( + `[test-group-report] invalid grouped report ${displayPath}: command must be test-group-report`, + ); + } + if (typeof report.groupBy !== "string" || !["area", "folder", "top"].includes(report.groupBy)) { + throw new Error( + `[test-group-report] invalid grouped report ${displayPath}: groupBy must be area, folder, or top`, + ); + } + validateCounter(report.totals, reportPath, "totals"); + const groups = validateCounterRows(report, reportPath, "groups"); + const configs = validateCounterRows(report, reportPath, "configs"); + const topFiles = validateTopFileRows(report, reportPath); + if (!Array.isArray(report.slowTests)) { + throw new Error( + `[test-group-report] invalid grouped report ${displayPath}: slowTests must be an array`, + ); + } + const runs = validateRunRows(report, reportPath); + if (groups.length === 0 && configs.length === 0 && topFiles.length === 0 && runs.length === 0) { + throw new Error(`[test-group-report] invalid grouped report ${displayPath}: no evidence rows`); + } +} + +/** + * Resolves JSON report and per-run artifact directories from an output path. + */ +export function resolveReportArtifactDirs(outputPath: string) { + const outputDir = path.dirname(outputPath); + const outputExt = path.extname(outputPath); + const outputStem = path.basename(outputPath, outputExt) || "group-report"; + const artifactDir = path.join(outputDir, outputStem); + return { + reportDir: path.join(artifactDir, "vitest-json"), + logDir: path.join(artifactDir, "logs"), + }; +} + +function withUniqueLabels<Plan extends { label: string }>(plans: Plan[]) { + const totals = new Map<string, number>(); + for (const plan of plans) { + totals.set(plan.label, (totals.get(plan.label) ?? 0) + 1); + } + const seen = new Map<string, number>(); + return plans.map((plan) => { + const total = totals.get(plan.label) ?? 0; + if (total <= 1) { + return plan; + } + const index = (seen.get(plan.label) ?? 0) + 1; + seen.set(plan.label, index); + return { + ...plan, + label: `${plan.label}-${index}`, + }; + }); +} + +function buildFullSuiteLeafRunPlans() { + const previousLeafShards = process.env.OPENCLAW_TEST_PROJECTS_LEAF_SHARDS; + process.env.OPENCLAW_TEST_PROJECTS_LEAF_SHARDS = "1"; + try { + return buildFullSuiteVitestRunPlans([], process.cwd()); + } finally { + if (previousLeafShards === undefined) { + delete process.env.OPENCLAW_TEST_PROJECTS_LEAF_SHARDS; + } else { + process.env.OPENCLAW_TEST_PROJECTS_LEAF_SHARDS = previousLeafShards; + } + } +} + +/** + * Resolves explicit or full-suite Vitest config plans for report generation. + */ +export function resolveRunPlans(args: TestGroupReportArgs): TestGroupRunPlan[] { + if (args.reports.length > 0) { + return []; + } + if (args.fullSuite) { + return withUniqueLabels( + buildFullSuiteLeafRunPlans().map((plan) => ({ + config: plan.config, + forwardedArgs: plan.forwardedArgs ?? [], + label: normalizeConfigLabel(plan.config), + })), + ); + } + const configs = args.configs.length > 0 ? args.configs : ["test/vitest/vitest.unit.config.ts"]; + return configs.map((config) => ({ + config, + forwardedArgs: [], + label: normalizeConfigLabel(config), + })); +} + +/** + * Builds env for full-suite report runs, including per-config cache paths. + */ +export function resolveFullSuiteVitestEnv( + args: Pick<TestGroupReportArgs, "fullSuite">, + env: NodeJS.ProcessEnv = process.env, + label = "", +): NodeJS.ProcessEnv { + if ( + !args.fullSuite || + env.OPENCLAW_VITEST_MAX_WORKERS?.trim() || + env.OPENCLAW_TEST_WORKERS?.trim() + ) { + return {}; + } + + return { + OPENCLAW_VITEST_MAX_WORKERS: label === "commands" ? "1" : "2", + }; +} + +/** + * Resolves bounded concurrency for grouped report run plans. + */ +export function resolveRunPlanConcurrency( + args: Pick<TestGroupReportArgs, "concurrency" | "fullSuite">, + runPlanCount: number, +) { + if (runPlanCount <= 1) { + return 1; + } + if (args.concurrency !== null) { + return Math.min(args.concurrency, runPlanCount); + } + if (args.fullSuite) { + return 1; + } + return Math.min(2, runPlanCount); +} + +function hasExplicitIsolationArg(args: string[]) { + return args.some( + (arg) => arg === "--isolate" || arg === "--no-isolate" || arg.startsWith("--isolate="), + ); +} + +/** + * Gives full-suite duration reports one process lifetime per test file. + * This prevents unrelated retained module graphs and GC pauses from being + * attributed to whichever assertion happens to run next in a shared worker. + */ +export function resolveReportVitestArgs( + args: Pick<TestGroupReportArgs, "fullSuite" | "vitestArgs">, +) { + if (!args.fullSuite || hasExplicitIsolationArg(args.vitestArgs)) { + return args.vitestArgs; + } + return [...args.vitestArgs, "--isolate=true"]; +} + +/** + * Builds concrete report run specs from parsed args and config plans. + */ +export function resolveReportRunSpecs( + args: TestGroupReportArgs, + runPlans: TestGroupRunPlan[], + params: { concurrency?: number; env?: NodeJS.ProcessEnv; cwd?: string } = {}, +): TestGroupRunSpec[] { + const concurrency = params.concurrency ?? resolveRunPlanConcurrency(args, runPlans.length); + const env = params.env ?? process.env; + const vitestArgs = resolveReportVitestArgs(args); + const specs = runPlans.map((plan) => ({ + ...plan, + env: resolveFullSuiteVitestEnv(args, env, plan.label), + vitestArgs, + })); + if (concurrency <= 1) { + return specs; + } + return applyParallelVitestCachePaths(specs, { + cwd: params.cwd ?? process.cwd(), + env, + }); +} + +function printRunLine(run: TestGroupRun) { + console.log( + `[test-group-report] ${run.label} status=${run.status} wall=${formatMs(run.elapsedMs)} rss=${formatBytesAsMb(run.maxRssBytes)} report=${run.reportPath}`, + ); +} + +function printSlowTestsForRun(entry: ReportInputEntry, maxTestMs: number | null) { + if (maxTestMs === null || !fs.existsSync(entry.reportPath)) { + return; + } + const input = readReportInputs([entry]).reports[0]; + if (!input) { + return; + } + const report = buildGroupedTestReport({ + groupBy: "area", + maxTestMs, + reports: [input], + }); + for (const test of report.slowTests) { + console.log( + `[test-group-report] slow-test config=${test.config} duration=${formatMs(test.durationMs)} file=${test.file} name=${test.fullName}`, + ); + } +} + +export async function runReportPlans(params: { + args: TestGroupReportArgs; + logDir: string; + reportDir: string; + runPlans: TestGroupRunPlan[]; + runVitestJsonReport?: (params: RunVitestParams) => Promise<TestGroupRun>; +}) { + const concurrency = resolveRunPlanConcurrency(params.args, params.runPlans.length); + const runSpecs = resolveReportRunSpecs(params.args, params.runPlans, { concurrency }); + const runVitest = params.runVitestJsonReport ?? runVitestJsonReport; + let failed = false; + let exitCode = 0; + + const results = await pMap( + runSpecs, + async (plan) => { + if (exitCode !== 0) { + return null; + } + const slug = sanitizePathSegment(plan.label); + const run = await runVitest({ + config: plan.config, + forwardedArgs: plan.forwardedArgs, + env: plan.env, + label: plan.label, + logPath: path.join(params.logDir, `${slug}.log`), + reportPath: path.join(params.reportDir, `${slug}.json`), + rss: params.args.rss, + timeoutMs: params.args.timeoutMs, + killGraceMs: params.args.killGraceMs, + vitestArgs: plan.vitestArgs, + }); + printRunLine(run); + let includeEntry = true; + if (run.status !== 0) { + failed = true; + if (!fs.existsSync(run.reportPath)) { + console.error( + `[test-group-report] missing JSON report for failed config; see ${run.logPath}`, + ); + includeEntry = false; + } else { + try { + readReportInput({ config: plan.label, reportPath: run.reportPath, run }); + console.error( + `[test-group-report] config failed; keeping partial report from ${run.reportPath}`, + ); + } catch (error) { + const reason = coerceErrorMessage(error); + console.error( + `[test-group-report] config failed; skipping unusable JSON report from ${run.reportPath} (${reason})`, + ); + includeEntry = false; + } + } + if (!params.args.allowFailures) { + exitCode = run.status || 1; + } + } + const entry = includeEntry ? { config: plan.label, reportPath: run.reportPath, run } : null; + if (entry) { + printSlowTestsForRun(entry, params.args.maxTestMs); + } + return { entry, run }; + }, + { concurrency, stopOnError: true }, + ); + + return { + failed, + exitCode, + runEntries: results.flatMap((result) => (result?.entry ? [result.entry] : [])), + runs: results.flatMap((result) => (result ? [result.run] : [])), + }; +} + +async function main() { + const args = parseTestGroupReportArgs(process.argv.slice(2)); + if (args.help) { + console.log(usage()); + return; + } + + const output = path.resolve( + args.output ?? (args.compare ? DEFAULT_COMPARE_OUTPUT : DEFAULT_OUTPUT), + ); + + if (args.compare) { + const beforePath = path.resolve(args.compare.before); + const afterPath = path.resolve(args.compare.after); + const comparison = buildGroupedTestComparison({ + before: readGroupedReport(beforePath), + after: readGroupedReport(afterPath), + beforePath, + afterPath, + }); + + fs.mkdirSync(path.dirname(output), { recursive: true }); + fs.writeFileSync(output, `${JSON.stringify(comparison, null, 2)}\n`, "utf8"); + console.log( + renderGroupedTestComparison(comparison, { limit: args.limit, topFiles: args.topFiles }), + ); + console.log(`[test-group-report:compare] wrote ${path.relative(process.cwd(), output)}`); + return; + } + + const { reportDir, logDir } = resolveReportArtifactDirs(output); + const runEntries = []; + const runs = []; + const runPlans = resolveRunPlans(args); + let failed = false; + let exitCode = 0; + + for (const reportPath of args.reports) { + runEntries.push({ + config: path.basename(reportPath).replace(/\.json$/u, ""), + reportPath: path.resolve(reportPath), + }); + } + + if (runPlans.length > 0) { + const result = await runReportPlans({ args, logDir, reportDir, runPlans }); + failed = result.failed; + exitCode = result.exitCode; + runEntries.push(...result.runEntries); + runs.push(...result.runs); + } + + if (exitCode !== 0) { + process.exit(exitCode); + } + + const reportInputsResult = readReportInputs(runEntries); + if (reportInputsResult.missing.length > 0) { + for (const entry of reportInputsResult.missing) { + console.error( + `[test-group-report] missing JSON report for ${entry.config}: ${entry.reportPath}`, + ); + } + process.exit(1); + } + if (reportInputsResult.invalid.length > 0) { + for (const { entry, reason } of reportInputsResult.invalid) { + console.error( + `[test-group-report] invalid JSON report for ${entry.config}: ${entry.reportPath} (${reason})`, + ); + } + process.exit(1); + } + const reportInputs = reportInputsResult.reports; + if (reportInputs.length === 0) { + console.error("[test-group-report] no valid JSON reports were available"); + process.exit(1); + } + const report = buildGroupedTestReport({ + groupBy: args.groupBy, + maxTestMs: args.maxTestMs ?? undefined, + reports: reportInputs, + }); + const envelope = { + ...report, + command: "test-group-report", + failed, + runs: runs.length > 0 ? runs : reportInputs.map((entry) => entry.run).filter(Boolean), + system: { + node: process.version, + platform: process.platform, + arch: process.arch, + cpuCount: os.availableParallelism?.() ?? os.cpus().length, + totalMemoryBytes: os.totalmem(), + }, + }; + + fs.mkdirSync(path.dirname(output), { recursive: true }); + fs.writeFileSync(output, `${JSON.stringify(envelope, null, 2)}\n`, "utf8"); + console.log(renderGroupedTestReport(report, { limit: args.limit, topFiles: args.topFiles })); + console.log(`[test-group-report] wrote ${path.relative(process.cwd(), output)}`); + + if (args.maxTestMs !== null && report.slowTests.length > 0) { + console.error( + `[test-group-report] ${report.slowTests.length} tests exceeded ${formatMs(args.maxTestMs)}`, + ); + process.exit(1); + } + + if (failed && !args.allowFailures) { + process.exit(1); + } +} + +const isMain = + typeof process.argv[1] === "string" && + process.argv[1].length > 0 && + import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href; + +if (isMain) { + main().catch((error: unknown) => { + console.error(coerceErrorMessage(error)); + process.exit(1); + }); +} diff --git a/scripts/test-install-sh-docker.sh b/scripts/test-install-sh-docker.sh new file mode 100644 index 0000000000000000000000000000000000000000..15d90adfaba6b22aa72c2e1bc1cc6b4fc3ffaaa6 --- /dev/null +++ b/scripts/test-install-sh-docker.sh @@ -0,0 +1,739 @@ +#!/usr/bin/env bash +# Bash 5.3+ can deadlock writing heredoc pipes on macOS before the reader starts. +if [[ ${OSTYPE:-} == darwin* && $BASH != /bin/bash ]] && ((BASH_VERSINFO[0] > 5 || (BASH_VERSINFO[0] == 5 && BASH_VERSINFO[1] >= 3))); then + exec /bin/bash "$0" "$@" +fi +set -euo pipefail + +HARNESS_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +ROOT_DIR="${OPENCLAW_INSTALL_SMOKE_SOURCE_DIR:-$HARNESS_ROOT}" +ROOT_DIR="$(cd "$ROOT_DIR" && pwd)" +# shellcheck source=./docker/install-sh-common/version-parse.sh +source "$HARNESS_ROOT/scripts/docker/install-sh-common/version-parse.sh" +source "$HARNESS_ROOT/scripts/lib/docker-build.sh" +source "$HARNESS_ROOT/scripts/lib/docker-e2e-package.sh" +DOCKER_COMMAND_TIMEOUT="${DOCKER_COMMAND_TIMEOUT:-${OPENCLAW_INSTALL_SMOKE_DOCKER_COMMAND_TIMEOUT:-600s}}" +INSTALL_SMOKE_DOCKER_RUN_TIMEOUT="${OPENCLAW_INSTALL_SMOKE_DOCKER_RUN_TIMEOUT:-2700s}" + +normalize_npm_pack_json_file() { + local pack_json_file="$1" + node --input-type=module - "$pack_json_file" <<'NODE' +import fs from "node:fs"; + +const packJsonFile = process.argv[2]; +const parsed = JSON.parse(fs.readFileSync(packJsonFile, "utf8")); +let entries = [parsed]; +if (Array.isArray(parsed)) { + entries = parsed; +} else if (parsed && typeof parsed === "object") { + const looksLikeEntry = + (typeof parsed.id === "string") || + (typeof parsed.name === "string") || + (typeof parsed.version === "string") || + (typeof parsed.filename === "string"); + if (!looksLikeEntry) { + const keyedEntries = Object.values(parsed).filter( + (entry) => entry && typeof entry === "object" && !Array.isArray(entry), + ); + if (keyedEntries.length > 0) { + entries = keyedEntries; + } + } +} +if (entries.length === 0) { + throw new Error("npm pack output did not contain a package result"); +} +fs.writeFileSync(packJsonFile, `${JSON.stringify(entries, null, 2)}\n`, "utf8"); +NODE +} + +run_install_smoke_container() { + DOCKER_COMMAND_TIMEOUT="$INSTALL_SMOKE_DOCKER_RUN_TIMEOUT" docker_e2e_docker_run_cmd run "$@" +} + +resolve_default_smoke_platform() { + local host_arch + if [[ -n "${OPENCLAW_INSTALL_SMOKE_PLATFORM:-}" ]]; then + printf "%s" "$OPENCLAW_INSTALL_SMOKE_PLATFORM" + return + fi + host_arch="$(uname -m)" + if [[ "${CI:-}" == "true" || "${GITHUB_ACTIONS:-}" == "true" ]]; then + case "$host_arch" in + arm64 | aarch64) + printf "linux/arm64" + return + ;; + esac + printf "linux/amd64" + return + fi + case "$host_arch" in + arm64 | aarch64) + printf "linux/arm64" + ;; + *) + printf "linux/amd64" + ;; + esac +} + +print_pack_audit() { + local label="$1" + local pack_json_file="$2" + node -e ' +const raw = require("node:fs").readFileSync(process.argv[2], "utf8") || "[]"; +const label = process.argv[1]; +const parsed = JSON.parse(raw); +const last = Array.isArray(parsed) ? parsed.at(-1) : null; +if (!last) { + process.exit(1); +} +const formatBytes = (value) => { + if (!Number.isFinite(value)) return "unknown"; + const units = ["B", "KiB", "MiB", "GiB"]; + let current = value; + let unit = 0; + while (current >= 1024 && unit < units.length - 1) { + current /= 1024; + unit += 1; + } + return `${current.toFixed(unit === 0 ? 0 : 1)} ${units[unit]}`; +}; +const fileCount = Number.isFinite(last.entryCount) + ? last.entryCount + : Array.isArray(last.files) + ? last.files.length + : "unknown"; +console.log( + `==> Pack audit (${label}): version=${last.version ?? "unknown"} tgz=${formatBytes(last.size)} unpacked=${formatBytes(last.unpackedSize)} files=${fileCount}`, +); +' "$label" "$pack_json_file" +} + +assert_pack_unpacked_size_budget() { + local label="$1" + local pack_json_file="$2" + node --input-type=module - "$label" "$pack_json_file" <<'NODE' +import { readFileSync } from "node:fs"; + +const label = process.argv[2]; +const packJsonFile = process.argv[3]; +const raw = readFileSync(packJsonFile, "utf8") || "[]"; +const parsed = JSON.parse(raw); +const budgetOverride = process.env.OPENCLAW_INSTALL_SMOKE_PACK_UNPACKED_BUDGET_BYTES; +// Both bundled fs-safe loader layouts need all native targets (~31 MiB). +// Include that payload while retaining the previous package-size headroom. +const budgetBytes = budgetOverride ? Number(budgetOverride) : 235 * 1024 * 1024; +if (!Number.isFinite(budgetBytes)) { + throw new Error( + `OPENCLAW_INSTALL_SMOKE_PACK_UNPACKED_BUDGET_BYTES must be numeric, got ${JSON.stringify( + budgetOverride, + )}`, + ); +} +const entries = Array.isArray(parsed) ? parsed : [parsed]; +const errors = []; +let checkedCount = 0; +for (const [index, entry] of entries.entries()) { + if ( + !entry || + typeof entry !== "object" || + Array.isArray(entry) || + typeof entry.unpackedSize !== "number" || + !Number.isFinite(entry.unpackedSize) + ) { + continue; + } + checkedCount += 1; + if (entry.unpackedSize > budgetBytes) { + const resultLabel = + typeof entry.filename === "string" && entry.filename.trim() + ? entry.filename.trim() + : `pack result #${index + 1}`; + errors.push( + `${resultLabel} unpackedSize ${entry.unpackedSize} bytes exceeds budget ${budgetBytes} bytes. Investigate duplicate channel shims, copied extension trees, or other accidental pack bloat before release.`, + ); + } +} +if (entries.length > 0 && checkedCount === 0) { + errors.push( + `${label} npm pack output did not include unpackedSize; install smoke cannot verify pack budget.`, + ); +} +for (const error of errors) { + console.error(`ERROR: ${error}`); +} +if (errors.length > 0) { + process.exit(1); +} +NODE +} + +print_pack_delta_audit() { + local baseline_pack_json_file="$1" + local update_pack_json_file="$2" + node -e ' +const fs = require("node:fs"); +const [baselinePath, updatePath] = process.argv.slice(1); +const readLast = (path) => { + const parsed = JSON.parse(fs.readFileSync(path, "utf8") || "[]"); + return Array.isArray(parsed) ? parsed.at(-1) : null; +}; +const baseline = readLast(baselinePath); +const update = readLast(updatePath); +if (!baseline || !update) { + process.exit(1); +} +const formatSignedBytes = (value) => { + if (!Number.isFinite(value)) return "unknown"; + const sign = value > 0 ? "+" : value < 0 ? "-" : ""; + let current = Math.abs(value); + const units = ["B", "KiB", "MiB", "GiB"]; + let unit = 0; + while (current >= 1024 && unit < units.length - 1) { + current /= 1024; + unit += 1; + } + return `${sign}${current.toFixed(unit === 0 ? 0 : 1)} ${units[unit]}`; +}; +const fileCount = (entry) => + Number.isFinite(entry.entryCount) + ? entry.entryCount + : Array.isArray(entry.files) + ? entry.files.length + : undefined; +const baselineFiles = fileCount(baseline); +const updateFiles = fileCount(update); +const fileDelta = + Number.isFinite(baselineFiles) && Number.isFinite(updateFiles) + ? `${updateFiles - baselineFiles >= 0 ? "+" : ""}${updateFiles - baselineFiles}` + : "unknown"; +console.log( + `==> Pack audit delta (${baseline.version ?? "baseline"} -> ${update.version ?? "update"}): tgz=${formatSignedBytes((update.size ?? NaN) - (baseline.size ?? NaN))} unpacked=${formatSignedBytes((update.unpackedSize ?? NaN) - (baseline.unpackedSize ?? NaN))} files=${fileDelta}`, +); +' "$baseline_pack_json_file" "$update_pack_json_file" +} + +read_pack_tarball_filename() { + local pack_json_file="$1" + node -e ' +const fs = require("node:fs"); +const path = require("node:path"); +const raw = fs.readFileSync(process.argv[1], "utf8") || "[]"; +const parsed = JSON.parse(raw); +const last = Array.isArray(parsed) ? parsed.at(-1) : null; +const filename = typeof last?.filename === "string" ? last.filename.trim() : ""; +if ( + !filename.endsWith(".tgz") || + filename.includes("\0") || + filename !== path.basename(filename) || + filename !== path.win32.basename(filename) +) { + console.error(`ERROR: npm pack reported unsafe tarball filename ${JSON.stringify(filename)}`); + process.exit(1); +} +process.stdout.write(filename); +' "$pack_json_file" +} + +SMOKE_IMAGE="${OPENCLAW_INSTALL_SMOKE_IMAGE:-openclaw-install-smoke:local}" +NONROOT_IMAGE="${OPENCLAW_INSTALL_NONROOT_IMAGE:-openclaw-install-nonroot:local}" +SMOKE_PLATFORM="$(resolve_default_smoke_platform)" +NONROOT_PLATFORM="${OPENCLAW_INSTALL_NONROOT_PLATFORM:-$SMOKE_PLATFORM}" +INSTALL_URL="${OPENCLAW_INSTALL_URL:-https://openclaw.bot/install.sh}" +CLI_INSTALL_URL="${OPENCLAW_INSTALL_CLI_URL:-https://openclaw.bot/install-cli.sh}" +PACKAGE_NAME="${OPENCLAW_INSTALL_PACKAGE:-openclaw}" +INSTALL_SMOKE_GROUP="${OPENCLAW_INSTALL_SMOKE_GROUP:-all}" +RUN_UPDATE_GROUP=0 +RUN_NONROOT_GROUP=0 + +case "$INSTALL_SMOKE_GROUP" in + all) + RUN_UPDATE_GROUP=1 + RUN_NONROOT_GROUP=1 + ;; + update) + RUN_UPDATE_GROUP=1 + ;; + nonroot) + RUN_NONROOT_GROUP=1 + ;; + *) + echo "ERROR: OPENCLAW_INSTALL_SMOKE_GROUP must be all, update, or nonroot; got ${INSTALL_SMOKE_GROUP}" >&2 + exit 2 + ;; +esac + +SKIP_NONROOT="${OPENCLAW_INSTALL_SMOKE_SKIP_NONROOT:-0}" +SKIP_SMOKE_IMAGE_BUILD="${OPENCLAW_INSTALL_SMOKE_SKIP_IMAGE_BUILD:-0}" +SKIP_NONROOT_IMAGE_BUILD="${OPENCLAW_INSTALL_NONROOT_SKIP_IMAGE_BUILD:-0}" +SKIP_UPDATE="${OPENCLAW_INSTALL_SMOKE_SKIP_UPDATE:-0}" +SKIP_NPM_GLOBAL="${OPENCLAW_INSTALL_SMOKE_SKIP_NPM_GLOBAL:-0}" +SKIP_FRESHNESS="${OPENCLAW_INSTALL_SMOKE_SKIP_FRESHNESS:-0}" +FRESHNESS_INSTALL_URL="${OPENCLAW_INSTALL_SMOKE_FRESHNESS_INSTALL_URL:-file:///tmp/openclaw-install.sh}" +# npm min-release-age is days; 10000 keeps the control failure independent of normal release cadence. +FRESHNESS_MIN_RELEASE_AGE="${OPENCLAW_INSTALL_FRESHNESS_MIN_RELEASE_AGE:-10000}" +FRESHNESS_NPM_VERSION="${OPENCLAW_INSTALL_FRESHNESS_NPM_VERSION:-11.19.0}" +UPDATE_BASELINE_VERSION="${OPENCLAW_INSTALL_SMOKE_UPDATE_BASELINE:-latest}" +UPDATE_PACKAGE_SPEC="${OPENCLAW_INSTALL_SMOKE_UPDATE_PACKAGE_SPEC:-}" +UPDATE_DIST_IMAGE="${OPENCLAW_INSTALL_SMOKE_UPDATE_DIST_IMAGE:-}" +UPDATE_SKIP_LOCAL_BUILD="${OPENCLAW_INSTALL_SMOKE_UPDATE_SKIP_LOCAL_BUILD:-0}" +UPDATE_HOST_ALIAS="${OPENCLAW_INSTALL_SMOKE_UPDATE_HOST:-host.docker.internal}" +UPDATE_PORT="${OPENCLAW_INSTALL_SMOKE_UPDATE_PORT:-}" +UPDATE_EXPECT_VERSION="${OPENCLAW_INSTALL_SMOKE_UPDATE_EXPECT_VERSION:-}" +FROZEN_PAYLOAD_DIR="${OPENCLAW_INSTALL_SMOKE_FROZEN_PAYLOAD_DIR:-}" +FROZEN_NODE_VERSION="${OPENCLAW_INSTALL_SMOKE_NODE_VERSION:-}" +LATEST_DIR="$(mktemp -d)" +LATEST_FILE="${LATEST_DIR}/latest" +UPDATE_DIR="$(mktemp -d)" +UPDATE_SERVER_PID="" +UPDATE_SERVER_LOG="${UPDATE_DIR}/http.log" +UPDATE_TGZ_FILE="" +BASELINE_TGZ_FILE="" +BASELINE_TAG_URL="" +FRESH_TAG_URL="" +UPDATE_TAG_URL="" +UPDATE_DOCKER_HOST_ARGS=() +NPM_CACHE_DIR="${OPENCLAW_INSTALL_SMOKE_NPM_CACHE_DIR:-}" +NPM_CACHE_OWNED=0 +NPM_CACHE_PREPARED=0 +NPM_CACHE_DOCKER_ARGS=() +INSTALL_SCRIPT_PATH="$ROOT_DIR/scripts/install.sh" +CLI_INSTALL_SCRIPT_PATH="$ROOT_DIR/scripts/install-cli.sh" +SMOKE_RUNNER_ENV_ARGS=() + +require_regular_payload_file() { + local file_path="$1" + local label="$2" + if [[ ! -f "$file_path" || -L "$file_path" ]]; then + echo "ERROR: frozen install-smoke ${label} must be a regular file: ${file_path}" >&2 + exit 1 + fi +} + +if [[ -n "$FROZEN_PAYLOAD_DIR" ]]; then + FROZEN_PAYLOAD_DIR="$(cd "$FROZEN_PAYLOAD_DIR" && pwd)" + if [[ -n "$UPDATE_PACKAGE_SPEC" ]]; then + echo "ERROR: frozen install-smoke payload cannot be combined with OPENCLAW_INSTALL_SMOKE_UPDATE_PACKAGE_SPEC" >&2 + exit 1 + fi + require_regular_payload_file "$FROZEN_PAYLOAD_DIR/candidate.tgz" "package" + require_regular_payload_file "$FROZEN_PAYLOAD_DIR/candidate-pack.json" "package metadata" + require_regular_payload_file "$FROZEN_PAYLOAD_DIR/install.sh" "installer" + require_regular_payload_file "$FROZEN_PAYLOAD_DIR/install-cli.sh" "CLI installer" + if [[ -z "$UPDATE_EXPECT_VERSION" ]]; then + echo "ERROR: frozen install-smoke payload requires OPENCLAW_INSTALL_SMOKE_UPDATE_EXPECT_VERSION" >&2 + exit 1 + fi + if [[ ! "$FROZEN_NODE_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "ERROR: frozen install-smoke payload requires a trusted OPENCLAW_INSTALL_SMOKE_NODE_VERSION" >&2 + exit 1 + fi + INSTALL_SCRIPT_PATH="$FROZEN_PAYLOAD_DIR/install.sh" + CLI_INSTALL_SCRIPT_PATH="$FROZEN_PAYLOAD_DIR/install-cli.sh" +fi + +INSTALL_SCRIPT_DOCKER_ARGS=( + -v "$INSTALL_SCRIPT_PATH:/tmp/openclaw-install.sh:ro" + -v "$CLI_INSTALL_SCRIPT_PATH:/tmp/openclaw-install-cli.sh:ro" +) +if [[ -n "$FROZEN_PAYLOAD_DIR" ]]; then + INSTALL_SCRIPT_DOCKER_ARGS+=( + -e "OPENCLAW_NODE_VERSION=$FROZEN_NODE_VERSION" + ) +fi + +for env_name in \ + OPENCLAW_INSTALL_ALLOW_LEGACY_SAME_VERSION_APPLY \ + OPENCLAW_INSTALL_ALLOW_LEGACY_UPDATE_WARNING \ + OPENCLAW_INSTALL_SELF_UPDATE_WARNING_FIXED_VERSION \ + OPENCLAW_INSTALL_SMOKE_COMMAND_TIMEOUT \ + OPENCLAW_INSTALL_SMOKE_HEARTBEAT_INTERVAL \ + OPENCLAW_INSTALL_SMOKE_PREVIOUS \ + OPENCLAW_INSTALL_SMOKE_SKIP_PREVIOUS; do + env_value="${!env_name:-}" + if [[ -n "$env_value" && "$env_value" != "undefined" && "$env_value" != "null" ]]; then + SMOKE_RUNNER_ENV_ARGS+=(-e "$env_name") + fi +done + +remove_owned_npm_cache() { + if [[ "$NPM_CACHE_OWNED" != "1" || -z "$NPM_CACHE_DIR" || ! -d "$NPM_CACHE_DIR" ]]; then + return + fi + + if rm -rf "$NPM_CACHE_DIR" 2>/dev/null; then + return + fi + if command -v sudo >/dev/null 2>&1 && sudo -n rm -rf "$NPM_CACHE_DIR" 2>/dev/null; then + return + fi + + echo "WARN: failed to remove temporary npm cache: $NPM_CACHE_DIR" >&2 +} + +cleanup() { + if [[ -n "$UPDATE_SERVER_PID" ]]; then + kill "$UPDATE_SERVER_PID" >/dev/null 2>&1 || true + wait "$UPDATE_SERVER_PID" >/dev/null 2>&1 || true + fi + remove_owned_npm_cache || true + rm -rf "$LATEST_DIR" "$UPDATE_DIR" || true +} + +trap cleanup EXIT + +allocate_host_port() { + node -e ' + const net = require("node:net"); + const server = net.createServer(); + server.listen(0, "127.0.0.1", () => { + const address = server.address(); + if (!address || typeof address === "string") { + process.exit(1); + } + process.stdout.write(String(address.port)); + server.close(); + }); + ' +} + +ensure_local_update_dist_import_closure() { + if node "$HARNESS_ROOT/scripts/check-package-dist-imports.mjs" "$ROOT_DIR"; then + return 0 + fi + if [[ "$UPDATE_SKIP_LOCAL_BUILD" == "1" ]]; then + echo "ERROR: reused Docker image package dist failed import-closure check; exact-image mode forbids a local rebuild" >&2 + return 1 + fi + echo "WARN: reused Docker image dist failed import-closure check; rebuilding local release artifacts" >&2 + ( + cd "$ROOT_DIR" + pnpm build + ) +} + +read_candidate_version() { + node -e ' +const fs = require("node:fs"); +const packageJson = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); +process.stdout.write(packageJson.version); +' "$ROOT_DIR/package.json" +} + +prepare_update_tarball() { + local pack_json_file + local baseline_pack_json_file + local -a package_args + local package_tgz + local packed_update_version + pack_json_file="${UPDATE_DIR}/pack.json" + baseline_pack_json_file="${UPDATE_DIR}/baseline-pack.json" + if [[ -n "$FROZEN_PAYLOAD_DIR" ]]; then + # The producer already built and normalized candidate bytes inside an isolated pinned image. + # Privileged consumers only copy the verified artifact; they never build or import candidate code. + echo "==> Reuse frozen candidate payload for update smoke" + cp "$FROZEN_PAYLOAD_DIR/candidate.tgz" "${UPDATE_DIR}/candidate.tgz" + cp "$FROZEN_PAYLOAD_DIR/candidate-pack.json" "$pack_json_file" + UPDATE_TGZ_FILE="candidate.tgz" + elif [[ -n "$UPDATE_PACKAGE_SPEC" ]]; then + echo "==> Pack update tgz from spec: $UPDATE_PACKAGE_SPEC" + quiet_npm pack "$UPDATE_PACKAGE_SPEC" --json --pack-destination "$UPDATE_DIR" >"$pack_json_file" + normalize_npm_pack_json_file "$pack_json_file" + else + echo "==> Build local release artifacts for update smoke" + if [[ -n "$UPDATE_DIST_IMAGE" ]]; then + docker_e2e_restore_package_dist_from_image "$UPDATE_DIST_IMAGE" + ensure_local_update_dist_import_closure + elif [[ "$UPDATE_SKIP_LOCAL_BUILD" != "1" ]]; then + ( + cd "$ROOT_DIR" + pnpm build + ) + fi + UPDATE_EXPECT_VERSION="$(read_candidate_version)" + package_args=( + --source-dir "$ROOT_DIR" + --output-dir "$UPDATE_DIR" + --pack-json "$pack_json_file" + --skip-build + ) + if [[ "${OPENCLAW_INSTALL_SMOKE_ALLOW_UNRELEASED_CHANGELOG:-true}" == "true" ]]; then + package_args+=(--allow-unreleased-changelog) + fi + package_tgz="$( + node "$HARNESS_ROOT/scripts/package-openclaw-for-docker.mjs" "${package_args[@]}" + )" + UPDATE_TGZ_FILE="$(basename "$package_tgz")" + fi + if [[ -z "$FROZEN_PAYLOAD_DIR" ]]; then + if [[ -z "$UPDATE_PACKAGE_SPEC" ]]; then + node "$HARNESS_ROOT/scripts/check-openclaw-package-tarball.mjs" \ + --require-bundled-workspace-deps \ + "${UPDATE_DIR}/${UPDATE_TGZ_FILE}" + else + UPDATE_TGZ_FILE="$(read_pack_tarball_filename "$pack_json_file")" + fi + fi + print_pack_audit "update" "$pack_json_file" + assert_pack_unpacked_size_budget "update" "$pack_json_file" + packed_update_version="$( + node -e ' +const raw = require("node:fs").readFileSync(process.argv[1], "utf8") || "[]"; +const parsed = JSON.parse(raw); +const last = Array.isArray(parsed) ? parsed.at(-1) : null; +if (!last || typeof last.version !== "string" || last.version.length === 0) { + process.exit(1); +} +process.stdout.write(last.version); +' "$pack_json_file" + )" + if [[ -z "$UPDATE_EXPECT_VERSION" ]]; then + UPDATE_EXPECT_VERSION="$packed_update_version" + elif [[ "$UPDATE_EXPECT_VERSION" != "$packed_update_version" ]]; then + echo "ERROR: packed update version ${packed_update_version} does not match expected ${UPDATE_EXPECT_VERSION}" >&2 + exit 1 + fi + + echo "==> Pack baseline tgz: ${PACKAGE_NAME}@${UPDATE_BASELINE_VERSION}" + quiet_npm pack "${PACKAGE_NAME}@${UPDATE_BASELINE_VERSION}" --json --pack-destination "$UPDATE_DIR" >"$baseline_pack_json_file" + normalize_npm_pack_json_file "$baseline_pack_json_file" + BASELINE_TGZ_FILE="$(read_pack_tarball_filename "$baseline_pack_json_file")" + UPDATE_BASELINE_VERSION="$( + node -e ' +const raw = require("node:fs").readFileSync(process.argv[1], "utf8") || "[]"; +const parsed = JSON.parse(raw); +const last = Array.isArray(parsed) ? parsed.at(-1) : null; +if (!last || typeof last.version !== "string" || last.version.length === 0) { + process.exit(1); +} +process.stdout.write(last.version); +' "$baseline_pack_json_file" + )" + print_pack_audit "baseline" "$baseline_pack_json_file" + print_pack_delta_audit "$baseline_pack_json_file" "$pack_json_file" +} + +prepare_update_host_access() { + local host_os + host_os="$(uname -s)" + UPDATE_DOCKER_HOST_ARGS=() + if [[ "$host_os" == "Linux" ]]; then + UPDATE_DOCKER_HOST_ARGS=(--add-host "${UPDATE_HOST_ALIAS}:host-gateway") + fi +} + +prepare_npm_cache() { + if [[ "$NPM_CACHE_PREPARED" == "1" ]]; then + return + fi + if [[ -z "$NPM_CACHE_DIR" ]]; then + NPM_CACHE_DIR="$(mktemp -d)" + NPM_CACHE_OWNED=1 + fi + mkdir -p "$NPM_CACHE_DIR" + chmod 0777 "$NPM_CACHE_DIR" + NPM_CACHE_DOCKER_ARGS=( + -v "${NPM_CACHE_DIR}:/npm-cache" + -e npm_config_cache=/npm-cache + -e NPM_CONFIG_CACHE=/npm-cache + ) + NPM_CACHE_PREPARED=1 +} + +start_update_server() { + if [[ -z "$UPDATE_PORT" ]]; then + UPDATE_PORT="$(allocate_host_port)" + fi + BASELINE_TAG_URL="http://${UPDATE_HOST_ALIAS}:${UPDATE_PORT}/${BASELINE_TGZ_FILE}" + FRESH_TAG_URL="http://${UPDATE_HOST_ALIAS}:${UPDATE_PORT}/${UPDATE_TGZ_FILE}" + UPDATE_TAG_URL="http://${UPDATE_HOST_ALIAS}:${UPDATE_PORT}/${UPDATE_TGZ_FILE}" + echo "==> Serve baseline tgz: $BASELINE_TAG_URL" + echo "==> Serve latest tgz: $FRESH_TAG_URL" + ( + cd "$UPDATE_DIR" + exec python3 -m http.server "$UPDATE_PORT" --bind 0.0.0.0 + ) >"$UPDATE_SERVER_LOG" 2>&1 & + UPDATE_SERVER_PID=$! + sleep 1 + if ! kill -0 "$UPDATE_SERVER_PID" >/dev/null 2>&1; then + echo "ERROR: failed to start update tgz server" >&2 + tail -n 50 "$UPDATE_SERVER_LOG" >&2 || true + exit 1 + fi +} + +if [[ "$RUN_UPDATE_GROUP" == "1" ]]; then + if [[ "$SKIP_SMOKE_IMAGE_BUILD" == "1" ]]; then + echo "==> Reuse prebuilt smoke image: $SMOKE_IMAGE" + else + echo "==> Build smoke image (upgrade, root, ${SMOKE_PLATFORM}): $SMOKE_IMAGE" + docker_build_run install-smoke-build \ + --platform "$SMOKE_PLATFORM" \ + -t "$SMOKE_IMAGE" \ + -f "$HARNESS_ROOT/scripts/docker/install-sh-smoke/Dockerfile" \ + "$HARNESS_ROOT/scripts/docker" + fi + + if [[ "$SKIP_UPDATE" == "1" ]]; then + echo "==> Skip update smoke (OPENCLAW_INSTALL_SMOKE_SKIP_UPDATE=1)" + else + prepare_update_tarball + prepare_update_host_access + prepare_npm_cache + start_update_server + + echo "==> Run installer smoke test (root): $FRESH_TAG_URL" + run_install_smoke_container --rm -t \ + --platform "$SMOKE_PLATFORM" \ + ${UPDATE_DOCKER_HOST_ARGS[@]+"${UPDATE_DOCKER_HOST_ARGS[@]}"} \ + ${NPM_CACHE_DOCKER_ARGS[@]+"${NPM_CACHE_DOCKER_ARGS[@]}"} \ + "${INSTALL_SCRIPT_DOCKER_ARGS[@]}" \ + ${SMOKE_RUNNER_ENV_ARGS[@]+"${SMOKE_RUNNER_ENV_ARGS[@]}"} \ + -v "${LATEST_DIR}:/out" \ + -e OPENCLAW_INSTALL_URL="$INSTALL_URL" \ + -e OPENCLAW_INSTALL_PACKAGE="$PACKAGE_NAME" \ + -e OPENCLAW_INSTALL_METHOD=npm \ + -e OPENCLAW_INSTALL_FRESH_VERSION="$UPDATE_EXPECT_VERSION" \ + -e OPENCLAW_INSTALL_FRESH_TAG_URL="$FRESH_TAG_URL" \ + -e OPENCLAW_INSTALL_LATEST_OUT="/out/latest" \ + -e OPENCLAW_NO_ONBOARD=1 \ + -e OPENCLAW_NO_PROMPT=1 \ + -e DEBIAN_FRONTEND=noninteractive \ + "$SMOKE_IMAGE" + + LATEST_VERSION="" + if [[ -f "$LATEST_FILE" ]]; then + LATEST_VERSION="$(cat "$LATEST_FILE")" + fi + public_latest_version="$(quiet_npm view "$PACKAGE_NAME" version 2>/dev/null || true)" + if [[ -n "$public_latest_version" ]]; then + LATEST_VERSION="$public_latest_version" + fi + + echo "==> Run update smoke (${UPDATE_BASELINE_VERSION} -> ${UPDATE_EXPECT_VERSION})" + run_install_smoke_container --rm -t \ + --platform "$SMOKE_PLATFORM" \ + ${UPDATE_DOCKER_HOST_ARGS[@]+"${UPDATE_DOCKER_HOST_ARGS[@]}"} \ + ${NPM_CACHE_DOCKER_ARGS[@]+"${NPM_CACHE_DOCKER_ARGS[@]}"} \ + ${SMOKE_RUNNER_ENV_ARGS[@]+"${SMOKE_RUNNER_ENV_ARGS[@]}"} \ + -e OPENCLAW_INSTALL_PACKAGE="$PACKAGE_NAME" \ + -e OPENCLAW_INSTALL_SMOKE_MODE=update \ + -e OPENCLAW_INSTALL_UPDATE_BASELINE="$UPDATE_BASELINE_VERSION" \ + -e OPENCLAW_INSTALL_UPDATE_BASELINE_TAG_URL="$BASELINE_TAG_URL" \ + -e OPENCLAW_INSTALL_UPDATE_EXPECT_VERSION="$UPDATE_EXPECT_VERSION" \ + -e OPENCLAW_INSTALL_UPDATE_TAG_URL="$UPDATE_TAG_URL" \ + -e OPENCLAW_NO_ONBOARD=1 \ + -e OPENCLAW_NO_PROMPT=1 \ + -e DEBIAN_FRONTEND=noninteractive \ + "$SMOKE_IMAGE" + + if [[ "$SKIP_NPM_GLOBAL" == "1" ]]; then + echo "==> Skip direct npm global smoke (OPENCLAW_INSTALL_SMOKE_SKIP_NPM_GLOBAL=1)" + else + echo "==> Run direct npm global smoke (${UPDATE_BASELINE_VERSION} -> ${UPDATE_EXPECT_VERSION})" + run_install_smoke_container --rm -t \ + --platform "$SMOKE_PLATFORM" \ + ${UPDATE_DOCKER_HOST_ARGS[@]+"${UPDATE_DOCKER_HOST_ARGS[@]}"} \ + ${NPM_CACHE_DOCKER_ARGS[@]+"${NPM_CACHE_DOCKER_ARGS[@]}"} \ + ${SMOKE_RUNNER_ENV_ARGS[@]+"${SMOKE_RUNNER_ENV_ARGS[@]}"} \ + -e OPENCLAW_INSTALL_PACKAGE="$PACKAGE_NAME" \ + -e OPENCLAW_INSTALL_SMOKE_MODE=npm-global \ + -e OPENCLAW_INSTALL_UPDATE_BASELINE="$UPDATE_BASELINE_VERSION" \ + -e OPENCLAW_INSTALL_UPDATE_BASELINE_TAG_URL="$BASELINE_TAG_URL" \ + -e OPENCLAW_INSTALL_UPDATE_EXPECT_VERSION="$UPDATE_EXPECT_VERSION" \ + -e OPENCLAW_INSTALL_UPDATE_TAG_URL="$UPDATE_TAG_URL" \ + -e OPENCLAW_NO_ONBOARD=1 \ + -e OPENCLAW_NO_PROMPT=1 \ + -e DEBIAN_FRONTEND=noninteractive \ + "$SMOKE_IMAGE" + fi + fi + + if [[ "$SKIP_FRESHNESS" == "1" ]]; then + echo "==> Skip installer npm freshness smoke (OPENCLAW_INSTALL_SMOKE_SKIP_FRESHNESS=1)" + else + prepare_npm_cache + echo "==> Run installer npm freshness smoke" + run_install_smoke_container --rm -t \ + --platform "$SMOKE_PLATFORM" \ + ${NPM_CACHE_DOCKER_ARGS[@]+"${NPM_CACHE_DOCKER_ARGS[@]}"} \ + "${INSTALL_SCRIPT_DOCKER_ARGS[@]}" \ + ${SMOKE_RUNNER_ENV_ARGS[@]+"${SMOKE_RUNNER_ENV_ARGS[@]}"} \ + -e OPENCLAW_INSTALL_URL="$FRESHNESS_INSTALL_URL" \ + -e OPENCLAW_INSTALL_PACKAGE="$PACKAGE_NAME" \ + -e OPENCLAW_INSTALL_SMOKE_MODE=freshness \ + -e OPENCLAW_INSTALL_FRESHNESS_VERSION="${OPENCLAW_INSTALL_FRESHNESS_VERSION:-latest}" \ + -e OPENCLAW_INSTALL_FRESHNESS_MIN_RELEASE_AGE="$FRESHNESS_MIN_RELEASE_AGE" \ + -e OPENCLAW_INSTALL_FRESHNESS_NPM_VERSION="$FRESHNESS_NPM_VERSION" \ + -e OPENCLAW_NO_ONBOARD=1 \ + -e OPENCLAW_NO_PROMPT=1 \ + -e DEBIAN_FRONTEND=noninteractive \ + "$SMOKE_IMAGE" + fi +else + echo "==> Skip update installer smoke group" +fi + +LATEST_VERSION="${LATEST_VERSION:-}" + +if [[ "$RUN_NONROOT_GROUP" != "1" ]]; then + echo "==> Skip non-root installer smoke group" + exit 0 +fi + +if [[ -z "$LATEST_VERSION" ]]; then + echo "==> Resolve public npm version for non-root installer assertion" + LATEST_VERSION="$(quiet_npm view "$PACKAGE_NAME" version)" +fi + +if [[ "$SKIP_NONROOT" == "1" ]]; then + echo "==> Skip non-root installer smoke (OPENCLAW_INSTALL_SMOKE_SKIP_NONROOT=1)" +else + if [[ "$SKIP_NONROOT_IMAGE_BUILD" == "1" ]]; then + echo "==> Reuse prebuilt non-root image: $NONROOT_IMAGE" + else + echo "==> Build non-root image (${NONROOT_PLATFORM}): $NONROOT_IMAGE" + docker_build_run install-nonroot-build \ + --platform "$NONROOT_PLATFORM" \ + -t "$NONROOT_IMAGE" \ + -f "$HARNESS_ROOT/scripts/docker/install-sh-nonroot/Dockerfile" \ + "$HARNESS_ROOT/scripts/docker" + fi + + echo "==> Run installer non-root test: $INSTALL_URL" + run_install_smoke_container --rm -t \ + --platform "$NONROOT_PLATFORM" \ + "${INSTALL_SCRIPT_DOCKER_ARGS[@]}" \ + -e OPENCLAW_INSTALL_URL="$INSTALL_URL" \ + -e OPENCLAW_INSTALL_PACKAGE="$PACKAGE_NAME" \ + -e OPENCLAW_INSTALL_METHOD=npm \ + -e OPENCLAW_INSTALL_EXPECT_VERSION="$LATEST_VERSION" \ + -e OPENCLAW_NO_ONBOARD=1 \ + -e OPENCLAW_NO_PROMPT=1 \ + -e DEBIAN_FRONTEND=noninteractive \ + "$NONROOT_IMAGE" +fi + +if [[ "${OPENCLAW_INSTALL_SMOKE_SKIP_CLI:-0}" == "1" ]]; then + echo "==> Skip CLI installer smoke (OPENCLAW_INSTALL_SMOKE_SKIP_CLI=1)" + exit 0 +fi + +if [[ "$SKIP_NONROOT" == "1" ]]; then + echo "==> Skip CLI installer smoke (non-root image skipped)" + exit 0 +fi + +echo "==> Run CLI installer non-root test (same image)" +run_install_smoke_container --rm -t \ + --platform "$NONROOT_PLATFORM" \ + --entrypoint /bin/bash \ + "${INSTALL_SCRIPT_DOCKER_ARGS[@]}" \ + -e OPENCLAW_INSTALL_URL="$INSTALL_URL" \ + -e OPENCLAW_INSTALL_CLI_URL="$CLI_INSTALL_URL" \ + -e OPENCLAW_NO_ONBOARD=1 \ + -e OPENCLAW_NO_PROMPT=1 \ + -e DEBIAN_FRONTEND=noninteractive \ + "$NONROOT_IMAGE" -lc 'set -o pipefail; curl -fsSL --connect-timeout 30 --max-time 300 -- "$OPENCLAW_INSTALL_CLI_URL" | bash -s -- --set-npm-prefix --no-onboard' diff --git a/scripts/test-ios-shell-gateway.mjs b/scripts/test-ios-shell-gateway.mjs new file mode 100644 index 0000000000000000000000000000000000000000..775a6a7398effcf6962c90448df8516385276767 --- /dev/null +++ b/scripts/test-ios-shell-gateway.mjs @@ -0,0 +1,262 @@ +// Synthetic loopback Gateway for the native iOS approval-navigation UI tests. +// This fixture never executes commands or resolves approvals. +// First terminal, from the repository root: node scripts/test-ios-shell-gateway.mjs +// Second terminal: pnpm ios:gen, then run the two cases on a fresh owned simulator: +// TEST_RUNNER_OPENCLAW_IOS_LIVE_GATEWAY=1 \ +// TEST_RUNNER_OPENCLAW_IOS_LIVE_SETUP_CODE='{"url":"ws://127.0.0.1:19876","token":"synthetic-navigation-token"}' \ +// TEST_RUNNER_OPENCLAW_IOS_APPROVAL_FIXTURE_URL=http://127.0.0.1:19876 \ +// xcodebuild test -project apps/ios/OpenClaw.xcodeproj -scheme OpenClawUITests \ +// -destination 'platform=iOS Simulator,id=<owned-simulator-udid>' \ +// -derivedDataPath /tmp/openclaw-ios-navigation-proof -jobs 4 -parallel-testing-enabled NO \ +// -only-testing:OpenClawUITests/OpenClawSnapshotUITests/testLiveGatewayApprovalNotificationsFromOverview \ +// -only-testing:OpenClawUITests/OpenClawSnapshotUITests/testLiveGatewayApprovalNotificationsFromSettings +// TEST_RUNNER_ forwards these opt-in settings to XCTest; no real Gateway credentials are used. +import { createServer } from "node:http"; +import { WebSocketServer, WebSocket } from "ws"; +const requests = []; +let partial = false; +const created = Date.now(); +const approval = { + id: "navigation-proof-approval", + urlPath: "/approval/navigation-proof-approval", + createdAtMs: created, + expiresAtMs: created + 3_600_000, + status: "pending", + presentation: { + kind: "exec", + commandText: "echo navigation-proof", + commandPreview: "Synthetic navigation check — no command executes", + allowedDecisions: ["allow-once", "deny"], + agentId: "main", + host: "gateway", + }, +}; +const sessions = Array.from({ length: 205 }, (_, i) => ({ + key: i === 0 ? "agent:main:main" : `agent:main:navigation-${i}`, + displayName: i === 0 ? "Navigation proof" : `Synthetic session ${i}`, + label: i === 0 ? "Navigation proof" : `Synthetic session ${i}`, + kind: "direct", + updatedAt: created - i * 1000, + totalTokens: 100 + i, + inputTokens: 80 + i, + outputTokens: 20, +})); +const methods = [ + "health", + "config.get", + "agents.list", + "sessions.list", + "chat.history", + "voicewake.get", + "approval.get", + "approval.resolve", + "cron.list", + "cron.status", + "system-presence", + "node.list", + "sessions.subscribe", + "sessions.unsubscribe", + "session.status", + "models.list", + "sessions.preview", + "chat.send", +]; +const server = createServer((req, res) => { + res.setHeader("content-type", "application/json"); + if (req.url === "/approval") { + let count = 0; + for (const ws of wss.clients) { + if (ws.readyState === WebSocket.OPEN && ws.proofRole === "operator") { + ws.send( + JSON.stringify({ + type: "event", + event: "exec.approval.requested", + payload: { id: approval.id }, + }), + ); + count++; + } + } + res.end(JSON.stringify({ sent: count })); + } else if (req.url === "/partial") { + partial = true; + res.end(JSON.stringify({ partial })); + } else if (req.url === "/complete") { + partial = false; + res.end(JSON.stringify({ partial })); + } else { + res.end(JSON.stringify({ requests, connections: wss.clients.size, partial })); + } +}); +const wss = new WebSocketServer({ server }); +wss.on("connection", (ws) => { + ws.send( + JSON.stringify({ + type: "event", + event: "connect.challenge", + payload: { nonce: "synthetic-proof-nonce", ts: Date.now() }, + }), + ); + ws.on("message", (raw) => { + const data = Array.isArray(raw) ? Buffer.concat(raw) : Buffer.from(raw); + const req = JSON.parse(data.toString("utf8")); + if (req.type !== "req") { + return; + } + const params = req.params ?? {}; + requests.push({ + method: req.method, + offset: params.offset, + limit: params.limit, + role: params.role, + }); + const reply = (payload) => + ws.send(JSON.stringify({ type: "res", id: req.id, ok: true, payload })); + const fail = (message) => + ws.send( + JSON.stringify({ + type: "res", + id: req.id, + ok: false, + error: { code: "INVALID_REQUEST", message }, + }), + ); + console.log(JSON.stringify(requests.at(-1))); + switch (req.method) { + case "connect": { + ws.proofRole = params.role; + reply({ + type: "hello-ok", + protocol: 3, + server: { version: "navigation-proof", connId: "synthetic" }, + features: { methods, events: ["exec.approval.requested", "tick"] }, + snapshot: { + presence: [], + health: { ok: true }, + stateVersion: { presence: 1, health: 1 }, + uptimeMs: 1000, + sessionDefaults: { + defaultAgentId: "main", + mainKey: "main", + mainSessionKey: "agent:main:main", + scope: "per-sender", + }, + }, + auth: { + role: params.role, + scopes: params.scopes ?? [], + deviceToken: `synthetic-${params.role}`, + }, + policy: { maxPayload: 1048576, maxBufferedBytes: 1048576, tickIntervalMs: 30000 }, + }); + break; + } + case "health": + reply({ + ok: true, + ts: Date.now(), + durationMs: 1, + channels: {}, + agents: [], + sessions: { count: 205 }, + }); + break; + case "config.get": + reply({ + config: { + agents: { defaults: { model: { primary: "openai/gpt-5" } } }, + gateway: { mode: "local" }, + }, + hash: "synthetic", + valid: true, + }); + break; + case "agents.list": + reply({ + defaultId: "main", + mainKey: "main", + scope: "per-sender", + agents: [{ id: "main", name: "Navigation proof" }], + }); + break; + case "sessions.list": { + const offset = params.offset ?? 0; + if (partial && offset > 0) { + fail("Synthetic later-page failure"); + break; + } + const rows = sessions.slice(offset, offset + (params.limit ?? 200)); + reply({ + ts: Date.now(), + count: rows.length, + totalCount: sessions.length, + offset, + nextOffset: offset + rows.length, + hasMore: offset + rows.length < sessions.length, + defaults: {}, + sessions: rows, + }); + break; + } + case "chat.history": + reply({ + sessionKey: params.sessionKey ?? "agent:main:main", + sessionId: "synthetic-session", + messages: [], + }); + break; + case "voicewake.get": + reply({ triggers: [] }); + break; + case "approval.get": + reply({ approval }); + break; + case "approval.resolve": + fail("Navigation proof never executes or approves commands"); + break; + case "cron.list": + reply({ jobs: [] }); + break; + case "cron.status": + reply({ enabled: true, jobs: 0 }); + break; + case "system-presence": + reply([]); + break; + case "node.list": + reply({ nodes: [] }); + break; + case "sessions.subscribe": + case "sessions.unsubscribe": + reply({ ok: true }); + break; + case "models.list": + reply({ models: [] }); + break; + case "sessions.preview": + reply({ ts: Date.now(), previews: [] }); + break; + default: + fail(`Unsupported synthetic method: ${req.method}`); + } + }); +}); +const tick = setInterval(() => { + for (const ws of wss.clients) { + if (ws.readyState === WebSocket.OPEN) { + ws.send(JSON.stringify({ type: "event", event: "tick", payload: { ts: Date.now() } })); + } + } +}, 10_000); +server.listen(19876, "127.0.0.1", () => + console.log("Synthetic Gateway listening on loopback:19876"), +); +process.on("SIGINT", () => { + clearInterval(tick); + // close() waits for existing peers; a connected simulator must not keep this fixture alive. + for (const ws of wss.clients) { + ws.terminate(); + } + wss.close(); + server.close(); +}); diff --git a/scripts/test-ios-sidebar-attention-gateway.mjs b/scripts/test-ios-sidebar-attention-gateway.mjs new file mode 100644 index 0000000000000000000000000000000000000000..3ec96bd9b35df0908a51793097c30d91da1f652b --- /dev/null +++ b/scripts/test-ios-sidebar-attention-gateway.mjs @@ -0,0 +1,461 @@ +// Synthetic loopback Gateway for SidebarAttentionUITests. No provider or command execution. +// Run: node scripts/test-ios-sidebar-attention-gateway.mjs +// Forward these settings through xcodebuild's TEST_RUNNER_ environment prefix: +// OPENCLAW_IOS_ATTENTION_FIXTURE_URL=http://127.0.0.1:19877 +// OPENCLAW_IOS_LIVE_SETUP_CODE={"url":"ws://127.0.0.1:19877","token":"synthetic-attention-token"} +import { createServer } from "node:http"; +import { WebSocket, WebSocketServer } from "ws"; + +const port = 19877; +const mainKey = "agent:main:main"; +const parentKey = "agent:main:attention-parent"; +const reviewKey = "agent:main:attention-review"; +const requests = []; +let questions = []; +let approvals = []; +let created; + +function reset() { + created = Date.now(); + questions = ["Which draft should we review first?", "Should the summary include a timeline?"].map( + (question, index) => ({ + id: `attention-question-${index + 1}`, + questions: [ + { + questionId: "choice", + header: "Review", + question, + options: [{ label: "Draft A" }, { label: "Draft B" }], + }, + ...(index === 0 + ? [ + { + questionId: "reviewer", + header: "Reviewer", + question: "Who should review the appendix?", + options: [{ label: "Research team" }, { label: "Editor" }], + }, + ] + : []), + ], + agentId: "main", + sessionKey: reviewKey, + createdAtMs: created + index, + expiresAtMs: created + 3_600_000, + status: "pending", + }), + ); + approvals = ["exec", "exec", "plugin", "plugin", "system-agent", "system-agent"].map( + (kind, index) => { + const presentation = + kind === "exec" + ? { + kind, + commandText: "echo synthetic-review", + commandPreview: + index === 0 + ? "Inspect the synthetic review folder" + : "Summarize the synthetic notes", + allowedDecisions: ["allow-once", "deny"], + agentId: "main", + host: "gateway", + } + : kind === "plugin" + ? { + kind, + title: `Review synthetic plugin action ${index - 1}`, + description: "A synthetic plugin request. No external action runs.", + severity: "info", + pluginId: "synthetic", + toolName: "review", + allowedDecisions: ["allow-once", "deny"], + agentId: "main", + } + : { + kind, + title: `Review synthetic proposal ${index - 3}`, + description: "A synthetic proposal. No state outside the fixture changes.", + proposalHash: `synthetic-proposal-${index}`, + allowedDecisions: [], + agentId: "main", + }; + return { + id: `attention-approval-${index + 1}`, + urlPath: `/approval/attention-approval-${index + 1}`, + createdAtMs: created + 100 + index, + expiresAtMs: created + 3_600_000, + status: "pending", + sourceSessionKey: reviewKey, + presentation, + }; + }, + ); + requests.length = 0; +} +reset(); + +const methods = [ + "health", + "config.get", + "agents.list", + "sessions.list", + "chat.history", + "voicewake.get", + "question.list", + "question.get", + "question.resolve", + "exec.approval.list", + "plugin.approval.list", + "openclaw.approval.list", + "approval.get", + "approval.resolve", + "cron.list", + "cron.status", + "system-presence", + "node.list", + "sessions.subscribe", + "sessions.unsubscribe", + "session.status", + "models.list", + "sessions.preview", + "sessions.groups.list", +]; +const events = [ + "question.requested", + "question.resolved", + "exec.approval.requested", + "exec.approval.resolved", + "plugin.approval.resolved", + "openclaw.approval.resolved", + "tick", +]; +function broadcast(event, payload) { + for (const ws of wss.clients) { + if (ws.readyState === WebSocket.OPEN && ws.proofRole === "operator") { + ws.send(JSON.stringify({ type: "event", event, payload })); + } + } +} +function settleQuestions(status, id) { + for (const question of questions.filter( + (entry) => entry.status === "pending" && (id === undefined || entry.id === id), + )) { + question.status = status; + if (status === "answered") { + question.answers = { + answers: Object.fromEntries( + question.questions.map((item) => [item.questionId, [item.options[0].label]]), + ), + }; + } + broadcast("question.resolved", { + id: question.id, + status, + ...(question.answers ? { answers: question.answers } : {}), + }); + } +} +function addParentQuestion() { + const question = { + id: "attention-question-parent", + questions: [ + { + questionId: "page", + header: "Website", + question: "Which page should we refresh first?", + options: [{ label: "Home page" }, { label: "About page" }], + }, + ], + agentId: "main", + sessionKey: parentKey, + createdAtMs: created - 100, + expiresAtMs: created + 3_600_000, + status: "pending", + }; + questions.push(question); + broadcast("question.requested", question); +} +function settleApprovals(status, id) { + for (const approval of approvals.filter( + (entry) => entry.status === "pending" && (id === undefined || entry.id === id), + )) { + approval.status = status; + approval.resolvedAtMs = Date.now(); + approval.reason = status === "expired" ? "timeout" : "run-aborted"; + const family = + approval.presentation.kind === "system-agent" ? "openclaw" : approval.presentation.kind; + broadcast(`${family}.approval.resolved`, { + id: approval.id, + decision: "deny", + resolvedAtMs: approval.resolvedAtMs, + }); + } +} +const server = createServer((req, res) => { + res.setHeader("content-type", "application/json"); + const path = new URL(req.url, `http://127.0.0.1:${port}`).pathname; + if (req.method === "POST") { + if (path === "/reset") { + reset(); + } else if (path === "/questions/answer") { + settleQuestions("answered"); + } else if (path === "/questions/answer-oldest") { + settleQuestions("answered", questions[0].id); + } else if (path === "/questions/add-parent") { + addParentQuestion(); + } else if (path === "/questions/cancel-parent") { + settleQuestions("cancelled", "attention-question-parent"); + } else if (path === "/questions/expire-newer") { + settleQuestions("expired", questions[1].id); + } else if (path === "/questions/cancel") { + settleQuestions("cancelled"); + } else if (path === "/questions/expire") { + settleQuestions("expired"); + } else if (path === "/approvals/cancel") { + settleApprovals("cancelled"); + } else if (path === "/approvals/expire") { + settleApprovals("expired"); + } else if (path === "/approvals/expire-last") { + settleApprovals("expired", approvals.at(-1).id); + } else { + res.writeHead(404); + res.end("{}"); + return; + } + } + res.end( + JSON.stringify({ + requests, + questions: questions.map(({ id, status }) => ({ id, status })), + approvals: approvals.map(({ id, status }) => ({ id, status })), + connections: wss.clients.size, + }), + ); +}); +const wss = new WebSocketServer({ server }); +wss.on("connection", (ws) => { + ws.send( + JSON.stringify({ + type: "event", + event: "connect.challenge", + payload: { nonce: "synthetic-attention-nonce", ts: Date.now() }, + }), + ); + ws.on("message", (raw) => { + const req = JSON.parse(Buffer.from(raw).toString("utf8")); + if (req.type !== "req") { + return; + } + const params = req.params ?? {}; + requests.push({ method: req.method, sessionKey: params.sessionKey, id: params.id }); + console.log(JSON.stringify(requests.at(-1))); + const reply = (payload) => + ws.send(JSON.stringify({ type: "res", id: req.id, ok: true, payload })); + const fail = (message) => + ws.send( + JSON.stringify({ + type: "res", + id: req.id, + ok: false, + error: { code: "INVALID_REQUEST", message }, + }), + ); + if (req.method.endsWith(".approval.list")) { + const kind = req.method.startsWith("openclaw") ? "system-agent" : req.method.split(".")[0]; + reply( + approvals + .filter((entry) => entry.status === "pending" && entry.presentation.kind === kind) + .map((entry) => ({ + id: entry.id, + approvalKind: kind, + createdAtMs: entry.createdAtMs, + expiresAtMs: entry.expiresAtMs, + request: { sessionKey: entry.sourceSessionKey, agentId: "main" }, + })), + ); + return; + } + switch (req.method) { + case "connect": + ws.proofRole = params.role; + reply({ + type: "hello-ok", + protocol: 3, + server: { version: "synthetic-attention", connId: "synthetic" }, + features: { methods, events }, + snapshot: { + presence: [], + health: { ok: true }, + stateVersion: { presence: 1, health: 1 }, + uptimeMs: 1000, + sessionDefaults: { + defaultAgentId: "main", + mainKey: "main", + mainSessionKey: mainKey, + scope: "per-sender", + }, + }, + auth: { + role: params.role, + scopes: params.scopes ?? [], + deviceToken: `synthetic-${params.role}`, + }, + policy: { maxPayload: 1048576, maxBufferedBytes: 1048576, tickIntervalMs: 30000 }, + }); + break; + case "health": + reply({ + ok: true, + ts: Date.now(), + durationMs: 1, + channels: {}, + agents: [], + sessions: { count: 3 }, + }); + break; + case "config.get": + reply({ + config: { agents: { defaults: {} }, gateway: { mode: "local" } }, + hash: "synthetic", + valid: true, + }); + break; + case "agents.list": + reply({ + defaultId: "main", + mainKey: "main", + scope: "per-sender", + agents: [{ id: "main", name: "Research assistant" }], + }); + break; + case "sessions.list": + reply({ + ts: Date.now(), + count: 3, + totalCount: 3, + offset: 0, + nextOffset: 3, + hasMore: false, + defaults: {}, + sessions: [ + { + key: mainKey, + displayName: "Home", + label: "Home", + kind: "direct", + updatedAt: created, + totalTokens: 120, + }, + { + key: parentKey, + displayName: "Website refresh", + label: "Website refresh", + category: "Research", + kind: "direct", + childSessions: [reviewKey], + updatedAt: created - 30000, + totalTokens: 60, + }, + { + key: reviewKey, + displayName: "Pending review", + label: "Pending review", + category: "Research", + kind: "direct", + updatedAt: created - 60000, + totalTokens: 84, + }, + ], + }); + break; + case "chat.history": + reply({ + sessionKey: params.sessionKey ?? mainKey, + sessionId: "synthetic-session", + messages: [ + { + role: "assistant", + content: [{ type: "text", text: "Your research workspace is ready." }], + timestamp: created, + }, + ], + }); + break; + case "question.list": + reply({ questions: questions.filter((entry) => entry.status === "pending") }); + break; + case "question.get": + reply({ question: questions.find((entry) => entry.id === params.id) }); + break; + case "question.resolve": { + const question = questions.find((entry) => entry.id === params.id); + if (!question) { + fail("Unknown synthetic question"); + break; + } + question.status = params.cancel === true ? "cancelled" : "answered"; + if (question.status === "answered") { + question.answers = params.answers; + } + const result = { + id: question.id, + status: question.status, + ...(question.answers ? { answers: question.answers } : {}), + }; + reply(result); + broadcast("question.resolved", result); + break; + } + case "approval.get": + reply({ approval: approvals.find((entry) => entry.id === params.id) }); + break; + case "approval.resolve": + fail("Use the fixture lifecycle endpoints; no approval actions execute"); + break; + case "voicewake.get": + reply({ triggers: [] }); + break; + case "cron.list": + reply({ jobs: [] }); + break; + case "cron.status": + reply({ enabled: true, jobs: 0 }); + break; + case "system-presence": + reply([]); + break; + case "node.list": + reply({ nodes: [] }); + break; + case "sessions.subscribe": + case "sessions.unsubscribe": + reply({ ok: true }); + break; + case "models.list": + reply({ models: [] }); + break; + case "sessions.groups.list": + reply({ groups: [{ name: "Research", position: 0 }] }); + break; + case "sessions.preview": + reply({ ts: Date.now(), previews: [] }); + break; + default: + fail(`Unsupported synthetic method: ${req.method}`); + } + }); +}); +const tick = setInterval(() => broadcast("tick", { ts: Date.now() }), 10_000); +server.listen(port, "127.0.0.1", () => + console.log(`Synthetic attention Gateway listening on loopback:${port}`), +); +for (const signal of ["SIGINT", "SIGTERM"]) { + process.on(signal, () => { + clearInterval(tick); + for (const ws of wss.clients) { + ws.terminate(); + } + wss.close(); + server.close(); + }); +} diff --git a/scripts/test-live-acp-bind-docker.sh b/scripts/test-live-acp-bind-docker.sh new file mode 100644 index 0000000000000000000000000000000000000000..c7363141666effcc89937f3c6489604aae89c5f9 --- /dev/null +++ b/scripts/test-live-acp-bind-docker.sh @@ -0,0 +1,356 @@ +#!/usr/bin/env bash +# Bash 5.3+ can deadlock writing heredoc pipes on macOS before the reader starts. +if [[ ${OSTYPE:-} == darwin* && $BASH != /bin/bash ]] && ((BASH_VERSINFO[0] > 5 || (BASH_VERSINFO[0] == 5 && BASH_VERSINFO[1] >= 3))); then + exec /bin/bash "$0" "$@" +fi +set -euo pipefail + +SCRIPT_ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +ROOT_DIR="${OPENCLAW_LIVE_DOCKER_REPO_ROOT:-$SCRIPT_ROOT_DIR}" +ROOT_DIR="$(cd "$ROOT_DIR" && pwd)" +TRUSTED_HARNESS_DIR="${OPENCLAW_LIVE_DOCKER_TRUSTED_HARNESS_DIR:-$SCRIPT_ROOT_DIR}" +if [[ -z "$TRUSTED_HARNESS_DIR" || ! -d "$TRUSTED_HARNESS_DIR" ]]; then + echo "ERROR: trusted live Docker harness directory not found: ${TRUSTED_HARNESS_DIR:-<empty>}." >&2 + exit 1 +fi +TRUSTED_HARNESS_DIR="$(cd "$TRUSTED_HARNESS_DIR" && pwd)" +source "$TRUSTED_HARNESS_DIR/scripts/lib/live-docker-auth.sh" +IMAGE_NAME="${OPENCLAW_IMAGE:-openclaw:local}" +LIVE_IMAGE_NAME="${OPENCLAW_LIVE_IMAGE:-${IMAGE_NAME}-live}" +CONFIG_DIR="${OPENCLAW_CONFIG_DIR:-$HOME/.openclaw}" +WORKSPACE_DIR="${OPENCLAW_WORKSPACE_DIR:-$HOME/.openclaw/workspace}" +PROFILE_FILE="$(openclaw_live_default_profile_file)" +ACP_AGENT_LIST_RAW="${OPENCLAW_LIVE_ACP_BIND_AGENTS:-${OPENCLAW_LIVE_ACP_BIND_AGENT:-claude,codex,gemini}}" +ACP_CLAUDE_AUTH_MODE="${OPENCLAW_LIVE_ACP_BIND_CLAUDE_AUTH:-auto}" +ACP_SETUP_TIMEOUT_SECONDS="$(openclaw_live_read_positive_int_env OPENCLAW_LIVE_ACP_BIND_SETUP_TIMEOUT_SECONDS 180)" +DOCKER_TRUSTED_HARNESS_CONTAINER_DIR="/trusted-harness" +DOCKER_TRUSTED_HARNESS_MOUNT=(-v "$TRUSTED_HARNESS_DIR":"$DOCKER_TRUSTED_HARNESS_CONTAINER_DIR":ro) + +openclaw_live_acp_bind_append_build_extension() { + local extension="${1:?extension required}" + local current="${OPENCLAW_DOCKER_BUILD_EXTENSIONS:-${OPENCLAW_EXTENSIONS:-}}" + case " $current " in + *" $extension "*) + ;; + *) + export OPENCLAW_DOCKER_BUILD_EXTENSIONS="${current:+$current }$extension" + ;; + esac +} + +openclaw_live_acp_bind_resolve_auth_provider() { + case "${1:-}" in + claude) printf '%s\n' "claude-cli" ;; + codex) printf '%s\n' "codex-cli" ;; + droid) printf '%s\n' "droid" ;; + gemini) printf '%s\n' "google-gemini-cli" ;; + opencode) printf '%s\n' "opencode" ;; + *) + echo "Unsupported OPENCLAW_LIVE_ACP_BIND agent: ${1:-} (expected claude, codex, droid, gemini, or opencode)" >&2 + return 1 + ;; + esac +} + +openclaw_live_acp_bind_resolve_agent_command() { + case "${1:-}" in + claude) printf '%s' "${OPENCLAW_LIVE_ACP_BIND_AGENT_COMMAND_CLAUDE:-${OPENCLAW_LIVE_ACP_BIND_AGENT_COMMAND:-}}" ;; + codex) printf '%s' "${OPENCLAW_LIVE_ACP_BIND_AGENT_COMMAND_CODEX:-${OPENCLAW_LIVE_ACP_BIND_AGENT_COMMAND:-}}" ;; + droid) printf '%s' "${OPENCLAW_LIVE_ACP_BIND_AGENT_COMMAND_DROID:-${OPENCLAW_LIVE_ACP_BIND_AGENT_COMMAND:-}}" ;; + gemini) printf '%s' "${OPENCLAW_LIVE_ACP_BIND_AGENT_COMMAND_GEMINI:-${OPENCLAW_LIVE_ACP_BIND_AGENT_COMMAND:-}}" ;; + opencode) printf '%s' "${OPENCLAW_LIVE_ACP_BIND_AGENT_COMMAND_OPENCODE:-${OPENCLAW_LIVE_ACP_BIND_AGENT_COMMAND:-}}" ;; + *) return 1 ;; + esac +} + +case "$ACP_CLAUDE_AUTH_MODE" in + auto | api-key | subscription) + ;; + *) + echo "ERROR: OPENCLAW_LIVE_ACP_BIND_CLAUDE_AUTH must be one of: auto, api-key, subscription." >&2 + exit 1 + ;; +esac + +openclaw_live_init_temp_dirs +openclaw_live_init_cli_tools_dir +openclaw_live_init_cache_home_dir +openclaw_live_acp_bind_load_factory_api_key_from_profile() { + [[ -z "${FACTORY_API_KEY:-}" ]] || return 0 + [[ -f "$PROFILE_FILE" && -r "$PROFILE_FILE" ]] || return 0 + [[ "$PROFILE_FILE" != "$HOME/.profile" ]] || return 0 + + local line value + line="$(sed -nE 's/^(export[[:space:]]+)?FACTORY_API_KEY=//p' "$PROFILE_FILE" | tail -n 1 || true)" + [[ -n "$line" ]] || return 0 + value="$line" + if [[ "$value" == \"*\" && "$value" == *\" ]]; then + value="${value#\"}" + value="${value%\"}" + elif [[ "$value" == \'*\' && "$value" == *\' ]]; then + value="${value#\'}" + value="${value%\'}" + fi + [[ -n "$value" ]] || return 0 + export FACTORY_API_KEY="$value" +} + +read -r -d '' LIVE_TEST_CMD <<'EOF' || true +set -euo pipefail +[ -f "$HOME/.profile" ] && [ -r "$HOME/.profile" ] && source "$HOME/.profile" || true +export NPM_CONFIG_PREFIX="${NPM_CONFIG_PREFIX:-$HOME/.npm-global}" +export npm_config_prefix="$NPM_CONFIG_PREFIX" +export XDG_CACHE_HOME="${XDG_CACHE_HOME:-$HOME/.cache}" +export COREPACK_HOME="${COREPACK_HOME:-$XDG_CACHE_HOME/node/corepack}" +export NPM_CONFIG_CACHE="${NPM_CONFIG_CACHE:-$XDG_CACHE_HOME/npm}" +export npm_config_cache="$NPM_CONFIG_CACHE" +mkdir -p "$NPM_CONFIG_PREFIX" "$HOME/.local/bin" "$XDG_CACHE_HOME" "$COREPACK_HOME" "$NPM_CONFIG_CACHE" +chmod 700 "$XDG_CACHE_HOME" "$COREPACK_HOME" "$NPM_CONFIG_CACHE" || true +export PATH="$HOME/.local/bin:$NPM_CONFIG_PREFIX/bin:$PATH" +trusted_scripts_dir="${OPENCLAW_LIVE_DOCKER_SCRIPTS_DIR:-/src/scripts}" +source "$trusted_scripts_dir/lib/live-docker-stage.sh" +openclaw_live_stage_mounted_auth +run_setup_command() { + openclaw_live_run_setup_command \ + "${OPENCLAW_LIVE_ACP_BIND_SETUP_TIMEOUT_SECONDS:?missing live ACP bind setup timeout seconds}" \ + "live ACP bind setup" \ + "$@" +} +agent="${OPENCLAW_LIVE_ACP_BIND_AGENT:-claude}" +case "$agent" in + claude) + claude_auth_mode="${OPENCLAW_LIVE_ACP_BIND_CLAUDE_AUTH:-auto}" + if [ "$claude_auth_mode" = "subscription" ]; then + unset ANTHROPIC_API_KEY + unset ANTHROPIC_API_KEY_OLD + unset ANTHROPIC_API_TOKEN + unset ANTHROPIC_AUTH_TOKEN + unset ANTHROPIC_OAUTH_TOKEN + fi + # Resolve through ACPX so pnpm cannot select an unrelated root or hoisted SDK. + claude_code_version="$( + node -e 'const path = require("node:path"); const { createRequire } = require("node:module"); const acpxRequire = createRequire(path.resolve("extensions/acpx/package.json")); const adapterPackagePath = acpxRequire.resolve("@agentclientprotocol/claude-agent-acp/package.json"); const adapterRequire = createRequire(adapterPackagePath); const sdkEntry = adapterRequire.resolve("@anthropic-ai/claude-agent-sdk"); const packagePath = path.join(path.dirname(sdkEntry), "package.json"); process.stdout.write(require(packagePath).claudeCodeVersion);' + )" + claude_package_json="$NPM_CONFIG_PREFIX/lib/node_modules/@anthropic-ai/claude-code/package.json" + real_claude="$NPM_CONFIG_PREFIX/bin/claude-real" + installed_claude_code_version="" + if [ -f "$claude_package_json" ]; then + installed_claude_code_version="$( + node -e 'process.stdout.write(require(process.argv[1]).version);' \ + "$claude_package_json" 2>/dev/null || true + )" + fi + if [ "$installed_claude_code_version" != "$claude_code_version" ]; then + rm -f "$NPM_CONFIG_PREFIX/bin/claude" "$real_claude" + run_setup_command npm install -g "@anthropic-ai/claude-code@$claude_code_version" + fi + if [ ! -x "$real_claude" ] && [ -x "$NPM_CONFIG_PREFIX/bin/claude" ]; then + mv "$NPM_CONFIG_PREFIX/bin/claude" "$real_claude" + fi + if [ -x "$real_claude" ]; then + cat > "$NPM_CONFIG_PREFIX/bin/claude" <<WRAP +#!/usr/bin/env bash +script_dir="\$(CDPATH= cd -- "\$(dirname -- "\$0")" && pwd)" +if [ "\${OPENCLAW_LIVE_ACP_BIND_CLAUDE_AUTH:-auto}" = "subscription" ]; then + unset ANTHROPIC_API_KEY ANTHROPIC_API_KEY_OLD ANTHROPIC_API_TOKEN + unset ANTHROPIC_AUTH_TOKEN ANTHROPIC_OAUTH_TOKEN +else + if [ -n "\${OPENCLAW_LIVE_ACP_BIND_ANTHROPIC_API_KEY:-}" ]; then + export ANTHROPIC_API_KEY="\${OPENCLAW_LIVE_ACP_BIND_ANTHROPIC_API_KEY}" + fi + if [ -n "\${OPENCLAW_LIVE_ACP_BIND_ANTHROPIC_API_KEY_OLD:-}" ]; then + export ANTHROPIC_API_KEY_OLD="\${OPENCLAW_LIVE_ACP_BIND_ANTHROPIC_API_KEY_OLD}" + fi +fi +exec "\$script_dir/claude-real" "\$@" +WRAP + chmod +x "$NPM_CONFIG_PREFIX/bin/claude" + fi + export CLAUDE_CODE_EXECUTABLE="$NPM_CONFIG_PREFIX/bin/claude" + echo "Using Claude Code $claude_code_version declared by the ACPX-owned Claude Agent SDK" + claude --version + claude auth status || true + ;; + codex) + if [ ! -x "$NPM_CONFIG_PREFIX/bin/codex" ]; then + run_setup_command npm install -g @openai/codex + fi + ;; + droid) + if ! command -v droid >/dev/null 2>&1; then + run_setup_command bash -lc 'curl -fsSL https://app.factory.ai/cli | sh' + export PATH="$HOME/.local/bin:$PATH" + fi + droid --version + if [ -z "${FACTORY_API_KEY:-}" ]; then + echo "ERROR: Droid Docker ACP bind requires FACTORY_API_KEY; Factory OAuth/keyring auth in ~/.factory is not portable into the container." >&2 + exit 1 + fi + ;; + gemini) + mkdir -p "$HOME/.gemini" + if [ ! -x "$NPM_CONFIG_PREFIX/bin/gemini" ]; then + run_setup_command npm install -g @google/gemini-cli + fi + openclaw_live_stage_gemini_auth + ;; + opencode) + if [ ! -x "$NPM_CONFIG_PREFIX/bin/opencode" ]; then + run_setup_command npm install -g opencode-ai + fi + export OPENCODE_CONFIG_CONTENT="$( + node -e 'process.stdout.write(JSON.stringify({model: process.env.OPENCLAW_LIVE_ACP_BIND_OPENCODE_MODEL || "opencode/kimi-k2.6"}))' + )" + ;; + *) + echo "Unsupported OPENCLAW_LIVE_ACP_BIND_AGENT: $agent" >&2 + exit 1 + ;; +esac +tmp_dir="$(mktemp -d)" +openclaw_live_stage_source_tree "$tmp_dir" +openclaw_live_stage_node_modules "$tmp_dir" +openclaw_live_link_runtime_tree "$tmp_dir" +openclaw_live_stage_state_dir "$tmp_dir/.openclaw-state" +openclaw_live_prepare_staged_config +cd "$tmp_dir" +export OPENCLAW_LIVE_ACP_BIND_AGENT_COMMAND="${OPENCLAW_LIVE_ACP_BIND_AGENT_COMMAND:-}" +openclaw_live_run_staged_script scripts/test-live -- ${OPENCLAW_LIVE_ACP_BIND_TEST_FILES:-src/gateway/gateway-acp-bind.live.test.ts} +EOF + +openclaw_live_acp_bind_append_build_extension acpx +OPENCLAW_LIVE_DOCKER_REPO_ROOT="$ROOT_DIR" "$TRUSTED_HARNESS_DIR/scripts/test-live-build-docker.sh" + +IFS=',' read -r -a ACP_AGENT_TOKENS <<<"$ACP_AGENT_LIST_RAW" +ACP_AGENTS=() +for token in "${ACP_AGENT_TOKENS[@]}"; do + agent="$(openclaw_live_trim "$token")" + [[ -n "$agent" ]] || continue + openclaw_live_acp_bind_resolve_auth_provider "$agent" >/dev/null + ACP_AGENTS+=("$agent") +done + +if ((${#ACP_AGENTS[@]} == 0)); then + echo "No ACP bind agents selected. Use OPENCLAW_LIVE_ACP_BIND_AGENTS=claude,codex,droid,gemini,opencode." >&2 + exit 1 +fi + +for ACP_AGENT in "${ACP_AGENTS[@]}"; do + AUTH_PROVIDER="$(openclaw_live_acp_bind_resolve_auth_provider "$ACP_AGENT")" + AGENT_COMMAND="$(openclaw_live_acp_bind_resolve_agent_command "$ACP_AGENT")" + + openclaw_live_collect_auth_for_providers "$AUTH_PROVIDER" + DOCKER_AUTH_PRESTAGED=0 + openclaw_live_init_managed_home + openclaw_live_init_profile_mount + openclaw_live_finalize_auth_mounts + + if [[ "$ACP_AGENT" == "droid" ]]; then + openclaw_live_acp_bind_load_factory_api_key_from_profile + fi + if [[ "$ACP_AGENT" == "droid" && -z "${FACTORY_API_KEY:-}" ]]; then + echo "==> Run ACP bind live test in Docker" + echo "==> Agent: $ACP_AGENT" + echo "==> Profile file: $PROFILE_STATUS" + echo "==> Auth dirs: ${AUTH_DIRS_CSV:-none}" + echo "==> Auth files: ${AUTH_FILES_CSV:-none}" + echo "ERROR: Droid Docker ACP bind requires FACTORY_API_KEY; Factory OAuth/keyring auth in ~/.factory is not portable into the container." >&2 + exit 1 + fi + CLAUDE_AUTH_MODE="$ACP_CLAUDE_AUTH_MODE" + if [[ "$ACP_AGENT" == "claude" && "$CLAUDE_AUTH_MODE" == "auto" ]]; then + if [[ -n "${CLAUDE_CODE_OAUTH_TOKEN:-}" || -f "$HOME/.claude/.credentials.json" ]]; then + CLAUDE_AUTH_MODE="subscription" + else + CLAUDE_AUTH_MODE="api-key" + fi + fi + + echo "==> Run ACP bind live test in Docker" + echo "==> Agent: $ACP_AGENT" + echo "==> Test files: ${OPENCLAW_LIVE_ACP_BIND_TEST_FILES:-src/gateway/gateway-acp-bind.live.test.ts}" + echo "==> Profile file: $PROFILE_STATUS" + echo "==> Auth dirs: ${AUTH_DIRS_CSV:-none}" + echo "==> Auth files: ${AUTH_FILES_CSV:-none}" + if [[ "$ACP_AGENT" == "claude" ]]; then + echo "==> Claude auth mode: $CLAUDE_AUTH_MODE" + fi + if openclaw_live_uses_managed_bind_dirs; then + openclaw_live_chown_bind_dirs_for_container_user \ + "$LIVE_IMAGE_NAME" \ + "$DOCKER_USER" \ + "$CLI_TOOLS_DIR" \ + "$CACHE_HOME_DIR" \ + "${DOCKER_HOME_DIR:-}" + fi + DOCKER_RUN_ARGS=() + openclaw_live_init_docker_run_args DOCKER_RUN_ARGS "${OPENCLAW_LIVE_ACP_BIND_DOCKER_RUN_TIMEOUT:-2700s}" + DOCKER_AUTH_ENV=() + if [[ "$ACP_AGENT" == "claude" && "$CLAUDE_AUTH_MODE" == "subscription" ]]; then + DOCKER_AUTH_ENV+=( + -e CLAUDE_CODE_OAUTH_TOKEN="${CLAUDE_CODE_OAUTH_TOKEN:-}" + -e OPENCLAW_LIVE_ACP_BIND_CLAUDE_AUTH="$CLAUDE_AUTH_MODE" + ) + elif [[ "$ACP_AGENT" == "claude" ]]; then + DOCKER_AUTH_ENV+=( + -e ANTHROPIC_API_KEY + -e ANTHROPIC_API_KEY_OLD + -e OPENCLAW_LIVE_ACP_BIND_ANTHROPIC_API_KEY="${ANTHROPIC_API_KEY:-}" + -e OPENCLAW_LIVE_ACP_BIND_ANTHROPIC_API_KEY_OLD="${ANTHROPIC_API_KEY_OLD:-}" + -e OPENCLAW_LIVE_ACP_BIND_CLAUDE_AUTH="$CLAUDE_AUTH_MODE" + ) + fi + DOCKER_RUN_ARGS+=(--rm -t \ + -u "$DOCKER_USER" \ + --entrypoint bash \ + -e GEMINI_API_KEY \ + -e GOOGLE_API_KEY \ + -e FACTORY_API_KEY \ + -e OPENAI_API_KEY \ + -e CODEX_API_KEY \ + -e ACPX_AUTH_OPENAI_API_KEY \ + -e ACPX_AUTH_CODEX_API_KEY \ + -e OPENCODE_API_KEY \ + -e OPENCODE_ZEN_API_KEY \ + -e OPENCODE_CONFIG_CONTENT \ + -e COREPACK_ENABLE_DOWNLOAD_PROMPT=0 \ + -e HOME=/home/node \ + -e NODE_OPTIONS="$(openclaw_live_container_node_options)" \ + -e OPENCLAW_SKIP_CHANNELS=1 \ + -e OPENCLAW_VITEST_FS_MODULE_CACHE=0 \ + -e OPENCLAW_DOCKER_AUTH_PRESTAGED="$DOCKER_AUTH_PRESTAGED" \ + -e OPENCLAW_DOCKER_AUTH_DIRS_RESOLVED="$AUTH_DIRS_CSV" \ + -e OPENCLAW_DOCKER_AUTH_FILES_RESOLVED="$AUTH_FILES_CSV" \ + -e OPENCLAW_LIVE_DOCKER_SCRIPTS_DIR="${DOCKER_TRUSTED_HARNESS_CONTAINER_DIR}/scripts" \ + -e OPENCLAW_LIVE_DOCKER_SOURCE_STAGE_MODE="${OPENCLAW_LIVE_DOCKER_SOURCE_STAGE_MODE:-copy}" \ + -e OPENCLAW_LIVE_TEST=1 \ + -e OPENCLAW_LIVE_ACP_BIND=1 \ + -e OPENCLAW_LIVE_ACP_BIND_AGENT="$ACP_AGENT" \ + -e OPENCLAW_LIVE_ACP_BIND_REQUIRE_CRON="${OPENCLAW_LIVE_ACP_BIND_REQUIRE_CRON:-}" \ + -e OPENCLAW_LIVE_ACP_BIND_TEST_FILES="${OPENCLAW_LIVE_ACP_BIND_TEST_FILES:-}" \ + -e OPENCLAW_LIVE_ACP_BIND_CODEX_MODEL="${OPENCLAW_LIVE_ACP_BIND_CODEX_MODEL:-}" \ + -e OPENCLAW_LIVE_ACP_BIND_SETUP_TIMEOUT_SECONDS="$ACP_SETUP_TIMEOUT_SECONDS" \ + -e OPENCLAW_LIVE_ACP_BIND_OPENCODE_MODEL="${OPENCLAW_LIVE_ACP_BIND_OPENCODE_MODEL:-opencode/kimi-k2.6}" \ + -e OPENCLAW_LIVE_ACP_SPAWN_DEFAULTS="${OPENCLAW_LIVE_ACP_SPAWN_DEFAULTS:-}" \ + -e OPENCLAW_LIVE_ACP_SPAWN_DEFAULTS_AGENT="${OPENCLAW_LIVE_ACP_SPAWN_DEFAULTS_AGENT:-}" \ + -e OPENCLAW_LIVE_ACP_SPAWN_DEFAULTS_CONNECT_TIMEOUT_MS="${OPENCLAW_LIVE_ACP_SPAWN_DEFAULTS_CONNECT_TIMEOUT_MS:-}" \ + -e OPENCLAW_LIVE_ACP_SPAWN_DEFAULTS_MODEL="${OPENCLAW_LIVE_ACP_SPAWN_DEFAULTS_MODEL:-}" \ + -e OPENCLAW_LIVE_ACP_SPAWN_DEFAULTS_THINKING="${OPENCLAW_LIVE_ACP_SPAWN_DEFAULTS_THINKING:-}" \ + -e OPENCLAW_LIVE_ACP_SPAWN_DEFAULTS_TIMEOUT_MS="${OPENCLAW_LIVE_ACP_SPAWN_DEFAULTS_TIMEOUT_MS:-}" \ + -e OPENCLAW_LIVE_ACP_BIND_AGENT_COMMAND="$AGENT_COMMAND") + openclaw_live_append_array DOCKER_RUN_ARGS DOCKER_AUTH_ENV + openclaw_live_append_array DOCKER_RUN_ARGS DOCKER_HOME_MOUNT + openclaw_live_append_array DOCKER_RUN_ARGS DOCKER_TRUSTED_HARNESS_MOUNT + DOCKER_RUN_ARGS+=(\ + -v "$CACHE_HOME_DIR":/home/node/.cache \ + -v "$ROOT_DIR":/src:ro \ + -v "$CONFIG_DIR":/home/node/.openclaw \ + -v "$WORKSPACE_DIR":/home/node/.openclaw/workspace \ + -v "$CLI_TOOLS_DIR":/home/node/.npm-global) + openclaw_live_append_array DOCKER_RUN_ARGS EXTERNAL_AUTH_MOUNTS + openclaw_live_append_array DOCKER_RUN_ARGS PROFILE_MOUNT + DOCKER_RUN_ARGS+=(\ + "$LIVE_IMAGE_NAME" \ + -lc "$LIVE_TEST_CMD") + "${DOCKER_RUN_ARGS[@]}" +done diff --git a/scripts/test-live-codex-harness-docker.sh b/scripts/test-live-codex-harness-docker.sh new file mode 100644 index 0000000000000000000000000000000000000000..575b255203e3f1c38003b2a30bef7be93387425c --- /dev/null +++ b/scripts/test-live-codex-harness-docker.sh @@ -0,0 +1,393 @@ +#!/usr/bin/env bash +# Bash 5.3+ can deadlock writing heredoc pipes on macOS before the reader starts. +if [[ ${OSTYPE:-} == darwin* && $BASH != /bin/bash ]] && ((BASH_VERSINFO[0] > 5 || (BASH_VERSINFO[0] == 5 && BASH_VERSINFO[1] >= 3))); then + exec /bin/bash "$0" "$@" +fi +set -euo pipefail + +SCRIPT_ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +ROOT_DIR="${OPENCLAW_LIVE_DOCKER_REPO_ROOT:-$SCRIPT_ROOT_DIR}" +ROOT_DIR="$(cd "$ROOT_DIR" && pwd)" +TRUSTED_HARNESS_DIR="${OPENCLAW_LIVE_DOCKER_TRUSTED_HARNESS_DIR:-${OPENCLAW_LIVE_CODEX_TRUSTED_HARNESS_DIR:-$SCRIPT_ROOT_DIR}}" +if [[ -z "$TRUSTED_HARNESS_DIR" || ! -d "$TRUSTED_HARNESS_DIR" ]]; then + echo "ERROR: trusted Codex harness directory not found: ${TRUSTED_HARNESS_DIR:-<empty>}." >&2 + exit 1 +fi +TRUSTED_HARNESS_DIR="$(cd "$TRUSTED_HARNESS_DIR" && pwd)" +source "$TRUSTED_HARNESS_DIR/scripts/lib/live-docker-auth.sh" +IMAGE_NAME="${OPENCLAW_IMAGE:-openclaw:local}" +LIVE_IMAGE_NAME="${OPENCLAW_LIVE_IMAGE:-${IMAGE_NAME}-live}" +CONFIG_DIR="${OPENCLAW_CONFIG_DIR:-$HOME/.openclaw}" +WORKSPACE_DIR="${OPENCLAW_WORKSPACE_DIR:-$HOME/.openclaw/workspace}" +PROFILE_FILE="$(openclaw_live_default_profile_file)" +CODEX_HARNESS_AUTH_MODE="${OPENCLAW_LIVE_CODEX_HARNESS_AUTH:-codex-auth}" +CODEX_CLI_PACKAGE_SPEC="${OPENCLAW_LIVE_CODEX_CLI_PACKAGE_SPEC:-}" +CODEX_HARNESS_SETUP_TIMEOUT_SECONDS="$(openclaw_live_read_positive_int_env OPENCLAW_LIVE_CODEX_HARNESS_SETUP_TIMEOUT_SECONDS 180)" +CODEX_HARNESS_TARGET_COUNT=1 +if [[ -n "${OPENCLAW_LIVE_CODEX_HARNESS_TARGETS:-}" ]]; then + IFS=',' read -r -a CODEX_HARNESS_TARGET_ITEMS <<<"$OPENCLAW_LIVE_CODEX_HARNESS_TARGETS" + CODEX_HARNESS_TARGET_COUNT="${#CODEX_HARNESS_TARGET_ITEMS[@]}" +fi +# Each target starts an isolated 15-minute Vitest suite. Preserve the old +# 35-minute single-target budget while scaling matrix runs linearly. +CODEX_HARNESS_DOCKER_RUN_TIMEOUT="${OPENCLAW_LIVE_CODEX_HARNESS_DOCKER_RUN_TIMEOUT:-$((2100 * CODEX_HARNESS_TARGET_COUNT))s}" +DOCKER_TRUSTED_HARNESS_MOUNT=() +DOCKER_TRUSTED_HARNESS_CONTAINER_DIR="" +DOCKER_CACHE_CONTAINER_DIR="/tmp/openclaw-cache" +DOCKER_CLI_TOOLS_CONTAINER_DIR="/tmp/openclaw-npm-global" +DOCKER_EXTRA_ENV_FILES=() +DOCKER_AUTH_PRESTAGED=0 + +openclaw_live_codex_harness_append_build_extension() { + local extension="${1:?extension required}" + local current="${OPENCLAW_DOCKER_BUILD_EXTENSIONS:-${OPENCLAW_EXTENSIONS:-}}" + case " $current " in + *" $extension "*) + ;; + *) + export OPENCLAW_DOCKER_BUILD_EXTENSIONS="${current:+$current }$extension" + ;; + esac +} + +case "$CODEX_HARNESS_AUTH_MODE" in + codex-auth | api-key) + ;; + *) + echo "ERROR: OPENCLAW_LIVE_CODEX_HARNESS_AUTH must be one of: codex-auth, api-key." >&2 + exit 1 + ;; +esac + +if [[ -f "$PROFILE_FILE" && -r "$PROFILE_FILE" ]]; then + set -a + # shellcheck disable=SC1090 + source "$PROFILE_FILE" + set +a +fi + +if [[ "$CODEX_HARNESS_AUTH_MODE" == "api-key" && -z "${OPENAI_API_KEY:-}" ]]; then + echo "ERROR: OPENCLAW_LIVE_CODEX_HARNESS_AUTH=api-key requires OPENAI_API_KEY." >&2 + exit 1 +fi +if [[ "$CODEX_HARNESS_AUTH_MODE" != "api-key" && ! -s "$HOME/.codex/auth.json" ]]; then + echo "ERROR: OPENCLAW_LIVE_CODEX_HARNESS_AUTH=codex-auth requires ~/.codex/auth.json before building the live Docker image." >&2 + if [[ -n "${OPENAI_API_KEY:-}" ]]; then + echo "If this is a Testbox/API-key run, set OPENCLAW_LIVE_CODEX_HARNESS_AUTH=api-key and run through openclaw-testbox-env." >&2 + fi + exit 1 +fi +if [[ -z "$CODEX_CLI_PACKAGE_SPEC" ]]; then + CODEX_CLI_PACKAGE_SPEC="$( + node -e ' + const pkg = require(process.argv[1]); + const version = pkg.dependencies?.["@openai/codex"]; + if (!version || typeof version !== "string") process.exit(1); + process.stdout.write(`@openai/codex@${version}`); + ' "$ROOT_DIR/extensions/codex/package.json" + )" +fi + +openclaw_live_init_temp_dirs +openclaw_live_init_cli_tools_dir +openclaw_live_init_cache_home_dir +openclaw_live_init_managed_home +if [[ "$CODEX_HARNESS_AUTH_MODE" == "api-key" ]]; then + if [[ -z "${DOCKER_HOME_DIR:-}" ]]; then + DOCKER_HOME_DIR="$(mktemp -d "${RUNNER_TEMP:-/tmp}/openclaw-docker-home.XXXXXX")" + TEMP_DIRS+=("$DOCKER_HOME_DIR") + openclaw_live_prepare_bind_dir_for_container_user "$DOCKER_HOME_DIR" + DOCKER_HOME_MOUNT=(-v "$DOCKER_HOME_DIR":/home/node) + fi + CONFIG_DIR="$(mktemp -d "${RUNNER_TEMP:-/tmp}/openclaw-docker-config.XXXXXX")" + WORKSPACE_DIR="$(mktemp -d "${RUNNER_TEMP:-/tmp}/openclaw-docker-workspace.XXXXXX")" + TEMP_DIRS+=("$CONFIG_DIR" "$WORKSPACE_DIR") + chmod 0777 "$DOCKER_HOME_DIR" "$CONFIG_DIR" "$WORKSPACE_DIR" || true + DOCKER_CACHE_CONTAINER_DIR="/home/node/.cache" + DOCKER_CLI_TOOLS_CONTAINER_DIR="/home/node/.npm-global" +fi + +if [[ "$CODEX_HARNESS_AUTH_MODE" == "api-key" ]]; then + PROFILE_MOUNT=() + PROFILE_STATUS="api-key-env" +else + openclaw_live_init_profile_mount +fi + +DOCKER_TRUSTED_HARNESS_CONTAINER_DIR="/trusted-harness" +DOCKER_TRUSTED_HARNESS_MOUNT=(-v "$TRUSTED_HARNESS_DIR":"$DOCKER_TRUSTED_HARNESS_CONTAINER_DIR":ro) + +AUTH_FILES=() +if [[ "$CODEX_HARNESS_AUTH_MODE" != "api-key" ]]; then + while IFS= read -r auth_file; do + [[ -n "$auth_file" ]] || continue + AUTH_FILES+=("$auth_file") + done < <(openclaw_live_collect_auth_files_from_csv "openai") +fi + +AUTH_DIRS=() +openclaw_live_finalize_auth_mounts + +DOCKER_AUTH_ENV=() +if [[ "$CODEX_HARNESS_AUTH_MODE" == "api-key" ]]; then + docker_env_dir="$(mktemp -d "${RUNNER_TEMP:-/tmp}/openclaw-codex-harness-env.XXXXXX")" + TEMP_DIRS+=("$docker_env_dir") + docker_env_file="$docker_env_dir/openai.env" + { + printf 'OPENAI_API_KEY=%s\n' "${OPENAI_API_KEY}" + printf 'CODEX_API_KEY=%s\n' "${CODEX_API_KEY:-$OPENAI_API_KEY}" + if [[ -n "${OPENAI_BASE_URL:-}" ]]; then + printf 'OPENAI_BASE_URL=%s\n' "${OPENAI_BASE_URL}" + fi + } >"$docker_env_file" + DOCKER_EXTRA_ENV_FILES+=(--env-file "$docker_env_file") +fi + +read -r -d '' LIVE_TEST_CMD <<'EOF' || true +set -euo pipefail +[ -f "$HOME/.profile" ] && [ -r "$HOME/.profile" ] && source "$HOME/.profile" || true +export NPM_CONFIG_PREFIX="${NPM_CONFIG_PREFIX:-$HOME/.npm-global}" +export npm_config_prefix="$NPM_CONFIG_PREFIX" +export XDG_CACHE_HOME="${XDG_CACHE_HOME:-$HOME/.cache}" +export COREPACK_HOME="${COREPACK_HOME:-$XDG_CACHE_HOME/node/corepack}" +export NPM_CONFIG_CACHE="${NPM_CONFIG_CACHE:-$XDG_CACHE_HOME/npm}" +export npm_config_cache="$NPM_CONFIG_CACHE" +cleanup_codex_live_mounts() { + chmod -R a+rwX "$HOME" "$NPM_CONFIG_PREFIX" "$XDG_CACHE_HOME" 2>/dev/null || true +} +trap cleanup_codex_live_mounts EXIT +if [ "${OPENCLAW_LIVE_CODEX_HARNESS_DEBUG:-}" = "1" ]; then + id + mount | grep -E 'openclaw-cache|openclaw-npm|/home/node' || true + ls -ld "$HOME" "$XDG_CACHE_HOME" "$NPM_CONFIG_PREFIX" 2>/dev/null || true +fi +# Force the Codex harness to use the staged `~/.codex` auth files. This lane +# is not meant to exercise raw OpenAI API-key routing unless the lane +# explicitly opts into API-key auth for CI. +if [ "${OPENCLAW_LIVE_CODEX_HARNESS_AUTH:-codex-auth}" != "api-key" ]; then + unset OPENAI_API_KEY OPENAI_BASE_URL +fi +mkdir -p "$NPM_CONFIG_PREFIX" "$XDG_CACHE_HOME" "$COREPACK_HOME" "$NPM_CONFIG_CACHE" +chmod 700 "$XDG_CACHE_HOME" "$COREPACK_HOME" "$NPM_CONFIG_CACHE" || true +export PATH="$NPM_CONFIG_PREFIX/bin:$PATH" +trusted_scripts_dir="${OPENCLAW_LIVE_DOCKER_SCRIPTS_DIR:-/src/scripts}" +source "$trusted_scripts_dir/lib/live-docker-stage.sh" +openclaw_live_stage_mounted_auth +run_setup_command() { + openclaw_live_run_setup_command \ + "${OPENCLAW_LIVE_CODEX_HARNESS_SETUP_TIMEOUT_SECONDS:?missing live Codex harness setup timeout seconds}" \ + "live Codex harness setup" \ + "$@" +} +if [ "${OPENCLAW_LIVE_CODEX_HARNESS_AUTH:-codex-auth}" != "api-key" ] && [ ! -s "$HOME/.codex/auth.json" ]; then + echo "ERROR: missing ~/.codex/auth.json for Codex harness live test." >&2 + exit 1 +fi +if [ "${OPENCLAW_LIVE_CODEX_HARNESS_AUTH:-codex-auth}" != "api-key" ]; then + node --import tsx "$trusted_scripts_dir/prepare-codex-ci-auth.ts" "$HOME/.codex/auth.json" +fi +run_setup_command npm install -g "$OPENCLAW_LIVE_CODEX_CLI_PACKAGE_SPEC" +"$NPM_CONFIG_PREFIX/bin/codex" --version +if [ "${OPENCLAW_LIVE_CODEX_HARNESS_AUTH:-codex-auth}" = "api-key" ]; then + printf '%s\n' "$OPENAI_API_KEY" | "$NPM_CONFIG_PREFIX/bin/codex" login --with-api-key >/dev/null +fi +tmp_dir="$(mktemp -d)" +openclaw_live_stage_source_tree "$tmp_dir" +openclaw_live_stage_node_modules "$tmp_dir" +openclaw_live_link_runtime_tree "$tmp_dir" +if [ ! -f "$tmp_dir/extensions/codex/openclaw.plugin.json" ]; then + echo "ERROR: staged Codex plugin not found for live harness." >&2 + exit 1 +fi +# Source Gateway and plugin must share one prepared-runtime owner; built artifacts own a +# separate lifecycle and are validated by the packaged-plugin Docker lane instead. +export OPENCLAW_BUNDLED_PLUGINS_DIR="$tmp_dir/extensions" +openclaw_live_stage_state_dir "$tmp_dir/.openclaw-state" +if [ -n "${OPENCLAW_LIVE_CODEX_TRUSTED_HARNESS_DIR:-}" ] && [ -d "$OPENCLAW_LIVE_CODEX_TRUSTED_HARNESS_DIR" ]; then + for harness_file in src/gateway/gateway-codex-harness.live-helpers.ts; do + if [ -f "$OPENCLAW_LIVE_CODEX_TRUSTED_HARNESS_DIR/$harness_file" ]; then + mkdir -p "$(dirname "$tmp_dir/$harness_file")" + cp "$OPENCLAW_LIVE_CODEX_TRUSTED_HARNESS_DIR/$harness_file" "$tmp_dir/$harness_file" + fi + done +fi +openclaw_live_prepare_staged_config +cd "$tmp_dir" +if [ "${OPENCLAW_LIVE_CODEX_HARNESS_USE_CI_SAFE_CODEX_CONFIG:-1}" = "1" ]; then + node --import tsx "$trusted_scripts_dir/prepare-codex-ci-config.ts" "$HOME/.codex/config.toml" "$tmp_dir" +fi +codex_preflight_log="$tmp_dir/codex-preflight.log" +codex_preflight_token="CODEX-PREFLIGHT-OK" +if ! "$NPM_CONFIG_PREFIX/bin/codex" exec \ + --json \ + --color never \ + --skip-git-repo-check \ + "Reply exactly: $codex_preflight_token" >"$codex_preflight_log" 2>&1; then + if grep -q "Failed to extract accountId from token" "$codex_preflight_log"; then + echo "ERROR: Codex auth cannot extract accountId from the available token; refresh OPENCLAW_CODEX_AUTH_JSON or use OPENCLAW_LIVE_CODEX_HARNESS_AUTH=api-key." >&2 + exit 1 + fi + tail -c 262144 "$codex_preflight_log" >&2 || true + exit 1 +fi +run_codex_harness_target() { + local model="${1:?model required}" + local thinking="${2:?thinking required}" + export OPENCLAW_LIVE_CODEX_HARNESS_MODEL="$model" + export OPENCLAW_LIVE_CODEX_HARNESS_THINKING="$thinking" + echo "==> Codex harness target: model=$model thinking=$thinking" + openclaw_live_run_staged_script scripts/test-live -- ${OPENCLAW_LIVE_CODEX_TEST_FILES:-src/gateway/gateway-codex-harness.live.test.ts} +} +if [ -n "${OPENCLAW_LIVE_CODEX_HARNESS_TARGETS:-}" ]; then + IFS=',' read -r -a harness_targets <<<"$OPENCLAW_LIVE_CODEX_HARNESS_TARGETS" + for harness_target in "${harness_targets[@]}"; do + model="${harness_target%%=*}" + thinking="${harness_target##*=}" + if [ -z "$model" ] || [ -z "$thinking" ] || [ "$model" = "$thinking" ]; then + echo "ERROR: invalid Codex harness target '$harness_target'; expected provider/model=thinking." >&2 + exit 1 + fi + run_codex_harness_target "$model" "$thinking" + done +else + run_codex_harness_target \ + "${OPENCLAW_LIVE_CODEX_HARNESS_MODEL:-openai/gpt-5.6-luna}" \ + "${OPENCLAW_LIVE_CODEX_HARNESS_THINKING:-low}" +fi +EOF + +openclaw_live_codex_harness_append_build_extension codex +# The release package image intentionally excludes externalized plugins such as +# Codex. This lane must rebuild the live image so the plugin-owned harness is +# present under the bundled plugin runtime directory. +OPENCLAW_SKIP_DOCKER_BUILD=0 +export OPENCLAW_SKIP_DOCKER_BUILD +OPENCLAW_LIVE_DOCKER_REPO_ROOT="$ROOT_DIR" "$TRUSTED_HARNESS_DIR/scripts/test-live-build-docker.sh" +if openclaw_live_uses_managed_bind_dirs; then + openclaw_live_chown_bind_dirs_for_container_user \ + "$LIVE_IMAGE_NAME" \ + "$DOCKER_USER" \ + "$CLI_TOOLS_DIR" \ + "$CACHE_HOME_DIR" \ + "$CONFIG_DIR" \ + "$WORKSPACE_DIR" \ + "${DOCKER_HOME_DIR:-}" +fi + +echo "==> Run Codex harness live test in Docker" +echo "==> Model: ${OPENCLAW_LIVE_CODEX_HARNESS_MODEL:-openai/gpt-5.6-luna}" +echo "==> Thinking: ${OPENCLAW_LIVE_CODEX_HARNESS_THINKING:-low}" +echo "==> Expected native effort: ${OPENCLAW_LIVE_CODEX_HARNESS_EXPECTED_EFFORT:-auto}" +echo "==> Targets: ${OPENCLAW_LIVE_CODEX_HARNESS_TARGETS:-single model}" +echo "==> Target count: $CODEX_HARNESS_TARGET_COUNT" +echo "==> Docker run timeout: $CODEX_HARNESS_DOCKER_RUN_TIMEOUT" +echo "==> Chat image probe: ${OPENCLAW_LIVE_CODEX_HARNESS_CHAT_IMAGE_PROBE:-0}" +echo "==> Image probe: ${OPENCLAW_LIVE_CODEX_HARNESS_IMAGE_PROBE:-1}" +echo "==> MCP probe: ${OPENCLAW_LIVE_CODEX_HARNESS_MCP_PROBE:-1}" +echo "==> Multi-session probe: ${OPENCLAW_LIVE_CODEX_HARNESS_MULTI_SESSION_PROBE:-0}" +echo "==> Subagent probe: ${OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_PROBE:-1}" +echo "==> Subagent count: ${OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_COUNT:-1}" +echo "==> Subagent-only fast path: ${OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_ONLY:-auto}" +echo "==> Guardian probe: ${OPENCLAW_LIVE_CODEX_HARNESS_GUARDIAN_PROBE:-1}" +echo "==> Code-mode-only probe: ${OPENCLAW_LIVE_CODEX_HARNESS_CODE_MODE_ONLY:-0}" +echo "==> Loop relay disabled: ${OPENCLAW_LIVE_CODEX_HARNESS_DISABLE_LOOP_RELAY:-0}" +echo "==> Resume stress: ${OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS:-0}" +echo "==> Resume stress history turns: ${OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS_HISTORY_TURNS:-4}" +echo "==> Resume stress restarts: ${OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS_RESTARTS:-3}" +echo "==> Compaction stress: ${OPENCLAW_LIVE_CODEX_HARNESS_COMPACTION_STRESS:-0}" +echo "==> Compaction stress turns: ${OPENCLAW_LIVE_CODEX_HARNESS_COMPACTION_STRESS_TURNS:-4}" +echo "==> Large output bytes: ${OPENCLAW_LIVE_CODEX_HARNESS_LARGE_OUTPUT_BYTES:-300000}" +echo "==> Auth mode: $CODEX_HARNESS_AUTH_MODE" +echo "==> Profile file: $PROFILE_STATUS" +echo "==> CI-safe Codex config: ${OPENCLAW_LIVE_CODEX_HARNESS_USE_CI_SAFE_CODEX_CONFIG:-1}" +echo "==> Test files: ${OPENCLAW_LIVE_CODEX_TEST_FILES:-src/gateway/gateway-codex-harness.live.test.ts}" +echo "==> Codex CLI package: $CODEX_CLI_PACKAGE_SPEC" +echo "==> Harness fallback: none" +echo "==> Auth files: ${AUTH_FILES_CSV:-none}" +DOCKER_RUN_ARGS=() +openclaw_live_init_docker_run_args DOCKER_RUN_ARGS "$CODEX_HARNESS_DOCKER_RUN_TIMEOUT" +DOCKER_RUN_ARGS+=(--rm -t \ + -u "$DOCKER_USER" \ + --entrypoint bash \ + -e COREPACK_ENABLE_DOWNLOAD_PROMPT=0 \ + -e HOME=/home/node \ + -e NPM_CONFIG_PREFIX="$DOCKER_CLI_TOOLS_CONTAINER_DIR" \ + -e npm_config_prefix="$DOCKER_CLI_TOOLS_CONTAINER_DIR" \ + -e XDG_CACHE_HOME="$DOCKER_CACHE_CONTAINER_DIR" \ + -e COREPACK_HOME="$DOCKER_CACHE_CONTAINER_DIR/node/corepack" \ + -e NPM_CONFIG_CACHE="$DOCKER_CACHE_CONTAINER_DIR/npm" \ + -e npm_config_cache="$DOCKER_CACHE_CONTAINER_DIR/npm" \ + -e NODE_OPTIONS="$(openclaw_live_container_node_options)" \ + -e OPENCLAW_AGENT_HARNESS_FALLBACK=none \ + -e OPENCLAW_DOCKER_AUTH_PRESTAGED="$DOCKER_AUTH_PRESTAGED" \ + -e OPENCLAW_CODEX_APP_SERVER_BIN="${OPENCLAW_CODEX_APP_SERVER_BIN:-codex}" \ + -e OPENCLAW_DOCKER_AUTH_FILES_RESOLVED="$AUTH_FILES_CSV" \ + -e OPENCLAW_LIVE_DOCKER_SOURCE_STAGE_MODE="${OPENCLAW_LIVE_DOCKER_SOURCE_STAGE_MODE:-copy}" \ + -e OPENCLAW_LIVE_CODEX_HARNESS_AUTH="$CODEX_HARNESS_AUTH_MODE" \ + -e OPENCLAW_LIVE_CODEX_HARNESS=1 \ + -e OPENCLAW_LIVE_CODEX_HARNESS_CHAT_IMAGE_PROBE="${OPENCLAW_LIVE_CODEX_HARNESS_CHAT_IMAGE_PROBE:-0}" \ + -e OPENCLAW_LIVE_CODEX_HARNESS_CODE_MODE_ONLY="${OPENCLAW_LIVE_CODEX_HARNESS_CODE_MODE_ONLY:-0}" \ + -e OPENCLAW_LIVE_CODEX_HARNESS_COMPACTION_STRESS="${OPENCLAW_LIVE_CODEX_HARNESS_COMPACTION_STRESS:-0}" \ + -e OPENCLAW_LIVE_CODEX_HARNESS_COMPACTION_STRESS_TURNS="${OPENCLAW_LIVE_CODEX_HARNESS_COMPACTION_STRESS_TURNS:-4}" \ + -e OPENCLAW_LIVE_CODEX_HARNESS_DEBUG="${OPENCLAW_LIVE_CODEX_HARNESS_DEBUG:-}" \ + -e OPENCLAW_LIVE_CODEX_HARNESS_DISABLE_LOOP_RELAY="${OPENCLAW_LIVE_CODEX_HARNESS_DISABLE_LOOP_RELAY:-0}" \ + -e OPENCLAW_LIVE_CODEX_HARNESS_GUARDIAN_PROBE="${OPENCLAW_LIVE_CODEX_HARNESS_GUARDIAN_PROBE:-1}" \ + -e OPENCLAW_LIVE_CODEX_HARNESS_IMAGE_PROBE="${OPENCLAW_LIVE_CODEX_HARNESS_IMAGE_PROBE:-1}" \ + -e OPENCLAW_LIVE_CODEX_HARNESS_LARGE_OUTPUT_BYTES="${OPENCLAW_LIVE_CODEX_HARNESS_LARGE_OUTPUT_BYTES:-300000}" \ + -e OPENCLAW_LIVE_CODEX_HARNESS_MCP_PROBE="${OPENCLAW_LIVE_CODEX_HARNESS_MCP_PROBE:-1}" \ + -e OPENCLAW_LIVE_CODEX_HARNESS_MULTI_SESSION_PROBE="${OPENCLAW_LIVE_CODEX_HARNESS_MULTI_SESSION_PROBE:-0}" \ + -e OPENCLAW_LIVE_CODEX_HARNESS_MODEL="${OPENCLAW_LIVE_CODEX_HARNESS_MODEL:-openai/gpt-5.6-luna}" \ + -e OPENCLAW_LIVE_CODEX_HARNESS_TARGETS="${OPENCLAW_LIVE_CODEX_HARNESS_TARGETS:-}" \ + -e OPENCLAW_LIVE_CODEX_HARNESS_THINKING="${OPENCLAW_LIVE_CODEX_HARNESS_THINKING:-low}" \ + -e OPENCLAW_LIVE_CODEX_HARNESS_EXPECTED_EFFORT="${OPENCLAW_LIVE_CODEX_HARNESS_EXPECTED_EFFORT:-}" \ + -e OPENCLAW_LIVE_CODEX_HARNESS_REQUIRE_GUARDIAN_EVENTS="${OPENCLAW_LIVE_CODEX_HARNESS_REQUIRE_GUARDIAN_EVENTS:-1}" \ + -e OPENCLAW_LIVE_CODEX_HARNESS_REQUEST_TIMEOUT_MS="${OPENCLAW_LIVE_CODEX_HARNESS_REQUEST_TIMEOUT_MS:-}" \ + -e OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS="${OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS:-0}" \ + -e OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS_HISTORY_TURNS="${OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS_HISTORY_TURNS:-4}" \ + -e OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS_RESTARTS="${OPENCLAW_LIVE_CODEX_HARNESS_RESUME_STRESS_RESTARTS:-3}" \ + -e OPENCLAW_LIVE_CODEX_HARNESS_SETUP_TIMEOUT_SECONDS="$CODEX_HARNESS_SETUP_TIMEOUT_SECONDS" \ + -e OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_ONLY="${OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_ONLY:-}" \ + -e OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_COUNT="${OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_COUNT:-1}" \ + -e OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_PROBE="${OPENCLAW_LIVE_CODEX_HARNESS_SUBAGENT_PROBE:-1}" \ + -e OPENCLAW_LIVE_CODEX_HARNESS_USE_CI_SAFE_CODEX_CONFIG="${OPENCLAW_LIVE_CODEX_HARNESS_USE_CI_SAFE_CODEX_CONFIG:-1}" \ + -e OPENCLAW_LIVE_CODEX_CLI_PACKAGE_SPEC="$CODEX_CLI_PACKAGE_SPEC" \ + -e OPENCLAW_CLI_BACKEND_LOG_OUTPUT="${OPENCLAW_CLI_BACKEND_LOG_OUTPUT:-}" \ + -e OPENCLAW_TEST_CONSOLE="${OPENCLAW_TEST_CONSOLE:-}" \ + -e OPENCLAW_LIVE_DOCKER_SCRIPTS_DIR="${DOCKER_TRUSTED_HARNESS_CONTAINER_DIR}/scripts" \ + -e OPENCLAW_LIVE_DOCKER_TRUSTED_HARNESS_DIR="$DOCKER_TRUSTED_HARNESS_CONTAINER_DIR" \ + -e OPENCLAW_LIVE_CODEX_TRUSTED_HARNESS_DIR="$DOCKER_TRUSTED_HARNESS_CONTAINER_DIR" \ + -e OPENCLAW_LIVE_CODEX_BIND="${OPENCLAW_LIVE_CODEX_BIND:-}" \ + -e OPENCLAW_LIVE_CODEX_BIND_MODEL="${OPENCLAW_LIVE_CODEX_BIND_MODEL:-}" \ + -e OPENCLAW_LIVE_CODEX_BIND_PROVIDER="${OPENCLAW_LIVE_CODEX_BIND_PROVIDER:-}" \ + -e OPENCLAW_LIVE_CODEX_BIND_REQUEST_TIMEOUT_MS="${OPENCLAW_LIVE_CODEX_BIND_REQUEST_TIMEOUT_MS:-}" \ + -e OPENCLAW_LIVE_CODEX_BIND_TIMEOUT_MS="${OPENCLAW_LIVE_CODEX_BIND_TIMEOUT_MS:-}" \ + -e OPENCLAW_LIVE_CODEX_TEST_FILES="${OPENCLAW_LIVE_CODEX_TEST_FILES:-}" \ + -e OPENCLAW_LIVE_TEST=1 \ + -e OPENCLAW_VITEST_FS_MODULE_CACHE=0) +openclaw_live_append_array DOCKER_RUN_ARGS DOCKER_AUTH_ENV +openclaw_live_append_array DOCKER_RUN_ARGS DOCKER_EXTRA_ENV_FILES +openclaw_live_append_array DOCKER_RUN_ARGS DOCKER_HOME_MOUNT +openclaw_live_append_array DOCKER_RUN_ARGS DOCKER_TRUSTED_HARNESS_MOUNT +DOCKER_RUN_ARGS+=(\ + -v "$ROOT_DIR":/src:ro \ + -v "$CONFIG_DIR":/home/node/.openclaw \ + -v "$WORKSPACE_DIR":/home/node/.openclaw/workspace) +if [[ "$CODEX_HARNESS_AUTH_MODE" != "api-key" ]]; then + DOCKER_RUN_ARGS+=(\ + -v "$CACHE_HOME_DIR":"$DOCKER_CACHE_CONTAINER_DIR" \ + -v "$CLI_TOOLS_DIR":"$DOCKER_CLI_TOOLS_CONTAINER_DIR") +fi +openclaw_live_append_array DOCKER_RUN_ARGS EXTERNAL_AUTH_MOUNTS +openclaw_live_append_array DOCKER_RUN_ARGS PROFILE_MOUNT +DOCKER_RUN_ARGS+=(\ + "$LIVE_IMAGE_NAME" \ + -lc "$LIVE_TEST_CMD") +if [[ "${OPENCLAW_LIVE_CODEX_HARNESS_DEBUG:-}" == "1" ]]; then + echo "==> Docker debug: host ids and mounted dirs" + id + ls -ld "$CACHE_HOME_DIR" "$CLI_TOOLS_DIR" "${DOCKER_HOME_DIR:-$HOME}" 2>/dev/null || true + printf '==> Docker debug args:' + printf ' %q' "${DOCKER_RUN_ARGS[@]}" + printf '\n' +fi +"${DOCKER_RUN_ARGS[@]}" diff --git a/scripts/test-live-gateway-models-docker.sh b/scripts/test-live-gateway-models-docker.sh new file mode 100644 index 0000000000000000000000000000000000000000..4b18f18970af20a609b5851dbc51d8ab70e91bdf --- /dev/null +++ b/scripts/test-live-gateway-models-docker.sh @@ -0,0 +1,143 @@ +#!/usr/bin/env bash +# Bash 5.3+ can deadlock writing heredoc pipes on macOS before the reader starts. +if [[ ${OSTYPE:-} == darwin* && $BASH != /bin/bash ]] && ((BASH_VERSINFO[0] > 5 || (BASH_VERSINFO[0] == 5 && BASH_VERSINFO[1] >= 3))); then + exec /bin/bash "$0" "$@" +fi +set -euo pipefail + +SCRIPT_ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +ROOT_DIR="${OPENCLAW_LIVE_DOCKER_REPO_ROOT:-$SCRIPT_ROOT_DIR}" +ROOT_DIR="$(cd "$ROOT_DIR" && pwd)" +TRUSTED_HARNESS_DIR="${OPENCLAW_LIVE_DOCKER_TRUSTED_HARNESS_DIR:-$SCRIPT_ROOT_DIR}" +if [[ -z "$TRUSTED_HARNESS_DIR" || ! -d "$TRUSTED_HARNESS_DIR" ]]; then + echo "ERROR: trusted live Docker harness directory not found: ${TRUSTED_HARNESS_DIR:-<empty>}." >&2 + exit 1 +fi +TRUSTED_HARNESS_DIR="$(cd "$TRUSTED_HARNESS_DIR" && pwd)" +source "$TRUSTED_HARNESS_DIR/scripts/lib/live-docker-auth.sh" +source "$TRUSTED_HARNESS_DIR/scripts/lib/frozen-target-compat.sh" +openclaw_resolve_frozen_live_cli_backend_package_mode "$ROOT_DIR" +IMAGE_NAME="${OPENCLAW_IMAGE:-openclaw:local}" +LIVE_IMAGE_NAME="${OPENCLAW_LIVE_IMAGE:-${IMAGE_NAME}-live}" +CONFIG_DIR="${OPENCLAW_CONFIG_DIR:-$HOME/.openclaw}" +WORKSPACE_DIR="${OPENCLAW_WORKSPACE_DIR:-$HOME/.openclaw/workspace}" +PROFILE_FILE="$(openclaw_live_default_profile_file)" +LIVE_GATEWAY_MAX_MODELS="$(openclaw_live_read_positive_int_env OPENCLAW_LIVE_GATEWAY_MAX_MODELS 8)" +LIVE_GATEWAY_STEP_TIMEOUT_MS="$(openclaw_live_read_positive_int_env OPENCLAW_LIVE_GATEWAY_STEP_TIMEOUT_MS 45000)" +LIVE_GATEWAY_MODEL_TIMEOUT_MS="$(openclaw_live_read_positive_int_env OPENCLAW_LIVE_GATEWAY_MODEL_TIMEOUT_MS 90000)" +DOCKER_AUTH_PRESTAGED=0 +DOCKER_TRUSTED_HARNESS_CONTAINER_DIR="/trusted-harness" +DOCKER_TRUSTED_HARNESS_MOUNT=(-v "$TRUSTED_HARNESS_DIR":"$DOCKER_TRUSTED_HARNESS_CONTAINER_DIR":ro) +openclaw_live_init_temp_dirs +openclaw_live_init_cli_tools_dir +openclaw_live_init_cache_home_dir +openclaw_live_init_managed_home +openclaw_live_init_profile_mount + +openclaw_live_collect_auth_for_providers "${OPENCLAW_LIVE_GATEWAY_PROVIDERS:-}" +openclaw_live_finalize_auth_mounts +CONTAINER_NODE_OPTIONS="$(openclaw_live_container_node_options)" + +read -r -d '' LIVE_TEST_CMD <<'EOF' || true +set -euo pipefail +[ -f "$HOME/.profile" ] && [ -r "$HOME/.profile" ] && source "$HOME/.profile" || true +export XDG_CACHE_HOME="${XDG_CACHE_HOME:-$HOME/.cache}" +export COREPACK_HOME="${COREPACK_HOME:-$XDG_CACHE_HOME/node/corepack}" +export NPM_CONFIG_CACHE="${NPM_CONFIG_CACHE:-$XDG_CACHE_HOME/npm}" +export npm_config_cache="$NPM_CONFIG_CACHE" +export NPM_CONFIG_PREFIX="$HOME/.npm-global" +export npm_config_prefix="$NPM_CONFIG_PREFIX" +export PATH="$NPM_CONFIG_PREFIX/bin:$PATH" +mkdir -p "$NPM_CONFIG_PREFIX" "$XDG_CACHE_HOME" "$COREPACK_HOME" "$NPM_CONFIG_CACHE" +chmod 700 "$XDG_CACHE_HOME" "$COREPACK_HOME" "$NPM_CONFIG_CACHE" || true +tmp_dir="$(mktemp -d)" +trusted_scripts_dir="${OPENCLAW_LIVE_DOCKER_SCRIPTS_DIR:-/src/scripts}" +source "$trusted_scripts_dir/lib/live-docker-stage.sh" +openclaw_live_stage_mounted_auth +openclaw_live_stage_source_tree "$tmp_dir" +openclaw_live_stage_node_modules "$tmp_dir" +openclaw_live_link_runtime_tree "$tmp_dir" +openclaw_live_stage_state_dir "$tmp_dir/.openclaw-state" +openclaw_live_prepare_staged_config +cd "$tmp_dir" +openclaw_live_prepare_cli_backend_docker_packages \ + "${OPENCLAW_LIVE_GATEWAY_PROVIDERS:-}" \ + "${OPENCLAW_LIVE_GATEWAY_MODELS:-}" +openclaw_live_stage_gemini_auth +openclaw_live_run_staged_script scripts/test-live -- src/gateway/gateway-models.profiles.live.test.ts +EOF + +OPENCLAW_LIVE_DOCKER_REPO_ROOT="$ROOT_DIR" "$TRUSTED_HARNESS_DIR/scripts/test-live-build-docker.sh" +if openclaw_live_uses_managed_bind_dirs; then + openclaw_live_chown_bind_dirs_for_container_user \ + "$LIVE_IMAGE_NAME" \ + "$DOCKER_USER" \ + "$CLI_TOOLS_DIR" \ + "$CACHE_HOME_DIR" \ + "${DOCKER_HOME_DIR:-}" +fi + +echo "==> Run gateway live model tests (profile keys)" +echo "==> Target: src/gateway/gateway-models.profiles.live.test.ts" +echo "==> Profile file: $PROFILE_STATUS" +echo "==> External auth dirs: ${AUTH_DIRS_CSV:-none}" +echo "==> External auth files: ${AUTH_FILES_CSV:-none}" +DOCKER_RUN_ARGS=() +FROZEN_TARGET_DOCKER_ENV=() +if [[ "${OPENCLAW_FROZEN_TARGET_LIVE_CLI_BACKEND_PACKAGE_MODE:-current}" == "legacy" ]]; then + FROZEN_TARGET_DOCKER_ENV+=( -e "OPENCLAW_FROZEN_TARGET_LIVE_CLI_BACKEND_PACKAGE_MODE=legacy" ) +fi +openclaw_live_init_docker_run_args DOCKER_RUN_ARGS "${OPENCLAW_LIVE_GATEWAY_DOCKER_RUN_TIMEOUT:-2100s}" +DOCKER_RUN_ARGS+=(--rm -t \ + -u "$DOCKER_USER" \ + --entrypoint bash \ + -e OPENAI_API_KEY \ + -e OPENAI_BASE_URL \ + -e ANTHROPIC_API_KEY \ + -e GEMINI_API_KEY \ + -e GOOGLE_API_KEY \ + -e MINIMAX_API_KEY \ + -e OPENROUTER_API_KEY \ + -e FIREWORKS_API_KEY \ + -e DEEPSEEK_API_KEY \ + -e XAI_API_KEY \ + -e ZAI_API_KEY \ + -e Z_AI_API_KEY \ + -e COREPACK_ENABLE_DOWNLOAD_PROMPT=0 \ + -e HOME=/home/node \ + -e NODE_OPTIONS="$CONTAINER_NODE_OPTIONS" \ + -e OPENCLAW_SKIP_CHANNELS=1 \ + -e OPENCLAW_SUPPRESS_NOTES=1 \ + -e OPENCLAW_DOCKER_AUTH_PRESTAGED="$DOCKER_AUTH_PRESTAGED" \ + -e OPENCLAW_DOCKER_AUTH_DIRS_RESOLVED="$AUTH_DIRS_CSV" \ + -e OPENCLAW_DOCKER_AUTH_FILES_RESOLVED="$AUTH_FILES_CSV" \ + -e OPENCLAW_LIVE_DOCKER_SCRIPTS_DIR="${DOCKER_TRUSTED_HARNESS_CONTAINER_DIR}/scripts" \ + -e OPENCLAW_LIVE_DOCKER_SOURCE_STAGE_MODE="${OPENCLAW_LIVE_DOCKER_SOURCE_STAGE_MODE:-copy}" \ + -e OPENCLAW_LIVE_TEST=1 \ + -e OPENCLAW_LIVE_TEST_QUIET="${OPENCLAW_LIVE_TEST_QUIET:-}" \ + -e OPENCLAW_LIVE_WRAPPER_HEARTBEAT_MS="${OPENCLAW_LIVE_WRAPPER_HEARTBEAT_MS:-}" \ + -e OPENCLAW_LIVE_REQUIRE_PROFILE_KEYS="${OPENCLAW_LIVE_REQUIRE_PROFILE_KEYS:-}" \ + -e OPENCLAW_LIVE_GATEWAY_MODELS="${OPENCLAW_LIVE_GATEWAY_MODELS:-modern}" \ + -e OPENCLAW_LIVE_GATEWAY_PROVIDERS="${OPENCLAW_LIVE_GATEWAY_PROVIDERS:-}" \ + -e OPENCLAW_LIVE_GATEWAY_THINKING="${OPENCLAW_LIVE_GATEWAY_THINKING:-}" \ + -e OPENCLAW_LIVE_GATEWAY_SMOKE="${OPENCLAW_LIVE_GATEWAY_SMOKE:-1}" \ + -e OPENCLAW_LIVE_GATEWAY_MAX_MODELS="$LIVE_GATEWAY_MAX_MODELS" \ + -e OPENCLAW_LIVE_GATEWAY_HEARTBEAT_MS="${OPENCLAW_LIVE_GATEWAY_HEARTBEAT_MS:-}" \ + -e OPENCLAW_LIVE_GATEWAY_STEP_TIMEOUT_MS="$LIVE_GATEWAY_STEP_TIMEOUT_MS" \ + -e OPENCLAW_LIVE_GATEWAY_MODEL_TIMEOUT_MS="$LIVE_GATEWAY_MODEL_TIMEOUT_MS" \ + -e OPENCLAW_VITEST_FS_MODULE_CACHE=0) +openclaw_live_append_array DOCKER_RUN_ARGS FROZEN_TARGET_DOCKER_ENV +openclaw_live_append_array DOCKER_RUN_ARGS DOCKER_HOME_MOUNT +openclaw_live_append_array DOCKER_RUN_ARGS DOCKER_TRUSTED_HARNESS_MOUNT +DOCKER_RUN_ARGS+=(\ + -v "$CLI_TOOLS_DIR":/home/node/.npm-global \ + -v "$CACHE_HOME_DIR":/home/node/.cache \ + -v "$ROOT_DIR":/src:ro \ + -v "$CONFIG_DIR":/home/node/.openclaw \ + -v "$WORKSPACE_DIR":/home/node/.openclaw/workspace) +openclaw_live_append_array DOCKER_RUN_ARGS EXTERNAL_AUTH_MOUNTS +openclaw_live_append_array DOCKER_RUN_ARGS PROFILE_MOUNT +DOCKER_RUN_ARGS+=(\ + "$LIVE_IMAGE_NAME" \ + -lc "$LIVE_TEST_CMD") +"${DOCKER_RUN_ARGS[@]}" diff --git a/scripts/test-live-shard.mts b/scripts/test-live-shard.mts new file mode 100644 index 0000000000000000000000000000000000000000..a103f57f5c05b08e366df158efded1edccb0d209 --- /dev/null +++ b/scripts/test-live-shard.mts @@ -0,0 +1,879 @@ +#!/usr/bin/env node +// Runs one named live-test shard with OPENCLAW_LIVE_TEST enabled. +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { asSafeIntegerInRange } from "../packages/normalization-core/src/number-coercion.ts"; +import { isRecord as isUnknownRecord } from "../packages/normalization-core/src/record-coerce.ts"; +import { parsePermissiveBooleanToken } from "./lib/arg-utils.mts"; +import { RUNTIME_POSTBUILD_STAMP_FILE } from "./lib/local-build-metadata-paths.mts"; +import { spawnPnpmRunner, type PnpmRunnerParams } from "./pnpm-runner.mts"; +import { + createVitestProcessCompletion, + installVitestProcessGroupCleanup, + shouldUseDetachedVitestProcessGroup, +} from "./vitest-process-group.mts"; + +const LIVE_TEST_SUFFIX = ".live.test.ts"; +const OPTIONAL_LIVE_SHARD_FILE_ENVS = new Map([ + [ + "extensions/codex/src/app-server/native-subagent-monitor.live.test.ts", + ["OPENCLAW_LIVE_CODEX_NATIVE_SUBAGENT"], + ], + [ + "extensions/codex/src/native-thread-coexistence.live.test.ts", + ["OPENCLAW_LIVE_CODEX_THREAD_COEXISTENCE"], + ], + ["src/agents/agent-mcp-style.cache.live.test.ts", ["OPENCLAW_LIVE_CACHE_TEST"]], + ["src/agents/cli-runner/bundle-mcp.gemini.live.test.ts", ["OPENCLAW_LIVE_CLI_MCP_GEMINI"]], + ["src/agents/embedded-agent-runner.cache.live.test.ts", ["OPENCLAW_LIVE_CACHE_TEST"]], + ["src/agents/live-cache-regression.live.test.ts", ["OPENCLAW_LIVE_CACHE_TEST"]], + ["src/agents/provider-headers.live.test.ts", ["OPENCLAW_LIVE_CACHE_TEST"]], + // Frozen release candidates before the announce-family move retain this path. + ["src/agents/subagent-announce.live.test.ts", ["OPENCLAW_LIVE_SUBAGENT_E2E"]], + [ + "src/agents/sessions/agent-session.openai-compaction.live.test.ts", + ["OPENCLAW_LIVE_OPENAI_COMPACTION"], + ], + ["src/agents/subagents/announce/subagent-announce.live.test.ts", ["OPENCLAW_LIVE_SUBAGENT_E2E"]], + ["src/agents/tools/image-tool.ollama.live.test.ts", ["OPENCLAW_LIVE_OLLAMA_IMAGE"]], + ["src/agents/tools/image-tool.providers.live.test.ts", ["OPENCLAW_LIVE_IMAGE_TOOL_TEST"]], + ["extensions/openai/realtime-meeting.live.test.ts", ["OPENCLAW_LIVE_GPT_LIVE"]], + [ + "extensions/openai/realtime-quicksilver-gateway-bridge.live.test.ts", + ["OPENCLAW_LIVE_GPT_LIVE"], + ], + ["extensions/openai/realtime-quicksilver.live.test.ts", ["OPENCLAW_LIVE_GPT_LIVE"]], + ["extensions/openai/realtime-talk-defaults.live.test.ts", ["OPENCLAW_LIVE_GPT_LIVE"]], + ["src/skills/workshop/experience-review.live.test.ts", ["OPENCLAW_LIVE_SKILL_EXPERIENCE_REVIEW"]], + ["src/system-agent/rescue-channel.live.test.ts", ["OPENCLAW_LIVE_SYSTEM_AGENT_RESCUE_CHANNEL"]], + ["src/gateway/android-node.capabilities.live.test.ts", ["OPENCLAW_LIVE_ANDROID_NODE"]], + ["src/gateway/gateway-acp-bind.live.test.ts", ["OPENCLAW_LIVE_ACP_BIND"]], + ["src/gateway/gateway-acp-spawn-defaults.live.test.ts", ["OPENCLAW_LIVE_ACP_SPAWN_DEFAULTS"]], + ["src/gateway/gateway-cli-backend.live.test.ts", ["OPENCLAW_LIVE_CLI_BACKEND"]], + ["src/gateway/gateway-codex-bind.live.test.ts", ["OPENCLAW_LIVE_CODEX_BIND"]], + ["src/gateway/gateway-codex-harness.live.test.ts", ["OPENCLAW_LIVE_CODEX_HARNESS"]], + ["src/gateway/gateway-openai-long-context.live.test.ts", ["OPENCLAW_LIVE_OPENAI_LONG_CONTEXT"]], + ["src/gateway/gateway-trajectory-export.live.test.ts", ["OPENCLAW_LIVE_CODEX_HARNESS"]], + ["src/infra/push-apns-http2.live.test.ts", ["OPENCLAW_LIVE_APNS_REACHABILITY"]], + ["test/e2e/crabbox-sandbox.live.test.ts", ["OPENCLAW_E2E_CRABBOX"]], + ["test/image-generation.infer-cli.live.test.ts", ["OPENCLAW_LIVE_INFER_CLI_TEST"]], +]); +const SKIPPED_ASSERTION_STATUSES = new Set(["disabled", "pending", "skipped", "todo"]); +const QA_RUNTIME_LIVE_TEST = "extensions/qa-lab/src/matrix-channel-driver.lifecycle.live.test.ts"; +const QA_RUNTIME_ARTIFACT = "dist/extensions/qa-lab/runtime-api.js"; +const SOURCE_PERFORMANCE_ARTIFACT = `dist/${RUNTIME_POSTBUILD_STAMP_FILE}`; +type LiveShardPreparation = { + env: NodeJS.ProcessEnv; + profile: string; + requiredArtifact: string; + runtimeEnv?: NodeJS.ProcessEnv; +}; + +/** Live-test shards included in release validation. */ +export const RELEASE_LIVE_TEST_SHARDS = Object.freeze([ + "native-live-src-agents", + "native-live-src-agents-zai-coding", + "native-live-src-gateway-core", + "native-live-src-gateway-profiles", + "native-live-src-gateway-backends", + "native-live-src-infra", + "native-live-test", + "native-live-extensions-a-k", + "native-live-extensions-l-n", + "native-live-extensions-moonshot", + "native-live-extensions-openai", + "native-live-extensions-o-z-other", + "native-live-extensions-xai", + "native-live-extensions-media-audio", + "native-live-extensions-media-music-google", + "native-live-extensions-media-music-minimax", + "native-live-extensions-media-video", +]); + +/** All live-test shards, including broader local-only shard aliases. */ +export const LIVE_TEST_SHARDS = Object.freeze([ + ...RELEASE_LIVE_TEST_SHARDS, + "native-live-extensions-o-z", + "native-live-extensions-media", + "native-live-extensions-media-music", +]); + +function walkFiles(rootDir: string) { + const files: string[] = []; + if (!fs.existsSync(rootDir)) { + return files; + } + const stack = [rootDir]; + for (let current = stack.pop(); current; current = stack.pop()) { + const entries = fs.readdirSync(current, { withFileTypes: true }); + for (const entry of entries) { + const fullPath = path.join(current, entry.name); + if (entry.isDirectory()) { + if ( + entry.name === "node_modules" || + entry.name === "dist" || + entry.name === "vendor" || + entry.name === "fixtures" + ) { + continue; + } + stack.push(fullPath); + continue; + } + if (entry.isFile()) { + files.push(fullPath); + } + } + } + return files; +} + +/** + * Lists all live test files from git/find fallback paths. + */ +export function collectAllLiveTestFiles(repoRoot = process.cwd()) { + const externalFiles = listExternalLiveTestFiles(repoRoot); + if (externalFiles) { + return externalFiles; + } + return ["src", "test", "extensions"] + .flatMap((dir) => walkFiles(path.join(repoRoot, dir))) + .map((file) => path.relative(repoRoot, file).split(path.sep).join("/")) + .filter((file) => file.endsWith(LIVE_TEST_SUFFIX)) + .toSorted((a, b) => a.localeCompare(b)); +} + +function listExternalLiveTestFiles(repoRoot: string) { + return listGitLiveTestFiles(repoRoot) ?? listFindLiveTestFiles(repoRoot); +} + +function listGitLiveTestFiles(repoRoot: string) { + const result = spawnSync("git", ["ls-files", "--", "src", "test", "extensions"], { + cwd: repoRoot, + encoding: "utf8", + maxBuffer: 1024 * 1024 * 4, + stdio: ["ignore", "pipe", "ignore"], + }); + if (result.status !== 0) { + return null; + } + return result.stdout + .split("\n") + .map((line) => line.trim()) + .filter((file) => file.endsWith(LIVE_TEST_SUFFIX)) + .toSorted((a, b) => a.localeCompare(b)); +} + +function listFindLiveTestFiles(repoRoot: string) { + const roots = ["src", "test", "extensions"].map((dir) => path.join(repoRoot, dir)); + const result = spawnSync( + "find", + [ + ...roots, + "(", + "-name", + "node_modules", + "-o", + "-name", + "dist", + "-o", + "-name", + "vendor", + "-o", + "-name", + "fixtures", + ")", + "-prune", + "-o", + "-type", + "f", + "-name", + `*${LIVE_TEST_SUFFIX}`, + "-print", + ], + { + cwd: repoRoot, + encoding: "utf8", + maxBuffer: 1024 * 1024 * 4, + stdio: ["ignore", "pipe", "ignore"], + }, + ); + if (result.status !== 0) { + return null; + } + return result.stdout + .split("\n") + .map((line) => line.trim()) + .filter((file) => file.length > 0) + .map((file) => path.relative(repoRoot, file).split(path.sep).join("/")) + .toSorted((a, b) => a.localeCompare(b)); +} + +function extensionKey(file: string) { + const relative = file.slice("extensions/".length); + return relative.split("/", 1)[0]?.toLowerCase() ?? ""; +} + +function isExtensionInRange(file: string, start: string, end: string) { + if (!file.startsWith("extensions/")) { + return false; + } + const key = extensionKey(file); + if (!key) { + return false; + } + const first = key[0]; + return first !== undefined && first >= start && first <= end; +} + +function isSourceGatewayLiveTest(file: string) { + return file.startsWith("src/gateway/") || file.startsWith("src/system-agent/"); +} + +function isGatewayBackendLiveTest(file: string) { + return ( + file === "src/gateway/gateway-acp-bind.live.test.ts" || + file === "src/gateway/gateway-cli-backend.live.test.ts" || + file === "src/gateway/gateway-codex-bind.live.test.ts" || + file === "src/gateway/gateway-codex-harness.live.test.ts" + ); +} + +function isGatewayProfilesLiveTest(file: string) { + return ( + file === "src/gateway/gateway-models.profiles.live.test.ts" || + file === "src/gateway/gateway-openai-long-context.live.test.ts" + ); +} + +function isExtensionMediaLiveTest(file: string) { + return ( + file === "extensions/music-generation-providers.live.test.ts" || + file === "extensions/minimax/minimax.live.test.ts" || + file === "extensions/openai/openai-tts.live.test.ts" || + file === "extensions/tts-local-cli/speech-provider.live.test.ts" || + file === "extensions/video-generation-providers.live.test.ts" || + file === "extensions/volcengine/tts.live.test.ts" || + file === "extensions/vydra/vydra.live.test.ts" + ); +} + +function isExtensionMediaMusicLiveTest(file: string) { + return file === "extensions/music-generation-providers.live.test.ts"; +} + +function isExtensionMediaVideoLiveTest(file: string) { + return file === "extensions/video-generation-providers.live.test.ts"; +} + +function isExtensionMediaAudioLiveTest(file: string) { + return ( + isExtensionMediaLiveTest(file) && + !isExtensionMediaMusicLiveTest(file) && + !isExtensionMediaVideoLiveTest(file) + ); +} + +function isXaiLiveTest(file: string) { + return file.startsWith("extensions/xai/"); +} + +function isMoonshotLiveTest(file: string) { + return file.startsWith("extensions/moonshot/"); +} + +/** + * Selects the live test files belonging to one shard name. + */ +export function selectLiveShardFiles(shard: string, files = collectAllLiveTestFiles()) { + switch (shard) { + case "native-live-src-agents": + return files.filter( + (file) => + file.startsWith("src/agents/") || + file.startsWith("src/llm/") || + file.startsWith("src/skills/"), + ); + case "native-live-src-agents-zai-coding": + return files.filter((file) => file === "src/agents/zai.live.test.ts"); + case "native-live-src-gateway": + return files.filter(isSourceGatewayLiveTest); + case "native-live-src-gateway-core": + return files.filter( + (file) => + isSourceGatewayLiveTest(file) && + !isGatewayBackendLiveTest(file) && + !isGatewayProfilesLiveTest(file), + ); + case "native-live-src-gateway-profiles": + return files.filter(isGatewayProfilesLiveTest); + case "native-live-src-gateway-backends": + return files.filter(isGatewayBackendLiveTest); + case "native-live-src-infra": + return files.filter((file) => file.startsWith("src/infra/")); + case "native-live-test": + return files.filter((file) => file.startsWith("test/")); + case "native-live-extensions-a-k": + return files.filter((file) => isExtensionInRange(file, "a", "k")); + case "native-live-extensions-l-n": + return files.filter( + (file) => + isExtensionInRange(file, "l", "n") && + !file.startsWith("extensions/openai/") && + !isMoonshotLiveTest(file) && + !isExtensionMediaLiveTest(file), + ); + case "native-live-extensions-moonshot": + return files.filter(isMoonshotLiveTest); + case "native-live-extensions-openai": + return files.filter( + (file) => file.startsWith("extensions/openai/") && !isExtensionMediaLiveTest(file), + ); + case "native-live-extensions-o-z": + return files.filter( + (file) => + isExtensionInRange(file, "o", "z") && + !file.startsWith("extensions/openai/") && + !isExtensionMediaLiveTest(file), + ); + case "native-live-extensions-o-z-other": + return files.filter( + (file) => + isExtensionInRange(file, "o", "z") && + !file.startsWith("extensions/openai/") && + !isExtensionMediaLiveTest(file) && + !isXaiLiveTest(file), + ); + case "native-live-extensions-xai": + return files.filter(isXaiLiveTest); + case "native-live-extensions-media": + return files.filter(isExtensionMediaLiveTest); + case "native-live-extensions-media-audio": + return files.filter(isExtensionMediaAudioLiveTest); + case "native-live-extensions-media-music": + case "native-live-extensions-media-music-google": + case "native-live-extensions-media-music-minimax": + return files.filter(isExtensionMediaMusicLiveTest); + case "native-live-extensions-media-video": + return files.filter(isExtensionMediaVideoLiveTest); + default: + throw new Error( + `Unknown live test shard '${shard}'. Expected one of: ${LIVE_TEST_SHARDS.join(", ")}`, + ); + } +} + +function usage(stream: NodeJS.WritableStream = process.stderr) { + stream.write( + `Usage: node scripts/test-live-shard.mjs <${LIVE_TEST_SHARDS.join("|")}> [--list]\n`, + ); +} + +/** + * Parses live-shard CLI args into shard name and Vitest passthrough args. + */ +export function parseLiveShardArgs(args: string[]) { + const separatorIndex = args.indexOf("--"); + const optionArgs = separatorIndex >= 0 ? args.slice(0, separatorIndex) : args; + const passthroughArgs = separatorIndex >= 0 ? args.slice(separatorIndex + 1) : []; + let shard = ""; + let listOnly = false; + for (const arg of optionArgs) { + if (arg === "--list") { + listOnly = true; + continue; + } + if (arg.startsWith("-")) { + throw new Error(`Unknown option: ${arg}`); + } + if (shard) { + throw new Error(`Unexpected argument: ${arg}`); + } + shard = arg; + } + return { shard, listOnly, passthroughArgs }; +} + +/** + * Builds pnpm/vitest args for selected live test files. + */ +export function buildLiveShardPnpmArgs(files: string[], passthroughArgs: string[]) { + return ["test:live", "--", ...files, ...passthroughArgs]; +} + +/** + * Resolves build profiles required by selected live tests. + */ +export function resolveLiveShardPreparation(files: string[]): LiveShardPreparation | null { + const gatewayProfiles = files.some(isGatewayProfilesLiveTest); + // Gateway/worker fixtures and vision requests load compiled runtime plugins. + // Build before Vitest; direct CLI launches cannot bootstrap a cold checkout. + if ( + files.some(isSourceGatewayLiveTest) || + files.some((file) => file.startsWith("test/e2e/qa-lab/runtime/")) || + files.includes("src/infra/heartbeat-runner.live.test.ts") || + files.includes("src/agents/tools/image-tool.providers.live.test.ts") || + files.includes("extensions/openai/openai.live.test.ts") + ) { + return { + env: {}, + profile: "sourcePerformance", + requiredArtifact: SOURCE_PERFORMANCE_ARTIFACT, + ...(gatewayProfiles + ? { + runtimeEnv: { + OPENCLAW_DISABLE_BONJOUR: "1", + OPENCLAW_GATEWAY_STARTUP_TRACE: "1", + OPENCLAW_LIVE_TEST_QUIET: "0", + OPENCLAW_LOG_LEVEL: "info", + OPENCLAW_PLUGIN_LIFECYCLE_TRACE: "1", + }, + } + : {}), + }; + } + if (files.includes(QA_RUNTIME_LIVE_TEST)) { + return { + env: { OPENCLAW_BUILD_PRIVATE_QA: "1" }, + profile: "qaRuntime", + requiredArtifact: QA_RUNTIME_ARTIFACT, + }; + } + return null; +} + +/** + * Builds the Vitest JSON report path used to prove that a live shard ran tests. + */ +export function buildLiveShardReportPath(shard: string, env = process.env) { + const reportDir = env.OPENCLAW_LIVE_SHARD_REPORT_DIR || ".artifacts/live-shards"; + return path.join(reportDir, `${shard}.vitest.json`); +} + +/** + * Adds reporters needed for both operator logs and machine-readable evidence. + */ +export function addLiveShardReportArgs(passthroughArgs: string[], reportPath: string) { + return [ + ...passthroughArgs, + "--reporter=default", + "--reporter=json", + `--outputFile.json=${reportPath}`, + ]; +} + +function readNonNegativeInt(value: unknown, label: string) { + if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 0) { + throw new Error(`Vitest report ${label} must be a non-negative integer.`); + } + return value; +} + +function normalizeReportFilePath(value: unknown, repoRoot = process.cwd()) { + const text = typeof value === "string" ? value : ""; + const repoRelative = path.isAbsolute(text) ? path.relative(repoRoot, path.resolve(text)) : text; + if (path.isAbsolute(repoRelative) || repoRelative.startsWith("..") || repoRelative === "") { + return text.split(path.sep).join("/"); + } + return repoRelative.split(path.sep).join("/"); +} + +function collectReportedLiveTestFiles(payload: unknown, repoRoot = process.cwd()) { + if (!isUnknownRecord(payload) || !Array.isArray(payload.testResults)) { + return null; + } + return new Set( + payload.testResults + .map((result) => + normalizeReportFilePath(isUnknownRecord(result) ? result.name : undefined, repoRoot), + ) + .filter((name) => name.length > 0), + ); +} + +function isDisabledOptInAssertion(assertion: Record<string, unknown>) { + if (assertion.status !== "passed") { + return false; + } + const fields = [ + assertion.fullName, + assertion.title, + ...(Array.isArray(assertion.ancestorTitles) ? assertion.ancestorTitles : []), + ]; + const text = fields + .filter((value) => typeof value === "string") + .join(" ") + .toLowerCase(); + return text.includes("disabled") && text.includes("opt-in"); +} + +function buildFilePassEvidence(result: Record<string, unknown>) { + const statuses: string[] = []; + const evidence = { + disabledOptInPassed: 0, + passed: 0, + statuses, + }; + if (Array.isArray(result.assertionResults)) { + for (const assertionValue of result.assertionResults) { + const assertion = isUnknownRecord(assertionValue) ? assertionValue : {}; + const status = typeof assertion.status === "string" ? assertion.status : ""; + if (status) { + evidence.statuses.push(status); + } + if (status === "passed") { + evidence.passed += 1; + if (isDisabledOptInAssertion(assertion)) { + evidence.disabledOptInPassed += 1; + } + } + } + return evidence; + } + evidence.passed = + asSafeIntegerInRange(result.numPassingTests, { min: 0 }) ?? + asSafeIntegerInRange(result.numPassedTests, { min: 0 }) ?? + 0; + return evidence; +} + +type FilePassEvidence = ReturnType<typeof buildFilePassEvidence>; + +function mergeFilePassEvidence(left: FilePassEvidence, right: FilePassEvidence) { + return { + disabledOptInPassed: left.disabledOptInPassed + right.disabledOptInPassed, + passed: left.passed + right.passed, + statuses: [...left.statuses, ...right.statuses], + }; +} + +function collectReportedLiveTestFileEvidence(payload: unknown, repoRoot = process.cwd()) { + if (!isUnknownRecord(payload) || !Array.isArray(payload.testResults)) { + return null; + } + const evidenceByFile = new Map<string, FilePassEvidence>(); + for (const result of payload.testResults) { + if (!isUnknownRecord(result)) { + continue; + } + const name = normalizeReportFilePath(result.name, repoRoot); + if (!name) { + continue; + } + const evidence = buildFilePassEvidence(result); + const existing = evidenceByFile.get(name); + evidenceByFile.set(name, existing ? mergeFilePassEvidence(existing, evidence) : evidence); + } + return evidenceByFile; +} + +function isDisabledOptionalLiveShardFile( + file: string, + evidence: FilePassEvidence | undefined, + env: NodeJS.ProcessEnv = process.env, +) { + const requiredEnvNames = OPTIONAL_LIVE_SHARD_FILE_ENVS.get(file); + if ( + !requiredEnvNames || + requiredEnvNames.some((name) => parsePermissiveBooleanToken(env[name]) === true) + ) { + return false; + } + const statuses = evidence?.statuses ?? []; + const nonSentinelStatuses = statuses.filter((status) => status !== "passed"); + return ( + statuses.length > 0 && + evidence?.passed === evidence?.disabledOptInPassed && + nonSentinelStatuses.every((status) => SKIPPED_ASSERTION_STATUSES.has(status)) + ); +} + +function countEnabledLivePasses( + file: string, + evidence: FilePassEvidence | undefined, + env: NodeJS.ProcessEnv = process.env, +) { + if ( + OPTIONAL_LIVE_SHARD_FILE_ENVS.has(file) && + isDisabledOptionalLiveShardFile(file, evidence, env) + ) { + return 0; + } + return Math.max(0, (evidence?.passed ?? 0) - (evidence?.disabledOptInPassed ?? 0)); +} + +/** + * Removes a previous JSON report before a shard run so stale success cannot be reused. + */ +export function removeLiveShardReportFile(reportPath: fs.PathLike) { + fs.rmSync(reportPath, { force: true }); +} + +/** + * Validates a Vitest JSON payload for live-shard proof. + */ +export function validateLiveShardReportPayload( + payload: unknown, + expectedFiles: string[] = [], + repoRoot = process.cwd(), + env = process.env, +) { + if (!isUnknownRecord(payload)) { + return { ok: false, reason: "Vitest report is not an object." }; + } + let passed; + let total; + try { + passed = readNonNegativeInt(payload.numPassedTests, "numPassedTests"); + total = readNonNegativeInt(payload.numTotalTests, "numTotalTests"); + } catch (error) { + return { ok: false, reason: error instanceof Error ? error.message : String(error) }; + } + if (passed > total) { + return { ok: false, reason: "Vitest report numPassedTests exceeds numTotalTests." }; + } + if (passed < 1) { + return { ok: false, reason: "Vitest report has no passing live tests." }; + } + if (expectedFiles.length > 0) { + const reportedFiles = collectReportedLiveTestFiles(payload, repoRoot); + const fileEvidence = collectReportedLiveTestFileEvidence(payload, repoRoot); + if (!reportedFiles || !fileEvidence) { + return { ok: false, reason: "Vitest report is missing testResults file evidence." }; + } + const missingFiles = expectedFiles + .map((file) => normalizeReportFilePath(file, repoRoot)) + .filter((file) => !reportedFiles.has(file)); + if (missingFiles.length > 0) { + return { + ok: false, + reason: `Vitest report missing selected live test file evidence: ${missingFiles.join(", ")}`, + }; + } + const enabledPassFiles = expectedFiles + .map((file) => normalizeReportFilePath(file, repoRoot)) + .filter((file) => countEnabledLivePasses(file, fileEvidence.get(file), env) > 0); + if (enabledPassFiles.length === 0) { + return { + ok: false, + reason: "Vitest report has no enabled selected live test files with passing assertions.", + }; + } + const noPassFiles = expectedFiles + .map((file) => normalizeReportFilePath(file, repoRoot)) + .filter((file) => { + const evidence = fileEvidence.get(file); + return ( + countEnabledLivePasses(file, evidence, env) < 1 && + !isDisabledOptionalLiveShardFile(file, evidence, env) + ); + }); + if (noPassFiles.length > 0) { + return { + ok: false, + reason: `Vitest report selected live test files had no passing assertions: ${noPassFiles.join(", ")}`, + }; + } + } + return { ok: true }; +} + +/** + * Reads and validates the live-shard Vitest JSON report. + */ +function validateLiveShardReport( + reportPath: fs.PathOrFileDescriptor, + expectedFiles: string[] = [], +) { + let payload; + try { + payload = JSON.parse(fs.readFileSync(reportPath, "utf8")); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + const displayPath = + typeof reportPath === "string" + ? reportPath + : typeof reportPath === "number" + ? String(reportPath) + : Buffer.isBuffer(reportPath) + ? reportPath.toString() + : fileURLToPath(reportPath); + return { ok: false, reason: `Unable to read Vitest report ${displayPath}: ${message}` }; + } + return validateLiveShardReportPayload(payload, expectedFiles); +} + +/** + * Builds spawn options for the live-shard Vitest child. + */ +export function buildLiveShardSpawnParams( + env = process.env, + platform = process.platform, + runtimeEnv?: NodeJS.ProcessEnv, +) { + return { + detached: shouldUseDetachedVitestProcessGroup(platform), + env: { ...env, ...runtimeEnv }, + stdio: "inherit", + } satisfies Pick<PnpmRunnerParams, "detached" | "env" | "stdio">; +} + +export function resolveLiveShardBuildEntrypoint(exists = fs.existsSync): string[] { + // Release harnesses run this trusted shard router from a frozen candidate + // checkout. Prefer its current TypeScript builder, then its native ancestor. + if (exists("scripts/build-all.mts")) { + return ["--import", "tsx", "scripts/build-all.mts"]; + } + if (exists("scripts/build-all.mjs")) { + return ["scripts/build-all.mjs"]; + } + throw new Error("Live test shard cannot find scripts/build-all.{mts,mjs}"); +} + +export function resolveLiveShardBuildProfile(profile: string, helpOutput: string): string { + return helpOutput.split("\n").includes(` ${profile}`) ? profile : "full"; +} + +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + const rawArgs = process.argv.slice(2); + const separatorIndex = rawArgs.indexOf("--"); + const optionArgs = separatorIndex >= 0 ? rawArgs.slice(0, separatorIndex) : rawArgs; + if (optionArgs.includes("--help") || optionArgs.includes("-h")) { + usage(process.stdout); + process.exit(0); + } + + let parsedArgs; + try { + parsedArgs = parseLiveShardArgs(rawArgs); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + usage(); + process.exit(2); + } + const { shard, listOnly, passthroughArgs } = parsedArgs; + if (!shard) { + usage(); + process.exit(2); + } + + let files; + try { + files = selectLiveShardFiles(shard); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + usage(); + process.exit(2); + } + if (files.length === 0) { + console.error(`Live test shard '${shard}' selected no files.`); + process.exit(2); + } + + if (listOnly) { + for (const file of files) { + console.log(file); + } + process.exit(0); + } + + // Some live tests exercise built private surfaces. Prepare their owning profile so + // shard routing cannot select a test whose required runtime artifact is absent. + const preparation = resolveLiveShardPreparation(files); + if (preparation) { + console.log( + `[test:live:shard] preparing ${preparation.profile} for ${preparation.requiredArtifact}`, + ); + const buildEntrypoint = resolveLiveShardBuildEntrypoint(); + const help = spawnSync(process.execPath, [...buildEntrypoint, "--help"], { + env: { ...process.env, ...preparation.env }, + encoding: "utf8", + }); + if (help.error) { + console.error(help.error); + process.exit(1); + } + if (help.signal) { + process.kill(process.pid, help.signal); + process.exit(1); + } + if ((help.status ?? 1) !== 0) { + process.exit(help.status ?? 1); + } + const buildProfile = resolveLiveShardBuildProfile(preparation.profile, help.stdout); + if (buildProfile !== preparation.profile) { + console.log( + `[test:live:shard] ${preparation.profile} is unavailable; preparing full build instead`, + ); + } + const result = spawnSync(process.execPath, [...buildEntrypoint, buildProfile], { + env: { ...process.env, ...preparation.env }, + stdio: "inherit", + }); + if (result.error) { + console.error(result.error); + process.exit(1); + } + if (result.signal) { + process.kill(process.pid, result.signal); + process.exit(1); + } + if ((result.status ?? 1) !== 0) { + process.exit(result.status ?? 1); + } + if (!fs.existsSync(preparation.requiredArtifact)) { + console.error( + `[test:live:shard] ${preparation.profile} did not produce ${preparation.requiredArtifact}`, + ); + process.exit(1); + } + } + + console.log(`[test:live:shard] ${shard}: ${files.length} file(s)`); + const reportPath = buildLiveShardReportPath(shard, process.env); + fs.mkdirSync(path.dirname(reportPath), { recursive: true }); + removeLiveShardReportFile(reportPath); + const spawnParams = buildLiveShardSpawnParams( + process.env, + process.platform, + preparation?.runtimeEnv, + ); + const child = spawnPnpmRunner({ + pnpmArgs: buildLiveShardPnpmArgs(files, addLiveShardReportArgs(passthroughArgs, reportPath)), + ...spawnParams, + }); + const cleanup = installVitestProcessGroupCleanup({ + child, + forceSignal: "SIGKILL", + forceSignalDelayMs: 100, + }); + createVitestProcessCompletion({ child, detached: spawnParams.detached }) + .finally(cleanup.teardown) + .then( + ({ code, signal }) => { + const forwardedSignal = cleanup.getForwardedSignal(); + if (forwardedSignal) { + process.kill(process.pid, forwardedSignal); + return; + } + if (signal) { + process.kill(process.pid, signal as NodeJS.Signals); + return; + } + if ((code ?? 1) === 0) { + const validation = validateLiveShardReport(reportPath, files); + if (!validation.ok) { + process.stderr.write(`[test:live:shard] ${validation.reason}\n`); + process.exit(1); + } + } + process.exit(code ?? 1); + }, + (error: unknown) => { + console.error(error); + process.exit(1); + }, + ); +} diff --git a/scripts/test-live-subagent-announce-docker.sh b/scripts/test-live-subagent-announce-docker.sh new file mode 100644 index 0000000000000000000000000000000000000000..3745b3113b38b29e4788e716889d606b6b53fdef --- /dev/null +++ b/scripts/test-live-subagent-announce-docker.sh @@ -0,0 +1,143 @@ +#!/usr/bin/env bash +# Bash 5.3+ can deadlock writing heredoc pipes on macOS before the reader starts. +if [[ ${OSTYPE:-} == darwin* && $BASH != /bin/bash ]] && ((BASH_VERSINFO[0] > 5 || (BASH_VERSINFO[0] == 5 && BASH_VERSINFO[1] >= 3))); then + exec /bin/bash "$0" "$@" +fi +set -euo pipefail + +SCRIPT_ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +ROOT_DIR="${OPENCLAW_LIVE_DOCKER_REPO_ROOT:-$SCRIPT_ROOT_DIR}" +ROOT_DIR="$(cd "$ROOT_DIR" && pwd)" +TRUSTED_HARNESS_DIR="${OPENCLAW_LIVE_DOCKER_TRUSTED_HARNESS_DIR:-$SCRIPT_ROOT_DIR}" +if [[ -z "$TRUSTED_HARNESS_DIR" || ! -d "$TRUSTED_HARNESS_DIR" ]]; then + echo "ERROR: trusted live Docker harness directory not found: ${TRUSTED_HARNESS_DIR:-<empty>}." >&2 + exit 1 +fi +TRUSTED_HARNESS_DIR="$(cd "$TRUSTED_HARNESS_DIR" && pwd)" +source "$TRUSTED_HARNESS_DIR/scripts/lib/live-docker-auth.sh" + +IMAGE_NAME="${OPENCLAW_IMAGE:-openclaw:local}" +LIVE_IMAGE_NAME="${OPENCLAW_LIVE_IMAGE:-${IMAGE_NAME}-live}" +CONFIG_DIR="${OPENCLAW_CONFIG_DIR:-$HOME/.openclaw}" +WORKSPACE_DIR="${OPENCLAW_WORKSPACE_DIR:-$HOME/.openclaw/workspace}" +PROFILE_FILE="$(openclaw_live_default_profile_file)" +DOCKER_EXTRA_ENV_FILES=() +DOCKER_TRUSTED_HARNESS_CONTAINER_DIR="/trusted-harness" +DOCKER_TRUSTED_HARNESS_MOUNT=(-v "$TRUSTED_HARNESS_DIR":"$DOCKER_TRUSTED_HARNESS_CONTAINER_DIR":ro) +openclaw_live_init_temp_dirs +openclaw_live_init_cache_home_dir +openclaw_live_init_managed_home +openclaw_live_init_profile_mount + +if [[ -n "${OPENAI_API_KEY:-}" || -n "${OPENAI_BASE_URL:-}" || -n "${GEMINI_API_KEY:-}" || -n "${GOOGLE_API_KEY:-}" ]]; then + docker_env_dir="$(mktemp -d "${RUNNER_TEMP:-/tmp}/openclaw-subagent-live-env.XXXXXX")" + TEMP_DIRS+=("$docker_env_dir") + docker_env_file="$docker_env_dir/provider.env" + { + if [[ -n "${OPENAI_API_KEY:-}" ]]; then + printf 'OPENCLAW_DOCKER_LIVE_OPENAI_API_KEY=%s\n' "${OPENAI_API_KEY}" + fi + if [[ -n "${OPENAI_BASE_URL:-}" ]]; then + printf 'OPENCLAW_DOCKER_LIVE_OPENAI_BASE_URL=%s\n' "${OPENAI_BASE_URL}" + fi + if [[ -n "${GEMINI_API_KEY:-}" ]]; then + printf 'OPENCLAW_DOCKER_LIVE_GEMINI_API_KEY=%s\n' "${GEMINI_API_KEY}" + fi + if [[ -n "${GOOGLE_API_KEY:-}" ]]; then + printf 'OPENCLAW_DOCKER_LIVE_GOOGLE_API_KEY=%s\n' "${GOOGLE_API_KEY}" + fi + } >"$docker_env_file" + DOCKER_EXTRA_ENV_FILES+=(--env-file "$docker_env_file") +fi + +CONTAINER_NODE_OPTIONS="$(openclaw_live_container_node_options)" + +read -r -d '' LIVE_TEST_CMD <<'EOF' || true +set -euo pipefail +[ -f "$HOME/.profile" ] && [ -r "$HOME/.profile" ] && source "$HOME/.profile" || true +if [ -n "${OPENCLAW_DOCKER_LIVE_OPENAI_API_KEY:-}" ]; then + export OPENAI_API_KEY="$OPENCLAW_DOCKER_LIVE_OPENAI_API_KEY" + unset OPENCLAW_DOCKER_LIVE_OPENAI_API_KEY +fi +if [ -n "${OPENCLAW_DOCKER_LIVE_OPENAI_BASE_URL:-}" ]; then + export OPENAI_BASE_URL="$OPENCLAW_DOCKER_LIVE_OPENAI_BASE_URL" + unset OPENCLAW_DOCKER_LIVE_OPENAI_BASE_URL +fi +if [ -n "${OPENCLAW_DOCKER_LIVE_GEMINI_API_KEY:-}" ]; then + export GEMINI_API_KEY="$OPENCLAW_DOCKER_LIVE_GEMINI_API_KEY" + unset OPENCLAW_DOCKER_LIVE_GEMINI_API_KEY +fi +if [ -n "${OPENCLAW_DOCKER_LIVE_GOOGLE_API_KEY:-}" ]; then + export GOOGLE_API_KEY="$OPENCLAW_DOCKER_LIVE_GOOGLE_API_KEY" + unset OPENCLAW_DOCKER_LIVE_GOOGLE_API_KEY +fi +export XDG_CACHE_HOME="${XDG_CACHE_HOME:-$HOME/.cache}" +export COREPACK_HOME="${COREPACK_HOME:-$XDG_CACHE_HOME/node/corepack}" +export NPM_CONFIG_CACHE="${NPM_CONFIG_CACHE:-$XDG_CACHE_HOME/npm}" +export npm_config_cache="$NPM_CONFIG_CACHE" +mkdir -p "$XDG_CACHE_HOME" "$COREPACK_HOME" "$NPM_CONFIG_CACHE" +chmod 700 "$XDG_CACHE_HOME" "$COREPACK_HOME" "$NPM_CONFIG_CACHE" || true +tmp_dir="$(mktemp -d)" +trusted_scripts_dir="${OPENCLAW_LIVE_DOCKER_SCRIPTS_DIR:-/src/scripts}" +source "$trusted_scripts_dir/lib/live-docker-stage.sh" +openclaw_live_stage_source_tree "$tmp_dir" +openclaw_live_stage_node_modules "$tmp_dir" +openclaw_live_link_runtime_tree "$tmp_dir" +openclaw_live_stage_state_dir "$tmp_dir/.openclaw-state" +openclaw_live_prepare_staged_config +cd "$tmp_dir" +announce_relative="$( + openclaw_live_resolve_unique_staged_file "$tmp_dir/src/agents" subagent-announce.live.test.ts +)" +announce_test="src/agents/$announce_relative" +OPENCLAW_LIVE_TEST=1 \ +OPENCLAW_LIVE_SUBAGENT_E2E=1 \ +OPENCLAW_VITEST_MAX_WORKERS="${OPENCLAW_VITEST_MAX_WORKERS:-1}" \ +openclaw_live_run_staged_script scripts/test-live -- "$announce_test" -- --reporter=verbose +EOF + +OPENCLAW_LIVE_DOCKER_REPO_ROOT="$ROOT_DIR" "$TRUSTED_HARNESS_DIR/scripts/test-live-build-docker.sh" +if openclaw_live_uses_managed_bind_dirs; then + openclaw_live_chown_bind_dirs_for_container_user \ + "$LIVE_IMAGE_NAME" \ + "$DOCKER_USER" \ + "$CACHE_HOME_DIR" \ + "${DOCKER_HOME_DIR:-}" +fi + +echo "==> Run subagent announce live test in Docker" +echo "==> Target: unique staged subagent-announce.live.test.ts" +echo "==> Model: ${OPENCLAW_LIVE_SUBAGENT_E2E_MODEL:-openai/gpt-5.6-luna}" +echo "==> Profile file: $PROFILE_STATUS" +DOCKER_RUN_ARGS=() +openclaw_live_init_docker_run_args DOCKER_RUN_ARGS "${OPENCLAW_LIVE_SUBAGENT_DOCKER_RUN_TIMEOUT:-1200s}" +DOCKER_RUN_ARGS+=(--rm -t \ + -u "$DOCKER_USER" \ + --entrypoint bash \ + -e COREPACK_ENABLE_DOWNLOAD_PROMPT=0 \ + -e HOME=/home/node \ + -e NODE_OPTIONS="$CONTAINER_NODE_OPTIONS" \ + -e OPENCLAW_SKIP_CHANNELS=1 \ + -e OPENCLAW_SUPPRESS_NOTES=1 \ + -e OPENCLAW_LIVE_DOCKER_SCRIPTS_DIR="${DOCKER_TRUSTED_HARNESS_CONTAINER_DIR}/scripts" \ + -e OPENCLAW_LIVE_DOCKER_SOURCE_STAGE_MODE="${OPENCLAW_LIVE_DOCKER_SOURCE_STAGE_MODE:-copy}" \ + -e OPENCLAW_LIVE_TEST=1 \ + -e OPENCLAW_LIVE_TEST_QUIET="${OPENCLAW_LIVE_TEST_QUIET:-}" \ + -e OPENCLAW_LIVE_WRAPPER_HEARTBEAT_MS="${OPENCLAW_LIVE_WRAPPER_HEARTBEAT_MS:-}" \ + -e OPENCLAW_LIVE_SUBAGENT_E2E=1 \ + -e OPENCLAW_LIVE_SUBAGENT_E2E_MODEL="${OPENCLAW_LIVE_SUBAGENT_E2E_MODEL:-}" \ + -e OPENCLAW_VITEST_FS_MODULE_CACHE=0 \ + -e OPENCLAW_VITEST_MAX_WORKERS="${OPENCLAW_VITEST_MAX_WORKERS:-1}") +openclaw_live_append_array DOCKER_RUN_ARGS DOCKER_EXTRA_ENV_FILES +openclaw_live_append_array DOCKER_RUN_ARGS DOCKER_HOME_MOUNT +openclaw_live_append_array DOCKER_RUN_ARGS DOCKER_TRUSTED_HARNESS_MOUNT +DOCKER_RUN_ARGS+=(\ + -v "$CACHE_HOME_DIR":/home/node/.cache \ + -v "$ROOT_DIR":/src:ro \ + -v "$CONFIG_DIR":/home/node/.openclaw \ + -v "$WORKSPACE_DIR":/home/node/.openclaw/workspace) +openclaw_live_append_array DOCKER_RUN_ARGS PROFILE_MOUNT +DOCKER_RUN_ARGS+=(\ + "$LIVE_IMAGE_NAME" \ + -lc "$LIVE_TEST_CMD") +"${DOCKER_RUN_ARGS[@]}" diff --git a/scripts/test-macos-health-render.sh b/scripts/test-macos-health-render.sh new file mode 100644 index 0000000000000000000000000000000000000000..81f7a73252846d8a07d7de00562361cb6bfd60a5 --- /dev/null +++ b/scripts/test-macos-health-render.sh @@ -0,0 +1,42 @@ +#!/bin/bash +set -euo pipefail + +# Run only on the disposable CI macOS worker after its normal Swift test build. +[[ "${CI:-}" == "true" && -n "${RUNNER_TEMP:-}" && -x /usr/bin/sandbox-exec ]] +render_repo="$(cd "$(dirname "$0")/.." && pwd -P)" +render_build="$(cd "$render_repo/apps/macos/.build/debug" && pwd -P)" +render_swift="$(xcrun --find swift)" +render_platform="$(xcrun --sdk macosx --show-sdk-platform-path)/Developer" +render_helper="$(dirname "$render_swift")/../libexec/swift/pm/swiftpm-testing-helper" +render_bundle="$render_build/OpenClawPackageTests.xctest/Contents/MacOS/OpenClawPackageTests" +[[ -x "$render_helper" && -f "$render_bundle" ]] +[[ -f "$render_platform/Library/Frameworks/Testing.framework/Testing" ]] +render_root="$(mktemp -d "$RUNNER_TEMP/openclaw-health-render.XXXXXX")" +render_root="$(cd "$render_root" && pwd -P)" +render_home="$render_root/home" +render_original_home="$(cd "$HOME" && pwd -P)" +render_canary="$(mktemp "$render_original_home/openclaw-health-denied.XXXXXX")" +trap 'rm -f "$render_canary"' EXIT +printf '%s\n' 'isolated-render-canary' > "$render_canary" +mkdir -p "$render_home/state" "$render_home/Library/Preferences" "$render_root/pngs" "$render_root/tmp" +printf 'artifact-path=%s\n' "$render_root/pngs" >> "$GITHUB_OUTPUT" + +# No inherited credentials, preferences daemons, network, or operator state. +# The compiled bundle runs directly; no package resolution or build runs here. +# Set SwiftPM's platform search paths after sandbox-exec; protected system +# executables otherwise remove DYLD variables before the test helper starts. +env -i \ + PATH=/usr/bin:/bin:/usr/sbin:/sbin \ + HOME="$render_home" CFFIXED_USER_HOME="$render_home" TMPDIR="$render_root/tmp/" \ + LLVM_PROFILE_FILE="$render_root/render-%p.profraw" \ + OPENCLAW_PROFILE=health-render \ + OPENCLAW_STATE_DIR="$render_home/state" OPENCLAW_CONFIG_PATH="$render_home/state/openclaw.json" \ + OPENCLAW_TEST_HEALTH_RENDER_DIR="$render_root/pngs" OPENCLAW_TEST_HEALTH_DENIED_FILE="$render_canary" \ + /usr/bin/sandbox-exec \ + -D "ORIGINAL_HOME=$render_original_home" -D "REPO=$render_repo" -D "ISOLATED_ROOT=$render_root" \ + -f "$render_repo/scripts/macos-health-render.sb" \ + /usr/bin/env \ + DYLD_FRAMEWORK_PATH="$render_platform/Library/Frameworks:$render_platform/Library/PrivateFrameworks" \ + DYLD_LIBRARY_PATH="$render_build:$render_platform/usr/lib" \ + "$render_helper" --test-bundle-path "$render_bundle" --testing-library swift-testing \ + --filter 'HealthStoreStateTests/`health settings render`' diff --git a/scripts/test-perf-budget.mts b/scripts/test-perf-budget.mts new file mode 100644 index 0000000000000000000000000000000000000000..c55f342a05dfc48019aed0aadf2fd444e0bc283d --- /dev/null +++ b/scripts/test-perf-budget.mts @@ -0,0 +1,136 @@ +// Runs a Vitest config and enforces wall-time regression budgets. +import { pathToFileURL } from "node:url"; +import { isRecord } from "@openclaw/normalization-core/record-coerce"; +import { + booleanFlag, + isStrictAffirmativeValue, + parseFlagArgs, + stringFlag, +} from "./lib/arg-utils.mts"; +import { budgetFloatFlag, readBudgetEnvNumber } from "./lib/budget-number-args.mts"; +import { coerceErrorMessage } from "./lib/error-format.mts"; +import { formatMs } from "./lib/vitest-report-cli-utils.mts"; +import { readJsonFile, runVitestJsonReport } from "./test-report-utils.mts"; + +function parseArgs(argv: readonly string[], env = process.env) { + const opts = parseFlagArgs( + argv, + { + config: "test/vitest/vitest.unit.config.ts", + maxWallMs: readBudgetEnvNumber("OPENCLAW_TEST_PERF_MAX_WALL_MS", env), + baselineWallMs: readBudgetEnvNumber("OPENCLAW_TEST_PERF_BASELINE_WALL_MS", env), + maxRegressionPct: readBudgetEnvNumber("OPENCLAW_TEST_PERF_MAX_REGRESSION_PCT", env) ?? 10, + reportOnly: isStrictAffirmativeValue(env.OPENCLAW_TEST_PERF_REPORT_ONLY), + }, + [ + stringFlag("--config", "config"), + budgetFloatFlag("--max-wall-ms", "maxWallMs"), + budgetFloatFlag("--baseline-wall-ms", "baselineWallMs"), + budgetFloatFlag("--max-regression-pct", "maxRegressionPct"), + booleanFlag("--report-only", "reportOnly", true), + ], + ); + if (opts.maxWallMs === null && opts.baselineWallMs === null && !opts.reportOnly) { + throw new Error( + "[test-perf-budget] provide --max-wall-ms, --baseline-wall-ms, or set --report-only for an explicit timing-only run", + ); + } + return opts; +} + +function collectPerfReportStats(reportPath: string) { + let report: unknown; + try { + report = readJsonFile(reportPath); + } catch (error) { + throw new Error( + `[test-perf-budget] failed to read Vitest JSON report ${reportPath}: ${coerceErrorMessage( + error, + )}`, + { cause: error }, + ); + } + + let totalFileDurationMs = 0; + let fileCount = 0; + const testResults = + isRecord(report) && Array.isArray(report.testResults) ? report.testResults : []; + for (const result of testResults.filter(isRecord)) { + if (typeof result.startTime === "number" && typeof result.endTime === "number") { + totalFileDurationMs += Math.max(0, result.endTime - result.startTime); + fileCount += 1; + } + } + if (fileCount === 0) { + throw new Error(`[test-perf-budget] Vitest JSON report contained no timed file results`); + } + return { fileCount, totalFileDurationMs }; +} + +function main() { + let opts; + try { + opts = parseArgs(process.argv.slice(2)); + } catch (error) { + console.error(coerceErrorMessage(error)); + process.exit(1); + } + + const startedAt = process.hrtime.bigint(); + const reportPath = runVitestJsonReport({ + config: opts.config, + prefix: "openclaw-vitest-perf", + }); + const elapsedMs = Number.parseFloat(String(process.hrtime.bigint() - startedAt)) / 1_000_000; + + let reportStats; + try { + reportStats = collectPerfReportStats(reportPath); + } catch (error) { + console.error(coerceErrorMessage(error)); + process.exit(1); + } + + const allowedByBaseline = + opts.baselineWallMs !== null + ? opts.baselineWallMs * (1 + (opts.maxRegressionPct ?? 0) / 100) + : null; + + let failed = false; + if (opts.maxWallMs !== null && elapsedMs > opts.maxWallMs) { + console.error( + `[test-perf-budget] wall time ${formatMs(elapsedMs)} exceeded max ${formatMs( + opts.maxWallMs, + )}.`, + ); + failed = true; + } + if (allowedByBaseline !== null && elapsedMs > allowedByBaseline) { + console.error( + `[test-perf-budget] wall time ${formatMs(elapsedMs)} exceeded baseline budget ${formatMs( + allowedByBaseline, + )} (baseline ${formatMs(opts.baselineWallMs ?? 0)}, +${String(opts.maxRegressionPct)}%).`, + ); + failed = true; + } + + console.log( + `[test-perf-budget] config=${opts.config} wall=${formatMs(elapsedMs)} file-sum=${formatMs( + reportStats.totalFileDurationMs, + )} files=${String(reportStats.fileCount)}`, + ); + + if (failed) { + process.exit(1); + } +} + +/** Test-facing parser helpers for budget validation. */ +export const testing = { + collectPerfReportStats, + parseArgs, +}; + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main(); +} diff --git a/scripts/test-projects-max.mts b/scripts/test-projects-max.mts new file mode 100644 index 0000000000000000000000000000000000000000..3e5e87b099134364147e545e130f6b1c9b43ce7f --- /dev/null +++ b/scripts/test-projects-max.mts @@ -0,0 +1,4 @@ +// Runs test-projects with an explicit high worker budget for max-throughput checks. +process.env.OPENCLAW_VITEST_MAX_WORKERS = "8"; + +await import("./test-projects.mts"); diff --git a/scripts/test-projects-run.mts b/scripts/test-projects-run.mts new file mode 100644 index 0000000000000000000000000000000000000000..55d1bdc5b9301dee164500557a159fadb60b301e --- /dev/null +++ b/scripts/test-projects-run.mts @@ -0,0 +1,597 @@ +// Dispatches Vitest project shards for explicit targets, changed files, or the +// full local suite. +import type { SpawnOptions } from "node:child_process"; +import fs from "node:fs"; +import { performance } from "node:perf_hooks"; +import pMap from "p-map"; +import { assertTestHomeSelection, combineTestHomeSelections } from "../test/test-home-policy.mts"; +import { loadPatternListFromEnv } from "../test/vitest/vitest.pattern-file.ts"; +import { formatMs } from "./lib/check-timing-summary.mts"; +import { signalExitCode } from "./lib/managed-child-process.mts"; +import { + prepareE2eVitestRuntime, + prepareVitestRuntime, + resolveVitestCliEntry, +} from "./lib/vitest-build-prerequisites.mts"; +import { createVitestCacheSlots } from "./lib/vitest-cache-slots.mts"; +import { hasNonRunVitestSubcommand } from "./lib/vitest-cli-mode.mts"; +import { parseVitestExecutionArgs } from "./lib/vitest-cli.mts"; +import { resolveVitestHomeSelection } from "./lib/vitest-home-selection.mts"; +import { isCiLikeEnv, resolveLocalFullSuiteProfile } from "./lib/vitest-local-scheduling.mts"; +import { resolveVitestNodeArgs, resolveVitestProcessEnv } from "./lib/vitest-process-env.mts"; +import type { exitVitestBySignal } from "./lib/vitest-process.mts"; +import { createVitestReportOwner, type VitestReportOwner } from "./lib/vitest-report-owner.mts"; +import { resolveVitestRuntimeCliSelections } from "./lib/vitest-runtime-selection.mts"; +import { + createShardTimingSample, + readShardTimings, + writeShardTimings, +} from "./lib/vitest-shard-timings.mts"; +import { getVitestWorkerDescriptor } from "./lib/vitest-worker-bootstrap.mts"; +import { createVitestWorkerRun, type VitestWorkerRun } from "./lib/vitest-worker-run.mts"; +import { resolveVitestSpawnParams, spawnWatchedVitestProcess } from "./run-vitest.mts"; +import { + applyDefaultMultiSpecVitestCachePaths, + applyDefaultVitestNoOutputTimeout, + applyFullExtensionsHeapBudget, + applyParallelVitestCachePaths, + buildFullSuiteVitestRunPlans, + createVitestPreflightPnpmArgs, + createVitestRunSpecs, + findUnmatchedExplicitTestTargets, + formatFailedShardDigest, + formatNoChangedTestTargetLines, + isTestFileTarget, + listFullExtensionVitestProjectConfigs, + orderFullSuiteSpecsForParallelRun, + parseTestProjectsArgs, + resolveParallelFullSuiteConcurrency, + resolveChangedTestTargetPlanForArgs, + resolveChangedTargetArgs, + shouldRetryVitestNoOutputTimeout, + type FailedVitestShard, + type VitestRunSpec as BaseVitestRunSpec, + type VitestCacheAssignment, + withRetryNoOutputTimeout, + writeVitestIncludeFile, +} from "./test-projects.test-support.mts"; + +type VitestRunSpec = BaseVitestRunSpec & { + timingIncludePatterns?: string[]; + continueOnFailure?: boolean; + reportIndex?: number; + workerRun?: VitestWorkerRun; + cacheAssignment?: VitestCacheAssignment; +}; +type VitestCommandOutcome = { + code: number; + noOutputTimedOut: boolean; + signal: NodeJS.Signals | null; + groupJoined: boolean; +}; + +type ShardTiming = NonNullable<ReturnType<typeof createShardTimingSample>>; + +function assertCacheLeaseJoined(spec: VitestRunSpec, result: VitestCommandOutcome) { + if ( + spec.cacheAssignment?.kind === "scheduler" && + spec.cacheAssignment.leased && + !result.groupJoined + ) { + throw new Error("Cannot continue a Vitest cache lease without verified group completion"); + } +} + +function printHelp() { + console.log(`Usage: node --import tsx scripts/test-projects.mts [--changed <base>] [--watch] [targets...] [-- vitest-args...] + +Runs the Vitest project shards that own the requested targets. With no targets, +this runs the full local suite. Use explicit targets for local edit loops.`); +} + +function cleanupVitestRunSpec(spec: VitestRunSpec) { + if (!spec.includeFilePath) { + return; + } + try { + fs.rmSync(spec.includeFilePath, { force: true }); + } catch { + // Best-effort cleanup for temp include lists. + } +} + +function runPnpmSpecCommand( + spec: VitestRunSpec, + pnpmArgs: string[], + workerRun?: VitestWorkerRun, + homeMode?: Parameters<typeof spawnWatchedVitestProcess>[0]["homeMode"], +) { + let noOutputTimedOut = false; + return new Promise<VitestCommandOutcome>((resolve, reject) => { + const { completion, getForwardedSignal } = spawnWatchedVitestProcess({ + workerRun, + homeMode, + pnpmArgs, + env: spec.env, + onNoOutputTimeout: () => { + noOutputTimedOut = true; + }, + spawnParams: { + cwd: process.cwd(), + ...resolveVitestSpawnParams(spec.env), + stdio: ["inherit", "pipe", "pipe"] satisfies SpawnOptions["stdio"], + }, + }); + + completion.then( + ({ code, signal, groupJoined }) => { + const exitSignal = getForwardedSignal() ?? signal; + resolve({ + code: exitSignal ? signalExitCode(exitSignal) : (code ?? 1), + noOutputTimedOut, + signal: exitSignal, + groupJoined, + }); + }, + (error: unknown) => { + reject(error instanceof Error ? error : new Error(String(error))); + }, + ); + }); +} + +async function runVitestSpec(spec: VitestRunSpec, reports: VitestReportOwner) { + let preflightJoined = true; + if (spec.includeFilePath && spec.includePatterns) { + writeVitestIncludeFile(spec.includeFilePath, spec.includePatterns, { + expandGlobs: !spec.watchMode, + }); + } + try { + if (spec.preflightPnpmArgs) { + console.error(`[test] preflight ${spec.config}`); + const preflightResult = await runPnpmSpecCommand( + spec, + spec.preflightPnpmArgs, + undefined, + "tooling", + ); + preflightJoined = preflightResult.groupJoined; + if (preflightResult.code !== 0 || preflightResult.signal) { + return preflightResult; + } + assertCacheLeaseJoined(spec, preflightResult); + } + const attempt = reports?.attempt(spec.reportIndex!, spec.pnpmArgs); + try { + const result = await runPnpmSpecCommand(spec, attempt?.args ?? spec.pnpmArgs, spec.workerRun); + attempt?.complete(result); + return { ...result, groupJoined: preflightJoined && result.groupJoined }; + } catch (error) { + attempt?.fail(error); + throw error; + } + } finally { + cleanupVitestRunSpec(spec); + } +} + +function applyDefaultParallelVitestWorkerBudget(specs: VitestRunSpec[], env: NodeJS.ProcessEnv) { + if (env.OPENCLAW_VITEST_MAX_WORKERS || env.OPENCLAW_TEST_WORKERS || isCiLikeEnv(env)) { + return specs; + } + const { vitestMaxWorkers } = resolveLocalFullSuiteProfile(env); + return specs.map((spec) => ({ + ...spec, + env: { + ...spec.env, + OPENCLAW_VITEST_MAX_WORKERS: String(vitestMaxWorkers), + }, + })); +} + +async function runLoggedVitestSpec(spec: VitestRunSpec, reports: VitestReportOwner) { + console.error(`[test] starting ${spec.config}`); + const startedAt = performance.now(); + let result = await runVitestSpec(spec, reports); + if (result.noOutputTimedOut && !spec.watchMode && shouldRetryVitestNoOutputTimeout(spec.env)) { + assertCacheLeaseJoined(spec, result); + console.error(`[test] retrying ${spec.config} after no-output timeout`); + const firstJoined = result.groupJoined; + result = await runVitestSpec(withRetryNoOutputTimeout(spec), reports); + result = { ...result, groupJoined: firstJoined && result.groupJoined }; + } + const durationMs = performance.now() - startedAt; + if (result.noOutputTimedOut && result.signal) { + console.error(`[test] ${spec.config} exceeded no-output timeout`); + return { + ...result, + code: result.code || 143, + signal: null, + timing: null, + }; + } + if (result.signal) { + console.error(`[test] ${spec.config} exited by signal ${result.signal}`); + return { ...result, timing: null }; + } + return { + ...result, + timing: createShardTimingSample(spec, durationMs), + }; +} + +function isFullExtensionsProjectRun(specs: VitestRunSpec[]) { + const fullExtensionProjectConfigs = new Set(listFullExtensionVitestProjectConfigs()); + return ( + specs.length > 1 && + specs.every( + (spec) => + !spec.watchMode && + spec.includePatterns === null && + fullExtensionProjectConfigs.has(spec.config), + ) + ); +} + +function printNoChangedTestTargets(args: string[], cwd: string, baseEnv: NodeJS.ProcessEnv) { + const plan = resolveChangedTestTargetPlanForArgs(args, cwd, undefined, { env: baseEnv }); + const skippedBroadFallbackPaths = plan?.skippedBroadFallbackPaths ?? []; + for (const line of formatNoChangedTestTargetLines(skippedBroadFallbackPaths)) { + console.error(line); + } +} + +async function runVitestSpecs( + specs: VitestRunSpec[], + concurrency: number, + reports: VitestReportOwner, + termination: { signal: NodeJS.Signals | null }, +) { + let exitCode = 0; + let stopScheduling = false; + const failures: FailedVitestShard[] = []; + const timings: ShardTiming[] = []; + const withCacheSlot = createVitestCacheSlots(concurrency); + await pMap( + specs, + async (spec, index) => { + if (stopScheduling || termination.signal) { + return; + } + let result: Awaited<ReturnType<typeof runLoggedVitestSpec>>; + try { + result = await withCacheSlot(spec, (assigned) => runLoggedVitestSpec(assigned, reports)); + } catch (error) { + stopScheduling = true; + throw error; + } + if (result.signal) { + // A forwarded termination signal must not admit replacement shards during shutdown. + termination.signal ??= result.signal; + stopScheduling = true; + } + if (result.code !== 0) { + exitCode ||= result.code; + if (concurrency === 1 && spec.continueOnFailure !== true) { + stopScheduling = true; + } + failures.push({ + code: result.code, + config: spec.config, + includePatterns: spec.includePatterns, + noOutputTimedOut: result.noOutputTimedOut, + order: index, + signal: result.signal, + }); + } + if (result.timing) { + timings.push(result.timing); + } + }, + // Join already-admitted shards even when another shard's group join fails. + { concurrency, stopOnError: false }, + ); + return { exitCode, failures, timings, stopScheduling }; +} + +export async function runTestProjects( + exitBySignal: typeof exitVitestBySignal, + args: string[] = process.argv.slice(2), + env: NodeJS.ProcessEnv = process.env, +) { + const suiteStartedAt = performance.now(); + if (args.length === 1 && (args[0] === "--help" || args[0] === "-h")) { + printHelp(); + return; + } + const baseEnv = resolveVitestProcessEnv(env); + const { targetArgs, forwardedArgs } = parseTestProjectsArgs(args, process.cwd()); + const unmatchedExplicitTargets = findUnmatchedExplicitTestTargets(args, process.cwd()); + if (unmatchedExplicitTargets.length > 0) { + for (const unmatched of unmatchedExplicitTargets) { + const suffix = unmatched.includePattern ? ` (tried: ${unmatched.includePattern})` : ""; + console.error( + `[test] explicit test target matched no test files: ${unmatched.target}${suffix}`, + ); + } + printTestSummary("failed", 1, performance.now() - suiteStartedAt); + process.exitCode = 1; + return; + } + const changedTargetArgs = + targetArgs.length === 0 + ? resolveChangedTargetArgs(args, process.cwd(), undefined, { env: baseEnv }) + : null; + const rawRunSpecs: VitestRunSpec[] = + targetArgs.length === 0 && changedTargetArgs === null + ? buildFullSuiteVitestRunPlans(args, process.cwd()).map((plan) => ({ + config: plan.config, + timingTargets: plan.timingTargets, + continueOnFailure: true, + env: baseEnv, + includeFilePath: null, + includePatterns: null, + pnpmArgs: [ + "exec", + "node", + ...resolveVitestNodeArgs(baseEnv), + resolveVitestCliEntry(), + ...(plan.watchMode ? [] : ["run"]), + "--config", + plan.config, + ...plan.forwardedArgs, + ], + preflightPnpmArgs: createVitestPreflightPnpmArgs(plan.config), + watchMode: plan.watchMode, + })) + : createVitestRunSpecs(args, { + baseEnv, + cwd: process.cwd(), + }); + const inheritedIncludePatterns = rawRunSpecs.some((spec) => !spec.includeFilePath) + ? loadPatternListFromEnv("OPENCLAW_VITEST_INCLUDE_FILE", baseEnv) + : null; + for (const spec of rawRunSpecs) { + // An owned include file replaces the inherited filter. Otherwise retain its + // identity beside CLI chunk targets without changing execution or cleanup. + if (!spec.includeFilePath && inheritedIncludePatterns !== null) { + spec.timingIncludePatterns = inheritedIncludePatterns; + } + } + const runSpecs: VitestRunSpec[] = applyDefaultMultiSpecVitestCachePaths( + applyDefaultVitestNoOutputTimeout( + applyFullExtensionsHeapBudget(rawRunSpecs, { env: baseEnv }), + { + env: baseEnv, + }, + ), + { cwd: process.cwd(), env: baseEnv }, + ); + + if (runSpecs.length === 0) { + printNoChangedTestTargets(args, process.cwd(), baseEnv); + printTestSummary("skipped", 0, performance.now() - suiteStartedAt); + return; + } + + const { parseCLI } = await import("vitest/node"); + if ( + targetArgs.length && + !runSpecs.some((spec) => spec.watchMode) && + !hasNonRunVitestSubcommand(forwardedArgs) + ) { + // Native parsing stays in the execution owner. Original filters distinguish + // explicit files from broad selections that also lower to literal include files. + const execution = parseVitestExecutionArgs(["run", ...forwardedArgs], parseCLI); + if ( + execution && + !execution.options.watch && + execution.options.run !== false && + execution.filter.length > 0 && + execution.filter.every( + (file) => isTestFileTarget(file) && /[/\\]/u.test(file) && !/[*?[\]{}]|[@+!]\(/u.test(file), + ) && + !Object.hasOwn(execution.options, "passWithNoTests") + ) { + for (const spec of runSpecs) { + const separator = spec.pnpmArgs.indexOf("--"); + spec.pnpmArgs.splice( + separator < 0 ? spec.pnpmArgs.length : separator, + 0, + "--passWithNoTests=false", + ); + } + } + } + + runSpecs.forEach((spec, index) => { + spec.reportIndex = index; + }); + const homeMode = combineTestHomeSelections( + runSpecs.map((spec) => resolveVitestHomeSelection(spec.pnpmArgs, { env: spec.env })), + ); + // Refuse a mixed real-home run before report setup or runtime preparation imports code. + assertTestHomeSelection(baseEnv, homeMode); + const reports = await createVitestReportOwner( + runSpecs.map((spec) => ({ + config: spec.config, + includePatterns: spec.includePatterns, + args: spec.pnpmArgs.slice(spec.pnpmArgs.indexOf(resolveVitestCliEntry()) + 1), + })), + process.cwd(), + ); + const termination: { signal: NodeJS.Signals | null } = { signal: null }; + const onSignal = (signal: NodeJS.Signals) => { + termination.signal ??= signal; + }; + process.on("SIGTERM", onSignal); + process.on("SIGINT", onSignal); + let workers: VitestWorkerRun | undefined; + let reportFailure: string | undefined; + let printCompletedSummary: (() => void) | undefined; + try { + const admitted = runSpecs.map((spec) => { + const cliArgs = spec.pnpmArgs.slice(spec.pnpmArgs.indexOf(resolveVitestCliEntry()) + 1); + const execution = parseVitestExecutionArgs(cliArgs, parseCLI); + // Metadata and invalid input must not install browser prerequisites either. + if (!execution) { + spec.preflightPnpmArgs = null; + } + return { spec, cliArgs, execution }; + }); + const runnable = admitted.filter(({ execution }) => execution !== null); + const e2eSpecs = runnable + .map(({ spec }) => spec) + .filter((spec) => spec.config === "test/vitest/vitest.e2e.config.ts"); + if (e2eSpecs.length > 0) { + const preparedEnv = await prepareE2eVitestRuntime(baseEnv); + for (const spec of e2eSpecs) { + spec.env = { ...spec.env, ...preparedEnv }; + } + } else { + const code = await prepareVitestRuntime( + runnable.flatMap(({ spec, cliArgs }) => { + const selections = resolveVitestRuntimeCliSelections(spec.config, cliArgs, spec.env); + // These selections are invocation-owned; their include files are not written yet. + for (const selection of selections) { + selection.includePatterns = spec.includePatterns; + } + return selections; + }), + baseEnv, + ); + if (code !== 0) { + printTestSummary("failed", 0, performance.now() - suiteStartedAt); + process.exitCode = code; + return; + } + } + + const compiled = runnable.filter( + ({ spec, execution }) => !spec.watchMode && !execution?.options.watch, + ); + if (compiled.length) { + workers = createVitestWorkerRun(baseEnv, getVitestWorkerDescriptor()); + for (const { spec } of compiled) { + spec.workerRun = workers; + } + } + const isFullSuiteRun = + targetArgs.length === 0 && + changedTargetArgs === null && + !runSpecs.some((spec) => spec.watchMode); + const isExplicitParallelMultiConfigRun = + Boolean(baseEnv.OPENCLAW_TEST_PROJECTS_PARALLEL) && + runSpecs.length > 1 && + !runSpecs.some((spec) => spec.watchMode); + const isParallelShardRun = + isFullSuiteRun || isFullExtensionsProjectRun(runSpecs) || isExplicitParallelMultiConfigRun; + let scheduledSpecs = runSpecs; + const concurrency = isParallelShardRun + ? resolveParallelFullSuiteConcurrency(runSpecs.length, baseEnv) + : 1; + if (isParallelShardRun) { + if (!isCiLikeEnv(baseEnv) && runSpecs.length > 1) { + console.warn( + `[test] warning: broad local run will start ${runSpecs.length} Vitest shards; use \`pnpm test:changed\` for routine checks.`, + ); + } + if (concurrency > 1) { + const shardTimings = readShardTimings(process.cwd(), baseEnv); + const orderedSpecs = orderFullSuiteSpecsForParallelRun(runSpecs, shardTimings).filter( + (spec): spec is VitestRunSpec => spec !== undefined, + ); + scheduledSpecs = applyDefaultParallelVitestWorkerBudget( + applyParallelVitestCachePaths(orderedSpecs, { + cwd: process.cwd(), + env: baseEnv, + }), + baseEnv, + ); + console.error( + `[test] running ${scheduledSpecs.length} Vitest shards with parallelism ${concurrency}`, + ); + } + } + + const result = await runVitestSpecs(scheduledSpecs, concurrency, reports, termination); + if (concurrency === 1 && termination.signal) { + return; + } + if (concurrency > 1 || !result.stopScheduling) { + writeShardTimings(result.timings, process.cwd(), baseEnv); + } + printCompletedSummary = () => + printTestSummary( + process.exitCode ? "failed" : "passed", + concurrency > 1 ? scheduledSpecs.length : result.timings.length, + performance.now() - suiteStartedAt, + concurrency > 1 ? "Vitest summaries above are per-shard, not aggregate totals." : undefined, + ); + if (concurrency > 1) { + for (const line of formatFailedShardDigest(result.failures)) { + console.error(line); + } + } + if (result.exitCode !== 0) { + process.exitCode = result.exitCode; + } + } catch (error) { + reportFailure = String(error); + throw error; + } finally { + try { + await workers?.dispose().catch((error: unknown) => { + reportFailure ??= String(error); + process.exitCode ||= 1; + console.error(error); + }); + if (reports) { + const reportCode = await reports.finish( + async (mergeArgs) => { + // Replay is source-only: selected configs load after all compiled + // borrowers close; report blobs own the exact executed selection. + const outcome = await runPnpmSpecCommand( + { ...runSpecs[0]!, env: baseEnv }, + [ + "exec", + "node", + ...resolveVitestNodeArgs(baseEnv), + resolveVitestCliEntry(), + ...mergeArgs, + ], + undefined, + homeMode, + ); + termination.signal ??= outcome.signal; + return outcome; + }, + termination.signal ? `Cancelled by ${termination.signal}` : reportFailure, + ); + if (reportCode) { + process.exitCode ||= reportCode; + } + } + printCompletedSummary?.(); + } finally { + process.off("SIGTERM", onSignal); + process.off("SIGINT", onSignal); + if (termination.signal) { + await exitBySignal(termination.signal); + } + } + } +} + +function printTestSummary( + status: "failed" | "passed" | "skipped", + shardCount: number, + durationMs: number, + detail?: string, +) { + const suffix = detail ? `; ${detail}` : ""; + console.error( + `[test] ${status} ${shardCount} Vitest shard${shardCount === 1 ? "" : "s"} in ${formatMs(durationMs)}${suffix}`, + ); +} diff --git a/scripts/test-projects-serial.mts b/scripts/test-projects-serial.mts new file mode 100644 index 0000000000000000000000000000000000000000..a8a2dac9d771ef1e5404e87622f293890295ba91 --- /dev/null +++ b/scripts/test-projects-serial.mts @@ -0,0 +1,5 @@ +// Runs test-projects serially with a one-worker Vitest budget. +process.env.OPENCLAW_TEST_PROJECTS_SERIAL = "1"; +process.env.OPENCLAW_VITEST_MAX_WORKERS = "1"; + +await import("./test-projects.mts"); diff --git a/scripts/test-projects.mts b/scripts/test-projects.mts new file mode 100644 index 0000000000000000000000000000000000000000..bf806915fadd6698ee2416c293bf58b5d85a6d1c --- /dev/null +++ b/scripts/test-projects.mts @@ -0,0 +1,5 @@ +// Importing this public entrypoint also starts the serial/max/import aliases. +import { runVitestCli } from "./lib/vitest-process.mts"; +import { runTestProjects } from "./test-projects-run.mts"; + +void runVitestCli("test", runTestProjects); diff --git a/scripts/test-projects.test-support.mts b/scripts/test-projects.test-support.mts new file mode 100644 index 0000000000000000000000000000000000000000..bab6d83caed16b9c7e0cd0ca77d6cbfcb739abe7 --- /dev/null +++ b/scripts/test-projects.test-support.mts @@ -0,0 +1,4919 @@ +// Test-project planning helpers used by scripts/run-vitest.mts, +// scripts/test-projects.mts, and focused tests. Exports are intentionally +// granular so project selection stays testable without spawning Vitest. +import { spawnSync, type SpawnSyncOptionsWithStringEncoding } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { + agentVitestProjectOwners, + embeddedAgentVitestProjectOwners, + isAgentsCoreIsolatedTestFile, + isAgentsSpawnProductionBoundaryTestFile, +} from "../test/vitest/vitest.agents-paths.mjs"; +import { isChannelSurfaceTestFile } from "../test/vitest/vitest.channel-paths.mjs"; +import { + cliProcessTestFiles, + isCliProcessTestFile, +} from "../test/vitest/vitest.cli-process-paths.mjs"; +import { + commandsLightTestFiles, + isCommandsLightTarget, + resolveCommandsLightIncludePattern, +} from "../test/vitest/vitest.commands-light-paths.mjs"; +import { + channelConfigContractPatterns, + channelRegistryContractPatterns, + channelSessionContractPatterns, + channelSurfaceContractPatterns, +} from "../test/vitest/vitest.contracts-paths.mjs"; +import { + DATABASE_WORKER_WATCH_OWNER_ENV_KEY, + DATABASE_WORKER_WATCH_TESTS_ENV_KEY, + databaseWorkerCoreFormerFastKinds, + databaseWorkerCoreTestFiles, + isDatabaseWorkerCoreTestFile, +} from "../test/vitest/vitest.database-worker-core-paths.mjs"; +import { codexExtensionTestRoots } from "../test/vitest/vitest.extension-codex-paths.mjs"; +import { + databaseWorkerExtensionTestFiles, + databaseWorkerExtensionTestRoots, +} from "../test/vitest/vitest.extension-database-workers-paths.mjs"; +import { matrixExtensionTestRoots } from "../test/vitest/vitest.extension-matrix-paths.mjs"; +import { telegramExtensionTestRoots } from "../test/vitest/vitest.extension-telegram-paths.mjs"; +import { + gatewayDatabaseWorkerTestFiles, + gatewayPluginTestFiles, +} from "../test/vitest/vitest.gateway-server-paths.mjs"; +import { intersectIncludePatterns } from "../test/vitest/vitest.include-patterns.ts"; +import { packageContractTestFiles } from "../test/vitest/vitest.package-contract-paths.mjs"; +import { resolveVitestFsModuleCacheRoot } from "../test/vitest/vitest.performance-config.ts"; +import { + isPluginSdkLightTarget, + pluginSdkLightTestFiles, + resolvePluginSdkLightIncludePattern, +} from "../test/vitest/vitest.plugin-sdk-paths.mjs"; +import { fullSuiteVitestShards, tuiPtyTestFiles } from "../test/vitest/vitest.test-shards.mjs"; +import { + isToolingIsolatedTestFile, + toolingIsolatedTestFiles, +} from "../test/vitest/vitest.tooling-isolated-paths.mjs"; +import { + isUiIsolatedTestFile, + uiIsolatedTestFiles, +} from "../test/vitest/vitest.ui-isolated-paths.mjs"; +import { + isControlUiSourcePath, + isPluginControlUiPath, + isUiBrowserTestFile, + uiTimingTestFiles, +} from "../test/vitest/vitest.ui-paths.mjs"; +import { + getUnitFastIsolatedTestFiles, + getUnitFastTestFiles, + getUnitFastTimerTestFiles, + resolveUnitFastIsolatedTestIncludePattern, + resolveUnitFastTestIncludePattern, + resolveUnitFastTimerTestIncludePattern, +} from "../test/vitest/vitest.unit-fast-paths.mjs"; +import { + isBoundaryTestFile, + isBundledPluginDependentUnitTestFile, + isUnitConfigTestFile, +} from "../test/vitest/vitest.unit-paths.mjs"; +import { + detectChangedLanes, + listChangedPathsFromGit as listChangedPathsFromGitSource, +} from "./changed-lanes.mts"; +import { parsePermissiveBooleanToken } from "./lib/arg-utils.mts"; +import { + getChangedPathFacts, + isTestFileTarget, + isTestSupportFileTarget, +} from "./lib/changed-path-facts.mjs"; +import { + GIT_LS_FILES_MAX_BUFFER_BYTES, + createExtensionTestProcessTargetChunks, + listTrackedTestPlanFiles, + resolveExtensionTestConfig, + splitExtensionTestProcessTargets, +} from "./lib/extension-test-plan.mts"; +import { + GATEWAY_SERVER_TEST_PROCESS_COUNT, + listGatewayServerTestTargets, + splitTestTargetChunks as splitTargetChunks, +} from "./lib/gateway-server-test-plan.mts"; +import { readTestSelectorSourceFacts } from "./lib/test-selector-source-facts.mts"; +// CI imports planning before dependency installation; execution owners stay outside this closure. +import { resolveVitestCliEntry } from "./lib/vitest-build-prerequisites.mts"; +import { + collectVitestFileFilters, + resolveBooleanModeFlag, + vitestOptionConsumesNextArg, +} from "./lib/vitest-cli-mode.mts"; +import { + isCiLikeEnv, + resolveLocalFullSuiteProfile, + type VitestHostInfo, +} from "./lib/vitest-local-scheduling.mts"; +import { + DEFAULT_VITEST_NO_OUTPUT_HEARTBEAT_MS, + resolveDefaultVitestNoOutputTimeoutMs, + resolveVitestNodeArgs, +} from "./lib/vitest-process-env.mts"; +import { + estimateVitestTestFileSeconds, + estimateVitestToolingFileSeconds, + resolveShardTimingKey, + type VitestShardTimingSpec, +} from "./lib/vitest-shard-metadata.mts"; + +export { isTestFileTarget } from "./lib/changed-path-facts.mjs"; + +type VitestRunPlan = { + config: string; + databaseWorkerWatchOwner?: string; + databaseWorkerWatchTests?: string[]; + forwardedArgs: string[]; + timingTargets?: string[]; + includePatterns: string[] | null; + watchMode: boolean; +}; + +export type VitestRunSpec = ReturnType<typeof createVitestRunSpecs>[number]; + +export type FailedVitestShard = { + code?: number | null; + config: string; + includePatterns?: string[] | null; + noOutputTimedOut?: boolean; + order?: number; + signal?: string | null; +}; + +type ChangedTestTargetOptions = { + cwd?: string; + env?: NodeJS.ProcessEnv; + broad?: boolean; + combineSiblingWithImportGraph?: boolean; + forceFullImportGraph?: boolean; + includeExtensionImpact?: boolean; + watchMode?: boolean; +}; + +type ChangedTestTargetPlan = { + mode: "none" | "broad" | "targets"; + targets: string[]; + skippedBroadFallbackPaths?: string[]; +}; + +type ImportGraphOptions = { tooling?: boolean }; +type VitestSpecShape = Pick<VitestRunSpec, "config" | "env"> & { + cacheAssignment?: VitestCacheAssignment; +}; +export type VitestCacheAssignment = + | { kind: "scheduler"; root: string; leased?: true } + | { kind: "caller" }; +type CacheAssignedSpec<T> = Omit<T, "env"> & { + env: NodeJS.ProcessEnv; + cacheAssignment?: VitestCacheAssignment; +}; +type WatchableVitestSpecShape = VitestSpecShape & Pick<VitestRunSpec, "watchMode">; +type ImportGraph = { + reverseImports: Map<string, string[]>; + testFiles: Set<string>; +}; +type ImportGraphEdges = { + file: string; + specifiers: string[]; + imports: Set<string>; + references: Set<string>; +}; +type UnmatchedExplicitTestTarget = { + target: string; + reason: "glob-matched-no-files" | "path-does-not-exist" | "target-matched-no-test-files"; + includePattern?: string; +}; + +const DEFAULT_VITEST_CONFIG = "test/vitest/vitest.unit.config.ts"; +const AGENTS_EMBEDDED_AGENT_TEST_ROOT = agentVitestProjectOwners.embedded.root; +const AGENTS_SPAWN_PRODUCTION_BOUNDARY_VITEST_CONFIG = + agentVitestProjectOwners.spawnProductionBoundary.config; +const AGENTS_CORE_ISOLATED_VITEST_CONFIG = agentVitestProjectOwners.coreIsolated.config; +const AGENTS_CORE_VITEST_CONFIG = agentVitestProjectOwners.core.config; +const AGENTS_EMBEDDED_AGENT_VITEST_CONFIG = agentVitestProjectOwners.embedded.config; +const AGENTS_EMBEDDED_AGENT_INCOMPLETE_TURN_VITEST_CONFIG = + agentVitestProjectOwners.embeddedIncompleteTurn.config; +const AGENTS_EMBEDDED_AGENT_OVERFLOW_COMPACTION_VITEST_CONFIG = + agentVitestProjectOwners.embeddedOverflowCompaction.config; +const AGENTS_EMBEDDED_AGENT_RUN_VITEST_CONFIG = agentVitestProjectOwners.embeddedRun.config; +const AGENTS_SUPPORT_VITEST_CONFIG = agentVitestProjectOwners.support.config; +const AGENTS_TOOLS_VITEST_CONFIG = agentVitestProjectOwners.tools.config; +const AGENTS_VITEST_CONFIG = agentVitestProjectOwners.all.config; +const ACP_VITEST_CONFIG = "test/vitest/vitest.acp.config.ts"; +const AUTO_REPLY_CORE_VITEST_CONFIG = "test/vitest/vitest.auto-reply-core.config.ts"; +const AUTO_REPLY_VITEST_CONFIG = "test/vitest/vitest.auto-reply.config.ts"; +const AUTO_REPLY_REPLY_VITEST_CONFIG = "test/vitest/vitest.auto-reply-reply.config.ts"; +const AUTO_REPLY_TOP_LEVEL_VITEST_CONFIG = "test/vitest/vitest.auto-reply-top-level.config.ts"; +const BOUNDARY_VITEST_CONFIG = "test/vitest/vitest.boundary.config.ts"; +const BUNDLED_VITEST_CONFIG = "test/vitest/vitest.bundled.config.ts"; +const CHANNEL_VITEST_CONFIG = "test/vitest/vitest.channels.config.ts"; +const CLI_PROCESS_VITEST_CONFIG = "test/vitest/vitest.cli-process.config.ts"; +const CLI_VITEST_CONFIG = "test/vitest/vitest.cli.config.ts"; +const COMMANDS_LIGHT_VITEST_CONFIG = "test/vitest/vitest.commands-light.config.ts"; +const COMMANDS_VITEST_CONFIG = "test/vitest/vitest.commands.config.ts"; +const CONTRACTS_CHANNEL_CONFIG_VITEST_CONFIG = + "test/vitest/vitest.contracts-channel-config.config.ts"; +const CONTRACTS_CHANNEL_REGISTRY_VITEST_CONFIG = + "test/vitest/vitest.contracts-channel-registry.config.ts"; +const CONTRACTS_CHANNEL_SESSION_VITEST_CONFIG = + "test/vitest/vitest.contracts-channel-session.config.ts"; +const CONTRACTS_CHANNEL_SURFACE_VITEST_CONFIG = + "test/vitest/vitest.contracts-channel-surface.config.ts"; +export const CONTRACTS_PLUGIN_VITEST_CONFIG = "test/vitest/vitest.contracts-plugin.config.ts"; +const CRON_VITEST_CONFIG = "test/vitest/vitest.cron.config.ts"; +const DAEMON_VITEST_CONFIG = "test/vitest/vitest.daemon.config.ts"; +const E2E_VITEST_CONFIG = "test/vitest/vitest.e2e.config.ts"; +const EXTENSION_ACTIVE_MEMORY_VITEST_CONFIG = + "test/vitest/vitest.extension-active-memory.config.ts"; +const EXTENSION_ACPX_VITEST_CONFIG = "test/vitest/vitest.extension-acpx.config.ts"; +const EXTENSION_BROWSER_VITEST_CONFIG = "test/vitest/vitest.extension-browser.config.ts"; +const EXTENSION_CODEX_VITEST_CONFIG = "test/vitest/vitest.extension-codex.config.ts"; +const EXTENSION_CODEX_APP_SERVER_ATTEMPT_LIGHT_VITEST_CONFIG = + "test/vitest/vitest.extension-codex-app-server-attempt-light.config.ts"; +const EXTENSION_CODEX_APP_SERVER_ATTEMPT_SUPPORT_VITEST_CONFIG = + "test/vitest/vitest.extension-codex-app-server-attempt-support.config.ts"; +const EXTENSION_CODEX_APP_SERVER_RUNTIME_VITEST_CONFIG = + "test/vitest/vitest.extension-codex-app-server-runtime.config.ts"; +const EXTENSION_CODEX_APP_SERVER_SUPPORT_VITEST_CONFIG = + "test/vitest/vitest.extension-codex-app-server-support.config.ts"; +const EXTENSION_CODEX_APP_SERVER_TOOLS_VITEST_CONFIG = + "test/vitest/vitest.extension-codex-app-server-tools.config.ts"; +const EXTENSION_CODEX_SURFACE_VITEST_CONFIG = + "test/vitest/vitest.extension-codex-surface.config.ts"; +const EXTENSION_DIFFS_VITEST_CONFIG = "test/vitest/vitest.extension-diffs.config.ts"; +const EXTENSION_DISCORD_VITEST_CONFIG = "test/vitest/vitest.extension-discord.config.ts"; +const EXTENSION_FEISHU_VITEST_CONFIG = "test/vitest/vitest.extension-feishu.config.ts"; +const EXTENSION_IMESSAGE_VITEST_CONFIG = "test/vitest/vitest.extension-imessage.config.ts"; +const EXTENSION_IRC_VITEST_CONFIG = "test/vitest/vitest.extension-irc.config.ts"; +const EXTENSION_LINE_VITEST_CONFIG = "test/vitest/vitest.extension-line.config.ts"; +const EXTENSION_MATTERMOST_VITEST_CONFIG = "test/vitest/vitest.extension-mattermost.config.ts"; +const EXTENSION_MEDIA_VITEST_CONFIG = "test/vitest/vitest.extension-media.config.ts"; +const EXTENSION_MATRIX_VITEST_CONFIG = "test/vitest/vitest.extension-matrix.config.ts"; +const EXTENSION_MEMORY_VITEST_CONFIG = "test/vitest/vitest.extension-memory.config.ts"; +const EXTENSION_MSTEAMS_VITEST_CONFIG = "test/vitest/vitest.extension-msteams.config.ts"; +const EXTENSION_MESSAGING_VITEST_CONFIG = "test/vitest/vitest.extension-messaging.config.ts"; +const EXTENSION_MISC_VITEST_CONFIG = "test/vitest/vitest.extension-misc.config.ts"; +const EXTENSION_PROVIDER_OPENAI_VITEST_CONFIG = + "test/vitest/vitest.extension-provider-openai.config.ts"; +const EXTENSION_PROVIDERS_VITEST_CONFIG = "test/vitest/vitest.extension-providers.config.ts"; +const EXTENSION_QA_VITEST_CONFIG = "test/vitest/vitest.extension-qa.config.ts"; +const EXTENSION_SIGNAL_VITEST_CONFIG = "test/vitest/vitest.extension-signal.config.ts"; +const EXTENSION_SLACK_VITEST_CONFIG = "test/vitest/vitest.extension-slack.config.ts"; +const EXTENSION_DATABASE_WORKERS_VITEST_CONFIG = + "test/vitest/vitest.extension-database-workers.config.ts"; +const EXTENSION_TELEGRAM_VITEST_CONFIG = "test/vitest/vitest.extension-telegram.config.ts"; +const EXTENSION_VOICE_CALL_VITEST_CONFIG = "test/vitest/vitest.extension-voice-call.config.ts"; +const EXTENSION_WHATSAPP_VITEST_CONFIG = "test/vitest/vitest.extension-whatsapp.config.ts"; +const EXTENSION_ZALO_VITEST_CONFIG = "test/vitest/vitest.extension-zalo.config.ts"; +const EXTENSIONS_VITEST_CONFIG = "test/vitest/vitest.extensions.config.ts"; +const FULL_AGENTIC_VITEST_CONFIG = "test/vitest/vitest.full-agentic.config.ts"; +const FULL_EXTENSIONS_VITEST_CONFIG = "test/vitest/vitest.full-extensions.config.ts"; +const GATEWAY_CLIENT_VITEST_CONFIG = "test/vitest/vitest.gateway-client.config.ts"; +const GATEWAY_CORE_VITEST_CONFIG = "test/vitest/vitest.gateway-core.config.ts"; +const GATEWAY_DATABASE_WORKERS_VITEST_CONFIG = + "test/vitest/vitest.gateway-database-workers.config.ts"; +const GATEWAY_METHODS_VITEST_CONFIG = "test/vitest/vitest.gateway-methods.config.ts"; +const GATEWAY_SERVER_VITEST_CONFIG = "test/vitest/vitest.gateway-server.config.ts"; +const GATEWAY_VITEST_CONFIG = "test/vitest/vitest.gateway.config.ts"; +const HOOKS_VITEST_CONFIG = "test/vitest/vitest.hooks.config.ts"; +const INFRA_VITEST_CONFIG = "test/vitest/vitest.infra.config.ts"; +const MEDIA_VITEST_CONFIG = "test/vitest/vitest.media.config.ts"; +const MEDIA_UNDERSTANDING_VITEST_CONFIG = "test/vitest/vitest.media-understanding.config.ts"; +const LOGGING_VITEST_CONFIG = "test/vitest/vitest.logging.config.ts"; +const PLUGIN_SDK_LIGHT_VITEST_CONFIG = "test/vitest/vitest.plugin-sdk-light.config.ts"; +const PLUGIN_SDK_VITEST_CONFIG = "test/vitest/vitest.plugin-sdk.config.ts"; +const PLUGINS_VITEST_CONFIG = "test/vitest/vitest.plugins.config.ts"; +const UNIT_FAST_VITEST_CONFIG = "test/vitest/vitest.unit-fast.config.ts"; +const UNIT_FAST_ISOLATED_VITEST_CONFIG = "test/vitest/vitest.unit-fast-isolated.config.ts"; +const UNIT_FAST_FAKE_TIMERS_VITEST_CONFIG = "test/vitest/vitest.unit-fast-fake-timers.config.ts"; +const UNIT_SECURITY_VITEST_CONFIG = "test/vitest/vitest.unit-security.config.ts"; +const UNIT_SRC_VITEST_CONFIG = "test/vitest/vitest.unit-src.config.ts"; +const UNIT_SUPPORT_VITEST_CONFIG = "test/vitest/vitest.unit-support.config.ts"; +const EXTENSION_TEST_PROCESS_ROOTS = new Map([ + [EXTENSION_CODEX_VITEST_CONFIG, codexExtensionTestRoots], + [EXTENSION_MATRIX_VITEST_CONFIG, matrixExtensionTestRoots], + [EXTENSION_TELEGRAM_VITEST_CONFIG, telegramExtensionTestRoots], + [ + EXTENSION_DATABASE_WORKERS_VITEST_CONFIG, + [...databaseWorkerExtensionTestRoots, ...databaseWorkerExtensionTestFiles], + ], +]); + +const FULL_SUITE_CONFIG_WEIGHT = new Map([ + [GATEWAY_VITEST_CONFIG, 180], + [GATEWAY_SERVER_VITEST_CONFIG, 180], + [GATEWAY_CORE_VITEST_CONFIG, 179], + [GATEWAY_CLIENT_VITEST_CONFIG, 178], + [GATEWAY_METHODS_VITEST_CONFIG, 177], + [COMMANDS_VITEST_CONFIG, 175], + [AGENTS_CORE_VITEST_CONFIG, 170], + [AGENTS_EMBEDDED_AGENT_VITEST_CONFIG, 169], + [AGENTS_EMBEDDED_AGENT_INCOMPLETE_TURN_VITEST_CONFIG, 169], + [AGENTS_EMBEDDED_AGENT_OVERFLOW_COMPACTION_VITEST_CONFIG, 169], + [AGENTS_EMBEDDED_AGENT_RUN_VITEST_CONFIG, 169], + [AGENTS_SUPPORT_VITEST_CONFIG, 168], + [AGENTS_TOOLS_VITEST_CONFIG, 167], + [EXTENSION_CODEX_VITEST_CONFIG, 168], + [EXTENSION_CODEX_APP_SERVER_ATTEMPT_LIGHT_VITEST_CONFIG, 82], + [EXTENSION_CODEX_APP_SERVER_ATTEMPT_SUPPORT_VITEST_CONFIG, 80], + [EXTENSION_CODEX_APP_SERVER_RUNTIME_VITEST_CONFIG, 88], + [EXTENSION_CODEX_APP_SERVER_TOOLS_VITEST_CONFIG, 78], + [EXTENSION_CODEX_APP_SERVER_SUPPORT_VITEST_CONFIG, 72], + [EXTENSION_CODEX_SURFACE_VITEST_CONFIG, 68], + [EXTENSION_VOICE_CALL_VITEST_CONFIG, 169], + [EXTENSIONS_VITEST_CONFIG, 168], + [EXTENSION_PROVIDER_OPENAI_VITEST_CONFIG, 167], + ["test/vitest/vitest.runtime-config.config.ts", 166], + [CONTRACTS_CHANNEL_CONFIG_VITEST_CONFIG, 85], + [CONTRACTS_CHANNEL_SURFACE_VITEST_CONFIG, 60], + [CONTRACTS_CHANNEL_SESSION_VITEST_CONFIG, 50], + [CONTRACTS_CHANNEL_REGISTRY_VITEST_CONFIG, 35], + [CONTRACTS_PLUGIN_VITEST_CONFIG, 20], + ["test/vitest/vitest.tasks.config.ts", 165], + [CHANNEL_VITEST_CONFIG, 164], + [UNIT_FAST_VITEST_CONFIG, 160], + [UNIT_FAST_ISOLATED_VITEST_CONFIG, 159], + [AUTO_REPLY_REPLY_VITEST_CONFIG, 155], + [INFRA_VITEST_CONFIG, 145], + ["test/vitest/vitest.secrets.config.ts", 140], + [CRON_VITEST_CONFIG, 135], + ["test/vitest/vitest.wizard.config.ts", 130], + [UNIT_SRC_VITEST_CONFIG, 125], + [EXTENSION_MATRIX_VITEST_CONFIG, 100], + [EXTENSION_DISCORD_VITEST_CONFIG, 98], + [EXTENSION_PROVIDERS_VITEST_CONFIG, 96], + [EXTENSION_TELEGRAM_VITEST_CONFIG, 94], + [EXTENSION_WHATSAPP_VITEST_CONFIG, 92], + [AUTO_REPLY_CORE_VITEST_CONFIG, 90], + [CLI_PROCESS_VITEST_CONFIG, 87], + [CLI_VITEST_CONFIG, 86], + [MEDIA_VITEST_CONFIG, 84], + [PLUGINS_VITEST_CONFIG, 82], + [BUNDLED_VITEST_CONFIG, 80], + [EXTENSION_SLACK_VITEST_CONFIG, 78], + [COMMANDS_LIGHT_VITEST_CONFIG, 48], + [PLUGIN_SDK_VITEST_CONFIG, 46], + [AUTO_REPLY_TOP_LEVEL_VITEST_CONFIG, 45], + [PLUGIN_SDK_LIGHT_VITEST_CONFIG, 38], + [DAEMON_VITEST_CONFIG, 36], + [BOUNDARY_VITEST_CONFIG, 34], + ["test/vitest/vitest.tooling.config.ts", 32], + ["test/vitest/vitest.tooling-isolated.config.ts", 1], + [UNIT_SECURITY_VITEST_CONFIG, 30], + [UNIT_SUPPORT_VITEST_CONFIG, 28], + [EXTENSION_ZALO_VITEST_CONFIG, 24], + [EXTENSION_IRC_VITEST_CONFIG, 20], + [EXTENSION_FEISHU_VITEST_CONFIG, 18], + [EXTENSION_MATTERMOST_VITEST_CONFIG, 16], + [EXTENSION_MESSAGING_VITEST_CONFIG, 14], + [EXTENSION_IMESSAGE_VITEST_CONFIG, 13], + [EXTENSION_LINE_VITEST_CONFIG, 12], + [EXTENSION_SIGNAL_VITEST_CONFIG, 11], + [EXTENSION_ACPX_VITEST_CONFIG, 10], + [EXTENSION_DIFFS_VITEST_CONFIG, 8], + [EXTENSION_ACTIVE_MEMORY_VITEST_CONFIG, 7], + [EXTENSION_MEMORY_VITEST_CONFIG, 6], + [EXTENSION_MSTEAMS_VITEST_CONFIG, 4], +]); + +function resolveSpecSortWeight( + spec: VitestShardTimingSpec, + shardTimings: ReadonlyMap<string, number>, +) { + const observed = shardTimings.get(resolveShardTimingKey(spec)); + if (observed !== undefined) { + return observed; + } + // Exact selections use their file costs; a whole-config sample would price a + // single worker proof like all tooling. Globs keep the whole-config fallback. + const includes = spec.timingTargets ?? spec.includePatterns; + const estimateFileSeconds = + spec.config === TOOLING_VITEST_CONFIG + ? estimateVitestToolingFileSeconds + : estimateVitestTestFileSeconds; + const seconds = + includes?.length && includes.every((file) => isTestFileTarget(file) && !isGlobTarget(file)) + ? includes.reduce((total, file) => total + estimateFileSeconds(file), 0) + : (FULL_SUITE_CONFIG_WEIGHT.get(spec.config) ?? 0); + return seconds * 1000; +} + +function interleaveSlowAndFastSpecs<T>(sortedSpecs: T[]) { + const ordered: T[] = []; + let slowIndex = 0; + let fastIndex = sortedSpecs.length - 1; + while (slowIndex <= fastIndex) { + const slowSpec = sortedSpecs[slowIndex]; + if (slowSpec !== undefined) { + ordered.push(slowSpec); + } + slowIndex += 1; + if (slowIndex <= fastIndex) { + const fastSpec = sortedSpecs[fastIndex]; + if (fastSpec !== undefined) { + ordered.push(fastSpec); + } + fastIndex -= 1; + } + } + return ordered; +} + +function uniqueOrdered<T>(values: T[]) { + return [...new Set(values)]; +} + +function isPathAtOrUnder(relative: string, root: string) { + return relative === root || relative.startsWith(`${root}/`); +} + +/** + * Orders full-suite specs so expensive shards start first in parallel runs. + */ +export function orderFullSuiteSpecsForParallelRun<T extends VitestShardTimingSpec>( + specs: T[], + shardTimings = new Map<string, number>(), +): T[] { + const sortedSpecs = specs.toSorted((a, b) => { + const weightDelta = + resolveSpecSortWeight(b, shardTimings) - resolveSpecSortWeight(a, shardTimings); + if (weightDelta !== 0) { + return weightDelta; + } + return a.config.localeCompare(b.config); + }); + return interleaveSlowAndFastSpecs(sortedSpecs); +} +const PROCESS_VITEST_CONFIG = "test/vitest/vitest.process.config.ts"; +const RUNTIME_CONFIG_VITEST_CONFIG = "test/vitest/vitest.runtime-config.config.ts"; +const SECRETS_VITEST_CONFIG = "test/vitest/vitest.secrets.config.ts"; +const SHARED_CORE_VITEST_CONFIG = "test/vitest/vitest.shared-core.config.ts"; +const TASKS_VITEST_CONFIG = "test/vitest/vitest.tasks.config.ts"; +const PACKAGE_CONTRACT_VITEST_CONFIG = "test/vitest/vitest.package-contract.config.ts"; +const TOOLING_DOCKER_VITEST_CONFIG = "test/vitest/vitest.tooling-docker.config.ts"; +const TOOLING_ISOLATED_VITEST_CONFIG = "test/vitest/vitest.tooling-isolated.config.ts"; +const TOOLING_VITEST_CONFIG = "test/vitest/vitest.tooling.config.ts"; +const TOOLING_DOCKER_TEST_TARGET = "test/scripts/docker-build-helper.test.ts"; +const BROAD_TOOLING_SCRIPT_TEST_PATTERNS = new Set([ + "test/scripts/**/*.test.ts", + "test/scripts/*.test.ts", +]); +const BROAD_TOOLING_SCRIPT_TEST_TARGET_CHUNK_SIZE = 60; +const FULL_SUITE_AGENTS_CORE_TEST_TARGET_CHUNK_COUNT = 6; +const FULL_SUITE_TOOLING_TEST_TARGET_CHUNK_SIZE = 2; +const FULL_SUITE_UNIT_FAST_TEST_TARGET_CHUNK_SIZE = 70; +const FULL_SUITE_UNIT_SRC_TEST_TARGET_CHUNK_SIZE = 150; +const TUI_VITEST_CONFIG = "test/vitest/vitest.tui.config.ts"; +const TUI_PTY_VITEST_CONFIG = "test/vitest/vitest.tui-pty.config.ts"; +const UI_VITEST_CONFIG = "test/vitest/vitest.ui.config.ts"; +const UI_BROWSER_VITEST_CONFIG = "test/vitest/vitest.ui-browser.config.ts"; +export const UI_E2E_VITEST_CONFIG = "test/vitest/vitest.ui-e2e.config.ts"; +const UI_ISOLATED_VITEST_CONFIG = "test/vitest/vitest.ui-isolated.config.ts"; +const UTILS_VITEST_CONFIG = "test/vitest/vitest.utils.config.ts"; +const WIZARD_VITEST_CONFIG = "test/vitest/vitest.wizard.config.ts"; +const INCLUDE_FILE_ENV_KEY = "OPENCLAW_VITEST_INCLUDE_FILE"; +const FS_MODULE_CACHE_PATH_ENV_KEY = "OPENCLAW_VITEST_FS_MODULE_CACHE_PATH"; +const FAILED_SHARD_DIGEST_LIMIT = 12; +const CHANGED_ARGS_PATTERN = /^--changed(?:=(.+))?$/u; +const VITEST_CONFIG_BY_KIND: Record<string, string> = { + unitFast: UNIT_FAST_VITEST_CONFIG, + unitFastIsolated: UNIT_FAST_ISOLATED_VITEST_CONFIG, + unitFastFakeTimers: UNIT_FAST_FAKE_TIMERS_VITEST_CONFIG, + boundary: BOUNDARY_VITEST_CONFIG, + toolingDocker: TOOLING_DOCKER_VITEST_CONFIG, + toolingIsolated: TOOLING_ISOLATED_VITEST_CONFIG, + tooling: TOOLING_VITEST_CONFIG, + contractsChannelSurface: CONTRACTS_CHANNEL_SURFACE_VITEST_CONFIG, + contractsChannelConfig: CONTRACTS_CHANNEL_CONFIG_VITEST_CONFIG, + contractsChannelRegistry: CONTRACTS_CHANNEL_REGISTRY_VITEST_CONFIG, + contractsChannelSession: CONTRACTS_CHANNEL_SESSION_VITEST_CONFIG, + contractsPlugin: CONTRACTS_PLUGIN_VITEST_CONFIG, + bundled: BUNDLED_VITEST_CONFIG, + gateway: GATEWAY_VITEST_CONFIG, + gatewayCore: GATEWAY_CORE_VITEST_CONFIG, + gatewayDatabaseWorkers: GATEWAY_DATABASE_WORKERS_VITEST_CONFIG, + gatewayClient: GATEWAY_CLIENT_VITEST_CONFIG, + gatewayMethods: GATEWAY_METHODS_VITEST_CONFIG, + gatewayServer: GATEWAY_SERVER_VITEST_CONFIG, + hooks: HOOKS_VITEST_CONFIG, + infra: INFRA_VITEST_CONFIG, + runtimeConfig: RUNTIME_CONFIG_VITEST_CONFIG, + cron: CRON_VITEST_CONFIG, + daemon: DAEMON_VITEST_CONFIG, + media: MEDIA_VITEST_CONFIG, + logging: LOGGING_VITEST_CONFIG, + packageContract: PACKAGE_CONTRACT_VITEST_CONFIG, + pluginSdkLight: PLUGIN_SDK_LIGHT_VITEST_CONFIG, + pluginSdk: PLUGIN_SDK_VITEST_CONFIG, + process: PROCESS_VITEST_CONFIG, + secrets: SECRETS_VITEST_CONFIG, + sharedCore: SHARED_CORE_VITEST_CONFIG, + tasks: TASKS_VITEST_CONFIG, + tui: TUI_VITEST_CONFIG, + tuiPty: TUI_PTY_VITEST_CONFIG, + mediaUnderstanding: MEDIA_UNDERSTANDING_VITEST_CONFIG, + acp: ACP_VITEST_CONFIG, + cliProcess: CLI_PROCESS_VITEST_CONFIG, + cli: CLI_VITEST_CONFIG, + commandLight: COMMANDS_LIGHT_VITEST_CONFIG, + command: COMMANDS_VITEST_CONFIG, + autoReply: AUTO_REPLY_VITEST_CONFIG, + autoReplyCore: AUTO_REPLY_CORE_VITEST_CONFIG, + autoReplyReply: AUTO_REPLY_REPLY_VITEST_CONFIG, + autoReplyTopLevel: AUTO_REPLY_TOP_LEVEL_VITEST_CONFIG, + agentCore: AGENTS_CORE_VITEST_CONFIG, + agentEmbedded: AGENTS_EMBEDDED_AGENT_VITEST_CONFIG, + agentEmbeddedIncompleteTurn: AGENTS_EMBEDDED_AGENT_INCOMPLETE_TURN_VITEST_CONFIG, + agentEmbeddedOverflowCompaction: AGENTS_EMBEDDED_AGENT_OVERFLOW_COMPACTION_VITEST_CONFIG, + agentEmbeddedRun: AGENTS_EMBEDDED_AGENT_RUN_VITEST_CONFIG, + agentSupport: AGENTS_SUPPORT_VITEST_CONFIG, + agentTools: AGENTS_TOOLS_VITEST_CONFIG, + agent: AGENTS_VITEST_CONFIG, + agentsSpawnProductionBoundary: AGENTS_SPAWN_PRODUCTION_BOUNDARY_VITEST_CONFIG, + agentsCoreIsolated: AGENTS_CORE_ISOLATED_VITEST_CONFIG, + agentsCore: AGENTS_CORE_VITEST_CONFIG, + agentsSupport: AGENTS_SUPPORT_VITEST_CONFIG, + agentsTools: AGENTS_TOOLS_VITEST_CONFIG, + plugin: PLUGINS_VITEST_CONFIG, + ui: UI_VITEST_CONFIG, + uiIsolated: UI_ISOLATED_VITEST_CONFIG, + uiBrowser: UI_BROWSER_VITEST_CONFIG, + uiTiming: "test/vitest/vitest.ui-timing.config.ts", + uiE2e: UI_E2E_VITEST_CONFIG, + unitSrc: UNIT_SRC_VITEST_CONFIG, + unitSecurity: UNIT_SECURITY_VITEST_CONFIG, + unitSupport: UNIT_SUPPORT_VITEST_CONFIG, + utils: UTILS_VITEST_CONFIG, + wizard: WIZARD_VITEST_CONFIG, + e2e: E2E_VITEST_CONFIG, + extensionActiveMemory: EXTENSION_ACTIVE_MEMORY_VITEST_CONFIG, + extensionAcpx: EXTENSION_ACPX_VITEST_CONFIG, + extensionCodex: EXTENSION_CODEX_VITEST_CONFIG, + extensionDiffs: EXTENSION_DIFFS_VITEST_CONFIG, + extensionBrowser: EXTENSION_BROWSER_VITEST_CONFIG, + extensionDiscord: EXTENSION_DISCORD_VITEST_CONFIG, + extensionFeishu: EXTENSION_FEISHU_VITEST_CONFIG, + extensionImessage: EXTENSION_IMESSAGE_VITEST_CONFIG, + extensionIrc: EXTENSION_IRC_VITEST_CONFIG, + extensionLine: EXTENSION_LINE_VITEST_CONFIG, + extensionMattermost: EXTENSION_MATTERMOST_VITEST_CONFIG, + extensionDatabaseWorkers: EXTENSION_DATABASE_WORKERS_VITEST_CONFIG, + extensionTelegram: EXTENSION_TELEGRAM_VITEST_CONFIG, + extensionVoiceCall: EXTENSION_VOICE_CALL_VITEST_CONFIG, + extensionWhatsApp: EXTENSION_WHATSAPP_VITEST_CONFIG, + extensionZalo: EXTENSION_ZALO_VITEST_CONFIG, + extensionMatrix: EXTENSION_MATRIX_VITEST_CONFIG, + extensionMedia: EXTENSION_MEDIA_VITEST_CONFIG, + extensionMemory: EXTENSION_MEMORY_VITEST_CONFIG, + extensionMisc: EXTENSION_MISC_VITEST_CONFIG, + extensionMsTeams: EXTENSION_MSTEAMS_VITEST_CONFIG, + extensionMessaging: EXTENSION_MESSAGING_VITEST_CONFIG, + extensionProviderOpenAi: EXTENSION_PROVIDER_OPENAI_VITEST_CONFIG, + extensionProvider: EXTENSION_PROVIDERS_VITEST_CONFIG, + extensionQa: EXTENSION_QA_VITEST_CONFIG, + extensionSignal: EXTENSION_SIGNAL_VITEST_CONFIG, + extensionSlack: EXTENSION_SLACK_VITEST_CONFIG, + extensionFull: FULL_EXTENSIONS_VITEST_CONFIG, + channel: CHANNEL_VITEST_CONFIG, + extension: EXTENSIONS_VITEST_CONFIG, +}; +const BROAD_CHANGED_FALLBACK_PATTERNS = [ + /^package\.json$/u, + /^pnpm-lock\.yaml$/u, + /^test\/setup(?:\.shared|\.extensions|-openclaw-runtime)?\.ts$/u, + /^vitest(?:\..+)?\.(?:config\.ts|paths\.mjs)$/u, + /^test\/vitest\/vitest\.(?:config|shared\.config|scoped-config|performance-config)\.ts$/u, + /^test\/helpers\//u, +]; +const PRECISE_SOURCE_TEST_TARGETS = new Map<string, string[]>([ + [ + "patches/vitest@5.0.0.patch", + [ + "test/scripts/run-vitest-profile.test.ts", + "test/scripts/run-vitest-state-cleanup.test.ts", + "test/scripts/vitest-fork-shutdown.test.ts", + "test/scripts/vitest-runner-task-updates.test.ts", + ], + ], + ["test/fixtures/vitest-fork-shutdown.mjs", ["test/scripts/vitest-fork-shutdown.test.ts"]], + ...["clock", "runner"].map<[string, string[]]>((part) => [ + `test/fixtures/vitest-runner-task-updates.${part}.mjs`, + ["test/scripts/vitest-runner-task-updates.test.ts"], + ]), + ...[ + "src/system-agent/setup-inference-turn.ts", + "src/agents/embedded-agent-runner/run/run-attempt-dispatch.ts", + ].map<[string, string[]]>((sourcePath) => [ + sourcePath, + [ + "src/agents/embedded-agent-runner/run.overflow-compaction.loop.test.ts", + "src/commands/onboard-guided.inference.e2e.test.ts", + ], + ]), + [ + "src/plugins/contracts/tts-contract-suites.ts", + [ + "src/plugins/contracts/core-extension-facade-boundary.test.ts", + "src/plugins/contracts/tts.contract.test.ts", + ], + ], + [ + "extensions/slack/src/monitor/enterprise-install.ts", + [ + "extensions/slack/src/monitor/enterprise-install.test.ts", + "extensions/slack/src/monitor/provider.auth-test-token.test.ts", + ], + ], + [ + "extensions/slack/src/channel-actions.ts", + [ + "extensions/slack/src/actions.reactions-limit.test.ts", + "extensions/slack/src/channel-actions-setup-status.contract.test.ts", + "extensions/slack/src/message-tools.test.ts", + ], + ], + [ + "src/gateway/worker-environments/worker-turn-launcher.ts", + [ + "src/gateway/worker-environments/worker-turn-launcher.test.ts", + "src/gateway/worker-environments/worker-turn-launcher-claim-admission.test.ts", + "src/gateway/worker-environments/worker-turn-launcher-failure-recovery.test.ts", + "src/gateway/worker-environments/worker-turn-launcher-reclaimed-placement.test.ts", + "src/gateway/worker-environments/worker-turn-launcher-remote-handoff.test.ts", + "src/gateway/worker-environments/worker-turn-launcher-terminal-results.test.ts", + ], + ], +]); +const DOCS_CONFIG_EXAMPLES_TEST_TARGET = "src/config/docs-config-examples.test.ts"; +const RUNTIME_SIDECAR_BASELINE_OWNER_TEST_TARGETS = ["src/plugins/bundled-plugin-metadata.test.ts"]; +const RUNTIME_SIDECAR_PATH_CONSUMER_TEST_TARGETS = [ + ...RUNTIME_SIDECAR_BASELINE_OWNER_TEST_TARGETS, + "src/infra/update-global.test.ts", + "src/infra/update-runner.test.ts", + "test/openclaw-npm-postpublish-verify.test.ts", +]; +const GITHUB_YAML_PINNING_GUARD_TEST_TARGETS = ["test/scripts/ci-workflow-guards.test.ts"]; +const GROUP_VISIBLE_REPLY_TEST_TARGETS = [ + "src/auto-reply/reply/dispatch-acp.test.ts", + "src/auto-reply/reply/dispatch-from-config.test.ts", + "src/auto-reply/reply/dispatch-from-config.delivery.test.ts", + "src/auto-reply/reply/dispatch-from-config.lifecycle.test.ts", + "src/auto-reply/reply/followup-runner.test.ts", + "src/auto-reply/reply/groups.test.ts", + "extensions/discord/src/monitor/message-handler.process.test.ts", + "extensions/slack/src/monitor.tool-result.test.ts", +]; +const GROUP_VISIBLE_REPLY_PROMPT_TEST_TARGETS = [ + "src/agents/system-prompt.test.ts", + ...GROUP_VISIBLE_REPLY_TEST_TARGETS, +]; +const CHANNEL_CONTRACT_REGISTRY_BACKED_TARGETS = [ + "directory", + "plugin", + "surfaces-only", + "threading", +].flatMap((suite) => + "abcdefgh" + .split("") + .map( + (shard) => + `src/channels/plugins/contracts/${suite}.registry-backed-shard-${shard}.contract.test.ts`, + ), +); +const CHANNEL_PLUGIN_SHAPE_PARITY_TEST_TARGET = + "src/channels/plugins/contracts/plugin-shape.contract.test.ts"; +const CHANNEL_PLUGIN_SHAPE_PARITY_WIRING_PATHS = new Set([ + "extensions/imessage/message-tool-api.ts", + "extensions/imessage/src/actions.ts", + "extensions/imessage/src/channel.ts", + "extensions/slack/message-tool-api.ts", + "extensions/slack/src/channel-actions.ts", + "extensions/slack/src/channel.ts", + "extensions/mattermost/gateway-auth-api.ts", + "extensions/mattermost/src/channel.ts", + "extensions/feishu/session-key-api.ts", + "extensions/feishu/src/channel.ts", + "extensions/telegram/session-key-api.ts", + "extensions/telegram/src/channel.ts", + "extensions/discord/session-key-api.ts", + "extensions/discord/thread-binding-api.ts", + "extensions/discord/src/channel.ts", + "extensions/matrix/thread-binding-api.ts", + "extensions/matrix/src/channel.ts", +]); +const TEST_HELPER_NORMALIZE_TEXT_TARGETS = [ + "src/auto-reply/reply/commands-status.test.ts", + "src/auto-reply/status.test.ts", + "src/tui/components/chat-log.test.ts", +]; +const HAPPY_PATH_PROMPT_SNAPSHOT_HELPER_TEST_TARGETS = ["test/scripts/prompt-snapshots.test.ts"]; +const APPCAST_TEST_TARGETS = ["test/appcast.test.ts", "test/scripts/make-appcast.test.ts"]; +const CODEX_VERSION_CONTRACT_TEST_TARGETS = [ + "extensions/codex/src/manifest.test.ts", + "extensions/openai/openai-provider.test.ts", + "test/scripts/codex-client-version-contract.test.ts", +]; +// The iframe script and native document load as assets outside the import graph. +const MERMAID_RENDERER_TEST_TARGETS = [ + "ui/src/components/markdown-mermaid.runtime.browser.test.ts", + "ui/src/components/markdown-mermaid-native.browser.test.ts", +]; +const SOURCE_TEST_TARGETS = new Map([ + ...PRECISE_SOURCE_TEST_TARGETS, + ["src/plugin-sdk/memory-host-events.ts", ["src/plugin-sdk/memory-host-events.test.ts"]], + ["src/plugin-sdk/persistent-dedupe.ts", ["src/plugin-sdk/memory-host-events.test.ts"]], + [ + "extensions/browser/src/browser/chrome-mcp-options.ts", + [ + "extensions/browser/src/browser/chrome-mcp.test.ts", + "test/scripts/ci-chrome-mcp-prewarm.test.ts", + ], + ], + [ + "scripts/prepare-apple-mermaid.mjs", + [ + "test/scripts/build-and-run-mac.test.ts", + "test/scripts/package-mac-app.test.ts", + "test/scripts/ci-workflow-guards.test.ts", + ], + ], + ["packages/mermaid-renderer/package.json", MERMAID_RENDERER_TEST_TARGETS], + ["packages/mermaid-renderer/vite.config.ts", MERMAID_RENDERER_TEST_TARGETS], + ["packages/mermaid-renderer/native/index.html", MERMAID_RENDERER_TEST_TARGETS], + ["packages/mermaid-renderer/src/renderer.ts", MERMAID_RENDERER_TEST_TARGETS], + ["packages/mermaid-renderer/src/frame.js", MERMAID_RENDERER_TEST_TARGETS], + ["packages/mermaid-renderer/src/native.ts", MERMAID_RENDERER_TEST_TARGETS], + [ + "packages/normalization-core/src/record-coerce.ts", + ["packages/normalization-core/src/record-coerce.test.ts", ...MERMAID_RENDERER_TEST_TARGETS], + ], + [ + "packages/normalization-core/package.json", + ["packages/normalization-core/src/package-exports.test.ts", ...MERMAID_RENDERER_TEST_TARGETS], + ], + ["extensions/codex/package.json", CODEX_VERSION_CONTRACT_TEST_TARGETS], + ["extensions/codex/src/app-server/version.ts", CODEX_VERSION_CONTRACT_TEST_TARGETS], + ["src/test-utils/openclaw-test-state.ts", ["src/test-utils/openclaw-test-state.test.ts"]], + [ + "src/channels/plugins/contracts/test-helpers/manifest.ts", + [ + ...CHANNEL_CONTRACT_REGISTRY_BACKED_TARGETS, + "src/channels/plugins/contracts/registry.contract.test.ts", + "src/channels/plugins/contracts/session-binding.registry-backed.contract.test.ts", + ], + ], + [ + "src/channels/plugins/contracts/test-helpers/registry-backed-contract-shards.ts", + CHANNEL_CONTRACT_REGISTRY_BACKED_TARGETS, + ], + ["test/helpers/normalize-text.ts", TEST_HELPER_NORMALIZE_TEXT_TARGETS], + [ + "test/helpers/agents/happy-path-prompt-snapshots.ts", + HAPPY_PATH_PROMPT_SNAPSHOT_HELPER_TEST_TARGETS, + ], + [ + "test/e2e/qa-lab/runtime/qa-otel-smoke-runtime.ts", + ["test/e2e/qa-lab/runtime/qa-otel-smoke.e2e.test.ts"], + ], + [ + "test/e2e/qa-lab/runtime/heartbeat-active-hours-runtime.ts", + ["test/e2e/qa-lab/runtime/heartbeat-active-hours-runtime.test.ts"], + ], + [ + "test/e2e/qa-lab/runtime/telegram-bot-token-runtime.ts", + ["test/e2e/qa-lab/runtime/telegram-bot-token-runtime.test.ts"], + ], + ["src/plugins/runtime-sidecar-paths-baseline.ts", RUNTIME_SIDECAR_BASELINE_OWNER_TEST_TARGETS], + ["src/plugins/runtime-sidecar-paths.ts", RUNTIME_SIDECAR_PATH_CONSUMER_TEST_TARGETS], + ["ui/config/control-ui-chunking.ts", ["ui/src/app/control-ui-chunking.test.ts"]], + ["ui/config/control-ui-locales.ts", ["ui/src/app/vite-config.node.test.ts"]], + [ + "src/plugin-sdk/test-helpers/directory-ids.ts", + [ + "extensions/discord/src/directory-contract.test.ts", + "extensions/slack/src/directory-contract.test.ts", + "extensions/telegram/src/directory-contract.test.ts", + ], + ], + [ + "src/plugin-sdk/channel-reply-pipeline.ts", + ["src/plugins/contracts/plugin-sdk-subpaths.test.ts", ...GROUP_VISIBLE_REPLY_TEST_TARGETS], + ], + ["src/plugin-sdk/reply-runtime.ts", ["src/plugins/contracts/plugin-sdk-subpaths.test.ts"]], + ["extensions/google-meet/index.ts", ["extensions/google-meet/index.test.ts"]], + [ + "extensions/google-meet/src/cli.ts", + [ + "extensions/google-meet/src/cli-artifacts.test.ts", + "extensions/google-meet/src/cli-runtime.test.ts", + "extensions/google-meet/src/cli.test.ts", + ], + ], + [ + "extensions/google-meet/src/cli-artifact-commands.ts", + ["extensions/google-meet/src/cli-artifacts.test.ts"], + ], + [ + "extensions/google-meet/src/cli-export.ts", + ["extensions/google-meet/src/cli-artifacts.test.ts"], + ], + [ + "extensions/google-meet/src/cli-space-commands.ts", + ["extensions/google-meet/src/cli-artifacts.test.ts"], + ], + [ + "extensions/google-meet/src/cli-runtime-commands.ts", + ["extensions/google-meet/src/cli-runtime.test.ts"], + ], + ["extensions/google-meet/src/cli-doctor.ts", ["extensions/google-meet/src/cli.test.ts"]], + [ + "extensions/google-meet/src/cli-command-context.ts", + [ + "extensions/google-meet/src/cli-artifacts.test.ts", + "extensions/google-meet/src/cli-runtime.test.ts", + "extensions/google-meet/src/cli.test.ts", + ], + ], + [ + "extensions/google-meet/src/cli-shared.ts", + [ + "extensions/google-meet/src/cli-artifacts.test.ts", + "extensions/google-meet/src/cli-runtime.test.ts", + "extensions/google-meet/src/cli.test.ts", + ], + ], + ["extensions/google-meet/src/create.ts", ["extensions/google-meet/index.test.ts"]], + ["extensions/google-meet/src/oauth.ts", ["extensions/google-meet/src/oauth.test.ts"]], + [ + "extensions/discord/src/monitor/message-handler.ts", + [ + "extensions/discord/src/channel-actions.contract.test.ts", + "extensions/discord/src/channel.message-adapter.test.ts", + "extensions/discord/src/channel.test.ts", + "extensions/discord/src/durable-delivery.test.ts", + "extensions/discord/src/monitor/message-handler.bot-self-filter.test.ts", + "extensions/discord/src/monitor/message-handler.queue.test.ts", + "extensions/discord/src/monitor/provider.skill-dedupe.test.ts", + "extensions/discord/src/monitor/provider.test.ts", + ], + ], + ["src/commands/doctor-memory-search.ts", ["src/commands/doctor-memory-search.test.ts"]], + [ + "src/agents/test-helpers/live-model-turn-probes.ts", + ["src/agents/live-model-turn-probes.test.ts"], + ], + [ + "src/plugins/provider-auth-choice.ts", + ["src/commands/auth-choice.apply.plugin-provider.test.ts", "src/commands/auth-choice.test.ts"], + ], + [ + "src/secrets/provider-env-vars.ts", + ["src/secrets/provider-env-vars.dynamic.test.ts", "src/secrets/provider-env-vars.test.ts"], + ], + [ + "packages/memory-host-sdk/src/host/embedding-defaults.ts", + ["extensions/memory-core/src/memory/embeddings.test.ts"], + ], + ["src/auto-reply/reply/dispatch-from-config.ts", GROUP_VISIBLE_REPLY_TEST_TARGETS], + ["src/auto-reply/reply/source-reply-delivery-mode.ts", GROUP_VISIBLE_REPLY_TEST_TARGETS], + [ + "src/auto-reply/reply/effective-reply-route.ts", + ["src/auto-reply/reply/effective-reply-route.test.ts", ...GROUP_VISIBLE_REPLY_TEST_TARGETS], + ], + ["src/auto-reply/reply/get-reply-run.ts", ["src/auto-reply/reply/followup-runner.test.ts"]], + ["src/auto-reply/reply/groups.ts", GROUP_VISIBLE_REPLY_TEST_TARGETS], + ["src/auto-reply/get-reply-options.types.ts", GROUP_VISIBLE_REPLY_TEST_TARGETS], + ["src/agents/system-prompt.ts", GROUP_VISIBLE_REPLY_PROMPT_TEST_TARGETS], + ["src/config/types.messages.ts", GROUP_VISIBLE_REPLY_TEST_TARGETS], + ["src/config/zod-schema.core.ts", GROUP_VISIBLE_REPLY_TEST_TARGETS], + ["src/auto-reply/reply/commands-acp.ts", ["src/auto-reply/reply/commands-acp.test.ts"]], + [ + "src/auto-reply/reply/dispatch-acp-command-bypass.ts", + ["src/auto-reply/reply/dispatch-acp-command-bypass.test.ts"], + ], +]); +const GENERATED_CHANGED_TEST_TARGET_PATTERNS = [ + /^extensions\/[^/]+\/src\/host\/.+\/\.bundle\.hash$/u, + /^extensions\/[^/]+\/src\/host\/.+\/[^/]+\.bundle\.js$/u, +]; +const SOURCE_ROOTS_FOR_IMPORT_GRAPH = [ + "src", + "extensions", + "packages", + "ui/src", + "ui/config", + "ui/public", + "test", +]; +const IMPORTABLE_FILE_EXTENSIONS = [".ts", ".tsx", ".mts", ".cts"]; +function importGraphPathspecs(roots: string[], suffixes: readonly string[]) { + return [ + ...roots.flatMap((root) => suffixes.map((suffix) => `:(glob)${root}/**/*${suffix}`)), + ...suffixes.map((suffix) => `:(glob)ui/*${suffix}`), + ]; +} +const IMPORT_GRAPH_GREP_PATHS = importGraphPathspecs( + SOURCE_ROOTS_FOR_IMPORT_GRAPH, + IMPORTABLE_FILE_EXTENSIONS, +); +const TOOLING_IMPORT_GRAPH_ROOTS = [...SOURCE_ROOTS_FOR_IMPORT_GRAPH, "scripts"]; +const TOOLING_IMPORTABLE_FILE_EXTENSIONS = [ + ...IMPORTABLE_FILE_EXTENSIONS, + ".js", + ".jsx", + ".mjs", + ".cjs", +]; +const TOOLING_IMPORT_GRAPH_GREP_PATHS = importGraphPathspecs( + TOOLING_IMPORT_GRAPH_ROOTS, + TOOLING_IMPORTABLE_FILE_EXTENSIONS, +); +const BROAD_CHANGED_ENV_KEY = "OPENCLAW_TEST_CHANGED_BROAD"; +const VITEST_NO_OUTPUT_TIMEOUT_ENV_KEY = "OPENCLAW_VITEST_NO_OUTPUT_TIMEOUT_MS"; +const VITEST_NO_OUTPUT_HEARTBEAT_ENV_KEY = "OPENCLAW_VITEST_NO_OUTPUT_HEARTBEAT_MS"; +const VITEST_NO_OUTPUT_RETRY_ENV_KEY = "OPENCLAW_VITEST_NO_OUTPUT_RETRY"; +/** Default no-output timeout applied to test-projects Vitest children. */ +export const DEFAULT_TEST_PROJECTS_VITEST_NO_OUTPUT_TIMEOUT_MS = String(900_000); +/** Default heartbeat interval applied to test-projects Vitest children. */ +export const DEFAULT_TEST_PROJECTS_VITEST_NO_OUTPUT_HEARTBEAT_MS = String( + DEFAULT_VITEST_NO_OUTPUT_HEARTBEAT_MS, +); + +export function formatNoChangedTestTargetLines(skippedBroadFallbackPaths: string[]) { + if (skippedBroadFallbackPaths.length === 0) { + return ["[test] no changed test targets; skipping Vitest."]; + } + + return [ + "[test] no precise changed test targets; skipping Vitest.", + `[test] ${skippedBroadFallbackPaths.length} changed path${ + skippedBroadFallbackPaths.length === 1 ? "" : "s" + } require broad Vitest fallback:`, + ...skippedBroadFallbackPaths.map((changedPath) => `[test] ${changedPath}`), + "[test] run `OPENCLAW_TEST_CHANGED_BROAD=1 pnpm test:changed` for broad coverage.", + ]; +} + +const EXPLICIT_SOURCE_FULL_IMPORT_GRAPH_THRESHOLD = 12; +function resolveTestProjectsVitestNoOutputTimeoutMs(config: string) { + const directRunnerTimeoutMs = resolveDefaultVitestNoOutputTimeoutMs(["run", "--config", config]); + return String( + Math.max(Number(DEFAULT_TEST_PROJECTS_VITEST_NO_OUTPUT_TIMEOUT_MS), directRunnerTimeoutMs), + ); +} +const VITEST_CONFIG_TARGET_KIND_BY_PATH = new Map<string, string>( + Object.entries(VITEST_CONFIG_BY_KIND).map(([kind, config]) => [config, kind]), +); +const RUNNABLE_VITEST_CONFIG_TARGETS = new Set([ + "ui/vitest.config.ts", + "vitest.config.ts", + DEFAULT_VITEST_CONFIG, + ...Object.values(VITEST_CONFIG_BY_KIND), + ...fullSuiteVitestShards.flatMap((shard) => [shard.config, ...shard.projects]), +]); +export const CHANNEL_CONTRACT_CONFIG_PATTERNS = new Map<string, readonly string[]>([ + [CONTRACTS_CHANNEL_SURFACE_VITEST_CONFIG, channelSurfaceContractPatterns], + [CONTRACTS_CHANNEL_CONFIG_VITEST_CONFIG, channelConfigContractPatterns], + [CONTRACTS_CHANNEL_REGISTRY_VITEST_CONFIG, channelRegistryContractPatterns], + [CONTRACTS_CHANNEL_SESSION_VITEST_CONFIG, channelSessionContractPatterns], +]); + +function normalizePathPattern(value: string) { + return value.replaceAll("\\", "/"); +} + +function listRepoFilesRecursive(root: string, cwd: string): string[] { + const entries = fs.readdirSync(root, { withFileTypes: true }); + return entries.flatMap((entry) => { + const absolute = path.join(root, entry.name); + if (entry.isDirectory()) { + return listRepoFilesRecursive(absolute, cwd); + } + if (!entry.isFile()) { + return []; + } + return [normalizePathPattern(path.relative(cwd, absolute))]; + }); +} + +let cachedBroadScriptTestTargets: string[] = []; +let cachedBroadScriptTestTargetsCwd: string | null = null; + +function listBroadScriptTestTargets(pattern: string, cwd: string) { + const root = path.join(cwd, "test/scripts"); + if (cachedBroadScriptTestTargetsCwd !== cwd) { + // Broad-target expansion can ask for the same process-stable checkout twice. + // Keep one inventory so planning does not repeat the directory walk. + cachedBroadScriptTestTargets = fs.existsSync(root) + ? listRepoFilesRecursive(root, cwd) + .filter((file) => file.endsWith(".test.ts")) + .toSorted((left, right) => left.localeCompare(right)) + : []; + cachedBroadScriptTestTargetsCwd = cwd; + } + return cachedBroadScriptTestTargets.filter((file) => path.matchesGlob(file, pattern)); +} + +function listBroadToolingScriptTestTargets(pattern: string, cwd: string) { + return listBroadScriptTestTargets(pattern, cwd).filter( + (file) => classifyTarget(file, cwd) === "tooling", + ); +} + +let cachedToolingFullSuiteTestTargets: string[] | null = null; +let cachedToolingFullSuiteTestTargetsCwd: string | null = null; + +function listToolingFullSuiteTestTargets(cwd: string) { + if (cachedToolingFullSuiteTestTargets && cachedToolingFullSuiteTestTargetsCwd === cwd) { + return cachedToolingFullSuiteTestTargets; + } + // The CLI plans against one process-stable checkout. Reuse its inventory when + // callers compare full-suite modes instead of walking the tree for every mode. + cachedToolingFullSuiteTestTargets = uniqueOrdered( + [path.join(cwd, "test"), path.join(cwd, "src", "scripts")].flatMap((root) => + fs.existsSync(root) ? listRepoFilesRecursive(root, cwd) : [], + ), + ) + // Explicit leaf targets bypass the config's live-test exclusion and produce an empty shard. + .filter( + (file) => + file.endsWith(".test.ts") && + !file.endsWith(".live.test.ts") && + classifyTarget(file, cwd) === "tooling", + ) + .toSorted((left, right) => left.localeCompare(right)); + cachedToolingFullSuiteTestTargetsCwd = cwd; + return cachedToolingFullSuiteTestTargets; +} + +function listUnitFastFullSuiteTestTargets() { + const timerTargets = new Set(getUnitFastTimerTestFiles()); + const isolatedTargets = new Set(getUnitFastIsolatedTestFiles()); + return getUnitFastTestFiles().filter( + (file) => !timerTargets.has(file) && !isolatedTargets.has(file), + ); +} + +let cachedUnitSrcFullSuiteTestTargets: string[] | null = null; +let cachedUnitSrcFullSuiteTestTargetsCwd: string | null = null; + +function listUnitSrcFullSuiteTestTargets(cwd: string) { + if (cachedUnitSrcFullSuiteTestTargets && cachedUnitSrcFullSuiteTestTargetsCwd === cwd) { + return cachedUnitSrcFullSuiteTestTargets; + } + const unitFastTargets = new Set(getUnitFastTestFiles()); + const srcDir = path.join(cwd, "src"); + cachedUnitSrcFullSuiteTestTargets = ( + fs.existsSync(srcDir) ? listRepoFilesRecursive(srcDir, cwd) : [] + ) + .filter( + (file) => + file.endsWith(".test.ts") && + isUnitConfigTestFile(file) && + !unitFastTargets.has(file) && + !path.matchesGlob(file, "src/acp/**") && + !path.matchesGlob(file, "src/security/**"), + ) + .toSorted((left, right) => left.localeCompare(right)); + cachedUnitSrcFullSuiteTestTargetsCwd = cwd; + return cachedUnitSrcFullSuiteTestTargets; +} + +function listAgentsCoreFullSuiteTestTargets(cwd: string) { + const isolatedTests = new Set([ + ...agentVitestProjectOwners.spawnProductionBoundary.include, + ...agentVitestProjectOwners.coreIsolated.include, + ]); + const agentsDir = path.join(cwd, "src/agents"); + if (!fs.existsSync(agentsDir)) { + return []; + } + return fs + .readdirSync(agentsDir, { withFileTypes: true }) + .filter((entry) => entry.isFile() && entry.name.endsWith(".test.ts")) + .map((entry) => `src/agents/${entry.name}`) + .filter((file) => !isolatedTests.has(file)) + .toSorted((left, right) => left.localeCompare(right)); +} + +function createBroadToolingScriptPlans(params: VitestRunPlan & { cwd: string }) { + const { config, forwardedArgs, includePatterns, watchMode, cwd } = params; + if (watchMode || config !== TOOLING_VITEST_CONFIG || !includePatterns) { + return null; + } + const pattern = includePatterns[0]; + const targets = + pattern !== undefined && + includePatterns.length === 1 && + BROAD_TOOLING_SCRIPT_TEST_PATTERNS.has(pattern) + ? listBroadToolingScriptTestTargets(pattern, cwd) + : includePatterns.every((target) => target.startsWith("test/scripts/")) + ? includePatterns + : []; + if (targets.length <= BROAD_TOOLING_SCRIPT_TEST_TARGET_CHUNK_SIZE) { + return null; + } + const chunkCount = Math.ceil(targets.length / BROAD_TOOLING_SCRIPT_TEST_TARGET_CHUNK_SIZE); + const chunks = splitTargetChunks(targets, chunkCount); + return chunks.length > 0 + ? chunks.map((chunk) => ({ + config, + forwardedArgs, + includePatterns: chunk, + watchMode, + })) + : null; +} + +function ownsIncludeSelection( + includePatterns: string[] | null, + ownedTargets?: ReadonlySet<string>, +) { + return ( + includePatterns !== null && + includePatterns.length > 0 && + includePatterns.every((pattern) => ownedTargets?.has(pattern)) + ); +} + +function resolveInheritedIncludeScope( + includePatterns: string[], + inheritedPatterns: string[], + ownedTargets?: ReadonlySet<string>, +) { + const owned = includePatterns.filter((pattern) => ownedTargets?.has(pattern)); + if (owned.length === includePatterns.length) { + return includePatterns; + } + return ( + intersectIncludePatterns( + uniqueOrdered([...inheritedPatterns, ...owned]), + includePatterns, + path.matchesGlob, + ) ?? includePatterns + ); +} + +function createBoundedExtensionPlans( + plan: VitestRunPlan, + env?: NodeJS.ProcessEnv, + ownedTargets?: ReadonlySet<string>, +) { + const { config, forwardedArgs, watchMode } = plan; + const roots = EXTENSION_TEST_PROCESS_ROOTS.get(config); + if (watchMode || !roots) { + return [plan]; + } + // Broad CI selections keep their inherited scope and process limits. + // Explicit owned selections do not depend on borrowed include metadata. + const includeFilePath = ownsIncludeSelection(plan.includePatterns, ownedTargets) + ? undefined + : env?.[INCLUDE_FILE_ENV_KEY]?.trim(); + if (includeFilePath) { + if (!fs.existsSync(includeFilePath)) { + return [{ ...plan, includePatterns: null }]; + } + const inheritedTargets = loadIncludePatternsForSpecFilter(env ?? {}) ?? []; + const scopedTargets = plan.includePatterns + ? resolveInheritedIncludeScope(plan.includePatterns, inheritedTargets, ownedTargets) + : inheritedTargets; + if (scopedTargets.length === 0) { + return []; + } + const chunks = splitExtensionTestProcessTargets(config, scopedTargets); + if (chunks.length <= 1) { + return [ + { + ...plan, + includePatterns: + !plan.includePatterns?.some((target) => ownedTargets?.has(target)) && + scopedTargets.length === inheritedTargets.length && + scopedTargets.every((target, index) => target === inheritedTargets[index]) + ? null + : scopedTargets, + }, + ]; + } + return chunks.map((includePatterns) => ({ + config, + forwardedArgs, + includePatterns, + watchMode, + })); + } + const chunks = createExtensionTestProcessTargetChunks( + config, + config === EXTENSION_DATABASE_WORKERS_VITEST_CONFIG && plan.includePatterns + ? plan.includePatterns + : roots, + forwardedArgs, + ); + if (chunks.length <= 1) { + return [plan]; + } + return chunks.map((includePatterns) => ({ + config, + forwardedArgs, + includePatterns, + watchMode, + })); +} + +function expandBroadToolingScriptTargets(targetArgs: string[], cwd: string, watchMode: boolean) { + if (watchMode) { + return targetArgs; + } + return uniqueOrdered( + targetArgs.flatMap((targetArg) => { + const pattern = toScopedIncludePattern(targetArg, cwd); + if (!BROAD_TOOLING_SCRIPT_TEST_PATTERNS.has(pattern)) { + return [targetArg]; + } + const targets = listBroadScriptTestTargets(pattern, cwd); + return targets.length > 0 ? targets : [targetArg]; + }), + ); +} + +function isExistingPathTarget(arg: string, cwd: string) { + return fs.existsSync(path.resolve(cwd, arg)); +} + +function isExistingFileTarget(arg: string, cwd: string) { + try { + return fs.statSync(path.resolve(cwd, arg)).isFile(); + } catch { + return false; + } +} + +function isExistingDirectoryTarget(arg: string, cwd: string) { + try { + return fs.statSync(path.resolve(cwd, arg)).isDirectory(); + } catch { + return false; + } +} + +function isGlobTarget(arg: string) { + return /[*?[\]{}]|[@+!]\(/u.test(arg); +} + +function isLikelyFileTarget(arg: string) { + return /(?:^|\/)[^/]+\.[A-Za-z0-9]+$/u.test(arg); +} + +function isPathLikeTargetArg(arg: string, cwd: string) { + if (!arg || arg === "--" || arg.startsWith("-")) { + return false; + } + const relative = toRepoRelativeTarget(arg, cwd); + return ( + isGlobTarget(arg) || + isTestFileTarget(arg) || + isVitestConfigPathLikeTarget(relative) || + isExistingPathTarget(arg, cwd) || + (path.posix.extname(relative) === "" && + /^(?:src|test|extensions|ui|packages|apps)\//u.test(relative)) || + Boolean(resolveExplicitTestPrefixTargets(arg, cwd)?.length) + ); +} + +function toRepoRelativeTarget(arg: string, cwd: string) { + if (isGlobTarget(arg)) { + return normalizePathPattern(arg.replace(/^\.\//u, "")); + } + const absolute = path.resolve(cwd, arg); + return normalizePathPattern(path.relative(cwd, absolute)); +} + +function explicitIncludeTargets(targetArgs: string[], cwd: string) { + return new Set( + targetArgs + .map((target) => toRepoRelativeTarget(target, cwd)) + .filter((target) => isTestFileTarget(target) && !isGlobTarget(target)), + ); +} + +function toScopedIncludePattern(arg: string, cwd: string) { + const relative = toRepoRelativeTarget(arg, cwd); + if (isGlobTarget(relative) || isTestFileTarget(relative)) { + return relative; + } + if (isExistingFileTarget(arg, cwd) || isLikelyFileTarget(relative)) { + const directory = normalizePathPattern(path.posix.dirname(relative)); + return directory === "." ? "**/*.test.ts" : `${directory}/**/*.test.ts`; + } + return `${relative.replace(/\/+$/u, "")}/**/*.test.ts`; +} + +const EXPLICIT_TEST_TARGET_ROOTS = ["src", "test", "extensions", "ui", "packages", "apps"]; +let cachedExplicitTestTargetFiles: string[] | null = null; +let cachedExplicitTestTargetFilesCwd: string | null = null; + +function listExplicitTestTargetFilesFromGit(cwd: string) { + const result = spawnSync( + "git", + [ + "ls-files", + "-z", + "--cached", + "--others", + "--exclude-standard", + "--", + ...EXPLICIT_TEST_TARGET_ROOTS, + ], + { + cwd, + encoding: "utf8", + maxBuffer: GIT_LS_FILES_MAX_BUFFER_BYTES, + stdio: ["ignore", "pipe", "pipe"], + }, + ); + if (result.status !== 0) { + return null; + } + return result.stdout.split("\0").filter((line) => line.length > 0 && isImportableGraphFile(line)); +} + +function listExplicitTestTargetFilesForCwd(cwd: string) { + if (cachedExplicitTestTargetFiles && cachedExplicitTestTargetFilesCwd === cwd) { + return cachedExplicitTestTargetFiles; + } + + cachedExplicitTestTargetFiles = + listExplicitTestTargetFilesFromGit(cwd) ?? + EXPLICIT_TEST_TARGET_ROOTS.flatMap((root) => listImportGraphFiles(cwd, root)); + cachedExplicitTestTargetFilesCwd = cwd; + return cachedExplicitTestTargetFiles; +} + +function resolveExplicitTestPrefixTargets(targetArg: string, cwd: string) { + if (isExistingPathTarget(targetArg, cwd) || isGlobTarget(targetArg)) { + return null; + } + const relative = toRepoRelativeTarget(targetArg, cwd).replace(/\/+$/u, ""); + if (!relative || isLikelyFileTarget(relative)) { + return null; + } + const directory = path.posix.dirname(relative); + const prefix = `${relative}.`; + // Bound filesystem probes to matching tests, even with a cached repo-wide inventory. + const targets = listExplicitTestTargetFilesForCwd(cwd).filter( + (file) => + path.posix.dirname(file) === directory && + file.startsWith(prefix) && + isTestFileTarget(file) && + fs.existsSync(path.join(cwd, file)), + ); + return targets.length > 0 ? targets.toSorted((left, right) => left.localeCompare(right)) : null; +} + +function includePatternMatchesAnyFile(pattern: string, files: string[]) { + return files.some((file) => file === pattern || path.matchesGlob(file, pattern)); +} + +function resolveExplicitSourceTestTargets( + targetArg: string, + cwd: string, + options: Pick<ChangedTestTargetOptions, "forceFullImportGraph" | "watchMode"> = {}, +) { + const relative = toRepoRelativeTarget(targetArg, cwd); + const kind = classifyTarget(targetArg, cwd); + if (shouldUseWholeConfigTarget(kind, targetArg, cwd)) { + return null; + } + if (!isExistingFileTarget(targetArg, cwd)) { + return null; + } + if (isTestFileTarget(relative)) { + return null; + } + const preciseTargets = [ + ...(resolvePreciseChangedTestTargets(relative, { + cwd, + forceFullImportGraph: options.forceFullImportGraph === true, + }) ?? []), + ...(options.watchMode ? [] : resolveKovaSchemaTestTargets(relative)), + ]; + if (preciseTargets.length > 0) { + return [...new Set(preciseTargets)].toSorted((left, right) => left.localeCompare(right)); + } + if (!isTestSupportFileTarget(relative)) { + return null; + } + return [ + ...new Set( + resolveAffectedTestsFromImportGraph(relative, cwd, { + forceFull: options.forceFullImportGraph === true, + }), + ), + ].toSorted((left, right) => left.localeCompare(right)); +} + +function expandExplicitSourceTestTargets(targetArgs: string[], cwd: string, watchMode: boolean) { + const sourceTargetCount = targetArgs.filter((targetArg) => { + const relative = toRepoRelativeTarget(targetArg, cwd); + return isExistingFileTarget(targetArg, cwd) && !isTestFileTarget(relative); + }).length; + const forceFullImportGraph = sourceTargetCount > EXPLICIT_SOURCE_FULL_IMPORT_GRAPH_THRESHOLD; + return targetArgs.flatMap((targetArg) => { + const relative = toRepoRelativeTarget(targetArg, cwd); + if (classifyTarget(targetArg, cwd) === "extensionFull") { + // The full aggregate already includes the dedicated database-worker project. + return [targetArg]; + } + const databaseWorkerTargets = databaseWorkerExtensionTestFiles.filter((file) => + isGlobTarget(relative) + ? path.matchesGlob(file, relative) + : isExistingDirectoryTarget(targetArg, cwd) && isPathAtOrUnder(file, relative), + ); + if (databaseWorkerTargets.length > 0) { + return [...databaseWorkerTargets, targetArg]; + } + if ( + (isPathAtOrUnder(relative, "ui") || isPluginControlUiPath(relative)) && + isGlobTarget(relative) + ) { + // Expand mixed browser globs before assigning files to their disjoint runners. + const targets = listExplicitTestTargetFilesForCwd(cwd).filter( + (file) => isTestFileTarget(file) && path.matchesGlob(file, relative), + ); + return targets.length > 0 ? targets : [targetArg]; + } + const prefixTargets = resolveExplicitTestPrefixTargets(targetArg, cwd); + if (prefixTargets) { + return prefixTargets; + } + if (relative === "src/commands" && isExistingDirectoryTarget(targetArg, cwd)) { + return [COMMANDS_LIGHT_VITEST_CONFIG, COMMANDS_VITEST_CONFIG]; + } + // Contract directory targets must fan out to the owning contract lanes; the + // generic channels/plugins projects exclude contracts/**, so routing a + // contracts directory there silently runs zero tests (passWithNoTests). + if (isExistingDirectoryTarget(targetArg, cwd)) { + if (isPathAtOrUnder(relative, "src/channels/plugins/contracts")) { + return [ + CONTRACTS_CHANNEL_SURFACE_VITEST_CONFIG, + CONTRACTS_CHANNEL_CONFIG_VITEST_CONFIG, + CONTRACTS_CHANNEL_REGISTRY_VITEST_CONFIG, + CONTRACTS_CHANNEL_SESSION_VITEST_CONFIG, + ]; + } + if (isPathAtOrUnder(relative, "src/plugins/contracts")) { + return [CONTRACTS_PLUGIN_VITEST_CONFIG]; + } + } + const exactDirectoryTargets = resolveExactSourceDirectoryTestTargets(targetArg, cwd); + if (exactDirectoryTargets) { + return exactDirectoryTargets; + } + const targets = resolveExplicitSourceTestTargets(targetArg, cwd, { + forceFullImportGraph, + watchMode, + }); + return targets && targets.length > 0 ? targets : [targetArg]; + }); +} + +const exactSourceDirectoryRoots = [ + "src/acp", + "src/agents", + "src/auto-reply", + "src/channels", + "src/cli", + "src/commands", + "src/config", + "src/cron", + "src/daemon", + "src/gateway", + "src/hooks", + "src/infra", + "src/logging", + "src/media", + "src/media-understanding", + "src/plugin-sdk", + "src/plugins", + "src/process", + "src/secrets", + "src/shared", + "src/tasks", + "src/tui", + "src/utils", + "src/wizard", + "ui/src", +]; + +function isExactSourceDirectoryTarget(relative: string) { + return exactSourceDirectoryRoots.some((root) => isPathAtOrUnder(relative, root)); +} + +function resolveExactSourceDirectoryTestTargets(targetArg: string, cwd: string) { + if (!isExistingDirectoryTarget(targetArg, cwd)) { + return null; + } + const relative = toRepoRelativeTarget(targetArg, cwd).replace(/\/+$/u, ""); + if (!isExactSourceDirectoryTarget(relative)) { + return null; + } + if (isCanonicalAgentOwnerDirectoryTarget(targetArg, cwd)) { + return [targetArg]; + } + const prefix = `${relative}/`; + const lightTargets = uniqueOrdered([ + ...getUnitFastTestFiles(), + ...pluginSdkLightTestFiles, + ...commandsLightTestFiles, + ]).filter((file) => file.startsWith(prefix)); + return lightTargets.length > 0 ? [...lightTargets, targetArg] : null; +} + +function isCanonicalAgentOwnerDirectoryTarget(targetArg: string, cwd: string) { + if (!isExistingDirectoryTarget(targetArg, cwd)) { + return false; + } + const kind = classifyTarget(targetArg, cwd); + if (kind === agentVitestProjectOwners.all.kind) { + return false; + } + const relative = toRepoRelativeTarget(targetArg, cwd).replace(/\/+$/u, ""); + return Object.values(agentVitestProjectOwners).some( + (owner) => owner.kind === kind && isPathAtOrUnder(relative, owner.root), + ); +} + +/** + * Finds explicit test path targets that do not match any known project plan. + */ +export function findUnmatchedExplicitTestTargets(args: string[], cwd = process.cwd()) { + const { targetArgs, watchMode } = parseTestProjectsArgs(args, cwd); + if (targetArgs.length === 0) { + return []; + } + + let candidateFiles: string[] | null = null; + const getCandidateFiles = () => { + candidateFiles ??= listExplicitTestTargetFilesForCwd(cwd); + return candidateFiles; + }; + const unmatched: UnmatchedExplicitTestTarget[] = []; + for (const targetArg of targetArgs) { + const relative = toRepoRelativeTarget(targetArg, cwd); + const absolute = path.resolve(cwd, targetArg); + // Existing exact tests need no inventory scan; lane assignment belongs to the run planner. + if (!isGlobTarget(relative) && isTestFileTarget(relative) && fs.existsSync(absolute)) { + continue; + } + if ( + resolveVitestConfigTargetKind(relative) || + (isVitestConfigFileTarget(relative) && isExistingFileTarget(targetArg, cwd)) + ) { + continue; + } + const kind = classifyTarget(targetArg, cwd); + if (shouldUseWholeConfigTarget(kind, targetArg, cwd)) { + continue; + } + if (isGlobTarget(relative)) { + if (!includePatternMatchesAnyFile(relative, getCandidateFiles())) { + unmatched.push({ + target: targetArg, + reason: "glob-matched-no-files", + }); + } + continue; + } + + if (!fs.existsSync(absolute)) { + if (resolveExplicitTestPrefixTargets(targetArg, cwd)) { + continue; + } + unmatched.push({ + target: targetArg, + reason: "path-does-not-exist", + ...(path.posix.extname(relative) === "" + ? { includePattern: `${relative}{,.*}.{test,spec}.{js,jsx,ts,tsx,mjs,cjs,mts,cts}` } + : {}), + }); + continue; + } + + const explicitSupportTargets = resolveExplicitSourceTestTargets(targetArg, cwd, { watchMode }); + if (explicitSupportTargets) { + if (explicitSupportTargets.length === 0) { + unmatched.push({ + target: targetArg, + reason: "target-matched-no-test-files", + }); + } + continue; + } + + const includePattern = toScopedIncludePattern(targetArg, cwd); + if (!includePatternMatchesAnyFile(includePattern, getCandidateFiles())) { + unmatched.push({ + target: targetArg, + reason: "target-matched-no-test-files", + includePattern, + }); + } + } + return unmatched; +} + +function isSkippedImportGraphDirectory(name: string) { + return name === ".git" || name === "dist" || name === "node_modules" || name === "vendor"; +} + +function listImportGraphFiles( + cwd: string, + directory: string, + files: string[] = [], + extensions: readonly string[] = IMPORTABLE_FILE_EXTENSIONS, + recursive = true, +) { + let entries; + try { + entries = fs.readdirSync(path.join(cwd, directory), { withFileTypes: true }); + } catch { + return files; + } + + for (const entry of entries) { + const relative = normalizePathPattern(path.posix.join(directory, entry.name)); + if (entry.isDirectory()) { + if (recursive && !isSkippedImportGraphDirectory(entry.name)) { + listImportGraphFiles(cwd, relative, files, extensions); + } + continue; + } + if (entry.isFile() && extensions.some((ext) => relative.endsWith(ext))) { + files.push(relative); + } + } + return files; +} + +function resolveImportSpecifier( + importer: string, + specifier: string, + fileSet: ReadonlySet<string>, + extensions: readonly string[] = IMPORTABLE_FILE_EXTENSIONS, +) { + if (!specifier.startsWith(".")) { + return null; + } + + const importerDir = path.posix.dirname(importer); + const base = normalizePathPattern(path.posix.normalize(path.posix.join(importerDir, specifier))); + const candidates = []; + const ext = path.posix.extname(base); + if (ext) { + candidates.push(base); + if ([".js", ".jsx", ".mjs", ".cjs"].includes(ext)) { + const withoutExt = base.slice(0, -ext.length); + candidates.push(...extensions.map((candidateExt) => `${withoutExt}${candidateExt}`)); + } + } else { + candidates.push( + ...extensions.map((candidateExt) => `${base}${candidateExt}`), + ...extensions.map((candidateExt) => `${base}/index${candidateExt}`), + ); + } + + return candidates.find((candidate) => fileSet.has(candidate)) ?? null; +} + +let cachedImportGraph: ImportGraph | null = null; +let cachedImportGraphCwd: string | null = null; +const cachedImportGraphFiles = new Map<string, string[]>(); +const cachedImportGraphGrepMatches = new Map<string, ImportGraphEdges[] | null>(); +const cachedImportGraphEdges = new Map<string, ImportGraphEdges>(); + +function isImportableGraphFile(relative: string) { + return IMPORTABLE_FILE_EXTENSIONS.some((ext) => relative.endsWith(ext)); +} + +function listImportGraphFilesForCwd(cwd: string, options: ImportGraphOptions = {}) { + const tooling = options.tooling === true; + const cacheKey = `${cwd}\0${tooling ? "tooling" : "source"}`; + if (cachedImportGraphFiles.has(cacheKey)) { + return cachedImportGraphFiles.get(cacheKey) ?? []; + } + const roots = tooling ? TOOLING_IMPORT_GRAPH_ROOTS : SOURCE_ROOTS_FOR_IMPORT_GRAPH; + const extensions = tooling ? TOOLING_IMPORTABLE_FILE_EXTENSIONS : IMPORTABLE_FILE_EXTENSIONS; + const files = listTrackedTestPlanFiles( + cwd, + tooling ? TOOLING_IMPORT_GRAPH_GREP_PATHS : IMPORT_GRAPH_GREP_PATHS, + ) ?? [ + ...roots.flatMap((root) => listImportGraphFiles(cwd, root, [], extensions)), + // Root configs are inputs; UI caches and generated sibling trees are not. + ...listImportGraphFiles(cwd, "ui", [], extensions, false), + ]; + cachedImportGraphFiles.set(cacheKey, files); + return files; +} + +function stripImportableGraphExtension( + relative: string, + extensions: readonly string[] = IMPORTABLE_FILE_EXTENSIONS, +) { + for (const ext of extensions) { + if (relative.endsWith(ext)) { + return relative.slice(0, -ext.length); + } + } + return relative; +} + +function resolveImportGraphSearchTerms( + relative: string, + extensions: readonly string[] = IMPORTABLE_FILE_EXTENSIONS, +) { + const withoutExtension = stripImportableGraphExtension(relative, extensions); + const basename = path.posix.basename(withoutExtension); + if (basename === "index" || basename.length < 3) { + return []; + } + const terms = []; + const segments = withoutExtension.split("/"); + if (segments.length > 1) { + terms.push(segments.slice(-2).join("/"), withoutExtension); + } + if (relative.startsWith("test/helpers/")) { + return [...new Set(terms)]; + } + terms.push(basename); + return [...new Set(terms)]; +} + +function readImportGraphEdges( + cwd: string, + files: string[], + fileSet: ReadonlySet<string>, + tooling = false, + terms: string[] = [], +) { + const cacheKey = (file: string) => `${cwd}\0${tooling}\0${file}`; + const requests = files + .map((file) => ({ file, parseImports: !cachedImportGraphEdges.has(cacheKey(file)) })) + .filter(({ parseImports }) => parseImports || terms.length > 0); + const extensions = tooling ? TOOLING_IMPORTABLE_FILE_EXTENSIONS : IMPORTABLE_FILE_EXTENSIONS; + return readTestSelectorSourceFacts(cwd, requests, terms, GIT_LS_FILES_MAX_BUFFER_BYTES).map( + ({ file, imports, matches, references }) => { + const resolve = (specifiers: string[]) => + new Set( + specifiers + .map((specifier) => resolveImportSpecifier(file, specifier, fileSet, extensions)) + .filter((imported) => imported !== null), + ); + const edges = cachedImportGraphEdges.get(cacheKey(file)) ?? { + file, + specifiers: imports, + imports: resolve(imports), + references: new Set<string>(), + }; + for (const reference of references) { + edges.references.add(reference); + } + cachedImportGraphEdges.set(cacheKey(file), edges); + return { edges, matches }; + }, + ); +} + +function listImportGraphGrepMatches( + cwd: string, + terms: string[], + options: ImportGraphOptions & { testFilesOnly?: boolean } = {}, +) { + const tooling = options.tooling === true; + const testFilesOnly = options.testFilesOnly === true; + // Narrow reference matches must not satisfy a later full import-frontier query. + const cacheKey = (term: string) => `${cwd}\0${tooling}\0${testFilesOnly}\0${term}`; + const matches = new Map( + terms.map((term) => [term, cachedImportGraphGrepMatches.get(cacheKey(term)) ?? null]), + ); + const missing = [...new Set(terms)].filter( + (term) => !cachedImportGraphGrepMatches.has(cacheKey(term)), + ); + if (missing.length === 0) { + return matches; + } + const roots = tooling ? TOOLING_IMPORT_GRAPH_ROOTS : SOURCE_ROOTS_FOR_IMPORT_GRAPH; + const extensions = tooling ? TOOLING_IMPORTABLE_FILE_EXTENSIONS : IMPORTABLE_FILE_EXTENSIONS; + // Literal-reference routing only consumes tests; keep import frontiers on the full scope. + const suffixes = testFilesOnly + ? extensions.flatMap((ext) => [`.test${ext}`, `.spec${ext}`]) + : extensions; + const grepPaths = importGraphPathspecs(roots, suffixes); + const spawnOptions: SpawnSyncOptionsWithStringEncoding = { + cwd, + encoding: "utf8", + // A frontier can exceed the platform argv limit; Git accepts stdin patterns. + input: missing.join("\n"), + maxBuffer: GIT_LS_FILES_MAX_BUFFER_BYTES, + stdio: ["pipe", "pipe", "pipe"], + }; + const result = spawnSync( + "git", + ["grep", "-l", "-z", "--fixed-strings", "-f", "-", "--", ...grepPaths], + spawnOptions, + ); + for (const term of missing) { + matches.set(term, []); + } + if (result.status !== 1) { + const trackedFiles = new Set(listImportGraphFilesForCwd(cwd, { tooling })); + // Source archives use the same filesystem inventory and native reader as the full graph. + const candidates = ( + result.status === 0 + ? result.stdout.split("\0").filter((file) => trackedFiles.has(file)) + : [...trackedFiles].filter((file) => !testFilesOnly || isTestFileTarget(file)) + ).toSorted((left, right) => left.localeCompare(right)); + // Per-term membership preserves the helper first-success rule. + // Cached edges need only term facts; full-graph acquisition reuses their parsing. + for (const { edges, matches: fileTerms } of readImportGraphEdges( + cwd, + candidates, + trackedFiles, + tooling, + missing, + )) { + for (const term of fileTerms) { + matches.get(term)?.push(edges); + } + } + } + for (const term of missing) { + cachedImportGraphGrepMatches.set(cacheKey(term), matches.get(term) ?? null); + } + return matches; +} + +function findDirectImporters( + importedFile: string, + matches: ReadonlyMap<string, ImportGraphEdges[] | null>, + extensions: readonly string[], +) { + const isTestHelper = importedFile.startsWith("test/helpers/"); + const terms = resolveImportGraphSearchTerms(importedFile, extensions); + if (terms.length === 0) { + return null; + } + + const importers: string[] = []; + for (const term of terms) { + const candidates = matches.get(term); + if (!candidates) { + return null; + } + for (const { file, imports } of candidates) { + if (file !== importedFile && !importers.includes(file) && imports.has(importedFile)) { + importers.push(file); + } + } + if (isTestHelper && importers.length > 0 && term.includes("/")) { + break; + } + } + return importers; +} + +/** Prove an entry is unshared using the canonical targeted reverse-import scan. */ +export function hasImportGraphConsumers( + changedPaths: string[], + cwd = process.cwd(), + options: ImportGraphOptions = {}, +) { + const tracked = listTrackedTestPlanFiles( + cwd, + options.tooling ? TOOLING_IMPORT_GRAPH_GREP_PATHS : IMPORT_GRAPH_GREP_PATHS, + ); + // An incomplete archive/filesystem inventory cannot prove that an entry is unshared. + if (tracked === null) { + return true; + } + cachedImportGraphFiles.set(`${cwd}\0${options.tooling ? "tooling" : "source"}`, tracked); + const files = new Set(tracked); + const extensions = options.tooling + ? TOOLING_IMPORTABLE_FILE_EXTENSIONS + : IMPORTABLE_FILE_EXTENSIONS; + const terms = changedPaths.flatMap((file) => resolveImportGraphSearchTerms(file, extensions)); + const matches = listImportGraphGrepMatches(cwd, terms, options); + return changedPaths.some((file) => { + const consumers = files.has(file) ? findDirectImporters(file, matches, extensions) : null; + return consumers === null || consumers.length > 0; + }); +} + +function resolveAffectedTestsFromTargetedImportScan( + changedPath: string, + cwd: string, + options: ImportGraphOptions & { direct?: boolean } = {}, +) { + const tooling = options.tooling === true; + const files = listImportGraphFilesForCwd(cwd, { tooling }); + const fileSet = new Set(files); + if (!fileSet.has(changedPath)) { + return []; + } + + const testFiles = new Set( + files.filter((file) => isTestFileTarget(file) && !file.endsWith(".live.test.ts")), + ); + let frontier = [changedPath]; + const seen = new Set(frontier); + const targets = []; + const extensions = tooling ? TOOLING_IMPORTABLE_FILE_EXTENSIONS : IMPORTABLE_FILE_EXTENSIONS; + while (frontier.length > 0) { + const terms = frontier.flatMap((file) => resolveImportGraphSearchTerms(file, extensions)); + const matches = listImportGraphGrepMatches(cwd, terms, { tooling }); + const next = []; + for (const current of frontier) { + const importers = findDirectImporters(current, matches, extensions); + if (importers === null) { + return null; + } + for (const importer of importers) { + if (seen.has(importer)) { + continue; + } + seen.add(importer); + if (testFiles.has(importer)) { + targets.push(importer); + } else if (options.direct !== true) { + next.push(importer); + } + } + } + frontier = next; + } + + return [...new Set(targets)].toSorted((left, right) => left.localeCompare(right)); +} + +function getImportGraph(cwd: string) { + if (cachedImportGraph && cachedImportGraphCwd === cwd) { + return cachedImportGraph; + } + + const files = listImportGraphFilesForCwd(cwd); + const fileSet = new Set(files); + const reverseImports = new Map<string, string[]>(); + const testFiles = new Set( + files.filter((file) => isTestFileTarget(file) && !file.endsWith(".live.test.ts")), + ); + + readImportGraphEdges(cwd, files, fileSet); + for (const file of files) { + const edges = cachedImportGraphEdges.get(`${cwd}\0false\0${file}`); + if (!edges) { + continue; + } + for (const imported of edges.imports) { + const importers = reverseImports.get(imported) ?? []; + importers.push(file); + reverseImports.set(imported, importers); + } + } + + cachedImportGraph = { reverseImports, testFiles }; + cachedImportGraphCwd = cwd; + return cachedImportGraph; +} + +/** Query relative imports/re-exports from targets without scanning unrelated source bodies. */ +export function hasImportGraphImpactOnTargets( + changedPaths: string[], + targetPaths: string[] | ((file: string) => boolean), + cwd = process.cwd(), + options: ImportGraphOptions = {}, +) { + const changed = new Set(changedPaths); + if (changed.size === 0) { + return false; + } + const files = listImportGraphFilesForCwd(cwd, options); + const targets = Array.isArray(targetPaths) ? targetPaths : files.filter(targetPaths); + if (targets.some((file) => changed.has(file))) { + return true; + } + // Staged deletions have left the index, but surviving importers still own their edges. + const fileSet = new Set([...files, ...changed]); + const extensions = options.tooling + ? TOOLING_IMPORTABLE_FILE_EXTENSIONS + : IMPORTABLE_FILE_EXTENSIONS; + const seen = new Set(targets); + let frontier = targets; + while (frontier.length > 0) { + readImportGraphEdges(cwd, frontier, fileSet, options.tooling); + const next: string[] = []; + for (const file of frontier) { + const edges = cachedImportGraphEdges.get(`${cwd}\0${options.tooling === true}\0${file}`); + // Resolve raw cached facts against this query's deleted-path identities too. + for (const specifier of edges?.specifiers ?? []) { + const dependency = resolveImportSpecifier(file, specifier, fileSet, extensions); + if (!dependency) { + continue; + } + if (changed.has(dependency)) { + return true; + } + if (!seen.has(dependency)) { + seen.add(dependency); + next.push(dependency); + } + } + } + frontier = next; + } + return false; +} + +function resolveAffectedTestsFromImportGraph( + changedPath: string, + cwd: string, + options: { forceFull?: boolean } = {}, +) { + if (options.forceFull !== true) { + const targetedTargets = resolveAffectedTestsFromTargetedImportScan(changedPath, cwd); + if (targetedTargets !== null) { + return targetedTargets; + } + } + + const { reverseImports, testFiles } = getImportGraph(cwd); + const queue = [changedPath]; + const seen = new Set(queue); + const targets = []; + + for (const current of queue) { + for (const importer of reverseImports.get(current) ?? []) { + if (seen.has(importer)) { + continue; + } + seen.add(importer); + if (testFiles.has(importer)) { + targets.push(importer); + } + queue.push(importer); + } + } + + return [...new Set(targets)].toSorted((left, right) => left.localeCompare(right)); +} + +function resolveVitestConfigTargetKind(relative: string) { + return VITEST_CONFIG_TARGET_KIND_BY_PATH.get(relative) ?? null; +} + +function isVitestConfigPathLikeTarget(relative: string) { + return ( + relative === "vitest.config.ts" || /^test\/vitest\/vitest\..+\.config\.ts$/u.test(relative) + ); +} + +function isVitestConfigFileTarget(relative: string) { + return RUNNABLE_VITEST_CONFIG_TARGETS.has(relative); +} + +function isVitestConfigTargetForKind(kind: string, targetArg: string, cwd: string) { + return resolveVitestConfigTargetKind(toRepoRelativeTarget(targetArg, cwd)) === kind; +} + +function isControlUiE2eTarget(relative: string) { + return ( + relative === "ui/src/test-helpers/control-ui-e2e.ts" || + relative === "ui/src/e2e" || + relative.startsWith("ui/src/e2e/") || + (isControlUiSourcePath(relative) && relative.endsWith(".e2e.test.ts")) + ); +} + +function resolveChannelContractTargetKind(relative: string) { + if (!relative.startsWith("src/channels/plugins/contracts/")) { + return null; + } + for (const [config, patterns] of CHANNEL_CONTRACT_CONFIG_PATTERNS) { + if (patterns.some((pattern) => path.matchesGlob(relative, pattern))) { + return VITEST_CONFIG_TARGET_KIND_BY_PATH.get(config); + } + } + return "contractsChannelSession"; +} + +function listChangedPathsFromGit(baseRef: string, cwd: string) { + return listChangedPathsFromGitSource({ base: baseRef, cwd }); +} + +function extractChangedBaseRef(args: string[]) { + for (let index = 0; index < args.length; index += 1) { + const arg = args[index]; + if (arg === undefined) { + continue; + } + const match = arg.match(CHANGED_ARGS_PATTERN); + if (!match) { + continue; + } + if (match[1]) { + return match[1]; + } + const nextArg = args[index + 1]; + return nextArg && nextArg !== "--" && !nextArg.startsWith("-") ? nextArg : "HEAD"; + } + return null; +} + +function stripChangedArgs(args: string[]) { + const strippedArgs = []; + for (let index = 0; index < args.length; index += 1) { + const arg = args[index]; + if (arg === undefined) { + continue; + } + const match = arg.match(CHANGED_ARGS_PATTERN); + if (!match) { + strippedArgs.push(arg); + continue; + } + if (!match[1]) { + const nextArg = args[index + 1]; + if (nextArg && nextArg !== "--" && !nextArg.startsWith("-")) { + index += 1; + } + } + } + return strippedArgs; +} + +function shouldKeepBroadChangedRun(changedPaths: string[]) { + return changedPaths.some((changedPath) => + PRECISE_SOURCE_TEST_TARGETS.has(changedPath) + ? false + : BROAD_CHANGED_FALLBACK_PATTERNS.some((pattern) => pattern.test(changedPath)), + ); +} + +function resolveToolingChangedTestTargets(changedPaths: string[], cwd = process.cwd()) { + const targets = []; + for (const changedPath of changedPaths) { + const testTargets = + SOURCE_TEST_TARGETS.get(changedPath) ?? resolveToolingTestTargets(changedPath, cwd); + if (!testTargets) { + return null; + } + targets.push(...testTargets); + if (CHANNEL_PLUGIN_SHAPE_PARITY_WIRING_PATHS.has(changedPath)) { + targets.push(CHANNEL_PLUGIN_SHAPE_PARITY_TEST_TARGET); + } + } + return [...new Set(targets)]; +} + +const TOOLING_SCRIPT_PATH_PATTERN = /^scripts\/(.+)\.(?:mjs|cjs|js|mts|cts|ts|sh|py|ps1)$/u; + +function resolveConventionalToolingTestTargets(changedPath: string, cwd = process.cwd()) { + const match = TOOLING_SCRIPT_PATH_PATTERN.exec(changedPath); + if (!match) { + return null; + } + const stem = match[1]; + if (stem === undefined) { + return null; + } + const basename = path.posix.basename(stem); + const dashedStem = stem.replaceAll("/", "-"); + const e2eLibStem = stem.startsWith("e2e/lib/") ? stem.slice("e2e/lib/".length) : null; + const e2eLibDashedStem = e2eLibStem?.replaceAll("/", "-"); + const e2eLibParts = e2eLibStem?.split("/") ?? []; + const e2eLibFamily = e2eLibParts.length > 1 ? e2eLibParts[0] : null; + const e2eLibFamilyCandidates = e2eLibFamily + ? [ + `test/scripts/${e2eLibFamily}.test.ts`, + `test/scripts/${e2eLibFamily}-client.test.ts`, + `test/scripts/${e2eLibFamily}-assertions.test.ts`, + `test/scripts/${e2eLibFamily}-probe.test.ts`, + ] + : []; + const candidates = [ + `test/scripts/${stem}.test.ts`, + `test/scripts/${dashedStem}.test.ts`, + `test/scripts/${basename}.test.ts`, + ...(e2eLibDashedStem + ? [`test/scripts/${e2eLibDashedStem}.test.ts`, `test/scripts/e2e-${e2eLibDashedStem}.test.ts`] + : []), + ...e2eLibFamilyCandidates, + `src/scripts/${stem}.test.ts`, + `src/scripts/${dashedStem}.test.ts`, + `src/scripts/${basename}.test.ts`, + ]; + const targets = candidates.filter((candidate) => fs.existsSync(path.join(cwd, candidate))); + return targets.length > 0 ? targets : null; +} + +function isToolingScriptPath(changedPath: string) { + return TOOLING_SCRIPT_PATH_PATTERN.test(changedPath); +} + +function resolveUpgradeSurvivorConfigRecipeTargets(changedPath: string) { + if (!/^scripts\/e2e\/lib\/upgrade-survivor\/config-recipe\/[^/]+\.json$/u.test(changedPath)) { + return null; + } + return ["test/scripts/upgrade-survivor-config-recipe.test.ts"]; +} + +function resolveDocsI18nBehaviorTargets(changedPath: string) { + if (!/^scripts\/docs-i18n\/testdata\/behavior\/[^/]+\/[^/]+$/u.test(changedPath)) { + return null; + } + return ["test/scripts/docs-i18n.test.ts"]; +} + +function resolveDocsI18nGoTargets(changedPath: string) { + if (!/^scripts\/docs-i18n\/(?:go\.(?:mod|sum)|[^/]+\.go)$/u.test(changedPath)) { + return null; + } + const targets = ["test/scripts/docs-i18n.test.ts"]; + if (changedPath === "scripts/docs-i18n/go.mod") { + targets.push("test/scripts/ci-workflow-guards.test.ts"); + } + return targets; +} + +function resolveK8sManifestTargets(changedPath: string) { + if (!/^scripts\/k8s\/manifests\/[^/]+\.yaml$/u.test(changedPath)) { + return null; + } + return ["test/scripts/k8s-manifests.test.ts"]; +} + +function resolveParallelsToolingTestTargets(changedPath: string) { + if ( + !/^scripts\/e2e\/parallels\/[^/]+\.ts$/u.test(changedPath) && + !/^scripts\/e2e\/parallels-(?:linux|macos|npm-update|windows)-smoke\.sh$/u.test(changedPath) + ) { + return null; + } + const targets = ["test/scripts/parallels-smoke-model.test.ts"]; + if ( + [ + "scripts/e2e/parallels/guest-transports.ts", + "scripts/e2e/parallels/host-command.ts", + "scripts/e2e/parallels/npm-update-scripts.ts", + "scripts/e2e/parallels/npm-update-smoke.ts", + ].includes(changedPath) + ) { + targets.push("test/scripts/parallels-npm-update-smoke.test.ts"); + } + if (changedPath === "scripts/e2e/parallels/update-job-timeout.ts") { + targets.push("test/scripts/parallels-update-job-timeout.test.ts"); + } + return targets; +} + +function resolveToolingTestOwnerTargets(...owners: string[]) { + return owners.map((owner) => (owner.includes("/") ? owner : `test/scripts/${owner}.test.ts`)); +} + +const packageAcceptance = "package-acceptance-workflow"; +const dockerBuild = "docker-build-helper"; +const dockerE2e = "docker-e2e-plan"; +const workflowGuards = "ci-workflow-guards"; +const pluginPrerelease = "plugin-prerelease-test-plan"; +const releaseCheck = "test/release-check.test.ts"; +const installDocker = "test-install-sh-docker"; +const changedScope = "src/scripts/ci-changed-scope.test.ts"; +const changedScopeTests = [ + "src/scripts/ci-changed-scope.contract-fixtures.test.ts", + "src/scripts/ci-changed-scope.control-ui.test.ts", + "src/scripts/ci-changed-scope.git-owner.test.ts", + "src/scripts/ci-changed-scope.native-i18n.test.ts", + changedScope, + "src/scripts/ci-changed-scope.windows.test.ts", +]; +const dockerCache = "src/docker-build-cache.test.ts"; +const dockerDigests = "src/docker-image-digests.test.ts"; +const openaiChatToolsE2e = "test/e2e/qa-lab/runtime/openai-compatible-chat-tools.e2e.test.ts"; +const npmPostpublish = "test/openclaw-npm-postpublish-verify.test.ts"; +const crossOsReleaseChecks = "openclaw-cross-os-release-checks"; +const runNode = "src/infra/run-node.test.ts"; +const pluginSdkEntryOwners = [ + "src/plugins/contracts/plugin-sdk-index.bundle.test.ts", + "src/plugins/contracts/plugin-sdk-package-contract-guardrails.test.ts", + "src/plugins/contracts/plugin-sdk-subpaths.test.ts", + "src/plugins/contracts/extension-package-project-boundaries.test.ts", + "plugin-sdk-surface-report", + "build-all", + releaseCheck, + "prepare-extension-package-boundary-artifacts", + "ts-topology", + "test/vitest/vitest.tooling.config.ts", +]; + +// Keep only genuinely ambiguous paths explicit; conventional discovery owns +// unambiguous scripts and direct imports without a second inventory. +const EXACT_TOOLING_TARGETS = new Map<string, string[]>([ + [".github/workflows/ci.yml", ["ci-platform-checkout", "ci-linux-git", "ci-git-owner"]], + [".github/workflows/docs-sync-publish.yml", ["docs-sync-publish"]], + [".github/workflows/docs-agent.yml", ["docs-agent-workflow"]], + ["scripts/generate-ci-git-owner.mts", ["ci-git-owner"]], + [ + ".github/workflows/openclaw-live-and-e2e-checks-reusable.yml", + [packageAcceptance, workflowGuards, "release-workflow-matrix-plan", installDocker], + ], + [ + ".github/workflows/plugin-clawhub-release.yml", + [packageAcceptance, "plugin-release-git-lifecycle", workflowGuards], + ], + [ + ".github/workflows/plugin-npm-release.yml", + [ + packageAcceptance, + "plugin-npm-extended-stable-workflow", + "plugin-release-git-lifecycle", + workflowGuards, + ], + ], + [".github/workflows/qa-live-transports-convex.yml", [packageAcceptance, workflowGuards]], + [".github/workflows/update-migration.yml", [packageAcceptance, workflowGuards]], + [ + ".github/actions/setup-node-env/action.yml", + ["setup-node-env-bun", packageAcceptance, workflowGuards], + ], + [".github/actions/setup-node-env/dependency-fingerprint.mjs", [workflowGuards]], + [".github/actions/setup-node-env/seed-bun-from-image.mjs", ["setup-node-env-bun"]], + [".github/actions/setup-pnpm-store-cache/action.yml", [packageAcceptance, workflowGuards]], + [".github/actions/setup-pnpm-store-cache/ensure-node.sh", ["setup-pnpm-store-cache-ensure-node"]], + ["test/e2e/qa-lab/runtime/mcp-channels-docker-client.ts", [dockerE2e, pluginPrerelease]], + ["scripts/e2e/lib/mcp-code-mode-probe-server.ts", ["mcp-code-mode-gateway-client"]], + ["scripts/e2e/cron-cli-docker.sh", [dockerBuild, "docker-e2e-observability"]], + ["scripts/ios-release-upload.sh", ["ios-release-wrapper-args", "ios-release-fastlane-gates"]], + ["scripts/release-verify-beta.ts", ["release-wrapper-scripts"]], + ["scripts/lib/bundled-plugin-build-entries.mjs", ["bundled-plugin-build-entries", releaseCheck]], + ["scripts/lib/docker-e2e-package.sh", [dockerBuild]], + [ + "scripts/lib/release-version.mjs", + [ + "test/release-version.test.ts", + "test/npm-publish-plan.test.ts", + "test/openclaw-npm-release-check.test.ts", + npmPostpublish, + "test/plugin-npm-release.test.ts", + "test/plugin-clawhub-release.test.ts", + "android-version", + "android-pin-version", + "docker-release-policy", + "docker-release-artifacts", + "full-release-validation-at-sha", + "ios-version", + "openclaw-npm-extended-stable-release", + "openclaw-npm-publish", + "npm-prepared-bundle", + "release-preflight", + "release-prepare", + "release-upgrade-baseline", + "release-version", + "upgrade-survivor-baselines", + "upgrade-survivor-config-recipe", + ], + ], + [ + "scripts/lib/release-context.mjs", + ["full-release-validation-at-sha", "release-candidate-checklist", packageAcceptance], + ], + [ + "scripts/lib/clawhub-bootstrap-artifact.mjs", + ["clawhub-bootstrap-artifact", "verify-clawhub-published-artifact"], + ], + [ + "scripts/lib/plugin-npm-release.ts", + ["test/plugin-npm-release.test.ts", "test/plugin-clawhub-release.test.ts"], + ], + [ + "scripts/lib/extension-source-classifier.mts", + [ + "extension-source-classifier", + "src/channels/plugins/contracts/channel-import-guardrails.test.ts", + ], + ], + ["scripts/build-stamp.mts", ["src/infra/build-stamp.test.ts"]], + ["scripts/run-vitest.mjs", ["run-vitest", "test-projects", "vitest-local-scheduling"]], + ["scripts/run-vitest.mts", ["run-vitest", "test-projects", "vitest-local-scheduling"]], + ["scripts/run-oxlint-shards.mts", ["run-oxlint", "lint-status"]], + ["scripts/run-oxlint.mts", ["run-oxlint", "lint-status"]], + ["scripts/run-oxlint.mjs", ["run-oxlint", "lint-status"]], + ["scripts/run-lint.mts", ["run-oxlint", "lint-status"]], + ["scripts/run-stylelint.mts", ["changed-lanes", "lint-status"]], + [ + "scripts/lib/failed-trailer.mts", + ["run-oxlint", "run-tsgo", "run-vitest", "changed-lanes", "lint-status"], + ], + ["scripts/lib/managed-child-process.mts", ["managed-child-process", "lint-status"]], + ["scripts/lib/dist-artifact-ownership.mts", ["dist-artifact-ownership", "lint-status"]], + ["scripts/docker-e2e-rerun.mts", ["docker-e2e-helper-cli"]], + ["scripts/openclaw-postpack.mjs", [TOOLING_VITEST_CONFIG]], + ["scripts/package-manifest.mjs", ["test/openclaw-prepack.test.ts"]], + ["scripts/openclaw-npm-prepublish-verify.ts", ["test/openclaw-npm-prepublish-verify.test.ts"]], + ["scripts/lib/docker-e2e-scenarios.mts", [dockerE2e, pluginPrerelease]], + ["scripts/lib/upgrade-survivor-policy.mjs", [dockerE2e]], + ["scripts/e2e/kitchen-sink-rpc-walk.mts", ["kitchen-sink-rpc-walk", pluginPrerelease]], + [ + "scripts/e2e/agents-delete-shared-workspace-docker.sh", + [dockerE2e, changedScope, "src/commands/agents.delete.test.ts"], + ], + [ + "scripts/e2e/browser-cdp-snapshot-docker.sh", + [dockerBuild, "browser-cdp-snapshot", "e2e-helper-env-limits"], + ], + [ + "scripts/e2e/config-reload-source-docker.sh", + [ + dockerE2e, + packageAcceptance, + "fixture-config", + "e2e-mock-config-limits", + "src/gateway/config-reload.test.ts", + ], + ], + [ + "scripts/e2e/gateway-network-docker.sh", + [dockerBuild, dockerE2e, packageAcceptance, "gateway-network-client", changedScope], + ], + ["scripts/e2e/npm-telegram-live-runner.ts", ["npm-telegram-live"]], + [ + "scripts/e2e/upgrade-survivor-docker.sh", + [ + dockerBuild, + dockerE2e, + packageAcceptance, + "upgrade-survivor-probe-gateway", + "upgrade-survivor-assertions", + "upgrade-survivor-mobile-pairing", + "upgrade-survivor-recovery-cleanup", + "openclaw-test-state", + ], + ], + [ + "scripts/e2e/lib/upgrade-survivor/run.sh", + [ + "upgrade-survivor-assertions", + "upgrade-survivor-mobile-pairing", + "upgrade-survivor-recovery-cleanup", + "upgrade-survivor-watchos-direct-node", + ], + ], + [ + "scripts/e2e/lib/upgrade-survivor/watchos-direct-node.mjs", + ["upgrade-survivor-watchos-direct-node"], + ], + [ + "scripts/e2e/lib/upgrade-survivor/mobile-pairing-client.mts", + ["upgrade-survivor-mobile-pairing"], + ], + [ + "scripts/e2e/lib/upgrade-survivor/recovery-cleanup-fixture.mjs", + ["upgrade-survivor-recovery-cleanup"], + ], + [ + "scripts/e2e/bundled-plugin-install-uninstall-docker.sh", + [dockerBuild, dockerE2e, pluginPrerelease, "bundled-plugin-install-uninstall-probe"], + ], + ["scripts/e2e/lib/plugin-update/corrupt-update-scenario.sh", ["plugin-update-unchanged-docker"]], + ["scripts/e2e/lib/plugin-update/probe.mjs", ["plugin-update-unchanged-docker"]], + ["scripts/e2e/lib/plugin-update/unchanged-scenario.sh", ["plugin-update-unchanged-docker"]], + [ + "scripts/e2e/update-corrupt-plugin-docker.sh", + [dockerBuild, dockerE2e, packageAcceptance, "plugin-update-unchanged-docker"], + ], + ["scripts/e2e/plugins-docker.sh", [dockerBuild, dockerE2e, "plugins-assertions"]], + [ + "scripts/e2e/release-user-journey-docker.sh", + [dockerBuild, dockerE2e, packageAcceptance, "release-user-journey-assertions"], + ], + [ + "scripts/e2e/openai-image-auth-docker.sh", + [ + dockerBuild, + dockerE2e, + "openai-image-auth-docker-client", + "extensions/openai/image-generation-provider.test.ts", + ], + ], + ["scripts/e2e/lib/openai-chat-tools/client.mjs", [openaiChatToolsE2e]], + [ + "scripts/e2e/lib/openai-web-search-minimal/client.mjs", + ["test/e2e/qa-lab/runtime/openai-web-search-minimal.e2e.test.ts"], + ], + ["scripts/e2e/qr-import-docker.sh", [dockerBuild]], + ["scripts/bundle-a2ui.mts", ["bundled-plugin-assets"]], +]); + +const SEMANTIC_TOOLING_TARGET_PATTERNS: Array<[RegExp, string[]]> = [ + [ + /^(?:git-hooks\/pre-commit|scripts\/pre-commit\/(?:guard-staged-content\.mjs|filter-staged-files\.mjs|format-staged\.sh|run-node-tool\.sh)|test\/git-hooks-pre-commit\.test-support\.ts)$/u, + ["test/git-hooks-pre-commit.test.ts", "test/git-hooks-pre-commit-boundaries.test.ts"], + ], + [/^scripts\/pr$/u, ["pr-merge", "pr-merge-outcome", "pr-operation-lock", "pr-wrappers"]], + [ + /^scripts\/pr-lib\/crabbox-gate-contract\.mjs$/u, + ["pr-crabbox-gate-publisher", "pr-crabbox-merge-bypass"], + ], + [ + /^scripts\/pr-lib\/crabbox-gate-plan\.mts$/u, + ["pr-crabbox-gate-plan", "pr-crabbox-gate-publisher", "pr-prepare-gates"], + ], + [ + /^scripts\/pr-lib\/crabbox-merge-bypass\.sh$/u, + ["pr-crabbox-merge-bypass", "pr-merge", "pr-merge-outcome"], + ], + [/^scripts\/lib\/windows-taskkill\.mjs$/u, ["managed-child-process", "run-with-env"]], + [ + /^scripts\/lib\/config-boundary-guard\.mts$/u, + [ + "src/plugins/contracts/config-boundary-guard.test.ts", + "src/plugins/contracts/deprecated-internal-config-api.test.ts", + ], + ], + [ + /^\.github\/workflows\/ci\.yml$/u, + [ + workflowGuards, + "changed-lanes", + "check-workflows", + "plugin-contract-test-plan", + pluginPrerelease, + "verify-pr-hosted-gates", + ], + ], + [ + /^\.github\/workflows\/ci-check-testbox\.yml$/u, + [workflowGuards, packageAcceptance, "changed-lanes"], + ], + [/^\.github\/workflows\/ci-check-arm-testbox\.yml$/u, [workflowGuards, packageAcceptance]], + [/^\.github\/workflows\/crabbox-hydrate\.yml$/u, [workflowGuards, packageAcceptance]], + [/^\.github\/workflows\/ci-build-artifacts-testbox\.yml$/u, [packageAcceptance, workflowGuards]], + [ + /^\.github\/workflows\/full-release-validation\.yml$/u, + [ + "src/dockerfile.test.ts", + "full-release-validation-state", + "full-release-validation-at-sha", + "full-release-candidate-reuse", + "find-reusable-release-validation", + "openclaw-npm-extended-stable-full-validation-workflow", + "release-no-push-workflow", + "release-ci-summary", + packageAcceptance, + pluginPrerelease, + "check-workflows", + ], + ], + [ + /^\.github\/workflows\/full-release-candidate\.yml$/u, + ["full-release-candidate-reuse", packageAcceptance, "check-workflows", workflowGuards], + ], + [ + /^\.github\/workflows\/openclaw-release-checks\.yml$/u, + [packageAcceptance, crossOsReleaseChecks, pluginPrerelease, installDocker], + ], + [ + /^\.github\/workflows\/docker-release(?:-prepare)?\.yml$/u, + [ + "src/dockerfile.test.ts", + "docker-channel-promote", + "docker-release-artifacts", + "vercel-container-registry-publish", + ], + ], + [/^\.github\/workflows\/install-smoke\.yml$/u, ["install-smoke-no-push-workflow", installDocker]], + [ + /^\.github\/workflows\/openclaw-performance\.yml$/u, + ["openclaw-performance-workflow", "openclaw-performance-git-lifecycle"], + ], + [/^\.github\/workflows\/linux-app-release\.yml$/u, ["release-workflow-git-lifecycle"]], + [ + /^\.github\/workflows\/macos-release\.yml$/u, + ["release-workflow-git-lifecycle", packageAcceptance], + ], + [ + /^\.github\/workflows\/npm-placeholder-bootstrap\.yml$/u, + ["release-workflow-git-lifecycle", "npm-placeholder-publication"], + ], + [/^\.github\/workflows\/plugin-prerelease\.yml$/u, [pluginPrerelease]], + [/^\.github\/workflows\/tui-pty\.yml$/u, [packageAcceptance]], + [ + /^\.github\/workflows\/openclaw-cross-os-release-checks-reusable\.yml$/u, + [crossOsReleaseChecks, "openclaw-cross-os-release-workflow", packageAcceptance], + ], + [ + /^\.github\/workflows\/openclaw-release-publish\.yml$/u, + [packageAcceptance, "docker-release-artifacts", "vercel-container-registry-publish"], + ], + [/^\.github\/workflows\/package-acceptance\.yml$/u, [packageAcceptance]], + [ + /^\.github\/workflows\/vercel-container-registry-publish\.yml$/u, + ["docker-channel-promote", "release-plan-producer", "vercel-container-registry-publish"], + ], + [ + /^\.github\/workflows\/plugin-clawhub-new\.yml$/u, + [packageAcceptance, "plugin-clawhub-new-workflow"], + ], + [ + /^\.github\/workflows\/openclaw-npm-release\.yml$/u, + [npmPostpublish, "openclaw-npm-extended-stable-workflow", packageAcceptance], + ], + [ + new RegExp( + [ + "^\\.github\\/workflows\\/(?:auto-response|clawsweeper-dispatch|labeler|", + "real-behavior-proof|stale)\\.yml$", + ].join(""), + "u", + ), + [workflowGuards], + ], + [ + new RegExp( + [ + "^\\.github\\/workflows\\/mantis-(?:discord-(?:smoke|status-reactions|", + "thread-attachment)|slack-desktop-smoke)\\.yml$", + ].join(""), + "u", + ), + [packageAcceptance, workflowGuards], + ], + [ + /^\.github\/(?:workflows\/mantis-web-ui-chat-proof\.yml|actions\/mantis-validate-trusted-ref\/action\.yml)$/u, + ["mantis-web-ui-chat-proof-workflow", packageAcceptance, workflowGuards], + ], + [/^\.github\/workflows\/android-release\.yml$/u, [packageAcceptance, workflowGuards]], + [ + /^\.github\/(?:actions\/(?:ensure-base-commit|git-owner|publish-generated-pr|mantis-validate-trusted-ref)\/|workflows\/(?:workflow-sanity|qa-profile-evidence|maturity-scorecard|docs-agent|docs-sync-publish|openclaw-performance|linux-app-release|macos-release|npm-placeholder-bootstrap|plugin-clawhub-release|plugin-npm-release|mantis-(?:discord-(?:smoke|status-reactions|thread-attachment)|slack-desktop-smoke|web-ui-chat-proof))\.yml$)/u, + [ + "ci-git-owner", + "ci-linux-git", + "ci-platform-checkout", + "src/scripts/ci-changed-scope.git-owner.test.ts", + ], + ], + [/^\.github\/actions\/publish-generated-pr\//u, [workflowGuards]], + [/^tsconfig\.scripts\.json$/u, ["changed-lanes", "test-projects"]], + [/^scripts\/test-projects\.test-support\.mts$/u, ["test-projects"]], + [/^scripts\/ci-changed-scope\.mjs$/u, [...changedScopeTests, "control-ui-i18n"]], + [/^scripts\/check-changed\.(?:mjs|mts)$/u, ["changed-lanes"]], + [/^scripts\/changed-lanes\.(?:mjs|mts)$/u, ["changed-lanes"]], + [ + /^scripts\/(?:lib\/tsx-cli-shim|tsx)\.mjs$/u, + ["direct-run-entrypoints", "lint-status", "local-check-runtime"], + ], + [ + new RegExp( + [ + "^scripts\\/(?:generate-prompt-snapshots|prompt-snapshot-files|", + "sync-codex-model-prompt-fixture)\\.ts$", + ].join(""), + "u", + ), + ["prompt-snapshots"], + ], + [/^scripts\/e2e\/npm-telegram-live-docker\.sh$/u, ["npm-telegram-live"]], + [ + /^scripts\/package-openclaw-for-docker\.m[jt]s$/u, + ["test/e2e/qa-lab/runtime/package-openclaw-for-docker.e2e.test.ts"], + ], + [/^scripts\/run-node\.(?:mjs|mts)$/u, [runNode]], + [/^scripts\/ios-write-swift-filelist\.m[jt]s$/u, ["ios-run"]], + [ + /^scripts\/pr-lib\/(?:merge(?:-outcome)?\.sh|merge-legacy-refusal\.mjs)$/u, + ["pr-merge", "pr-merge-outcome"], + ], + [/^scripts\/plugin-clawhub-publish\.sh$/u, ["test/plugin-clawhub-release.test.ts"]], + [/^scripts\/openclaw-npm-postpublish-verify\.ts$/u, [npmPostpublish]], + [ + /^scripts\/install\.ps1$/u, + ["install-ps1", "website-installer-sync-workflow", crossOsReleaseChecks, changedScope], + ], + [ + /^scripts\/(?:crabbox-wrapper(?:-providers)?|crabbox-routing-policy|testbox-lease-freshness)\.mts$/u, + ["crabbox-wrapper"], + ], + [/^scripts\/crabbox-wrapper\.mjs$/u, ["crabbox-wrapper"]], + [ + /^scripts\/copy-bundled-plugin-metadata\.(?:mjs|mts)$/u, + ["src/plugins/copy-bundled-plugin-metadata.test.ts", runNode], + ], + [ + /^scripts\/github\/run-openclaw-cross-os-release-checks\.sh$/u, + ["openclaw-cross-os-release-workflow"], + ], + [ + /^scripts\/(?:write-plugin-sdk-entry-dts\.ts|lib\/local-check-runtime\.mts)$/u, + [ + "test/scripts/write-plugin-sdk-entry-dts.test.ts", + "build-all", + "declaration-stage", + "tsdown-build", + "prepare-extension-package-boundary-artifacts", + ], + ], + [/^scripts\/pr-lib\/worktree\.sh$/u, ["test/vitest/vitest.tooling.config.ts"]], + [/^scripts\/dev\/gateway-smoke\.ts$/u, ["test/e2e/qa-lab/runtime/gateway-smoke.e2e.test.ts"]], + [/^scripts\/e2e\/mock-openai-server\.mjs$/u, ["e2e-mock-config-limits"]], + [/^apps\/android\/scripts\/build-release-artifacts\.ts$/u, ["android-release-artifacts"]], + [ + new RegExp( + [ + "^scripts\\/(?:auth-monitor|mobile-reauth|setup-auth-system|", + "termux-(?:auth-widget|quick-auth|sync-widget))\\.sh$|", + "^scripts\\/systemd\\/openclaw-auth-monitor\\.(?:service|timer)$", + ].join(""), + "u", + ), + ["auth-monitor"], + ], + [/^scripts\/native-app-i18n\.ts$/u, ["native-app-i18n", workflowGuards]], + [ + /^scripts\/github\/(?:dependency-guard|guard-shared)\.mjs$/u, + ["dependency-guard-script", "dependency-guard-workflow"], + ], + [ + /^scripts\/github\/(?:security-sensitive-guard|guard-shared)\.mjs$/u, + ["security-sensitive-guard-script", "security-sensitive-guard-workflow"], + ], + [/^scripts\/plugin-clawhub-release-check\.ts$/u, ["release-wrapper-scripts"]], + [ + /^scripts\/generate-runtime-sidecar-paths-baseline\.ts$/u, + ["src/plugins/bundled-plugin-metadata.test.ts"], + ], + [ + /^scripts\/lib\/guard-inventory-utils\.mjs$/u, + [ + "test/extension-import-boundaries.test.ts", + "test/plugin-extension-import-boundary.test.ts", + "test/architecture-smells.test.ts", + "test/test-helper-extension-import-boundary.test.ts", + "extension-import-boundary-checker", + "web-fetch-provider-boundary", + "src/plugins/contracts/plugin-sdk-subpaths.test.ts", + ], + ], + [ + /^scripts\/check-workflows\.mts$/u, + ["check-composite-action-input-interpolation", "check-no-conflict-markers", workflowGuards], + ], + [/^apps\/android\/fastlane\/Fastfile$/u, ["android-release-fastlane-gates"]], + [/^apps\/ios\/fastlane\/Fastfile$/u, ["ios-release-fastlane-gates"]], + [/^scripts\/ios-release-cut\.(?:sh|ts)$/u, ["ios-release-plan"]], + [/^scripts\/ios-release-prepare\.sh$/u, ["ios-release-prepare", "ios-release-wrapper-args"]], + [ + /^scripts\/lib\/bundled-runtime-sidecar-paths\.json$/u, + [ + "src/plugins/bundled-plugin-metadata.test.ts", + "src/infra/update-global.test.ts", + "src/infra/update-runner.test.ts", + npmPostpublish, + ], + ], + [/^scripts\/lib\/android-version\.ts$/u, ["android-version", "android-pin-version"]], + [ + /^scripts\/lib\/docker-e2e-plan\.(?:mjs|mts)$/u, + [dockerE2e, "docker-all-scheduler", pluginPrerelease], + ], + [ + new RegExp( + [ + "^scripts\\/lib\\/plugin-sdk-(?:deprecated-barrel-subpaths|entrypoints|", + "private-local-only-subpaths)\\.json$", + ].join(""), + "u", + ), + pluginSdkEntryOwners, + ], + [ + /^scripts\/lib\/plugin-sdk-deprecated-public-subpaths\.json$/u, + [ + "check-deprecated-api-usage", + "src/plugins/contracts/plugin-sdk-package-contract-guardrails.test.ts", + "plugin-sdk-surface-report", + "build-all", + ], + ], + [/^scripts\/lib\/plugin-sdk-entries\.(?:mjs|mts)$/u, pluginSdkEntryOwners], + [ + /^scripts\/lib\/official-external-channel-(?:catalog|seed)\.json$/u, + [ + "src/plugins/official-external-plugin-catalog.test.ts", + releaseCheck, + "test/official-channel-catalog.test.ts", + ], + ], + [ + /^scripts\/lib\/official-external-provider-catalog\.json$/u, + ["src/plugins/official-external-plugin-catalog.test.ts", releaseCheck], + ], + [ + /^scripts\/lib\/official-external-plugin-catalog\.json$/u, + ["src/plugins/official-external-plugin-catalog.test.ts", releaseCheck], + ], + [ + /^scripts\/lib\/workspace-bootstrap-smoke\.mts$/u, + [releaseCheck, "test/openclaw-npm-release-check.test.ts"], + ], + [/^scripts\/lib\/extension-test-plan\.(?:mjs|mts)$/u, ["test-extension"]], + [ + /^scripts\/lib\/recommended-tool-installs\.json$/u, + ["src/plugins/recommended-tool-installs.test.ts", releaseCheck], + ], + [/^scripts\/docker\/install-sh-common\/version-parse\.sh$/u, [installDocker]], + [ + /^scripts\/lib\/local-build-metadata(?:-paths)?\.(?:mjs|mts)$/u, + [ + "src/infra/build-stamp.test.ts", + "runtime-postbuild-stamp", + runNode, + "src/infra/package-dist-inventory.test.ts", + releaseCheck, + "test/openclaw-npm-release-check.test.ts", + "check-gateway-watch-regression", + "check-openclaw-package-tarball", + crossOsReleaseChecks, + ], + ], + [ + /^scripts\/lib\/package-dist-imports\.mjs$/u, + [ + "check-package-dist-imports", + "check-openclaw-package-tarball", + "postinstall-bundled-plugins", + releaseCheck, + ], + ], + [ + /^scripts\/lib\/build-metadata\.sh$/u, + [ + "src/docker-setup.e2e.test.ts", + "apple-release-source-check", + "ios-version", + "package-mac-app", + installDocker, + ], + ], + [/^scripts\/lib\/plistbuddy\.sh$/u, ["create-dmg", "package-mac-app", "package-mac-dist"]], + [/^scripts\/lib\/swift-toolchain\.sh$/u, ["package-mac-app", "package-mac-dist"]], + [/^scripts\/stage-cua-driver-macos\.sh$/u, ["package-mac-app"]], + [ + /^scripts\/(stage-cloudflared-macos\.sh|lib\/cloudflared-macos\.json)$/u, + ["stage-cloudflared-macos", "package-mac-app"], + ], + [ + /^scripts\/lib\/npm-publish-plan\.mjs$/u, + [ + "test/npm-publish-plan.test.ts", + "test/openclaw-npm-release-check.test.ts", + npmPostpublish, + "test/plugin-npm-release.test.ts", + "test/plugin-clawhub-release.test.ts", + "release-upgrade-baseline", + "android-version", + "ios-version", + "upgrade-survivor-baselines", + "upgrade-survivor-config-recipe", + ], + ], + [/^scripts\/lib\/npm-pack-budget\.mts$/u, [releaseCheck, installDocker]], + [ + /^scripts\/lib\/actions-artifact-archive\.mjs$/u, + ["full-release-candidate-reuse", "plugin-publication-artifact"], + ], + [ + /^scripts\/(?:lib\/)?full-release-candidate-reuse\.(?:mjs|d\.mts)$/u, + ["full-release-candidate-reuse"], + ], + [ + /^scripts\/lib\/static-extension-assets\.(?:mjs|mts)$/u, + ["bundled-plugin-assets", "runtime-postbuild", runNode, "plugin-npm-runtime-build-args"], + ], + [ + /^scripts\/lib\/plugin-npm-runtime-build\.(?:mjs|mts)$/u, + ["plugin-npm-runtime-build-args", "test/plugin-npm-runtime-build.test.ts"], + ], + [/^scripts\/lib\/run-node\.(?:mjs|mts)$/u, [runNode]], + [ + /^\.agents\/skills\/openclaw-changelog-update\/scripts\/verify-release-notes\.mjs$/u, + ["release-notes-ledger", "verify-release-notes"], + ], + [ + /^scripts\/e2e\/lib\/docker-stats\/assert-resource-ceiling\.mjs$/u, + ["docker-stats-resource-ceiling"], + ], + [ + /^scripts\/e2e\/mcp-channels-docker\.sh$/u, + [dockerBuild, "docker-e2e-observability", dockerE2e, pluginPrerelease], + ], + [ + /^scripts\/e2e\/cron-mcp-cleanup-docker\.sh$/u, + [dockerBuild, "docker-e2e-observability", dockerE2e, pluginPrerelease], + ], + [ + /^scripts\/e2e\/(?:mcp-code-mode-gateway-(?:live-)?docker|lib\/mcp-code-mode\/scenario)\.sh$/u, + [ + dockerBuild, + dockerE2e, + pluginPrerelease, + "mcp-code-mode-gateway-client", + "session-log-mentions", + ], + ], + [ + /^scripts\/e2e\/agent-bundle-mcp-tools-docker\.sh$/u, + [ + dockerBuild, + dockerE2e, + pluginPrerelease, + "src/agents/agent-bundle-mcp-runtime.test.ts", + "src/agents/agent-bundle-mcp-tools.materialize.test.ts", + ], + ], + [/^scripts\/lib\/tsgo-sparse-guard\.(?:mjs|mts)$/u, ["run-tsgo", "changed-lanes"]], + [ + new RegExp( + [ + "^(?:scripts\\/materialize-clawhub-cli\\.sh|", + "\\.github\\/release\\/clawhub-cli\\/package(?:-lock)?\\.json)$", + ].join(""), + "u", + ), + [packageAcceptance, "plugin-clawhub-new-workflow"], + ], + [ + new RegExp( + [ + "^(?:scripts\\/materialize-vercel-cli\\.sh|", + "\\.github\\/release\\/vercel-cli\\/package(?:-lock)?\\.json)$", + ].join(""), + "u", + ), + ["test/scripts/vercel-container-registry-publish.test.ts"], + ], + [ + /^scripts\/lib\/generated-text-asset\.mts$/u, + ["build-diffs-viewer-runtime", "bundled-plugin-assets"], + ], + [/^scripts\/check-plugin-npm-runtime-builds\.mts$/u, ["plugin-npm-runtime-build-args"]], + [ + /^scripts\/install\.sh$/u, + [ + "install-sh", + installDocker, + "website-installer-sync-workflow", + crossOsReleaseChecks, + changedScope, + ], + ], + [ + /^scripts\/sparkle-build\.ts$/u, + ["test/appcast.test.ts", releaseCheck, "package-mac-app", "package-mac-dist"], + ], + [ + /^test\/vitest\/vitest\.contracts-paths\.mjs$/u, + [ + "test-projects", + "test/vitest-projects-config.test.ts", + "test/vitest/vitest.contracts-channel-surface.config.ts", + "test/vitest/vitest.contracts-channel-config.config.ts", + "test/vitest/vitest.contracts-channel-registry.config.ts", + "test/vitest/vitest.contracts-channel-session.config.ts", + ], + ], + [/^scripts\/e2e\/openai-chat-tools-docker\.sh$/u, [openaiChatToolsE2e, dockerE2e]], + [ + /^scripts\/e2e\/session-runtime-context-docker\.sh$/u, + [ + dockerE2e, + "src/agents/embedded-agent-runner/run/runtime-context-prompt.test.ts", + "src/agents/embedded-agent-runner/transcript-rewrite.test.ts", + ], + ], + [ + new RegExp( + [ + "^scripts\\/e2e\\/(?!(?:config-reload-source|", + "kitchen-sink-(?:plugin|rpc)|npm-telegram-live|onboard|openai-chat-tools|", + "plugin-lifecycle-matrix|release-media-memory|session-runtime-context|", + "update-corrupt-plugin)-docker\\.sh$).+-docker\\.sh$", + ].join(""), + "u", + ), + [dockerBuild, dockerE2e], + ], + [/^scripts\/e2e\/kitchen-sink-rpc-docker\.sh$/u, [dockerBuild, pluginPrerelease]], + [/^scripts\/e2e\/kitchen-sink-plugin-docker\.sh$/u, [dockerBuild, pluginPrerelease]], + [/^scripts\/e2e\/plugin-lifecycle-matrix-docker\.sh$/u, [dockerBuild]], + [ + /^scripts\/e2e\/release-media-memory-docker\.sh$/u, + [dockerE2e, "release-media-memory-scenario"], + ], + [/^scripts\/e2e\/codex-media-path-docker\.sh$/u, ["codex-media-path-client"]], + [/^scripts\/e2e\/live-plugin-tool-docker\.sh$/u, ["live-plugin-tool-assertions"]], + [/^scripts\/e2e\/onboard-docker\.sh$/u, [dockerBuild, "openclaw-test-state"]], + [ + new RegExp( + [ + "^scripts\\/e2e\\/(?:codex-npm-plugin-live|config-reload-source|", + "gateway-network|npm-onboard-channel-agent|doctor-install-switch|", + "update-channel-switch|skill-install|upgrade-survivor|", + "update-corrupt-plugin|release-(?:plugin-marketplace|typed-onboarding|", + "upgrade-user-journey|user-journey)|plugin-binding-command-escape)-docker", + "\\.sh$", + ].join(""), + "u", + ), + [packageAcceptance], + ], + [ + /^scripts\/e2e\/npm-onboard-channel-agent-docker\.sh$/u, + ["npm-onboard-channel-agent-assertions"], + ], + [/^scripts\/docker\/cleanup-smoke\/Dockerfile$/u, [dockerCache, dockerDigests, dockerBuild]], + [ + /^scripts\/docker\/install-sh-(?:e2e|nonroot|smoke)\/Dockerfile$/u, + [dockerCache, dockerDigests, installDocker], + ], + [ + /^scripts\/docker\/sandbox\/Dockerfile$/u, + [dockerCache, dockerDigests, "src/dockerfile.test.ts"], + ], + [ + /^scripts\/docker\/sandbox\/Dockerfile\.browser$/u, + [dockerCache, dockerDigests, "src/agents/sandbox/browser.create.test.ts"], + ], + [/^scripts\/docker\/sandbox\/Dockerfile\.common$/u, [dockerCache]], + [/^scripts\/e2e\/Dockerfile$/u, [dockerCache, dockerDigests, dockerBuild, dockerE2e]], + [/^scripts\/e2e\/Dockerfile\.qr-import$/u, [dockerCache, dockerDigests, dockerBuild]], + [ + /^scripts\/e2e\/plugin-binding-command-escape\.Dockerfile$/u, + [dockerDigests, dockerBuild, dockerE2e], + ], + [ + new RegExp( + [ + "^(?:scripts\\/e2e|test\\/e2e\\/qa-lab\\/runtime)\\/agent-bundle-mcp-tools-doc", + "ker(?:-client)?\\.(?:sh|ts)$", + ].join(""), + "u", + ), + [ + "src/agents/agent-bundle-mcp-runtime.test.ts", + "src/agents/agent-bundle-mcp-tools.materialize.test.ts", + ], + ], + [ + /^test\/e2e\/qa-lab\/runtime\/system-agent-first-run-docker-client\.ts$/u, + [ + "src/cli/program/register.onboard.test.ts", + "src/cli/run-main.test.ts", + "src/cli/run-main.exit.test.ts", + "src/commands/system-agent-with-inference.test.ts", + "src/system-agent/assistant.configured.test.ts", + "src/system-agent/assistant.test.ts", + "src/system-agent/system-agent.test.ts", + "src/system-agent/operations.test.ts", + "src/system-agent/overview.test.ts", + "src/system-agent/audit.test.ts", + ], + ], + [ + /^scripts\/e2e\/system-agent-first-run-spec\.json$/u, + ["src/system-agent/operations.test.ts", "src/system-agent/audit.test.ts"], + ], + [ + /^scripts\/e2e\/system-agent-rescue-docker-client\.ts$/u, + [ + "src/system-agent/rescue-policy.test.ts", + "src/system-agent/rescue-message.test.ts", + "src/system-agent/operations.test.ts", + "src/system-agent/audit.test.ts", + ], + ], + [ + /^scripts\/e2e\/session-runtime-context-docker(?:-client)?\.(?:sh|ts)$/u, + [ + "src/agents/embedded-agent-runner/run/runtime-context-prompt.test.ts", + "src/agents/embedded-agent-runner/transcript-rewrite.test.ts", + ], + ], + [ + new RegExp( + [ + "^(?:scripts\\/e2e\\/(?!(?:cron-mcp-cleanup-docker-client\\.ts)$)(?:agent-bu", + "ndle-mcp-tools|mcp-[^/]+-docker|cron-mcp-cleanup-docker|", + "bundled-plugin-install-uninstall|plugin-update-unchanged).+|", + "test\\/e2e\\/qa-lab\\/runtime\\/mcp-channels-docker-client\\.ts)$", + ].join(""), + "u", + ), + [pluginPrerelease, dockerE2e], + ], + [ + new RegExp( + [ + "^(?:scripts\\/e2e\\/mcp-code-mode-gateway-(?:docker|live-docker|", + "docker-client).+|scripts\\/mcp-code-mode-gateway-e2e\\.ts)$", + ].join(""), + "u", + ), + ["mcp-code-mode-gateway-client", "session-log-mentions"], + ], + [ + /^scripts\/e2e\/(?:mcp-channels|cron-cli|cron-mcp-cleanup)-docker\.sh$/u, + ["docker-e2e-observability"], + ], + [ + /^scripts\/e2e\/cron-mcp-cleanup-docker-client\.ts$/u, + [ + "cron-mcp-cleanup-docker-client", + "src/gateway/server.cron.test.ts", + "src/gateway/server-methods/agent.test.ts", + "src/cron/isolated-agent/run.fast-mode.test.ts", + "src/cron/active-jobs-manual-run.test.ts", + ], + ], + [/^scripts\/e2e\/cron-mcp-cleanup-docker\.sh$/u, ["cron-mcp-cleanup-docker-client"]], + [ + /^test\/e2e\/qa-lab\/runtime\/mcp-channels\.fixture\.ts$/u, + ["test/e2e/qa-lab/runtime/mcp-gateway-transport.e2e.test.ts", "cron-mcp-cleanup-docker-client"], + ], + [ + /^scripts\/e2e\/lib\/auth-profile-store-assertions\.mjs$/u, + ["release-scenarios-assertions", "npm-onboard-channel-agent-assertions"], + ], + [ + /^scripts\/lib\/plugin-npm-package-manifest\.(?:mjs|mts)$/u, + ["plugin-npm-package-manifest-args", "test/plugin-npm-package-manifest.test.ts"], + ], + [ + /^scripts\/e2e\/lib\/release-assertion-files\.mjs$/u, + ["release-scenarios-assertions", "release-user-journey-assertions"], + ], + [/^scripts\/e2e\/lib\/fixtures\/config\.mjs$/u, ["fixture-config"]], + [/^scripts\/e2e\/lib\/fixtures\/plugins\.mjs$/u, ["fixture-plugin-commands"]], + [ + /^scripts\/e2e\/lib\/codex-app-server-fixture\.mjs$/u, + ["codex-media-path-client", "test/e2e/qa-lab/runtime/codex-auth-product-proof.e2e.test.ts"], + ], + [ + /^scripts\/e2e\/lib\/incremental-line-reader\.mjs$/u, + [ + "incremental-line-reader", + "config-reload-log-scanner", + "codex-media-path-client", + "test/e2e/qa-lab/runtime/codex-auth-product-proof.e2e.test.ts", + ], + ], + [ + /^scripts\/e2e\/lib\/fixture\.mjs$/u, + ["fixture-config", "fixtures-workspace", "fixture-plugin-commands"], + ], + [ + /^test\/e2e\/qa-lab\/runtime\/openai-image-auth-docker-client\.ts$/u, + [ + "openai-image-auth-docker-client", + "extensions/openai/image-generation-provider.test.ts", + "src/image-generation/openai-compatible-image-provider.test.ts", + ], + ], + [ + /^test\/e2e\/qa-lab\/runtime\/codex-auth-app-server\.fixture\.mjs$/u, + ["test/e2e/qa-lab/runtime/codex-auth-product-proof.e2e.test.ts"], + ], + [ + /^scripts\/e2e\/lib\/openai-web-search-minimal\/(?:mock-server\.mjs|scenario\.sh)$/u, + [ + "test/e2e/qa-lab/runtime/openai-web-search-minimal.e2e.test.ts", + "test/e2e/qa-lab/runtime/openai-web-search-minimal-assertions.e2e.test.ts", + ], + ], + [ + /^scripts\/e2e\/lib\/openwebui\/http-probe\.mjs$/u, + ["test/e2e/qa-lab/runtime/openwebui-probe.e2e.test.ts"], + ], + [ + /^scripts\/e2e\/(?:browser-cdp-snapshot-docker\.sh|lib\/env-limits\.mjs)$/u, + ["e2e-helper-env-limits"], + ], + [/^scripts\/e2e\/skill-install-docker\.sh$/u, ["e2e-shell-tempfiles"]], + [/^scripts\/e2e\/npm-onboard-channel-agent-docker\.sh$/u, [pluginPrerelease]], + [ + /^scripts\/e2e\/lib\/clawhub-fixture-server\.cjs$/u, + ["clawhub-fixture-server", pluginPrerelease], + ], + [ + /^scripts\/e2e\/lib\/codex-media-path\/jsonl-request-tail\.mts$/u, + ["codex-media-path-client", "test/e2e/qa-lab/runtime/codex-auth-product-proof.e2e.test.ts"], + ], + [ + /^scripts\/e2e\/lib\/codex-npm-plugin-live\/assertions\.mjs$/u, + ["codex-install-assertions", dockerBuild], + ], + [ + /^scripts\/e2e\/(?:lib\/openai-chat-tools\/scenario\.sh|openai-chat-tools-docker\.sh)$/u, + [openaiChatToolsE2e], + ], + [/^scripts\/e2e\/lib\/openai-chat-tools\/write-config\.mjs$/u, [openaiChatToolsE2e]], + [ + /^scripts\/e2e\/openai-web-search-minimal-docker\.sh$/u, + [ + "test/e2e/qa-lab/runtime/openai-web-search-minimal.e2e.test.ts", + "test/e2e/qa-lab/runtime/openai-web-search-minimal-assertions.e2e.test.ts", + ], + ], + [ + /^scripts\/e2e\/openwebui-docker\.sh$/u, + ["test/e2e/qa-lab/runtime/openwebui-probe.e2e.test.ts", "fixture-config"], + ], +]; +function resolveSemanticToolingTargets(changedPath: string) { + return SEMANTIC_TOOLING_TARGET_PATTERNS.flatMap(([pattern, owners]) => + pattern.test(changedPath) ? resolveToolingTestOwnerTargets(...owners) : [], + ); +} + +function isGithubWorkflowOrActionYaml(changedPath: string) { + return ( + /^\.github\/workflows\/[^/]+\.ya?ml$/u.test(changedPath) || + /^\.github\/actions\/.+\.ya?ml$/u.test(changedPath) + ); +} + +function resolveGithubYamlGuardTargets(changedPath: string) { + if (isGithubWorkflowOrActionYaml(changedPath)) { + return GITHUB_YAML_PINNING_GUARD_TEST_TARGETS; + } + return null; +} + +function resolveDirectToolingReferenceTests(changedPath: string, cwd: string) { + return ( + listImportGraphGrepMatches(cwd, [changedPath], { tooling: true, testFilesOnly: true }).get( + changedPath, + ) ?? [] + ) + .filter( + ({ file, references }) => + file !== "test/scripts/test-projects.test.ts" && + !file.endsWith(".live.test.ts") && + isTestFileTarget(file) && + references.has(changedPath), + ) + .map(({ file }) => file); +} + +function resolveToolingTestTargets(changedPath: string, cwd = process.cwd()) { + if ( + /^test\/scripts\/(?:ci-(?:checkout|git-owner|linux-git|platform-checkout|windows-process-census)\.test(?:-support)?\.ts|generated-publisher\.test-support\.ts|openclaw-performance-(?:workflow\.test(?:-support)?|git-lifecycle\.test)\.ts|plugin-release-git-lifecycle\.test\.ts|release-workflow-git-lifecycle\.test\.ts|fixtures\/(?:ci-platform-checkout\.mjs|ci-checkout-auth\.py|ci-windows-process-census\.(?:mjs|py)))$/u.test( + changedPath, + ) + ) { + return resolveToolingTestOwnerTargets( + "ci-git-owner", + "ci-linux-git", + "ci-platform-checkout", + "openclaw-performance-workflow", + "openclaw-performance-git-lifecycle", + "plugin-release-git-lifecycle", + "release-workflow-git-lifecycle", + workflowGuards, + ); + } + if (changedPath.startsWith("test/scripts/") && isTestFileTarget(changedPath)) { + return [changedPath]; + } + if (BROAD_CHANGED_FALLBACK_PATTERNS.some((pattern) => pattern.test(changedPath))) { + return null; + } + // Test-runner declarations still share their implementation owner. Script + // declaration sidecars are gone, so this no longer aliases scripts/*.d.mts. + const implementationPath = + !changedPath.startsWith("scripts/") && changedPath.endsWith(".d.mts") + ? changedPath.replace(/\.d\.mts$/u, ".mjs") + : changedPath; + const githubYaml = isGithubWorkflowOrActionYaml(implementationPath); + const exactOwners = EXACT_TOOLING_TARGETS.get(implementationPath); + if (exactOwners && !githubYaml) { + return resolveToolingTestOwnerTargets(...exactOwners); + } + const exactTargets = exactOwners ? resolveToolingTestOwnerTargets(...exactOwners) : []; + const semanticTargets = resolveSemanticToolingTargets(implementationPath); + const facts = getChangedPathFacts(changedPath); + const hasToolingOwner = + exactTargets.length > 0 || + semanticTargets.length > 0 || + facts.surface === "rootTooling" || + changedPath === "Dockerfile" || + changedPath === ".crabbox.yaml" || + changedPath.startsWith(".agents/") || + isToolingScriptPath(implementationPath) || + (facts.surface === "app" && /\/(?:fastlane|scripts)\//u.test(changedPath)) || + (facts.surface === "extension" && /\/(?:scripts\/|package\.json$)/u.test(changedPath)) || + (facts.surface === "rootTest" && changedPath.startsWith("test/e2e/qa-lab/")); + if (!hasToolingOwner) { + return null; + } + const crossOsReleaseTargets = + implementationPath === "scripts/openclaw-cross-os-release-checks.ts" || + implementationPath.startsWith("scripts/lib/cross-os-release-checks/") + ? ["test/scripts/openclaw-cross-os-release-checks.test.ts"] + : null; + const explicitTargets = + (changedPath === "Dockerfile" + ? [ + "src/docker-build-cache.test.ts", + "src/docker-image-digests.test.ts", + "src/dockerfile.test.ts", + "test/scripts/test-install-sh-docker.test.ts", + ] + : changedPath === ".crabbox.yaml" + ? ["test/scripts/package-acceptance-workflow.test.ts"] + : null) ?? + crossOsReleaseTargets ?? + resolveUpgradeSurvivorConfigRecipeTargets(implementationPath) ?? + resolveDocsI18nBehaviorTargets(implementationPath) ?? + resolveDocsI18nGoTargets(implementationPath) ?? + resolveK8sManifestTargets(implementationPath) ?? + resolveParallelsToolingTestTargets(implementationPath); + const githubYamlGuardTargets = resolveGithubYamlGuardTargets(implementationPath); + const conventionalTargets = resolveConventionalToolingTestTargets(implementationPath, cwd); + const hasDirectOwner = Boolean( + exactTargets.length || + explicitTargets?.length || + githubYamlGuardTargets?.length || + semanticTargets.length || + conventionalTargets?.length, + ); + const importGraphResult = + !hasDirectOwner && + TOOLING_IMPORTABLE_FILE_EXTENSIONS.some((ext) => implementationPath.endsWith(ext)) + ? resolveAffectedTestsFromTargetedImportScan(implementationPath, cwd, { + tooling: true, + direct: true, + }) + : []; + const importGraphTargets = importGraphResult ?? []; + const referenceTargets = + githubYaml || (semanticTargets.length === 0 && !hasDirectOwner) + ? resolveDirectToolingReferenceTests(implementationPath, cwd) + : []; + const targets = [ + ...(!hasDirectOwner && isRoutableChangedTarget(changedPath) && isTestFileTarget(changedPath) + ? [changedPath] + : []), + ...exactTargets, + ...(explicitTargets ?? []), + ...semanticTargets, + ...(conventionalTargets ?? []), + ...importGraphTargets, + ...referenceTargets, + ...(githubYamlGuardTargets ?? []), + // Root aliases also control native bundling; keep the existing tooling owners. + ...(changedPath === "tsconfig.json" ? MERMAID_RENDERER_TEST_TARGETS : []), + ]; + if (targets.length > 0) { + return uniqueOrdered(targets); + } + return isToolingScriptPath(implementationPath) || facts.surface === "rootTooling" + ? [TOOLING_VITEST_CONFIG] + : null; +} + +function shouldUseBroadChangedTargets(env = process.env) { + return parsePermissiveBooleanToken(env[BROAD_CHANGED_ENV_KEY]) === true; +} + +function isRoutableChangedTarget(changedPath: string) { + if (GENERATED_CHANGED_TEST_TARGET_PATTERNS.some((pattern) => pattern.test(changedPath))) { + return false; + } + if (changedPath.endsWith(".live.test.ts")) { + return false; + } + const surface = getChangedPathFacts(changedPath).surface; + return ( + ["source", "package", "extension", "rootTest"].includes(surface) || + changedPath === "ui" || + changedPath.startsWith("ui/") || + ["src", "test", "extensions", "packages"].includes(changedPath) + ); +} + +function resolveSiblingTestTarget(changedPath: string, cwd: string) { + if (!/\.[cm]?tsx?$/u.test(changedPath) || isTestFileTarget(changedPath)) { + return null; + } + const withoutExtension = changedPath.replace(/\.[cm]?tsx?$/u, ""); + const sibling = `${withoutExtension}.test.ts`; + return fs.existsSync(path.join(cwd, sibling)) ? sibling : null; +} + +function shouldCombineSiblingTestWithImportGraph(changedPath: string) { + const sourcePrefix = `${changedPath.replace(/\.[cm]?tsx?$/u, "")}.`; + return ( + changedPath.startsWith("test/helpers/") || + databaseWorkerExtensionTestFiles.some((file) => file.startsWith(sourcePrefix)) + ); +} + +function shouldRouteChangedTargetWithoutImportGraph(changedPath: string) { + return changedPath.endsWith(".live.test.ts") || isControlUiSourcePath(changedPath); +} + +function resolvePromptSnapshotFixtureTargets(changedPath: string) { + if ( + !/^test\/fixtures\/agents\/prompt-snapshots\/.+\.(?:json|md(?:\.diff)?)$/u.test(changedPath) + ) { + return null; + } + return ["test/scripts/prompt-snapshots.test.ts"]; +} + +function resolvePackageFixtureTargets(changedPath: string, cwd: string) { + const match = /^packages\/([^/]+)\/test\/fixtures\/([^/]+)\/.+$/u.exec(changedPath); + const packageName = match?.[1]; + const fixtureFamily = match?.[2]; + if (!packageName || !fixtureFamily) { + return null; + } + const owner = `packages/${packageName}/src/${fixtureFamily}.test.ts`; + return fs.existsSync(path.join(cwd, owner)) ? [owner] : null; +} + +function resolveAppcastTargets(changedPath: string) { + return changedPath === "appcast.xml" ? APPCAST_TEST_TARGETS : null; +} + +function resolveKovaSchemaTestTargets(changedPath: string) { + // The workflow fixture reads schema bytes, so imports cannot express this dependency. + return changedPath === "src/config/zod-schema.agent-defaults.ts" || + changedPath === "src/config/zod-schema.agent-defaults-base.ts" + ? ["test/scripts/openclaw-performance-workflow.test.ts"] + : []; +} + +function resolvePreciseChangedTestTargets( + changedPath: string, + options: ChangedTestTargetOptions & { skipImportGraph?: boolean }, +) { + const cwd = options.cwd ?? process.cwd(); + const pluginSdkInclude = resolvePluginSdkLightIncludePattern(changedPath); + const mappedTargets = + SOURCE_TEST_TARGETS.get(changedPath) ?? + (pluginSdkInclude + ? pluginSdkLightTestFiles.filter((file) => path.matchesGlob(file, pluginSdkInclude)) + : null) ?? + (/^extensions\/[^/]+\/openclaw\.plugin\.json$/u.test(changedPath) + ? [changedPath, DOCS_CONFIG_EXAMPLES_TEST_TARGET] + : null) ?? + resolveToolingTestTargets(changedPath, cwd) ?? + resolveAppcastTargets(changedPath) ?? + resolvePromptSnapshotFixtureTargets(changedPath) ?? + resolvePackageFixtureTargets(changedPath, cwd); + if (mappedTargets) { + return mappedTargets; + } + if (isRoutableChangedTarget(changedPath) && isTestFileTarget(changedPath)) { + return [changedPath]; + } + const siblingTest = resolveSiblingTestTarget(changedPath, cwd); + if ( + siblingTest && + !shouldCombineSiblingTestWithImportGraph(changedPath) && + options.combineSiblingWithImportGraph !== true + ) { + return [siblingTest]; + } + if (shouldRouteChangedTargetWithoutImportGraph(changedPath)) { + return isControlUiSourcePath(changedPath) ? [changedPath] : null; + } + if (options.skipImportGraph === true) { + return null; + } + const facts = getChangedPathFacts(changedPath); + if ( + facts.surface === "source" || + facts.surface === "package" || + facts.surface === "extension" || + changedPath.startsWith("test/helpers/") || + changedPath.startsWith("ui/src/") || + changedPath.startsWith("ui/config/") + ) { + const affectedTests = resolveAffectedTestsFromImportGraph(changedPath, cwd, { + forceFull: options.forceFullImportGraph === true, + }); + if (affectedTests.length > 0) { + return siblingTest ? uniqueOrdered([siblingTest, ...affectedTests]) : affectedTests; + } + } + return siblingTest ? [siblingTest] : null; +} + +function isDeletedChangedTestTarget(changedPath: string, cwd: string) { + return isTestFileTarget(changedPath) && !fs.existsSync(path.join(cwd, changedPath)); +} + +/** + * Maps changed repo paths to the smallest useful Vitest target plan. + */ +export function resolveChangedTestTargetPlan( + changedPaths: string[], + options: ChangedTestTargetOptions = {}, +): ChangedTestTargetPlan { + if (changedPaths.length === 0) { + return { mode: "none", targets: [] }; + } + const cwd = options.cwd ?? process.cwd(); + const executableChangedPaths = changedPaths.filter( + (changedPath) => !isDeletedChangedTestTarget(changedPath, cwd), + ); + const toolingTargets = resolveToolingChangedTestTargets(executableChangedPaths, cwd); + if (toolingTargets) { + return { mode: "targets", targets: toolingTargets }; + } + const changedLanes = detectChangedLanes(executableChangedPaths); + const env = options.env ?? {}; + const useBroadFallback = options.broad ?? shouldUseBroadChangedTargets(env); + const skipImportGraph = changedLanes.lanes.all && !useBroadFallback; + const targets = []; + const skippedBroadFallbackPaths = []; + for (const changedPath of executableChangedPaths) { + const needsPluginShapeParity = CHANNEL_PLUGIN_SHAPE_PARITY_WIRING_PATHS.has(changedPath); + const preciseTargets = resolvePreciseChangedTestTargets(changedPath, { + ...options, + skipImportGraph, + }); + if (preciseTargets) { + targets.push(...preciseTargets); + if (needsPluginShapeParity) { + targets.push(CHANNEL_PLUGIN_SHAPE_PARITY_TEST_TARGET); + } + continue; + } + const needsBroadFallback = shouldKeepBroadChangedRun([changedPath]) || changedLanes.lanes.all; + if (needsBroadFallback) { + if (useBroadFallback) { + return { mode: "broad", targets: [] }; + } + skippedBroadFallbackPaths.push(changedPath); + continue; + } + if (isRoutableChangedTarget(changedPath)) { + targets.push(changedPath); + } + if (needsPluginShapeParity) { + targets.push(CHANNEL_PLUGIN_SHAPE_PARITY_TEST_TARGET); + } + } + if ( + useBroadFallback && + options.includeExtensionImpact !== false && + changedLanes.extensionImpactFromCore + ) { + targets.push("extensions"); + } + const plan: ChangedTestTargetPlan = { + mode: "targets", + targets: uniqueOrdered([ + ...targets, + ...(options.watchMode ? [] : executableChangedPaths.flatMap(resolveKovaSchemaTestTargets)), + ]), + }; + if (skippedBroadFallbackPaths.length > 0) { + plan.skippedBroadFallbackPaths = [...new Set(skippedBroadFallbackPaths)]; + } + return plan; +} + +export function listFullExtensionVitestProjectConfigs() { + return ( + fullSuiteVitestShards.find((shard) => shard.config === FULL_EXTENSIONS_VITEST_CONFIG) + ?.projects ?? [] + ); +} + +export function resolveChangedTargetArgs( + args: string[], + cwd = process.cwd(), + listChangedPaths: (baseRef: string, cwd: string) => string[] = listChangedPathsFromGit, + options: ChangedTestTargetOptions = {}, +) { + const plan = resolveChangedTestTargetPlanForArgs(args, cwd, listChangedPaths, options); + if (!plan) { + return null; + } + if (plan.mode === "broad") { + return null; + } + return plan.targets; +} + +export function resolveChangedTestTargetPlanForArgs( + args: string[], + cwd = process.cwd(), + listChangedPaths: (baseRef: string, cwd: string) => string[] = listChangedPathsFromGit, + options: ChangedTestTargetOptions = {}, +) { + const baseRef = extractChangedBaseRef(args); + if (!baseRef) { + return null; + } + const changedPaths = listChangedPaths(baseRef, cwd); + return resolveChangedTestTargetPlan(changedPaths, { + cwd, + ...options, + watchMode: parseTestProjectsArgs(args, cwd).watchMode, + }); +} + +function classifyTarget(arg: string, cwd: string, beforeDatabaseWorkerOwnership = false) { + const relative = toRepoRelativeTarget(arg, cwd); + if (!beforeDatabaseWorkerOwnership && databaseWorkerExtensionTestFiles.includes(relative)) { + return "extensionDatabaseWorkers"; + } + const configTargetKind = resolveVitestConfigTargetKind(relative); + if (configTargetKind) { + return configTargetKind; + } + if (gatewayPluginTestFiles.includes(relative)) { + return "gatewayMethods"; + } + if (beforeDatabaseWorkerOwnership) { + const formerFastKind = databaseWorkerCoreFormerFastKinds.get(relative); + if (formerFastKind) { + return formerFastKind; + } + } else if (isDatabaseWorkerCoreTestFile(relative)) { + return "infra"; + } + if (gatewayDatabaseWorkerTestFiles.includes(relative)) { + return beforeDatabaseWorkerOwnership ? "gateway" : "gatewayDatabaseWorkers"; + } + if (isAgentsCoreIsolatedTestFile(relative)) { + return agentVitestProjectOwners.coreIsolated.kind; + } + if (isAgentsSpawnProductionBoundaryTestFile(relative)) { + return agentVitestProjectOwners.spawnProductionBoundary.kind; + } + if (isControlUiE2eTarget(relative)) { + return "uiE2e"; + } + if (packageContractTestFiles.includes(relative)) { + return "packageContract"; + } + if (isUiIsolatedTestFile(relative)) { + return "uiIsolated"; + } + if (uiTimingTestFiles.includes(relative)) { + return "uiTiming"; + } + if (isUiBrowserTestFile(relative)) { + return "uiBrowser"; + } + if (isPathAtOrUnder(relative, "ui") || isPluginControlUiPath(relative)) { + return "ui"; + } + if (relative.startsWith("src/tui/tui-pty-") || tuiPtyTestFiles.includes(relative)) { + return "tuiPty"; + } + if (relative.endsWith(".e2e.test.ts")) { + return "e2e"; + } + if ( + relative === "src/gateway/gateway.test.ts" || + relative === "src/gateway/server.startup-matrix-migration.integration.test.ts" + ) { + return "e2e"; + } + const channelContractKind = resolveChannelContractTargetKind(relative); + if (channelContractKind) { + return channelContractKind; + } + if (relative.startsWith("src/plugins/contracts/")) { + return "contractsPlugin"; + } + // These tests share stateful runner mocks and must keep the dedicated serial + // owner even when their contents also qualify for a unit-fast lane. + if (agentVitestProjectOwners.embeddedIncompleteTurn.include.includes(relative)) { + return agentVitestProjectOwners.embeddedIncompleteTurn.kind; + } + // Explicit isolation ownership wins over inferred unit-fast eligibility. + // Otherwise a thin wrapper can move a stateful tooling test into a shared worker. + if (isToolingIsolatedTestFile(relative)) { + return "toolingIsolated"; + } + if (resolveUnitFastTimerTestIncludePattern(relative)) { + return "unitFastFakeTimers"; + } + if (resolveUnitFastIsolatedTestIncludePattern(relative)) { + return "unitFastIsolated"; + } + if (resolveUnitFastTestIncludePattern(relative)) { + return "unitFast"; + } + if (relative === "extensions") { + return "extensionFull"; + } + if (getChangedPathFacts(relative).surface === "extension") { + const extensionRoot = relative.split("/").slice(0, 2).join("/"); + return VITEST_CONFIG_TARGET_KIND_BY_PATH.get(resolveExtensionTestConfig(extensionRoot))!; + } + if (isChannelSurfaceTestFile(relative)) { + return "channel"; + } + if (isBoundaryTestFile(relative)) { + return "boundary"; + } + if (relative === TOOLING_DOCKER_TEST_TARGET) { + return "toolingDocker"; + } + if ( + relative.startsWith("test/") || + relative === "src/scripts" || + relative.startsWith("src/scripts/") || + relative === "src/config/doc-baseline.integration.test.ts" || + relative === "src/config/schema.base.generated.test.ts" || + relative === "src/config/schema.help.quality.test.ts" + ) { + return "tooling"; + } + if (isBundledPluginDependentUnitTestFile(relative)) { + return "bundled"; + } + if (isPathAtOrUnder(relative, "src/channels")) { + return "channel"; + } + if (isPathAtOrUnder(relative, "src/gateway")) { + return "gateway"; + } + if ( + isPathAtOrUnder(relative, "packages/gateway-client") || + isPathAtOrUnder(relative, "packages/gateway-protocol") + ) { + return "gatewayClient"; + } + if (isPathAtOrUnder(relative, "src/hooks")) { + return "hooks"; + } + if (isPathAtOrUnder(relative, "src/infra")) { + return "infra"; + } + if (isPathAtOrUnder(relative, "src/config")) { + return "runtimeConfig"; + } + if (isPathAtOrUnder(relative, "src/cron")) { + return "cron"; + } + if (isPathAtOrUnder(relative, "src/daemon")) { + return "daemon"; + } + if (isPathAtOrUnder(relative, "src/media-understanding")) { + return "mediaUnderstanding"; + } + if (isPathAtOrUnder(relative, "src/media")) { + return "media"; + } + if (isPathAtOrUnder(relative, "src/logging")) { + return "logging"; + } + if (isPathAtOrUnder(relative, "src/plugin-sdk")) { + return isPluginSdkLightTarget(relative) ? "pluginSdkLight" : "pluginSdk"; + } + if (isPathAtOrUnder(relative, "src/process")) { + return "process"; + } + if (isPathAtOrUnder(relative, "src/secrets")) { + return "secrets"; + } + if (isPathAtOrUnder(relative, "src/shared")) { + return "sharedCore"; + } + if (isPathAtOrUnder(relative, "src/tasks")) { + return "tasks"; + } + if (isPathAtOrUnder(relative, "src/tui")) { + return "tui"; + } + if (isPathAtOrUnder(relative, "src/acp")) { + return "acp"; + } + if (isCliProcessTestFile(relative)) { + return "cliProcess"; + } + if (isPathAtOrUnder(relative, "src/cli")) { + return "cli"; + } + if (isPathAtOrUnder(relative, "src/commands")) { + return isCommandsLightTarget(relative) ? "commandLight" : "command"; + } + if (isPathAtOrUnder(relative, "src/auto-reply")) { + return "autoReply"; + } + if (isPathAtOrUnder(relative, agentVitestProjectOwners.all.root)) { + // Focused runs must preserve the full suite's isolated harness and hook-timeout contracts. + if ( + relative === agentVitestProjectOwners.all.root || + relative === AGENTS_EMBEDDED_AGENT_TEST_ROOT + ) { + return agentVitestProjectOwners.all.kind; + } + if (agentVitestProjectOwners.embeddedOverflowCompaction.include.includes(relative)) { + return agentVitestProjectOwners.embeddedOverflowCompaction.kind; + } + if (isPathAtOrUnder(relative, agentVitestProjectOwners.embeddedRun.root)) { + return agentVitestProjectOwners.embeddedRun.kind; + } + if (isPathAtOrUnder(relative, AGENTS_EMBEDDED_AGENT_TEST_ROOT)) { + return isGlobTarget(relative) + ? agentVitestProjectOwners.all.kind + : agentVitestProjectOwners.embedded.kind; + } + if (isPathAtOrUnder(relative, agentVitestProjectOwners.tools.root)) { + return agentVitestProjectOwners.tools.kind; + } + if (isGlobTarget(relative)) { + const owner = + relative.slice(agentVitestProjectOwners.all.root.length + 1).split("/", 1)[0] ?? ""; + return isGlobTarget(owner) + ? agentVitestProjectOwners.all.kind + : agentVitestProjectOwners.support.kind; + } + return isTestFileTarget(relative) && + path.posix.dirname(relative) === agentVitestProjectOwners.core.root + ? agentVitestProjectOwners.core.kind + : agentVitestProjectOwners.support.kind; + } + if (isPathAtOrUnder(relative, "src/plugins")) { + return "plugin"; + } + if (isPathAtOrUnder(relative, "src/utils")) { + return "utils"; + } + if (isPathAtOrUnder(relative, "src/wizard")) { + return "wizard"; + } + return "default"; +} + +function resolveLightLaneIncludePatterns(kind: string, targetArg: string, cwd: string) { + const relative = toRepoRelativeTarget(targetArg, cwd); + if (kind === "unitFast") { + const includePattern = resolveUnitFastTestIncludePattern(relative); + return includePattern ? [includePattern] : null; + } + if (kind === "unitFastFakeTimers") { + const includePattern = resolveUnitFastTimerTestIncludePattern(relative); + return includePattern ? [includePattern] : null; + } + if (kind === "unitFastIsolated") { + const includePattern = resolveUnitFastIsolatedTestIncludePattern(relative); + return includePattern ? [includePattern] : null; + } + if (kind === "pluginSdkLight") { + const includePattern = resolvePluginSdkLightIncludePattern(relative); + return includePattern ? [includePattern] : null; + } + if (kind === "commandLight") { + const includePattern = resolveCommandsLightIncludePattern(relative); + return includePattern ? [includePattern] : null; + } + return null; +} + +function shouldUseWholeConfigTarget(kind: string, targetArg: string, cwd: string) { + if (isVitestConfigTargetForKind(kind, targetArg, cwd)) { + return true; + } + if (kind === "uiE2e") { + const relative = toRepoRelativeTarget(targetArg, cwd); + return relative === "ui/src/test-helpers/control-ui-e2e.ts"; + } + if (kind !== "ui") { + return false; + } + const relative = toRepoRelativeTarget(targetArg, cwd); + // Source files need whole-project coverage; existing directories already define a test scope. + if (isTestFileTarget(relative) || isExistingDirectoryTarget(targetArg, cwd)) { + return false; + } + if (isPluginControlUiPath(relative) && !isLikelyFileTarget(relative)) { + return false; + } + return isControlUiSourcePath(relative); +} + +function createVitestArgs( + params: Pick<VitestRunPlan, "config" | "forwardedArgs" | "watchMode"> & { + env?: NodeJS.ProcessEnv; + }, +) { + return [ + "exec", + "node", + ...resolveVitestNodeArgs(params.env), + resolveVitestCliEntry(), + ...(params.watchMode ? [] : ["run"]), + "--config", + params.config, + ...(params.config === UI_E2E_VITEST_CONFIG ? ["--configLoader", "runner"] : []), + ...params.forwardedArgs, + ]; +} + +export function createVitestPreflightPnpmArgs(config: string) { + if (config !== UI_E2E_VITEST_CONFIG) { + return null; + } + return ["exec", "node", "--import", "tsx", "scripts/ensure-playwright-chromium.mts"]; +} + +export function parseTestProjectsArgs(args: string[], cwd = process.cwd()) { + const forwardedArgs: string[] = []; + const nonTargetArgs: string[] = []; + const targetArgs: string[] = []; + let watchMode = false; + let passthrough = false; + + for (let index = 0; index < args.length; index++) { + const arg = args[index]!; + if (arg === "--") { + // The project wrapper consumes separators; direct native and batch calls retain theirs. + passthrough = targetArgs.length > 0; + continue; + } + const watch = resolveBooleanModeFlag(args, index, "watch", "-w"); + if (watch) { + watchMode = watch.value; + } + // Preserve bare wrapper --watch's existing omission of the named run command. + const next = args[index + 1]; + if (!passthrough && arg === "--watch" && next !== "true" && next !== "false") { + continue; + } + forwardedArgs.push(arg); + if (vitestOptionConsumesNextArg(arg, next)) { + forwardedArgs.push(next!); + // Preserve operand occurrences even when their value also names a target. + nonTargetArgs.push(arg, next!); + index++; + } else if (!passthrough && isPathLikeTargetArg(arg, cwd)) { + targetArgs.push(arg); + } else { + nonTargetArgs.push(arg); + } + } + + return { forwardedArgs, nonTargetArgs, targetArgs, watchMode }; +} + +export function buildVitestRunPlans( + args: string[], + cwd = process.cwd(), + listChangedPaths: (baseRef: string, cwd: string) => string[] = listChangedPathsFromGit, + options: ChangedTestTargetOptions = {}, +) { + const { + forwardedArgs, + nonTargetArgs: remainingArgs, + targetArgs, + watchMode, + } = parseTestProjectsArgs(args, cwd); + const changedTargetArgs = + targetArgs.length === 0 ? resolveChangedTargetArgs(args, cwd, listChangedPaths, options) : null; + const requestedTargetArgs = changedTargetArgs ?? targetArgs; + const ownedTargets = explicitIncludeTargets(targetArgs, cwd); + if ( + watchMode && + requestedTargetArgs.some((target) => { + const relative = toRepoRelativeTarget(target, cwd); + return ( + (isPathAtOrUnder(relative, "ui") || isPluginControlUiPath(relative)) && + isGlobTarget(relative) + ); + }) + ) { + throw new Error( + "watch mode with UI glob targets is not supported; use a literal test path, directory, or dedicated UI suite", + ); + } + const activeTargetArgs = expandBroadToolingScriptTargets( + expandExplicitSourceTestTargets(requestedTargetArgs, cwd, watchMode), + cwd, + watchMode, + ); + const activeForwardedArgs = + changedTargetArgs !== null ? stripChangedArgs(forwardedArgs) : forwardedArgs; + if (changedTargetArgs !== null && activeTargetArgs.length === 0) { + return []; + } + if (activeTargetArgs.length === 0) { + return [ + { + config: DEFAULT_VITEST_CONFIG, + forwardedArgs: activeForwardedArgs, + includePatterns: null, + watchMode, + }, + ]; + } + + const nonTargetArgs = changedTargetArgs !== null ? activeForwardedArgs : remainingArgs; + const classifiedTargets = activeTargetArgs.map((targetArg) => ({ + targetArg, + relative: toRepoRelativeTarget(targetArg, cwd), + kind: classifyTarget(targetArg, cwd), + })); + const hasGatewayAggregateTarget = classifiedTargets.some(({ kind }) => kind === "gateway"); + const explicitConfigTargets = classifiedTargets.map(({ relative }) => relative); + const impliedDatabaseWorkerTargets = databaseWorkerCoreTestFiles.filter((file) => + [...requestedTargetArgs, ...activeTargetArgs].some((targetArg) => { + const relative = toRepoRelativeTarget(targetArg, cwd); + return ( + (isTestFileTarget(relative) || + isGlobTarget(relative) || + isExistingDirectoryTarget(targetArg, cwd)) && + includePatternMatchesAnyFile(toScopedIncludePattern(targetArg, cwd), [file]) + ); + }), + ); + const hasPackageFileTarget = classifiedTargets.some( + ({ kind, relative }) => + kind === "packageContract" && relative !== PACKAGE_CONTRACT_VITEST_CONFIG, + ); + // The wrapper removes its separator; remaining positional filters can widen + // a package file run beyond the package inventory. Option operands cannot. + const hasE2eTarget = + classifiedTargets.some(({ kind }) => kind === "e2e") || + (!explicitConfigTargets.includes(PACKAGE_CONTRACT_VITEST_CONFIG) && + hasPackageFileTarget && + collectVitestFileFilters(["run", ...nonTargetArgs]).length > 0); + if ( + explicitConfigTargets.every(isVitestConfigFileTarget) && + impliedDatabaseWorkerTargets.length === 0 + ) { + if (watchMode && explicitConfigTargets.length > 1) { + throw new Error( + "watch mode with mixed test suites is not supported; target one suite at a time or use a dedicated suite command", + ); + } + return explicitConfigTargets + .filter((config) => !hasE2eTarget || config !== PACKAGE_CONTRACT_VITEST_CONFIG) + .filter( + (config) => !hasGatewayAggregateTarget || config !== GATEWAY_DATABASE_WORKERS_VITEST_CONFIG, + ) + .flatMap((config) => + createBoundedExtensionPlans( + { + config, + forwardedArgs: nonTargetArgs, + includePatterns: null, + watchMode, + }, + options.env, + ), + ); + } + + const groupedTargets = new Map<string, string[]>(); + for (const { targetArg, relative, kind: targetKind } of classifiedTargets) { + if (!watchMode && relative === AGENTS_EMBEDDED_AGENT_TEST_ROOT) { + // The recursive parent spans four harness owners; keep every isolated project intact. + for (const { kind, include: targets } of embeddedAgentVitestProjectOwners) { + const current = groupedTargets.get(kind) ?? []; + for (const target of targets) { + if (!current.includes(target)) { + current.push(target); + } + } + groupedTargets.set(kind, current); + } + continue; + } + + // A requested Gateway aggregate already owns its worker tests. Watch also + // keeps that aggregate; mixed E2E selections retain their serial build owner. + const kind = + targetKind === "gatewayDatabaseWorkers" && (watchMode || hasGatewayAggregateTarget) + ? "gateway" + : hasE2eTarget && targetKind === "packageContract" + ? "e2e" + : targetKind; + const current = groupedTargets.get(kind) ?? []; + current.push( + ...(hasE2eTarget && relative === PACKAGE_CONTRACT_VITEST_CONFIG + ? packageContractTestFiles + : kind === "gateway" && relative === GATEWAY_DATABASE_WORKERS_VITEST_CONFIG + ? gatewayDatabaseWorkerTestFiles + : [targetArg]), + ); + groupedTargets.set(kind, current); + } + if (impliedDatabaseWorkerTargets.length > 0) { + const current = groupedTargets.get("infra") ?? []; + for (const target of impliedDatabaseWorkerTargets) { + if (!current.includes(target)) { + current.push(target); + } + } + groupedTargets.set("infra", current); + } + const toolingTargets = groupedTargets.get("tooling") ?? []; + if ( + !watchMode && + toolingTargets.some((targetArg) => + includePatternMatchesAnyFile(toScopedIncludePattern(targetArg, cwd), [ + TOOLING_DOCKER_TEST_TARGET, + ]), + ) + ) { + const current = groupedTargets.get("toolingDocker") ?? []; + if (!current.includes(TOOLING_DOCKER_TEST_TARGET)) { + current.push(TOOLING_DOCKER_TEST_TARGET); + groupedTargets.set("toolingDocker", current); + } + } + const impliedToolingIsolatedTargets = !watchMode + ? toolingIsolatedTestFiles.filter((file) => + toolingTargets.some((targetArg) => + includePatternMatchesAnyFile(toScopedIncludePattern(targetArg, cwd), [file]), + ), + ) + : []; + if (impliedToolingIsolatedTargets.length > 0) { + const current = groupedTargets.get("toolingIsolated") ?? []; + for (const target of impliedToolingIsolatedTargets) { + if (!current.includes(target)) { + current.push(target); + } + } + groupedTargets.set("toolingIsolated", current); + } + const uiTargets = groupedTargets.get("ui") ?? []; + const impliedUiTimingTargets = uiTimingTestFiles.filter((file) => + uiTargets.some((targetArg) => + includePatternMatchesAnyFile(toScopedIncludePattern(targetArg, cwd), [file]), + ), + ); + if (impliedUiTimingTargets.length > 0) { + groupedTargets.set("uiTiming", [ + ...new Set([...(groupedTargets.get("uiTiming") ?? []), ...impliedUiTimingTargets]), + ]); + } + const broadUiTargets = uiTargets.filter( + (targetArg) => !isTestFileTarget(toRepoRelativeTarget(targetArg, cwd)), + ); + if (broadUiTargets.length > 0) { + const browserTargets = listExplicitTestTargetFilesForCwd(cwd).filter( + (file) => + isUiBrowserTestFile(file) && + broadUiTargets.some( + (targetArg) => + shouldUseWholeConfigTarget("ui", targetArg, cwd) || + includePatternMatchesAnyFile(toScopedIncludePattern(targetArg, cwd), [file]), + ), + ); + if (browserTargets.length > 0) { + groupedTargets.set("uiBrowser", [ + ...new Set([...(groupedTargets.get("uiBrowser") ?? []), ...browserTargets]), + ]); + } + } + const impliedUiIsolatedTargets = uiIsolatedTestFiles.filter((file) => + uiTargets.some((targetArg) => + includePatternMatchesAnyFile(toScopedIncludePattern(targetArg, cwd), [file]), + ), + ); + if (impliedUiIsolatedTargets.length > 0) { + const current = groupedTargets.get("uiIsolated") ?? []; + for (const target of impliedUiIsolatedTargets) { + if (!current.includes(target)) { + current.push(target); + } + } + groupedTargets.set("uiIsolated", current); + } + // Source-child ownership can cross shared suites (for example state tests). + // Match every active target so broad selections cannot silently omit excluded children. + const impliedCliProcessTargets = cliProcessTestFiles.filter((file) => + activeTargetArgs.some((targetArg) => + includePatternMatchesAnyFile(toScopedIncludePattern(targetArg, cwd), [file]), + ), + ); + if (impliedCliProcessTargets.length > 0) { + const current = groupedTargets.get("cliProcess") ?? []; + for (const target of impliedCliProcessTargets) { + if (!current.includes(target)) { + current.push(target); + } + } + groupedTargets.set("cliProcess", current); + } + + const impliedExtensionWorkerTargets = classifiedTargets + .filter(({ kind }) => kind === "extensionDatabaseWorkers") + .map(({ relative }) => relative); + const impliedWatchWorkerTargets = [ + ...impliedDatabaseWorkerTargets, + ...impliedExtensionWorkerTargets, + ]; + const previousWatchKinds = watchMode + ? new Set(classifiedTargets.map(({ targetArg }) => classifyTarget(targetArg, cwd, true))) + : new Set<string>(); + if (watchMode && (groupedTargets.size > 1 || previousWatchKinds.size > 1)) { + if (impliedWatchWorkerTargets.length > 0 && previousWatchKinds.size === 1) { + const previousKind = [...previousWatchKinds][0]!; + const wholeOwner = classifiedTargets.some(({ targetArg }) => + shouldUseWholeConfigTarget(previousKind, targetArg, cwd), + ); + return [ + { + config: "test/vitest/vitest.database-worker-watch.config.ts", + databaseWorkerWatchOwner: VITEST_CONFIG_BY_KIND[previousKind] ?? DEFAULT_VITEST_CONFIG, + databaseWorkerWatchTests: wholeOwner + ? [...databaseWorkerCoreTestFiles, ...databaseWorkerExtensionTestFiles].filter( + (file) => classifyTarget(file, cwd, true) === previousKind, + ) + : impliedWatchWorkerTargets, + forwardedArgs: nonTargetArgs, + includePatterns: wholeOwner + ? null + : uniqueOrdered(activeTargetArgs.map((target) => toScopedIncludePattern(target, cwd))), + watchMode: true, + }, + ]; + } + throw new Error( + "watch mode with mixed test suites is not supported; target one suite at a time or use a dedicated suite command", + ); + } + + const orderedKinds = Object.keys(VITEST_CONFIG_BY_KIND); + orderedKinds.splice(orderedKinds.indexOf("boundary"), 0, "default"); + const plans: VitestRunPlan[] = []; + for (const kind of orderedKinds) { + const grouped = groupedTargets.get(kind); + if (!grouped || grouped.length === 0) { + continue; + } + if (kind === "extensionFull") { + const configs = watchMode + ? [FULL_EXTENSIONS_VITEST_CONFIG] + : listFullExtensionVitestProjectConfigs(); + for (const config of configs) { + const plan = { + config, + forwardedArgs: nonTargetArgs, + includePatterns: null, + watchMode, + }; + plans.push(...createBoundedExtensionPlans(plan, options.env)); + } + continue; + } + const config = VITEST_CONFIG_BY_KIND[kind] ?? DEFAULT_VITEST_CONFIG; + const useCliTargetArgs = + kind === "e2e" || + kind === "packageContract" || + grouped.every((targetArg) => isCanonicalAgentOwnerDirectoryTarget(targetArg, cwd)) || + (kind === "default" && + grouped.every((targetArg) => isTestFileTarget(toRepoRelativeTarget(targetArg, cwd)))); + const useWholeConfigTarget = grouped.some((targetArg) => + shouldUseWholeConfigTarget(kind, targetArg, cwd), + ); + const scopedTargetArgs = + useCliTargetArgs && !useWholeConfigTarget ? uniqueOrdered(grouped) : []; + const forwardedPlanArgs = [...nonTargetArgs, ...scopedTargetArgs]; + const unitCliIncludes = + kind === "default" && + useCliTargetArgs && + !watchMode && + !options.env?.[INCLUDE_FILE_ENV_KEY]?.trim() && + grouped.every((targetArg) => !path.isAbsolute(targetArg)) + ? uniqueOrdered(grouped.map((targetArg) => toRepoRelativeTarget(targetArg, cwd))) + : null; + // CI needs explicit selection metadata. Keep the CLI filters too: the unit + // config and runner use them for exclude and empty-selection policy. + const scopedUnitIncludes = + unitCliIncludes?.length && + unitCliIncludes.every( + (file) => + !isGlobTarget(file) && isUnitConfigTestFile(file) && isExistingFileTarget(file, cwd), + ) + ? unitCliIncludes + : null; + const includePatterns = useCliTargetArgs + ? scopedUnitIncludes + : useWholeConfigTarget + ? null + : uniqueOrdered( + grouped.flatMap((targetArg) => { + const lightLanePatterns = resolveLightLaneIncludePatterns(kind, targetArg, cwd); + return lightLanePatterns ?? [toScopedIncludePattern(targetArg, cwd)]; + }), + ); + const broadToolingScriptPlans = createBroadToolingScriptPlans({ + config, + cwd, + forwardedArgs: forwardedPlanArgs, + includePatterns, + watchMode, + }); + if (broadToolingScriptPlans) { + plans.push(...broadToolingScriptPlans); + continue; + } + const processRoots = EXTENSION_TEST_PROCESS_ROOTS.get(config); + const boundedExtensionRoots = grouped.flatMap((targetArg) => { + const root = toRepoRelativeTarget(targetArg, cwd); + return processRoots?.includes(root) && isExistingDirectoryTarget(targetArg, cwd) + ? [root] + : []; + }); + const boundedRootsCoverGroupedTargets = grouped.every((targetArg) => { + const relativeTarget = toRepoRelativeTarget(targetArg, cwd); + return boundedExtensionRoots.some( + (root) => relativeTarget === root || relativeTarget.startsWith(`${root}/`), + ); + }); + const boundedExtensionPlans = + kind === "extensionDatabaseWorkers" || + (boundedExtensionRoots.length > 0 && boundedRootsCoverGroupedTargets) + ? createBoundedExtensionPlans( + { + config, + forwardedArgs: forwardedPlanArgs, + includePatterns, + watchMode, + }, + options.env, + ownedTargets, + ) + : null; + if (boundedExtensionPlans) { + plans.push(...boundedExtensionPlans); + continue; + } + plans.push({ + config, + forwardedArgs: forwardedPlanArgs, + includePatterns, + watchMode, + }); + } + return plans; +} + +export function buildFullSuiteVitestRunPlans(args: string[], cwd = process.cwd()): VitestRunPlan[] { + const { forwardedArgs, targetArgs, watchMode } = parseTestProjectsArgs(args, cwd); + if (watchMode) { + return [ + { + config: "vitest.config.ts", + forwardedArgs, + includePatterns: null, + watchMode, + }, + ]; + } + const parallelShardCount = parsePositiveInt( + process.env.OPENCLAW_TEST_PROJECTS_PARALLEL, + "OPENCLAW_TEST_PROJECTS_PARALLEL", + ); + const expandToProjectConfigs = + process.env.OPENCLAW_TEST_PROJECTS_LEAF_SHARDS === "1" || + (parallelShardCount !== null && parallelShardCount > 1) || + shouldExpandLocalFullSuiteShardsByDefault(process.env); + return fullSuiteVitestShards.flatMap((shard) => { + if ( + process.env.OPENCLAW_TEST_SKIP_FULL_EXTENSIONS_SHARD === "1" && + shard.config === FULL_EXTENSIONS_VITEST_CONFIG + ) { + return []; + } + // The remote Testbox full gate runs every agentic and extension project in one process tree. + // Bound project and worker lifetimes before either aggregate reaches V8's heap limit. + const expandShard = + expandToProjectConfigs || + (process.env.OPENCLAW_TESTBOX_REMOTE_RUN === "1" && + (shard.config === FULL_AGENTIC_VITEST_CONFIG || + shard.config === FULL_EXTENSIONS_VITEST_CONFIG)); + const configs = expandShard ? shard.projects : [shard.config]; + return configs.flatMap((config) => { + if (expandShard && targetArgs.length === 0) { + let chunks: string[][] = []; + if (config === AGENTS_CORE_VITEST_CONFIG) { + // A single non-isolated agents-core process grows until its worker can + // exit under the full-suite memory load. Bound each process lifetime. + chunks = splitTargetChunks( + listAgentsCoreFullSuiteTestTargets(cwd), + FULL_SUITE_AGENTS_CORE_TEST_TARGET_CHUNK_COUNT, + ); + } else if (config === UNIT_FAST_VITEST_CONFIG) { + const targets = listUnitFastFullSuiteTestTargets(); + const chunkCount = Math.ceil( + targets.length / FULL_SUITE_UNIT_FAST_TEST_TARGET_CHUNK_SIZE, + ); + chunks = splitTargetChunks(targets, chunkCount); + } else if (config === UNIT_SRC_VITEST_CONFIG) { + // The 600+ file process can run for 20 minutes and has produced late, + // non-reproducible mock-transform failures under Bun. Bound the worker + // lifetime while preserving the complete file inventory. + const targets = listUnitSrcFullSuiteTestTargets(cwd); + const chunkCount = Math.ceil(targets.length / FULL_SUITE_UNIT_SRC_TEST_TARGET_CHUNK_SIZE); + chunks = splitTargetChunks(targets, chunkCount); + } else if (config === TOOLING_VITEST_CONFIG) { + // Tooling tests spawn package managers and native helpers. Keep native + // process lifetime short enough that unrelated files cannot crash together. + const targets = listToolingFullSuiteTestTargets(cwd); + const chunkCount = Math.ceil(targets.length / FULL_SUITE_TOOLING_TEST_TARGET_CHUNK_SIZE); + chunks = splitTargetChunks(targets, chunkCount); + } else if (config === GATEWAY_SERVER_VITEST_CONFIG) { + chunks = splitTargetChunks( + listGatewayServerTestTargets(cwd), + GATEWAY_SERVER_TEST_PROCESS_COUNT, + ); + } else { + const roots = EXTENSION_TEST_PROCESS_ROOTS.get(config); + if (roots) { + chunks = createExtensionTestProcessTargetChunks(config, roots, forwardedArgs); + } + } + if (chunks.length > 0) { + return chunks.map((targets) => ({ + config, + forwardedArgs: [...forwardedArgs, ...targets], + timingTargets: targets, + includePatterns: null, + watchMode: false, + })); + } + } + return [ + { + config, + forwardedArgs, + includePatterns: null, + watchMode: false, + }, + ]; + }); + }); +} + +function shouldUseLocalFullSuiteParallelByDefault(env = process.env) { + if (hasConservativeVitestWorkerBudget(env)) { + return false; + } + return env.OPENCLAW_TEST_PROJECTS_SERIAL !== "1" && !isCiLikeEnv(env); +} + +function shouldExpandLocalFullSuiteShardsByDefault(env = process.env) { + return !isCiLikeEnv(env); +} + +function parsePositiveInt(value: string | undefined, label: string) { + const text = value?.trim(); + if (!text) { + return null; + } + if (!/^\d+$/u.test(text)) { + throw new Error(`${label} must be a positive integer; got: ${value}`); + } + const parsed = Number(text); + if (!Number.isSafeInteger(parsed) || parsed <= 0) { + throw new Error(`${label} must be a positive integer; got: ${value}`); + } + return parsed; +} + +function hasConservativeVitestWorkerBudget(env: NodeJS.ProcessEnv) { + const workerBudget = parsePositiveInt( + env.OPENCLAW_VITEST_MAX_WORKERS ?? env.OPENCLAW_TEST_WORKERS, + env.OPENCLAW_VITEST_MAX_WORKERS === undefined + ? "OPENCLAW_TEST_WORKERS" + : "OPENCLAW_VITEST_MAX_WORKERS", + ); + return workerBudget !== null && workerBudget <= 1; +} + +const FULL_EXTENSIONS_CONFIG = "test/vitest/vitest.full-extensions.config.ts"; +const FULL_EXTENSIONS_MIN_HEAP_MB = 8192; + +function ensureMaxOldSpaceSize(nodeOptions: string | undefined, minimumMb: number) { + const normalized = nodeOptions?.trim() ?? ""; + const matches = Array.from( + normalized.matchAll(/(^|\s)--max[-_]old[-_]space[-_]size(?:=|\s+)(\d+)(?=\s|$)/gu), + ); + const match = matches.at(-1); + if (!match) { + return [normalized, `--max-old-space-size=${minimumMb}`].filter(Boolean).join(" "); + } + const currentMb = Number(match[2]); + if (Number.isSafeInteger(currentMb) && currentMb >= minimumMb) { + return normalized; + } + const start = match.index; + const replacement = match[0].replace(/\d+$/u, String(minimumMb)); + return `${normalized.slice(0, start)}${replacement}${normalized.slice(start + match[0].length)}`; +} + +export function applyFullExtensionsHeapBudget<T extends VitestSpecShape>( + specs: T[], + params: { env?: NodeJS.ProcessEnv } = {}, +): Array<Omit<T, "env"> & { env: NodeJS.ProcessEnv }> { + const baseEnv = params.env ?? {}; + return specs.map((spec) => + spec.config === FULL_EXTENSIONS_CONFIG + ? { + ...spec, + env: { + ...spec.env, + NODE_OPTIONS: ensureMaxOldSpaceSize( + spec.env?.NODE_OPTIONS ?? baseEnv.NODE_OPTIONS, + FULL_EXTENSIONS_MIN_HEAP_MB, + ), + }, + } + : spec, + ); +} + +export function resolveParallelFullSuiteConcurrency( + specCount: number, + envInput?: NodeJS.ProcessEnv, + hostInfo?: VitestHostInfo, +) { + let env = envInput; + env ??= process.env; + const override = parsePositiveInt( + env.OPENCLAW_TEST_PROJECTS_PARALLEL, + "OPENCLAW_TEST_PROJECTS_PARALLEL", + ); + if (override !== null) { + return Math.min(override, specCount); + } + if (env.OPENCLAW_TEST_PROJECTS_SERIAL === "1") { + return 1; + } + if (isCiLikeEnv(env)) { + return 1; + } + if (hasConservativeVitestWorkerBudget(env)) { + return 1; + } + if ( + env.OPENCLAW_TEST_PROJECTS_LEAF_SHARDS !== "1" && + !shouldUseLocalFullSuiteParallelByDefault(env) + ) { + return 1; + } + return Math.min(resolveLocalFullSuiteProfile(env, hostInfo).shardParallelism, specCount); +} + +function sanitizeVitestCachePathSegment(value: string) { + return ( + value + .replace(/[^a-zA-Z0-9._-]+/gu, "-") + .replace(/^-+|-+$/gu, "") + .slice(0, 180) || "default" + ); +} + +export function applyParallelVitestCachePaths<T extends VitestSpecShape>( + specs: T[], + params: { cwd?: string; env?: NodeJS.ProcessEnv } = {}, +): Array<CacheAssignedSpec<T>> { + const baseEnv = params.env ?? process.env; + const cwd = params.cwd ?? process.cwd(); + const configuredCacheRoot = baseEnv[FS_MODULE_CACHE_PATH_ENV_KEY]?.trim() || undefined; + // CI publishes a persistent cache root, not a writer-safe leaf. Every + // concurrent Vitest process still needs its own live directory below it. + const cacheRoot = configuredCacheRoot ?? resolveVitestFsModuleCacheRoot(cwd); + return specs.map((spec, index) => { + const specCachePath = spec.env?.[FS_MODULE_CACHE_PATH_ENV_KEY]?.trim(); + if (specCachePath && specCachePath !== configuredCacheRoot) { + return { ...spec, cacheAssignment: spec.cacheAssignment ?? { kind: "caller" } }; + } + const cacheSegment = sanitizeVitestCachePathSegment(`${index}-${spec.config}`); + return { + ...spec, + cacheAssignment: { kind: "scheduler", root: cacheRoot }, + env: { + ...spec.env, + [FS_MODULE_CACHE_PATH_ENV_KEY]: path.join(cacheRoot, cacheSegment), + }, + }; + }); +} + +export function applyDefaultMultiSpecVitestCachePaths<T extends WatchableVitestSpecShape>( + specs: T[], + params: { cwd?: string; env?: NodeJS.ProcessEnv } = {}, +): Array<CacheAssignedSpec<T>> { + if (specs.length <= 1 || specs.some((spec) => spec.watchMode)) { + return specs; + } + // Same-config process lifetimes run one after another and must keep the + // restored CI seed. Isolating them would make every Telegram file pay a + // silent cold import. + if (specs.every((spec) => spec.config === specs[0]?.config)) { + return specs; + } + return applyParallelVitestCachePaths(specs, params); +} + +export function applyDefaultVitestNoOutputTimeout<T extends WatchableVitestSpecShape>( + specs: T[], + params: { env?: NodeJS.ProcessEnv } = {}, +): Array<Omit<T, "env"> & { env: NodeJS.ProcessEnv }> { + const baseEnv = params.env ?? process.env; + if ( + Object.hasOwn(baseEnv, VITEST_NO_OUTPUT_TIMEOUT_ENV_KEY) && + Object.hasOwn(baseEnv, VITEST_NO_OUTPUT_HEARTBEAT_ENV_KEY) + ) { + return specs; + } + return specs.map((spec) => { + if (spec.watchMode) { + return spec; + } + const env = spec.env ?? {}; + const nextEnv = { ...env }; + if ( + !Object.hasOwn(baseEnv, VITEST_NO_OUTPUT_TIMEOUT_ENV_KEY) && + !Object.hasOwn(env, VITEST_NO_OUTPUT_TIMEOUT_ENV_KEY) + ) { + nextEnv[VITEST_NO_OUTPUT_TIMEOUT_ENV_KEY] = resolveTestProjectsVitestNoOutputTimeoutMs( + spec.config, + ); + } + if ( + !Object.hasOwn(baseEnv, VITEST_NO_OUTPUT_HEARTBEAT_ENV_KEY) && + !Object.hasOwn(env, VITEST_NO_OUTPUT_HEARTBEAT_ENV_KEY) + ) { + nextEnv[VITEST_NO_OUTPUT_HEARTBEAT_ENV_KEY] = + DEFAULT_TEST_PROJECTS_VITEST_NO_OUTPUT_HEARTBEAT_MS; + } + return { + ...spec, + env: nextEnv, + }; + }); +} + +export function shouldRetryVitestNoOutputTimeout(env = process.env) { + const value = env[VITEST_NO_OUTPUT_RETRY_ENV_KEY]?.trim().toLowerCase(); + if (value === undefined && isCiLikeEnv(env)) { + return false; + } + return !["0", "false", "no", "off"].includes(value ?? ""); +} + +// Shards may pin a short no-output window so the known warm-cache stall dies +// fast (see AGENTS_CORE_RUNTIME_ENV in ci-node-test-plan.mts). A cold Vitest +// module cache makes those same imports legitimately silent for minutes, so +// the retry attempt must get the full watchdog window or it re-dies at the +// short limit and the job fails without ever running a test. +const RETRY_NO_OUTPUT_TIMEOUT_FLOOR_MS = 300_000; + +export function withRetryNoOutputTimeout<T extends { env?: NodeJS.ProcessEnv }>(spec: T): T { + const current = Number(spec.env?.[VITEST_NO_OUTPUT_TIMEOUT_ENV_KEY]); + if (!Number.isFinite(current) || current <= 0 || current >= RETRY_NO_OUTPUT_TIMEOUT_FLOOR_MS) { + return spec; + } + return { + ...spec, + env: { + ...spec.env, + [VITEST_NO_OUTPUT_TIMEOUT_ENV_KEY]: String(RETRY_NO_OUTPUT_TIMEOUT_FLOOR_MS), + }, + }; +} + +export function createVitestRunSpecs( + args: string[], + params: { baseEnv?: NodeJS.ProcessEnv; cwd?: string } = {}, +) { + const cwd = params.cwd ?? process.cwd(); + const baseEnv = params.baseEnv ?? process.env; + const ownedTargets = explicitIncludeTargets(parseTestProjectsArgs(args, cwd).targetArgs, cwd); + const plans = filterPlansForContractIncludeFile( + buildVitestRunPlans(args, cwd, listChangedPathsFromGit, { env: baseEnv }), + baseEnv, + ownedTargets, + ); + const inheritedIncludes = plans.some( + (plan) => + plan.includePatterns !== null && !ownsIncludeSelection(plan.includePatterns, ownedTargets), + ) + ? loadIncludePatternsForSpecFilter(baseEnv) + : null; + return plans.flatMap((originalPlan, index) => { + const plan = + originalPlan.includePatterns && inheritedIncludes + ? { + ...originalPlan, + includePatterns: resolveInheritedIncludeScope( + originalPlan.includePatterns, + inheritedIncludes, + ownedTargets, + ), + } + : originalPlan; + if (inheritedIncludes && plan.includePatterns?.length === 0) { + return []; + } + const includeFilePath = plan.includePatterns + ? path.join(os.tmpdir(), `openclaw-vitest-include-${randomUUID()}-${index}.json`) + : null; + return { + config: plan.config, + timingTargets: plan.timingTargets, + env: + includeFilePath || plan.databaseWorkerWatchOwner + ? { + ...baseEnv, + ...(includeFilePath ? { [INCLUDE_FILE_ENV_KEY]: includeFilePath } : {}), + ...(plan.databaseWorkerWatchOwner + ? { + [DATABASE_WORKER_WATCH_OWNER_ENV_KEY]: plan.databaseWorkerWatchOwner, + [DATABASE_WORKER_WATCH_TESTS_ENV_KEY]: JSON.stringify( + plan.databaseWorkerWatchTests, + ), + } + : {}), + } + : baseEnv, + includeFilePath, + includePatterns: plan.includePatterns, + pnpmArgs: createVitestArgs(plan), + preflightPnpmArgs: createVitestPreflightPnpmArgs(plan.config), + watchMode: plan.watchMode, + }; + }); +} + +function loadIncludePatternsForSpecFilter(env: NodeJS.ProcessEnv) { + const filePath = env[INCLUDE_FILE_ENV_KEY]?.trim(); + if (!filePath) { + return null; + } + const parsed: unknown = JSON.parse(fs.readFileSync(filePath, "utf8")); + if (!Array.isArray(parsed)) { + return []; + } + return parsed.filter((value): value is string => typeof value === "string" && value.length > 0); +} + +function includePatternMatchesConfig(candidate: string, configPatterns: readonly string[]) { + return configPatterns.some( + (pattern) => path.matchesGlob(candidate, pattern) || path.matchesGlob(pattern, candidate), + ); +} + +function filterPlansForContractIncludeFile( + plans: VitestRunPlan[], + env: NodeJS.ProcessEnv, + ownedTargets?: ReadonlySet<string>, +) { + if ( + !plans.some( + (plan) => + CHANNEL_CONTRACT_CONFIG_PATTERNS.has(plan.config) && + !ownsIncludeSelection(plan.includePatterns, ownedTargets), + ) + ) { + return plans; + } + const includePatterns = loadIncludePatternsForSpecFilter(env); + if (!includePatterns) { + return plans; + } + return plans.filter((plan) => { + if (plan.includePatterns?.some((pattern) => ownedTargets?.has(pattern))) { + return true; + } + const configPatterns = CHANNEL_CONTRACT_CONFIG_PATTERNS.get(plan.config); + if (!configPatterns) { + return true; + } + return includePatterns.some((candidate) => + includePatternMatchesConfig(candidate, configPatterns), + ); + }); +} + +function expandVitestIncludePatterns(includePatterns: string[], cwd: string) { + const candidateFiles = includePatterns.some(isGlobTarget) + ? listExplicitTestTargetFilesForCwd(cwd).toSorted() + : []; + return uniqueOrdered( + includePatterns.flatMap((pattern) => { + if (!isGlobTarget(pattern)) { + return [pattern]; + } + return candidateFiles.filter((file) => path.matchesGlob(file, pattern)); + }), + ); +} + +export function writeVitestIncludeFile( + filePath: string, + includePatterns: string[], + options: { cwd?: string; expandGlobs?: boolean } = {}, +) { + // Shared Vitest projects intersect this file with their ownership globs. + // One-shot runs emit concrete paths; watch runs retain globs for new files. + const expandedPatterns = + options.expandGlobs === false + ? includePatterns + : expandVitestIncludePatterns(includePatterns, options.cwd ?? process.cwd()); + fs.writeFileSync(filePath, `${JSON.stringify(expandedPatterns, null, 2)}\n`); +} + +function shellQuote(value: string) { + const text = value; + if (text === "") { + return "''"; + } + if (/^[A-Za-z0-9_./:=@%+-]+$/u.test(text)) { + return text; + } + return `'${text.replaceAll("'", "'\\''")}'`; +} + +function formatFailedShardRerunCommand(failure: FailedVitestShard) { + const includePatterns = failure.includePatterns ?? []; + if (includePatterns.length > 0) { + return ["pnpm", "test", ...includePatterns.map(shellQuote), "--", "--reporter=verbose"].join( + " ", + ); + } + return [ + "node", + "scripts/run-vitest.mjs", + "run", + "--config", + shellQuote(failure.config), + "--reporter=verbose", + ].join(" "); +} + +function formatFailedShardStatus(failure: FailedVitestShard) { + const details = []; + if (failure.code !== undefined && failure.code !== null) { + details.push(`exit ${failure.code}`); + } + if (failure.signal) { + details.push(`signal ${failure.signal}`); + } + if (failure.noOutputTimedOut) { + details.push("no-output timeout"); + } + return details.length > 0 ? ` (${details.join(", ")})` : ""; +} + +export function formatFailedShardDigest(failures: FailedVitestShard[]) { + if (failures.length === 0) { + return []; + } + + const orderedFailures = failures.toSorted((left, right) => { + const leftOrder = typeof left.order === "number" ? left.order : Number.MAX_SAFE_INTEGER; + const rightOrder = typeof right.order === "number" ? right.order : Number.MAX_SAFE_INTEGER; + return leftOrder - rightOrder || left.config.localeCompare(right.config); + }); + const shown = orderedFailures.slice(0, FAILED_SHARD_DIGEST_LIMIT); + const lines = [`[test] failed shard digest (${failures.length}):`]; + for (const failure of shown) { + const includePatterns = failure.includePatterns ?? []; + const includes = + includePatterns.length > 0 ? ` includes=${includePatterns.map(shellQuote).join(",")}` : ""; + lines.push(`[test] - ${failure.config}${formatFailedShardStatus(failure)}${includes}`); + lines.push(`[test] rerun: ${formatFailedShardRerunCommand(failure)}`); + } + if (shown.length < failures.length) { + lines.push(`[test] - ... ${failures.length - shown.length} more failed shard(s) omitted`); + } + return lines; +} diff --git a/scripts/test-update-cli-startup-bench.mts b/scripts/test-update-cli-startup-bench.mts new file mode 100644 index 0000000000000000000000000000000000000000..1377ca0fa71e76895d4324ea14460f8641c9bb04 --- /dev/null +++ b/scripts/test-update-cli-startup-bench.mts @@ -0,0 +1,78 @@ +// Refreshes the checked-in CLI startup benchmark fixture. +import { spawnSync } from "node:child_process"; +import { parseFlagArgs, stringFlag, intFlag } from "./lib/arg-utils.mts"; + +const CLI_STARTUP_BENCH_FIXTURE_PATH = "test/fixtures/cli-startup-bench.json"; + +if (process.argv.slice(2).includes("--help")) { + console.log( + [ + "Usage: node --import tsx scripts/test-update-cli-startup-bench.mts [options]", + "", + "Refresh the checked-in CLI benchmark fixture.", + "", + "Options:", + " --out <path> Output path (default: test/fixtures/cli-startup-bench.json)", + " --entry <path> CLI entry to benchmark (default: openclaw.mjs)", + " --preset <name> startup | real | all (default: all)", + " --runs <n> Measured runs per case (default: 5)", + " --warmup <n> Warmup runs per case (default: 1)", + " --timeout-ms <ms> Per-run timeout (default: 30000)", + " --help Show this help text", + "", + "Example:", + " node --import tsx scripts/test-update-cli-startup-bench.mts --preset all --runs 3 --warmup 1", + ].join("\n"), + ); + process.exit(0); +} + +const opts = parseFlagArgs( + process.argv.slice(2), + { + out: CLI_STARTUP_BENCH_FIXTURE_PATH, + entry: "openclaw.mjs", + preset: "all", + runs: 5, + warmup: 1, + timeoutMs: 30_000, + }, + [ + stringFlag("--out", "out"), + stringFlag("--entry", "entry"), + stringFlag("--preset", "preset"), + intFlag("--runs", "runs", { min: 1 }), + intFlag("--warmup", "warmup", { min: 0 }), + intFlag("--timeout-ms", "timeoutMs", { min: 1 }), + ], +); + +const args = [ + "--import", + "tsx", + "scripts/bench-cli-startup.ts", + "--entry", + opts.entry, + "--preset", + opts.preset, + "--runs", + String(opts.runs), + "--warmup", + String(opts.warmup), + "--timeout-ms", + String(opts.timeoutMs), + "--output", + opts.out, +]; + +const run = spawnSync(process.execPath, args, { + cwd: process.cwd(), + stdio: "inherit", + env: process.env, +}); + +if (run.status !== 0) { + process.exit(run.status ?? 1); +} + +console.log(`[test-update-cli-startup-bench] wrote fixture to ${opts.out}`); diff --git a/scripts/testbox-lease-freshness.mts b/scripts/testbox-lease-freshness.mts new file mode 100644 index 0000000000000000000000000000000000000000..ac963572bad0ef1c9c69f3e1fe0526f240b30484 --- /dev/null +++ b/scripts/testbox-lease-freshness.mts @@ -0,0 +1,150 @@ +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { + existsSync, + mkdirSync, + readFileSync, + readdirSync, + renameSync, + statSync, + writeFileSync, +} from "node:fs"; +import { resolve } from "node:path"; +import { isRecord } from "@openclaw/normalization-core/record-coerce"; + +const STATE_VERSION = 1; +const DEPENDENCY_INPUTS = ["package.json", "pnpm-lock.yaml", "pnpm-workspace.yaml", ".npmrc"]; +const ENVIRONMENT_INPUTS = [ + ".crabbox.yaml", + ".github/actions/prepare-testbox-shell", + ".node-version", + "scripts/crabbox-wrapper.mjs", + "scripts/crabbox-wrapper.mts", + "scripts/crabbox-source-capsule.mts", + "scripts/crabbox-source-receiver.mts", +]; + +function optionValue(args: readonly string[], name: string, fallback = "") { + const shortName = name.replace(/^--/u, "-"); + for (const [index, argument = ""] of args.entries()) { + if (argument === name || argument === shortName) { + return args[index + 1] ?? fallback; + } + if (argument.startsWith(`${name}=`) || argument.startsWith(`${shortName}=`)) { + return argument.slice(argument.indexOf("=") + 1); + } + } + return fallback; +} + +function git(repoRoot: string, args: readonly string[]) { + return execFileSync("git", ["-C", repoRoot, ...args], { + encoding: "utf8", + env: { ...process.env, GIT_CONFIG_GLOBAL: "/dev/null" }, + }).trim(); +} + +function listFiles(path: string): string[] { + if (!existsSync(path)) { + return []; + } + if (statSync(path).isFile()) { + return [path]; + } + return readdirSync(path, { withFileTypes: true }) + .flatMap((entry) => listFiles(resolve(path, entry.name))) + .toSorted((left, right) => left.localeCompare(right)); +} + +function digestInputs(repoRoot: string, inputs: readonly string[]) { + const hash = createHash("sha256"); + for (const input of inputs) { + for (const path of listFiles(resolve(repoRoot, input))) { + hash.update(path.slice(repoRoot.length)); + hash.update("\0"); + hash.update(readFileSync(path)); + hash.update("\0"); + } + } + return hash.digest("hex"); +} + +function buildTestboxLeaseFingerprint(repoRoot: string, args: readonly string[]) { + const workflow = optionValue( + args, + "--blacksmith-workflow", + ".github/workflows/ci-check-testbox.yml", + ); + let baseSha; + try { + baseSha = git(repoRoot, ["merge-base", "HEAD", "refs/remotes/origin/main"]); + } catch { + baseSha = git(repoRoot, ["rev-parse", "HEAD"]); + } + return { + version: STATE_VERSION, + baseSha, + headSha: git(repoRoot, ["rev-parse", "HEAD"]), + dependencyDigest: digestInputs(repoRoot, [...DEPENDENCY_INPUTS, "patches"]), + environmentDigest: digestInputs(repoRoot, [...ENVIRONMENT_INPUTS, workflow]), + workflow, + job: optionValue(args, "--blacksmith-job", "check"), + ref: optionValue(args, "--blacksmith-ref", "main"), + }; +} + +export function testboxLeaseStaleReasons(saved: unknown, current: unknown) { + if (!isRecord(saved) || saved.version !== STATE_VERSION || !isRecord(current)) { + return ["state schema"]; + } + return ["baseSha", "dependencyDigest", "environmentDigest", "workflow", "job", "ref"].filter( + (key) => saved[key] !== current[key], + ); +} + +export function prepareTestboxLeaseFreshness({ + args, + env, + provider, + repoRoot, +}: { + args: readonly string[]; + env: NodeJS.ProcessEnv; + provider: string; + repoRoot: string; +}) { + const id = optionValue(args, "--id"); + if (provider !== "blacksmith-testbox" || args[0] !== "run" || !id?.startsWith("tbx_")) { + return null; + } + const configuredStateDir = env.OPENCLAW_TESTBOX_LEASE_STATE_DIR?.trim(); + if (env.VITEST && !configuredStateDir) { + return null; + } + const stateDir = resolve(configuredStateDir || resolve(repoRoot, ".crabbox", "testbox-leases")); + const path = resolve(stateDir, `${id}.json`); + const current = buildTestboxLeaseFingerprint(repoRoot, args); + if (existsSync(path)) { + const saved: unknown = JSON.parse(readFileSync(path, "utf8")); + const staleReasons = testboxLeaseStaleReasons(saved, current); + if (staleReasons.length > 0 && env.OPENCLAW_TESTBOX_ALLOW_STALE !== "1") { + throw new Error( + `Testbox ${id} is stale (${staleReasons.join(", ")}); stop it and warm a fresh lease, or set OPENCLAW_TESTBOX_ALLOW_STALE=1 for an intentional diagnostic reuse`, + ); + } + return { current, path }; + } + return { current, path }; +} + +export function recordTestboxLeaseFreshness( + prepared: ReturnType<typeof prepareTestboxLeaseFreshness> | undefined, +) { + if (!prepared) { + return; + } + mkdirSync(resolve(prepared.path, ".."), { recursive: true }); + const temporaryPath = `${prepared.path}.tmp-${process.pid}`; + writeFileSync(temporaryPath, `${JSON.stringify(prepared.current, null, 2)}\n`); + renameSync(temporaryPath, prepared.path); +} diff --git a/scripts/tool-display.ts b/scripts/tool-display.ts new file mode 100644 index 0000000000000000000000000000000000000000..836348936abab6d76d2264109098f6d8c14a3ffd --- /dev/null +++ b/scripts/tool-display.ts @@ -0,0 +1,110 @@ +// Tool Display script supports OpenClaw repository automation. +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { TOOL_DISPLAY_CONFIG } from "../src/agents/tool-display-config.js"; + +type ToolDisplayConfig = typeof TOOL_DISPLAY_CONFIG; + +const scriptDir = path.dirname(fileURLToPath(import.meta.url)); +const repoRoot = path.resolve(scriptDir, ".."); +const outputPath = path.join( + repoRoot, + "apps/shared/OpenClawKit/Sources/OpenClawKit/Resources/tool-display.json", +); +const toolSources = [ + path.join(repoRoot, "src/agents/apply-patch.ts"), + path.join(repoRoot, "src/agents/bash-tools.exec-run.ts"), + path.join(repoRoot, "src/agents/bash-tools.process.ts"), + path.join(repoRoot, "src/auto-reply/reply/acp-projector.ts"), +]; + +const args = new Set(process.argv.slice(2)); +const shouldCheck = args.has("--check"); +const shouldWrite = args.has("--write"); + +if (!shouldCheck && !shouldWrite) { + console.error("Usage: node --import tsx scripts/tool-display.ts --check|--write"); + process.exit(1); +} + +const expected = serializeToolDisplayConfig(); +ensureCoreToolCoverage(); + +if (shouldWrite) { + fs.mkdirSync(path.dirname(outputPath), { recursive: true }); + fs.writeFileSync(outputPath, expected); + process.stdout.write(`wrote ${path.relative(repoRoot, outputPath)}\n`); + process.exit(0); +} + +if (!fs.existsSync(path.dirname(outputPath))) { + process.stdout.write( + `skip tool-display snapshot check; missing ${path.relative(repoRoot, path.dirname(outputPath))}\n`, + ); + process.exit(0); +} + +if (!fs.existsSync(outputPath)) { + console.error( + `missing generated snapshot: ${path.relative(repoRoot, outputPath)}\nrun: pnpm tool-display:write`, + ); + process.exit(1); +} + +const actual = fs.readFileSync(outputPath, "utf8"); +if (actual !== expected) { + console.error( + `tool-display snapshot is stale: ${path.relative(repoRoot, outputPath)}\nrun: pnpm tool-display:write`, + ); + process.exit(1); +} + +process.stdout.write("tool-display snapshot is up to date\n"); + +function ensureCoreToolCoverage() { + const toolNames = new Set<string>(); + for (const sourcePath of toolSources) { + collectToolNamesFromFile(sourcePath, toolNames); + } + for (const entry of fs.readdirSync(path.join(repoRoot, "src/agents/tools"))) { + if (!entry.endsWith(".ts") || entry.endsWith(".test.ts")) { + continue; + } + collectToolNamesFromFile(path.join(repoRoot, "src/agents/tools", entry), toolNames); + } + const missing = [...toolNames].filter((name) => !TOOL_DISPLAY_CONFIG.tools[name]).toSorted(); + if (missing.length > 0) { + console.error( + `tool-display metadata missing for runtime tools: ${missing.join(", ")}\nupdate: src/agents/tool-display-config.ts`, + ); + process.exit(1); + } +} + +function collectToolNamesFromFile(sourcePath: string, names: Set<string>) { + const source = fs.readFileSync(sourcePath, "utf8"); + for (const match of source.matchAll(/\bname:\s*"([A-Za-z0-9_-]+)"/g)) { + const name = match[1]?.trim(); + if (name) { + names.add(name); + } + } +} + +function serializeToolDisplayConfig(config: ToolDisplayConfig = TOOL_DISPLAY_CONFIG): string { + const tools = Object.entries(config.tools); + return [ + "{", + ` "version": ${JSON.stringify(config.version)},`, + ` "fallback": ${JSON.stringify(config.fallback)},`, + ' "tools": {', + ...tools.map( + ([name, spec], index) => + ` ${JSON.stringify(name)}: ${JSON.stringify(spec)}${index === tools.length - 1 ? "" : ","}`, + ), + " }", + "}", + "", + ].join("\n"); +} diff --git a/scripts/tsdown-build.mts b/scripts/tsdown-build.mts new file mode 100644 index 0000000000000000000000000000000000000000..a73f45743ee5f849ef71ba5c0ae2935d7d3fce38 --- /dev/null +++ b/scripts/tsdown-build.mts @@ -0,0 +1,2007 @@ +#!/usr/bin/env node + +// Runs the tsdown build with output cleanup, stale chunk pruning, and bounded +// child-process diagnostics. +import { + spawn, + spawnSync, + type StdioOptions, + type SpawnSyncOptionsWithStringEncoding, + type SpawnSyncReturns, +} from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { isPathInside } from "@openclaw/fs-safe/path"; +import { decodeMountInfoPath } from "../packages/normalization-core/src/mountinfo-path.ts"; +import { BUNDLED_PLUGIN_BUILD_ENV_NAMES } from "./lib/bundled-plugin-build-entries.mjs"; +import { BUNDLED_PLUGIN_PATH_PREFIX } from "./lib/bundled-plugin-paths.mjs"; +import { isDirectRunUrl } from "./lib/direct-run.mjs"; +import { + resolveDistArtifactLockPath, + withDistArtifactOwnership, +} from "./lib/dist-artifact-ownership.mts"; +import { toErrorObject } from "./lib/error-format.mts"; +import { + inspectManagedProcessGroup, + signalExitCode, + terminateManagedChild, + waitForManagedProcessGroupExit, +} from "./lib/managed-child-process.mts"; +import { parsePositiveInt } from "./lib/numeric-options.mjs"; +import { assertRealOutputRoot } from "./lib/output-root-guard.mjs"; +import { sanitizeBundlerHelperDtsExportTree } from "./lib/sanitize-bundler-helper-dts-exports.mts"; +import { + TSDOWN_PACKAGE_CONFIG_GROUP, + TSDOWN_UNIFIED_CONFIG_GROUP, + TSDOWN_UNIFIED_DTS_CONFIG_GROUPS, +} from "./lib/tsdown-config-groups.mts"; +import { + TSDOWN_PACKAGE_OUTPUT_ROOTS, + tsdownPackageOutputRoot, +} from "./lib/tsdown-output-roots.mts"; +import { resolvePnpmRunner } from "./pnpm-runner.mts"; + +const logLevel = process.env.OPENCLAW_BUILD_VERBOSE ? "info" : "warn"; +const INEFFECTIVE_DYNAMIC_IMPORT_MARKER = "[INEFFECTIVE_DYNAMIC_IMPORT]"; +const ANSI_ESCAPE_RE = new RegExp(String.raw`\u001B\[[0-9;]*m`, "g"); +const DEPENDENCY_PATH_MARKERS = ["node_modules/", "openclaw-pnpm-node-modules/"]; +const HASHED_ROOT_JS_RE = /^(?<base>.+)-[A-Za-z0-9_-]+\.js$/u; +const DEFAULT_CAPTURE_BYTES = 8 * 1024 * 1024; +const DEFAULT_HEARTBEAT_MS = 30_000; +const DEFAULT_TSDOWN_MAX_OLD_SPACE_MB = 12288; +const DEFAULT_WINDOWS_TSDOWN_MAX_OLD_SPACE_MB = 8192; +export const TSDOWN_MAX_OLD_SPACE_MB_ENV = "OPENCLAW_TSDOWN_MAX_OLD_SPACE_MB"; +const DOCKER_TSDOWN_MAX_OLD_SPACE_MB_ENV = "OPENCLAW_DOCKER_BUILD_TSDOWN_MAX_OLD_SPACE_MB"; +const TSDOWN_CGROUP_MEMORY_HEADROOM_MB = 768; +const DEFAULT_CGROUP_V2_MOUNT_PATH = "/sys/fs/cgroup"; +const DEFAULT_CGROUP_V1_MEMORY_MOUNT_PATH = "/sys/fs/cgroup/memory"; +const PROC_SELF_CGROUP_PATH = "/proc/self/cgroup"; +const PROC_SELF_LIMITS_PATH = "/proc/self/limits"; +const PROC_SELF_MOUNTINFO_PATH = "/proc/self/mountinfo"; +const PROC_SELF_STATUS_PATH = "/proc/self/status"; +const SERIALIZED_MAIN_CONFIG_GROUPS = [ + TSDOWN_PACKAGE_CONFIG_GROUP, + TSDOWN_UNIFIED_CONFIG_GROUP, + ...TSDOWN_UNIFIED_DTS_CONFIG_GROUPS, +]; +// The v2 high limit throttles reclaim, so a heap sized above it can stall the build instead of +// OOM-ing. Cgroup v1's soft limit is only advisory and must not reject an otherwise viable build. +const CGROUP_V2_MEMORY_LIMIT_FILES = ["memory.max", "memory.high"]; +const CGROUP_V1_MEMORY_LIMIT_FILES = ["memory.limit_in_bytes"]; +const PROC_MEMINFO_PATH = "/proc/meminfo"; +const tsdownStdio = () => ["ignore", "pipe", "pipe"] satisfies ["ignore", "pipe", "pipe"]; +// Build descendants get a short cleanup window; a timed-out build must not hold CI for seconds. +const TERMINATION_GRACE_MS = 250; +const POST_FORCE_KILL_WAIT_MS = 250; +const ROOT_TSDOWN_OUTPUT_ROOTS = ["dist", "dist-runtime"]; +const PRESERVED_TSDOWN_OUTPUT_FILES = ["dist/cli-startup-metadata.json"]; +const PRESERVE_CLI_STARTUP_METADATA_ENV = "OPENCLAW_PRESERVE_CLI_STARTUP_METADATA"; +const GENERATED_SOURCE_DECLARATION_PATHSPEC = ":(glob)extensions/**/*.d.ts"; +export const TSDOWN_DECLARATION_EXTENSIONS = [".d.ts", ".d.mts", ".d.cts"]; +const SOURCE_DECLARATION_SOURCE_EXTENSIONS = [".ts", ".tsx", ".mts", ".cts", ".js", ".mjs", ".cjs"]; +const RUN_NODE_SKIP_DTS_BUILD_ENV = "OPENCLAW_RUN_NODE_SKIP_DTS_BUILD"; + +const TSDOWN_SOURCE_EXTENSIONS = [ + ".cjs", + ".cts", + ".js", + ".json", + ".json5", + ".mjs", + ".mts", + ".sql", + ".ts", + ".tsx", + ".yaml", + ".yml", +]; + +export const TSDOWN_DECLARATION_TOOL_INPUTS = [ + "package.json", + "pnpm-lock.yaml", + "tsconfig.json", + "scripts/tsdown-build.mts", + "scripts/build-all.mts", + "scripts/lib/build-artifact-cache.mts", + "scripts/lib/dist-artifact-ownership.mts", + "scripts/lib/managed-child-process.mts", + "scripts/lib/vitest-resource-ownership.mts", + "scripts/lib/direct-run.mjs", + "scripts/lib/repo-root.mjs", + "scripts/lib/local-check-runtime.mts", + "scripts/tsx.mjs", + "scripts/lib/tsx-cli-shim.mjs", + "scripts/lib/bundled-plugin-build-entries.mjs", + "scripts/lib/bundled-plugin-paths.mjs", + "scripts/lib/optional-bundled-clusters.mjs", + "scripts/lib/plugin-sdk-entries.mts", + "scripts/lib/plugin-sdk-entrypoints.json", + "scripts/lib/plugin-sdk-private-local-only-subpaths.json", + "scripts/lib/plugin-sdk-deprecated-public-subpaths.json", + "scripts/lib/plugin-sdk-deprecated-barrel-subpaths.json", + "scripts/lib/root-package-bundled-plugin-excludes.mjs", + "scripts/lib/tsdown-config-groups.mts", + "scripts/lib/tsdown-declaration-boundary.mts", + "scripts/lib/tsdown-output-roots.mts", +]; +export const TSDOWN_PACKAGES_CACHE_INPUT = { + path: "packages", + extensions: TSDOWN_SOURCE_EXTENSIONS, + excludeDirectories: ["dist", "node_modules"], +}; +export const TSDOWN_UNIFIED_CACHE_ENV = [ + "OPENCLAW_BUILD_PRIVATE_QA", + ...BUNDLED_PLUGIN_BUILD_ENV_NAMES, +]; + +type OutputRootParams = { + cwd?: string; + env?: NodeJS.ProcessEnv; + fs?: typeof fs; + pathImpl?: Pick<typeof path, "dirname" | "parse" | "resolve">; + roots?: string[]; +}; + +export type MemoryLimitParams = { + availableMemoryBytes?: number; + cgroupMemoryLimitBytes?: number; + cgroupMemoryLimitPaths?: string[]; + constrainedMemoryBytes?: number; + env?: NodeJS.ProcessEnv; + fs?: { readFileSync(filePath: string, encoding: "utf8"): string }; + physicalMemoryBytes?: number; + platform?: string; + processResidentMemoryBytes?: number; + procMeminfoPath?: string; + procMemTotalBytes?: number; +}; + +type CgroupMount = { mountPoint: string; observed: boolean; root: string }; + +type ResolvedMemoryLimitParams = MemoryLimitParams & { resolvedMaxOldSpaceMb?: number }; + +type TsdownBuildParams = ResolvedMemoryLimitParams & { + args?: string[]; + comSpec?: string; + nodeExecPath?: string; + npmExecPath?: string; +}; + +type TsdownBuildResult = ReturnType<ReturnType<typeof createTsdownOutputScanner>["finish"]> & { + error: Error | null; + signal: string | null; + status: number | null; + timedOut: boolean; +}; + +type TsdownBuildInvocation = { + command: string; + args: string[]; + options: { + stdio: string[]; + shell: boolean; + windowsVerbatimArguments?: boolean; + env: NodeJS.ProcessEnv; + }; +}; + +function removeDistPluginNodeModulesSymlinks(rootDir: string) { + const extensionsDir = path.join(rootDir, "extensions"); + if (!fs.existsSync(extensionsDir)) { + return; + } + + for (const dirent of fs.readdirSync(extensionsDir, { withFileTypes: true })) { + if (!dirent.isDirectory()) { + continue; + } + const nodeModulesPath = path.join(extensionsDir, dirent.name, "node_modules"); + try { + if (fs.lstatSync(nodeModulesPath).isSymbolicLink()) { + fs.rmSync(nodeModulesPath, { force: true, recursive: true }); + } + } catch { + // Skip missing or unreadable paths so the build can proceed. + } + } +} + +export function pruneStaleRuntimeSymlinks(params: Pick<OutputRootParams, "cwd" | "fs"> = {}) { + const cwd = params.cwd ?? process.cwd(); + const fsImpl = params.fs ?? fs; + const distRoot = path.join(cwd, "dist"); + const distRuntimeRoot = path.join(cwd, "dist-runtime"); + assertRealOutputRoot(distRoot, { fs: fsImpl }); + assertRealOutputRoot(distRuntimeRoot, { fs: fsImpl }); + // runtime-postbuild stages plugin-owned node_modules into dist/ and links the + // dist-runtime overlay back to that tree. Remove only those symlinks up front + // so tsdown's clean step cannot traverse stale runtime overlays on rebuilds. + removeDistPluginNodeModulesSymlinks(distRoot); + removeDistPluginNodeModulesSymlinks(distRuntimeRoot); +} + +/** + * Removes build output roots while preserving explicitly protected artifacts. + */ +function assertTsdownCleanOutputRoots(params: OutputRootParams = {}) { + const pathImpl = params.pathImpl ?? path; + const cwd = pathImpl.resolve(params.cwd ?? process.cwd()); + const fsImpl = params.fs ?? fs; + const roots = params.roots ?? listTsdownOutputRoots(); + const rootPaths = roots.map((root) => pathImpl.resolve(cwd, root)); + for (const rootPath of rootPaths) { + if (pathImpl.parse(rootPath).root === rootPath) { + throw new Error( + "Cannot clean a filesystem root. Please specify a dedicated output directory.", + ); + } + if (isPathInside(rootPath, cwd)) { + throw new Error( + "Cannot clean the current working directory or one of its ancestors. Please specify a dedicated output directory.", + ); + } + if (isPathInside(rootPath, resolveDistArtifactLockPath(cwd))) { + throw new Error("Cannot clean the checkout's dist artifact ownership location."); + } + // A safe final component is insufficient: recursive removal follows symlinked parents. + // Validate every component below the nearest common ancestor before any mutation begins. + let candidatePath = rootPath; + while (!isPathInside(candidatePath, cwd)) { + assertRealOutputRoot(candidatePath, { fs: fsImpl }); + const parentPath = pathImpl.dirname(candidatePath); + if (parentPath === candidatePath) { + break; + } + candidatePath = parentPath; + } + } + return rootPaths; +} + +export function cleanTsdownOutputRoots(params: OutputRootParams = {}) { + const pathImpl = params.pathImpl ?? path; + const cwd = pathImpl.resolve(params.cwd ?? process.cwd()); + const fsImpl = params.fs ?? fs; + const env = params.env ?? process.env; + const roots = params.roots ?? listTsdownOutputRoots(); + // Validate the complete mutation set before traversing protected children or + // cleaning any earlier root; otherwise a later symlink can leave a partial build. + const rootPaths = assertTsdownCleanOutputRoots({ cwd, fs: fsImpl, pathImpl, roots }); + const protectedDeclarationPaths = + env[RUN_NODE_SKIP_DTS_BUILD_ENV] === "1" + ? listExistingGeneratedDeclarationOutputPaths(cwd, fsImpl, roots) + : new Set<string>(); + const protectedPaths = new Set([ + ...protectedDeclarationPaths, + ...listExistingPreservedOutputPaths(cwd, env, fsImpl), + ]); + for (const rootPath of rootPaths) { + try { + if (hasProtectedChild(rootPath, protectedPaths)) { + cleanOutputRootExcept(rootPath, protectedPaths, fsImpl); + } else { + fsImpl.rmSync(rootPath, { force: true, recursive: true }); + } + } catch { + // Best-effort cleanup. tsdown will recreate the output tree it needs. + } + } +} + +function hasProtectedChild(rootPath: string, protectedPaths: Set<string>) { + const rootWithSeparator = `${path.resolve(rootPath)}${path.sep}`; + for (const protectedPath of protectedPaths) { + if (protectedPath.startsWith(rootWithSeparator)) { + return true; + } + } + return false; +} + +function cleanOutputRootExcept(rootPath: string, protectedPaths: Set<string>, fsImpl: typeof fs) { + let entries; + try { + entries = fsImpl.readdirSync(rootPath, { withFileTypes: true }); + } catch { + return; + } + + for (const entry of entries) { + const entryPath = path.join(rootPath, entry.name); + const resolvedEntryPath = path.resolve(entryPath); + if (protectedPaths.has(resolvedEntryPath)) { + continue; + } + try { + if (entry.isDirectory()) { + cleanOutputRootExcept(entryPath, protectedPaths, fsImpl); + fsImpl.rmdirSync(entryPath); + } else { + fsImpl.rmSync(entryPath, { force: true }); + } + } catch { + // Keep best-effort semantics; protected children can keep a directory non-empty. + } + } +} + +function listExistingGeneratedDeclarationOutputPaths( + cwd: string, + fsImpl: typeof fs, + roots: string[], +) { + const protectedPaths = new Set<string>(); + for (const root of roots) { + collectDeclarationOutputPaths(path.resolve(cwd, root), protectedPaths, fsImpl); + } + return protectedPaths; +} + +function listExistingPreservedOutputPaths(cwd: string, env: NodeJS.ProcessEnv, fsImpl: typeof fs) { + // Vite owns and cleans this subtree; tsdown cannot recreate its assets. + const protectedPaths = new Set([path.resolve(cwd, "dist/control-ui")]); + // Mac packaging owns replacement of signed bundles. Rebuilding its JS must + // leave the previous app (including its private runtime) usable on failure. + const pendingDirectories = [path.join(cwd, "dist")]; + while (pendingDirectories.length > 0) { + const directory = pendingDirectories.pop()!; + if (!fsImpl.existsSync(directory)) { + continue; + } + for (const entry of fsImpl.readdirSync(directory, { withFileTypes: true })) { + if (!entry.isDirectory()) { + continue; + } + const child = path.join(directory, entry.name); + if (entry.name.endsWith(".app")) { + protectedPaths.add(child); + } else { + pendingDirectories.push(child); + } + } + } + if (env[PRESERVE_CLI_STARTUP_METADATA_ENV] !== "1") { + return protectedPaths; + } + for (const relativePath of PRESERVED_TSDOWN_OUTPUT_FILES) { + const absolutePath = path.resolve(cwd, relativePath); + try { + if (fsImpl.statSync(absolutePath).isFile()) { + protectedPaths.add(absolutePath); + } + } catch { + // Missing preserved outputs are normal on first build. + } + } + return protectedPaths; +} + +/** Publish generated declarations without claiming runtime assets or protected subtrees. */ +export function listReplaceableTsdownDeclarationOutputs(params: OutputRootParams = {}) { + const cwd = path.resolve(params.cwd ?? process.cwd()); + const fsImpl = params.fs ?? fs; + const roots = params.roots ?? listTsdownOutputRoots(); + assertTsdownCleanOutputRoots({ ...params, cwd, fs: fsImpl, roots }); + const protectedPaths = [ + ...listExistingPreservedOutputPaths(cwd, params.env ?? process.env, fsImpl), + ]; + return [...listExistingGeneratedDeclarationOutputPaths(cwd, fsImpl, roots)] + .filter( + (file) => + !protectedPaths.some( + (protectedPath) => + file === protectedPath || file.startsWith(`${protectedPath}${path.sep}`), + ), + ) + .toSorted(); +} + +function collectDeclarationOutputPaths( + rootPath: string, + protectedPaths: Set<string>, + fsImpl: typeof fs, +) { + let entries; + try { + entries = fsImpl.readdirSync(rootPath, { withFileTypes: true }); + } catch { + return; + } + + for (const entry of entries) { + const entryPath = path.join(rootPath, entry.name); + if (entry.isDirectory()) { + collectDeclarationOutputPaths(entryPath, protectedPaths, fsImpl); + } else if (TSDOWN_DECLARATION_EXTENSIONS.some((extension) => entry.name.endsWith(extension))) { + protectedPaths.add(path.resolve(entryPath)); + } + } +} + +export function pruneStaleRootChunkFiles(params: Pick<OutputRootParams, "cwd" | "fs"> = {}) { + const cwd = params.cwd ?? process.cwd(); + const fsImpl = params.fs ?? fs; + const roots = listTsdownOutputRoots().map((root) => path.join(cwd, root)); + for (const root of roots) { + assertRealOutputRoot(root, { fs: fsImpl }); + } + for (const root of roots) { + let entries; + try { + entries = fsImpl.readdirSync(root, { withFileTypes: true }); + } catch { + continue; + } + + for (const entry of entries) { + if (!entry.isFile()) { + continue; + } + if (!HASHED_ROOT_JS_RE.test(entry.name)) { + continue; + } + try { + fsImpl.rmSync(path.join(root, entry.name), { force: true }); + } catch { + // Best-effort cleanup. The subsequent build will overwrite any stragglers. + } + } + } +} + +export function listTsdownOutputRoots() { + return [...ROOT_TSDOWN_OUTPUT_ROOTS, ...TSDOWN_PACKAGE_OUTPUT_ROOTS]; +} + +function readForwardedOptions(args: string[], names: string[]) { + const values: string[] = []; + for (const [index, arg] of args.entries()) { + for (const name of names) { + if (arg === name) { + const value = args[index + 1]; + if (!value || value.startsWith("-")) { + throw new Error(`tsdown build requires one concrete ${names.join("/")} value`); + } + values.push(value); + } else if (arg.startsWith(`${name}=`)) { + const value = arg.slice(name.length + 1); + if (!value) { + throw new Error(`tsdown build requires one concrete ${names.join("/")} value`); + } + values.push(value); + } + } + } + return values; +} +const readForwardedOption = (args: string[], names: string[]) => + readForwardedOptions(args, names)[0]; +function readForwardedScalarOption(args: string[], names: string[], label: string) { + const values: string[] = []; + for (const [index, arg] of args.entries()) { + for (const name of names) { + if (arg === name) { + const value = args[index + 1]; + if (!value || value.startsWith("-")) { + throw new Error(`tsdown build requires one concrete ${label} value`); + } + values.push(value); + } else if (arg.startsWith(`${name}=`)) { + const value = arg.slice(name.length + 1); + if (!value) { + throw new Error(`tsdown build requires one concrete ${label} value`); + } + values.push(value); + } + } + } + if (values.length > 1) { + throw new Error(`tsdown build accepts only one ${label} value`); + } + return values[0]; +} +const isFilterFlag = (arg: string | undefined) => arg === "--filter" || arg === "-F"; +const isFilterArg = (arg: string) => + isFilterFlag(arg) || arg.startsWith("--filter=") || arg.startsWith("-F="); +const isConfigArg = (arg: string) => + arg === "--config" || + arg.startsWith("--config=") || + arg === "-c" || + arg.startsWith("-c=") || + arg === "--no-config"; +const isWatchArg = (arg: string) => + arg === "--watch" || arg.startsWith("--watch=") || arg === "-w" || arg.startsWith("-w="); +const isUnifiedDtsGroup = (value: string | undefined) => + TSDOWN_UNIFIED_DTS_CONFIG_GROUPS.some((group) => group === value); + +/** Limits cleanup to the output roots owned by an explicitly filtered build. */ +export function resolveTsdownCleanOutputRoots(args: string[] = []) { + const config = readForwardedOption(args, ["--config", "-c"]); + const outDir = readForwardedScalarOption(args, ["--out-dir", "-d"], "--out-dir/-d"); + const filters = readForwardedOptions(args, ["--filter", "-F"]); + const configPath = config ? path.resolve(config) : undefined; + const aiConfigPath = path.resolve("tsdown.ai.config.ts"); + const aiRoot = tsdownPackageOutputRoot("ai"); + const packageRoots = TSDOWN_PACKAGE_OUTPUT_ROOTS.filter((root) => root !== aiRoot); + const selectsRootCwd = filters.includes("."); + const selectedMainRoots = [ + ...(filters.includes(TSDOWN_PACKAGE_CONFIG_GROUP) ? packageRoots : []), + ...(filters.some( + (filter) => filter === TSDOWN_UNIFIED_CONFIG_GROUP || isUnifiedDtsGroup(filter), + ) + ? ROOT_TSDOWN_OUTPUT_ROOTS + : []), + ]; + + if (outDir !== undefined) { + return [outDir]; + } + if (configPath === aiConfigPath) { + return [aiRoot]; + } + if (selectsMainConfig(args)) { + return filters.length === 0 || selectsRootCwd || selectedMainRoots.length === 0 + ? [...ROOT_TSDOWN_OUTPUT_ROOTS, ...packageRoots] + : selectedMainRoots; + } + if (!config && selectsRootCwd) { + return listTsdownOutputRoots(); + } + if (!config && filters.length > 0 && selectedMainRoots.length > 0) { + return [aiRoot, ...selectedMainRoots]; + } + return listTsdownOutputRoots(); +} + +export function sanitizeTsdownBuildOutputRoots(args: string[] = [], cwd = process.cwd()): void { + const roots = resolveTsdownCleanOutputRoots(args); + const rootPaths = assertTsdownCleanOutputRoots({ cwd, roots }); + for (const rootPath of rootPaths) { + sanitizeBundlerHelperDtsExportTree(rootPath); + } +} + +function wrapperOwnsTsdownCleanup(args: string[]) { + if (readForwardedScalarOption(args, ["--out-dir", "-d"], "--out-dir/-d") !== undefined) { + return true; + } + const config = readForwardedOption(args, ["--config", "-c"]); + if (config === undefined) { + return true; + } + return path.resolve(config) === path.resolve("tsdown.ai.config.ts") || selectsMainConfig(args); +} + +type GitLsFilesResult = Pick<SpawnSyncReturns<string>, "status"> & { stdout?: string }; + +export function pruneUntrackedGeneratedSourceDeclarations( + params: { + cwd?: string; + fs?: typeof fs; + spawnSync?: ( + command: string, + args: string[], + options: SpawnSyncOptionsWithStringEncoding, + ) => GitLsFilesResult; + } = {}, +) { + const cwd = params.cwd ?? process.cwd(); + const fsImpl = params.fs ?? fs; + const spawnSyncImpl = params.spawnSync ?? spawnSync; + let result; + try { + result = spawnSyncImpl( + "git", + ["ls-files", "--others", "--exclude-standard", "--", GENERATED_SOURCE_DECLARATION_PATHSPEC], + { + cwd, + encoding: "utf8", + stdio: ["ignore", "pipe", "ignore"], + }, + ); + } catch { + return 0; + } + if (result.status !== 0 || typeof result.stdout !== "string") { + return 0; + } + + let removed = 0; + for (const rawPath of result.stdout.split(/\r?\n/u)) { + const relativePath = rawPath.trim().replaceAll("\\", "/"); + if (!relativePath.startsWith("extensions/") || !relativePath.endsWith(".d.ts")) { + continue; + } + const declarationPath = path.join(cwd, relativePath); + const sourceBase = declarationPath.slice(0, -".d.ts".length); + const hasMatchingSource = SOURCE_DECLARATION_SOURCE_EXTENSIONS.some((extension) => + fsImpl.existsSync(`${sourceBase}${extension}`), + ); + if (!hasMatchingSource) { + continue; + } + try { + fsImpl.rmSync(declarationPath, { force: true }); + removed += 1; + } catch { + // Best-effort cleanup; tsdown will still report any remaining stale files. + } + } + return removed; +} + +function findFatalUnresolvedImport(lines: string[]) { + for (const line of lines) { + if (!line.includes("[UNRESOLVED_IMPORT]")) { + continue; + } + + const normalizedLine = line.replace(ANSI_ESCAPE_RE, ""); + if ( + !normalizedLine.includes(BUNDLED_PLUGIN_PATH_PREFIX) && + !DEPENDENCY_PATH_MARKERS.some((marker) => normalizedLine.includes(marker)) + ) { + return normalizedLine; + } + } + + return null; +} + +function parsePositiveIntegerEnv(value: string | undefined, name: string) { + if (typeof value !== "string" || value.trim() === "") { + return null; + } + return parsePositiveInt(value, name); +} + +function parseNonNegativeIntegerEnv(value: string | undefined, name: string) { + if (typeof value !== "string" || value.trim() === "") { + return null; + } + const text = value.trim(); + if (!/^\d+$/u.test(text)) { + throw new Error(`${name} must be a non-negative integer`); + } + const parsed = Number(text); + if (!Number.isSafeInteger(parsed)) { + throw new Error(`${name} must be a non-negative safe integer`); + } + return parsed; +} + +function parseCgroupMemoryLimitBytes(value: string) { + const trimmed = value.trim(); + if (trimmed === "" || trimmed === "max" || !/^\d+$/u.test(trimmed)) { + return null; + } + const parsed = BigInt(trimmed); + if (parsed < 0n || parsed > BigInt(Number.MAX_SAFE_INTEGER)) { + return null; + } + return Number(parsed); +} + +function isMissingFileError(error: unknown) { + return typeof error === "object" && error !== null && "code" in error && error.code === "ENOENT"; +} + +function readProcessRlimitMemoryBytes(params: MemoryLimitParams) { + if ((params.platform ?? process.platform) !== "linux") { + return null; + } + try { + const rawLimits = (params.fs ?? fs).readFileSync(PROC_SELF_LIMITS_PATH, "utf8"); + let tightestLimitBytes: number | null = null; + for (const match of rawLimits.matchAll( + /^Max (?:address space|data size)\s+(?<soft>\d+|unlimited)\s+/gmu, + )) { + const softLimit = match.groups?.soft; + if (!softLimit || softLimit === "unlimited") { + continue; + } + const parsed = parseCgroupMemoryLimitBytes(softLimit); + if (parsed !== null && (tightestLimitBytes === null || parsed < tightestLimitBytes)) { + tightestLimitBytes = parsed; + } + } + return tightestLimitBytes; + } catch { + return null; + } +} + +function parseCgroupInactiveFileBytes(value: string, isV1: boolean) { + const match = isV1 + ? (value.match(/^total_inactive_file\s+(\d+)$/mu) ?? value.match(/^inactive_file\s+(\d+)$/mu)) + : value.match(/^inactive_file\s+(\d+)$/mu); + return match?.[1] ? parseCgroupMemoryLimitBytes(match[1]) : 0; +} + +function readProcessResidentMemoryBytes(params: MemoryLimitParams) { + const configured = params.processResidentMemoryBytes; + if (configured !== undefined && Number.isFinite(configured) && configured >= 0) { + return Math.trunc(configured); + } + try { + const match = (params.fs ?? fs) + .readFileSync(PROC_SELF_STATUS_PATH, "utf8") + .match(/^VmRSS:\s+(\d+)\s+kB$/mu); + const bytes = match?.[1] ? BigInt(match[1]) * 1024n : null; + return bytes !== null && bytes <= BigInt(Number.MAX_SAFE_INTEGER) ? Number(bytes) : null; + } catch { + return null; + } +} + +// Controller mount points are host layout, not constants: v1 controllers may be co-mounted at +// the cgroup root instead of a per-controller directory. Read them where the kernel records +// them so a slice budget is never missed because a path was assumed. +function resolveCgroupMountPoints(params: MemoryLimitParams = {}) { + const fsImpl = params.fs ?? fs; + let rawMountinfo = ""; + try { + rawMountinfo = fsImpl.readFileSync(PROC_SELF_MOUNTINFO_PATH, "utf8"); + } catch { + // Unreadable off Linux; the documented defaults still apply. + } + + // One hierarchy can be visible through several mounts, and only some of them expose a subtree + // containing this process, so every view is kept as a candidate rather than the last one seen. + const unified: CgroupMount[] = []; + const v1Memory: CgroupMount[] = []; + for (const line of rawMountinfo.split("\n")) { + // mountinfo separates its variable optional fields from the fstype with a lone "-". + const [fields, describe] = line.split(" - "); + // mountinfo fields 4 and 5 are the mount root and mount point. + const mountFields = (fields ?? "").split(" "); + const rawRoot = mountFields[3]; + const rawMountPoint = mountFields[4]; + const [fsType, , superOptions] = (describe ?? "").split(" "); + if (!rawMountPoint || !rawRoot) { + continue; + } + // The kernel escapes space, tab, newline, and backslash in these two fields, so + // matching them verbatim would miss any cgroup path containing one of them. + const root = decodeMountInfoPath(rawRoot); + const mountPoint = decodeMountInfoPath(rawMountPoint); + if (fsType === "cgroup2") { + unified.push({ mountPoint, observed: true, root }); + } else if (fsType === "cgroup" && (superOptions ?? "").split(",").includes("memory")) { + v1Memory.push({ mountPoint, observed: true, root }); + } + } + return { + unified: + unified.length > 0 + ? unified + : [{ mountPoint: DEFAULT_CGROUP_V2_MOUNT_PATH, observed: false, root: "/" }], + v1Memory: + v1Memory.length > 0 + ? v1Memory + : [{ mountPoint: DEFAULT_CGROUP_V1_MEMORY_MOUNT_PATH, observed: false, root: "/" }], + }; +} + +// mountinfo field 4 is the subtree a cgroupfs mount exposes, so /proc/self/cgroup records are +// relative to it: under a container mount the visible leaf is the mount point itself, not the +// host-absolute path. A record outside that subtree is not reachable through this mount, and +// probing the mount root instead would size the build from an unrelated cgroup's limit. +function relativeCgroupPath(mountRoot: string, cgroupPath: string) { + if (cgroupPath.split("/").includes("..")) { + return null; + } + if (mountRoot === "/") { + return cgroupPath; + } + const mountRootSegments = mountRoot.split("/").filter(Boolean); + if (mountRootSegments.length > 0 && mountRootSegments.every((segment) => segment === "..")) { + // The visible root is an ancestor, but the process's hidden child name cannot be + // reconstructed. Treat it as unresolved instead of mistaking the parent for the leaf. + return null; + } + if (mountRootSegments.includes("..")) { + return null; + } + // A namespace-root record proves nothing about a mount rooted elsewhere: the kernel + // contract does not make "/" plus an arbitrary subtree a match, so adopting that pair + // could cap the heap from an unrelated cgroup. Fail closed to host sizing instead. + if (cgroupPath === "/") { + return null; + } + if (cgroupPath === mountRoot) { + return "/"; + } + return cgroupPath.startsWith(`${mountRoot}/`) ? cgroupPath.slice(mountRoot.length) : null; +} + +// A systemd slice budget lives on the process's own cgroup, never on a hierarchy root, so +// probing only the root misses every limit outside a namespaced container. Legacy and hybrid +// hosts publish that same budget through the v1 memory controller instead of the `0::` record, +// so both hierarchies are walked leaf-to-root; depth 0 is the root probe. +function resolveCgroupMemoryLimitPaths(params: MemoryLimitParams = {}) { + const fsImpl = params.fs ?? fs; + let rawCgroup = ""; + let cgroupRecordReadFailed = false; + try { + rawCgroup = fsImpl.readFileSync(PROC_SELF_CGROUP_PATH, "utf8"); + } catch { + cgroupRecordReadFailed = (params.platform ?? process.platform) === "linux"; + } + + const paths: string[] = []; + const addHierarchy = ( + mounts: CgroupMount[], + limitFiles: string[], + cgroupPath: string, + hierarchyFile?: string, + ) => { + const initialPathCount = paths.length; + let addedObservedPath = false; + let hierarchyMetadataUnreadable = false; + for (const mount of mounts) { + const mountInitialPathCount = paths.length; + const mountRootSegments = mount.root.split("/").filter(Boolean); + if (mountRootSegments.length > 0 && mountRootSegments.every((segment) => segment === "..")) { + continue; + } + const relative = relativeCgroupPath(mount.root, cgroupPath ?? mount.root); + if (relative === null) { + continue; + } + const segments = relative.split("/").filter(Boolean); + for (let depth = segments.length; depth >= 0; depth -= 1) { + if (hierarchyFile && depth < segments.length) { + try { + const hierarchyPath = path.join( + mount.mountPoint, + ...segments.slice(0, depth), + hierarchyFile, + ); + const hierarchyMode = fsImpl.readFileSync(hierarchyPath, "utf8").trim(); + if (hierarchyMode === "0") { + break; + } + if (hierarchyMode !== "1") { + hierarchyMetadataUnreadable = true; + break; + } + } catch { + hierarchyMetadataUnreadable = true; + break; + } + } + for (const limitFile of limitFiles) { + paths.push(path.join(mount.mountPoint, ...segments.slice(0, depth), limitFile)); + } + } + addedObservedPath ||= mount.observed && paths.length > mountInitialPathCount; + } + return { + added: paths.length > initialPathCount, + addedObservedPath, + hierarchyMetadataUnreadable, + }; + }; + + const mounts = resolveCgroupMountPoints(params); + let sawMemoryRecord = false; + let sawObservedV2Mapping = false; + let sawObservedV2Root = false; + let sawUnreadableV1HierarchyMetadata = false; + let sawV1MemoryRecord = false; + let sawRejectedCgroupMapping = false; + let sawUnresolvedCgroupLimit = false; + for (const line of rawCgroup.split("\n")) { + const record = /^\d+:([^:]*):(.*)$/u.exec(line); + if (!record) { + continue; + } + const controllers = record[1] ?? ""; + if (controllers === "") { + sawMemoryRecord = true; + const cgroupPath = record[2] ?? ""; + sawObservedV2Root ||= + cgroupPath === "/" && mounts.unified.some((mount) => mount.observed && mount.root === "/"); + const resolved = addHierarchy(mounts.unified, CGROUP_V2_MEMORY_LIMIT_FILES, cgroupPath); + sawObservedV2Mapping ||= resolved.addedObservedPath; + sawRejectedCgroupMapping ||= !resolved.added; + sawUnresolvedCgroupLimit ||= !resolved.added; + } else if (controllers.split(",").includes("memory")) { + sawMemoryRecord = true; + sawV1MemoryRecord = true; + const resolved = addHierarchy( + mounts.v1Memory, + CGROUP_V1_MEMORY_LIMIT_FILES, + record[2] ?? "", + "memory.use_hierarchy", + ); + sawUnreadableV1HierarchyMetadata ||= resolved.hierarchyMetadataUnreadable; + sawRejectedCgroupMapping ||= !resolved.added; + sawUnresolvedCgroupLimit ||= !resolved.added; + } + } + // Only probe the mounts blind when this process has no memory cgroup record at all; a record + // that no mount can represent means the limit is unreadable here, not that the root applies. + if (!sawMemoryRecord) { + for (const mount of mounts.unified) { + addHierarchy([mount], CGROUP_V2_MEMORY_LIMIT_FILES, mount.root); + } + for (const mount of mounts.v1Memory) { + addHierarchy([mount], CGROUP_V1_MEMORY_LIMIT_FILES, mount.root); + } + } + return { + paths, + cgroupRecordReadFailed, + sawMemoryRecord, + sawObservedV2Mapping, + sawObservedUnconstrainedV2Root: sawObservedV2Root && !sawV1MemoryRecord, + sawRejectedCgroupMapping, + sawUnresolvedCgroupLimit, + sawUnreadableV1HierarchyMetadata, + sawV1MemoryRecord, + }; +} + +function readCgroupMemoryLimitBytes(params: MemoryLimitParams = {}) { + const configuredLimit = params.cgroupMemoryLimitBytes; + if (configuredLimit !== undefined && Number.isFinite(configuredLimit) && configuredLimit >= 0) { + return { limitBytes: Math.trunc(configuredLimit), unresolved: false, usageKnown: false }; + } + + const fsImpl = params.fs ?? fs; + const resolvedPaths = params.cgroupMemoryLimitPaths + ? { + cgroupRecordReadFailed: false, + paths: params.cgroupMemoryLimitPaths, + sawMemoryRecord: false, + sawObservedV2Mapping: false, + sawObservedUnconstrainedV2Root: false, + sawRejectedCgroupMapping: false, + sawUnresolvedCgroupLimit: false, + sawUnreadableV1HierarchyMetadata: false, + sawV1MemoryRecord: false, + } + : resolveCgroupMemoryLimitPaths(params); + // libuv folds cgroup v1's advisory soft limit into constrainedMemory(). Preserve its separate + // process rlimit candidate while the owner walk reads only authoritative cgroup hard limits. + const rlimitMemoryBytes = readProcessRlimitMemoryBytes(params); + const constrainedMemoryBytes = + resolvedPaths.sawV1MemoryRecord || + resolvedPaths.sawRejectedCgroupMapping || + resolvedPaths.sawUnresolvedCgroupLimit || + resolvedPaths.sawUnreadableV1HierarchyMetadata + ? 0 + : (params.constrainedMemoryBytes ?? + (params.fs === undefined ? process.constrainedMemory() : 0)); + // An ancestor may bound the leaf, so the tightest limit in the chain wins. + let tightestLimitBytes = + Number.isFinite(constrainedMemoryBytes) && constrainedMemoryBytes > 0 + ? Math.trunc(constrainedMemoryBytes) + : null; + if ( + rlimitMemoryBytes !== null && + (tightestLimitBytes === null || rlimitMemoryBytes < tightestLimitBytes) + ) { + tightestLimitBytes = rlimitMemoryBytes; + } + const processResidentMemoryBytes = readProcessResidentMemoryBytes(params); + let readControllerLimit = false; + let readV1HardLimit = false; + let sawDisabledV2MemoryController = false; + let sawUnreadableControllerFile = false; + let usageKnown = true; + for (const limitPath of resolvedPaths.paths) { + try { + const rawLimit = fsImpl.readFileSync(limitPath, "utf8"); + const trimmedLimit = rawLimit.trim(); + readControllerLimit ||= trimmedLimit === "max" || /^\d+$/u.test(trimmedLimit); + // A controller cannot be bound to v1 and v2 simultaneously. Reading the v1 hard-limit + // file therefore resolves memory ownership even when its value is the unlimited sentinel. + if (path.basename(limitPath) === "memory.limit_in_bytes" && /^\d+$/u.test(trimmedLimit)) { + readV1HardLimit = true; + } + const limitBytes = parseCgroupMemoryLimitBytes(rawLimit); + if (limitBytes === null) { + continue; + } + let availableBytes = limitBytes; + try { + const isV1 = path.basename(limitPath) === "memory.limit_in_bytes"; + const cgroupDir = path.dirname(limitPath); + const usageBytes = parseCgroupMemoryLimitBytes( + fsImpl.readFileSync( + path.join(cgroupDir, isV1 ? "memory.usage_in_bytes" : "memory.current"), + "utf8", + ), + ); + usageKnown &&= usageBytes !== null; + if (usageBytes !== null) { + let inactiveFileBytes = 0; + try { + inactiveFileBytes = + parseCgroupInactiveFileBytes( + fsImpl.readFileSync(path.join(cgroupDir, "memory.stat"), "utf8"), + isV1, + ) ?? 0; + } catch { + // Missing stats make all charged usage non-reclaimable for admission. + } + // Total controller usage includes kernel and unreclaimable file charges. Credit only + // inactive file pages and this wrapper's resident set before sizing its child. + const competingBytes = Math.max( + 0, + usageBytes - + Math.min(usageBytes, inactiveFileBytes) - + (processResidentMemoryBytes ?? 0), + ); + availableBytes = Math.max(0, limitBytes - competingBytes); + } + } catch { + // Keep the serial heap cap, but an unobserved shared budget cannot admit overlap. + usageKnown = false; + } + if (tightestLimitBytes === null || availableBytes < tightestLimitBytes) { + tightestLimitBytes = availableBytes; + } + } catch (error) { + if (!isMissingFileError(error)) { + sawUnreadableControllerFile = true; + continue; + } + if (path.basename(limitPath) === "memory.limit_in_bytes") { + continue; + } + try { + const controllers = fsImpl + .readFileSync(path.join(path.dirname(limitPath), "cgroup.controllers"), "utf8") + .trim() + .split(/\s+/u) + .filter(Boolean); + sawDisabledV2MemoryController ||= !controllers.includes("memory"); + } catch (controllerError) { + sawUnreadableControllerFile ||= !isMissingFileError(controllerError); + } + } + } + + return { + limitBytes: tightestLimitBytes, + usageKnown, + unresolved: + resolvedPaths.cgroupRecordReadFailed || + sawUnreadableControllerFile || + resolvedPaths.sawUnreadableV1HierarchyMetadata || + (resolvedPaths.sawUnresolvedCgroupLimit && !readV1HardLimit) || + (resolvedPaths.sawMemoryRecord && + !readControllerLimit && + !resolvedPaths.sawObservedUnconstrainedV2Root && + !( + resolvedPaths.sawObservedV2Mapping && + sawDisabledV2MemoryController && + !sawUnreadableControllerFile + )), + }; +} + +function parseProcMemoryBytes(value: string, field: "MemAvailable" | "MemTotal") { + const match = value.match(new RegExp(`^${field}:\\s+(\\d+)\\s+kB$`, "imu")); + const kibibytes = match?.[1]; + if (!kibibytes) { + return null; + } + const parsed = BigInt(kibibytes) * 1024n; + if (parsed < 0n || parsed > BigInt(Number.MAX_SAFE_INTEGER)) { + return null; + } + return Number(parsed); +} + +function readProcMemTotalBytes(params: MemoryLimitParams = {}) { + const configuredTotal = params.procMemTotalBytes; + if (configuredTotal && Number.isFinite(configuredTotal) && configuredTotal > 0) { + return Math.trunc(configuredTotal); + } + + const fsImpl = params.fs ?? fs; + try { + return parseProcMemoryBytes( + fsImpl.readFileSync(params.procMeminfoPath ?? PROC_MEMINFO_PATH, "utf8"), + "MemTotal", + ); + } catch { + return null; + } +} + +function readPhysicalMemoryTotalBytes(params: MemoryLimitParams = {}) { + const totalBytes = params.physicalMemoryBytes ?? os.totalmem(); + return Number.isFinite(totalBytes) && totalBytes > 0 ? Math.trunc(totalBytes) : null; +} + +function readHostAvailableMemoryBytes(params: MemoryLimitParams) { + if (params.availableMemoryBytes !== undefined) { + return Number.isFinite(params.availableMemoryBytes) && params.availableMemoryBytes >= 0 + ? Math.trunc(params.availableMemoryBytes) + : null; + } + if ((params.platform ?? process.platform) === "linux") { + try { + return parseProcMemoryBytes( + (params.fs ?? fs).readFileSync(params.procMeminfoPath ?? PROC_MEMINFO_PATH, "utf8"), + "MemAvailable", + ); + } catch { + return null; + } + } + return null; +} + +function readTsdownMemoryCapacity(params: MemoryLimitParams) { + const cgroupMemory = readCgroupMemoryLimitBytes(params); + if (cgroupMemory.unresolved) { + return { ...cgroupMemory, limitBytes: null, availableBytes: null }; + } + const physicalTotalBytes = readProcMemTotalBytes(params) ?? readPhysicalMemoryTotalBytes(params); + const hostAvailableBytes = readHostAvailableMemoryBytes(params); + const physicalLimitBytes = + hostAvailableBytes === null || physicalTotalBytes === null + ? (hostAvailableBytes ?? physicalTotalBytes) + : Math.min(hostAvailableBytes, physicalTotalBytes); + const limitBytes = + cgroupMemory.limitBytes === null || physicalLimitBytes === null + ? (cgroupMemory.limitBytes ?? physicalLimitBytes) + : Math.min(cgroupMemory.limitBytes, physicalLimitBytes); + return { ...cgroupMemory, limitBytes, availableBytes: hostAvailableBytes }; +} + +function resolveTsdownMemoryBudget(params: ResolvedMemoryLimitParams = {}) { + if (params.resolvedMaxOldSpaceMb !== undefined) { + return { maxOldSpaceMb: params.resolvedMaxOldSpaceMb, unresolvedCgroupMemory: false }; + } + const defaultMaxOldSpaceMb = + (params.platform ?? process.platform) === "win32" + ? DEFAULT_WINDOWS_TSDOWN_MAX_OLD_SPACE_MB + : DEFAULT_TSDOWN_MAX_OLD_SPACE_MB; + const envOverride = parsePositiveIntegerEnv( + (params.env ?? process.env)[TSDOWN_MAX_OLD_SPACE_MB_ENV], + TSDOWN_MAX_OLD_SPACE_MB_ENV, + ); + if (envOverride !== null) { + return { maxOldSpaceMb: envOverride, unresolvedCgroupMemory: false }; + } + const { limitBytes, unresolved } = readTsdownMemoryCapacity(params); + if (unresolved) { + return { maxOldSpaceMb: 1, unresolvedCgroupMemory: true }; + } + if (limitBytes === null) { + return { maxOldSpaceMb: defaultMaxOldSpaceMb, unresolvedCgroupMemory: false }; + } + + const limitMb = Math.floor(limitBytes / 1024 / 1024); + // Never exceed the budget just discovered: a floor applied on top of a real limit produces + // a heap the cgroup cannot honour, which is an OOM kill rather than a smaller build. + const cgroupCap = Math.max(1, limitMb - TSDOWN_CGROUP_MEMORY_HEADROOM_MB); + return { + maxOldSpaceMb: Math.min(defaultMaxOldSpaceMb, cgroupCap), + unresolvedCgroupMemory: false, + }; +} + +/** Independently staged misses may overlap within the two largest compiler budgets. */ +export function resolveStagedDeclarationConcurrency( + groups: readonly { name: string; maxOldSpaceMb: number }[], + params: MemoryLimitParams & { availableParallelism?: number } = {}, +): 1 | 2 { + if ( + groups.length < 2 || + groups.some((group) => !isUnifiedDtsGroup(group.name)) || + new Set(groups.map((group) => group.name)).size !== groups.length || + (params.availableParallelism ?? os.availableParallelism()) < 2 + ) { + return 1; + } + // Frozen or explicit per-child heaps do not establish available batch capacity. + // Unknown available memory stays serial; retain native headroom for each child. + const capacity = readTsdownMemoryCapacity(params); + const requiredBytes = groups + .map((group) => group.maxOldSpaceMb) + .toSorted((left, right) => right - left) + .slice(0, 2) + .reduce((sum, heap) => sum + (heap + TSDOWN_CGROUP_MEMORY_HEADROOM_MB) * 1024 * 1024, 0); + return !capacity.unresolved && + capacity.usageKnown && + capacity.availableBytes !== null && + capacity.limitBytes !== null && + capacity.limitBytes >= requiredBytes + ? 2 + : 1; +} + +const resolveTsdownMaxOldSpaceMb = (params: ResolvedMemoryLimitParams = {}) => + resolveTsdownMemoryBudget(params).maxOldSpaceMb; + +/** + * Measured against this repo by running the full eleven-invocation build inside real cgroups. + * A 5GiB slice resolves this heap, completes, and peaks at 4730MiB. A 4GiB slice (3328MB heap) + * and a 2816MiB slice (2048MB heap) are both killed in the third, unified-runtime invocation, + * which also runs when declarations are disabled. Roughly 380MiB of the peak is rolldown, a + * native addon which --max-old-space-size does not govern at all. + */ +const MEASURED_MIN_TSDOWN_HEAP_MB = 4352; + +/** + * Describes a host that cannot fit the build, or null when it can. Reported before any + * output is cleaned: a host that cannot rebuild must not also lose the build it already + * has. Fatal by default because continuing either aborts partway through the invocation + * list or, when the heap outruns a container limit, thrashes at the ceiling instead of + * failing, which starves every other process on the machine. + */ +export function describeInsufficientTsdownHeap( + params: ResolvedMemoryLimitParams = {}, + budget = resolveTsdownMemoryBudget(params), +) { + const { maxOldSpaceMb } = budget; + if (maxOldSpaceMb >= MEASURED_MIN_TSDOWN_HEAP_MB) { + return null; + } + const env = params.env ?? process.env; + const explicitHeapMb = parsePositiveIntegerEnv( + env[TSDOWN_MAX_OLD_SPACE_MB_ENV], + TSDOWN_MAX_OLD_SPACE_MB_ENV, + ); + const heapOverrideEnv = Object.hasOwn(env, "OPENCLAW_INTERNAL_DOCKER_BUILD_PLUGIN_IDS") + ? DOCKER_TSDOWN_MAX_OLD_SPACE_MB_ENV + : TSDOWN_MAX_OLD_SPACE_MB_ENV; + const fatal = explicitHeapMb === null; + const outcome = fatal + ? budget.unresolvedCgroupMemory + ? [ + "Stopping before any build output is removed. Pick one:", + " - run the build where the process cgroup limit is visible", + ` - set ${heapOverrideEnv}=<MB> to explicitly attempt the build anyway`, + ] + : [ + "Stopping before any build output is removed. Pick one:", + " - give this machine or container more memory", + ` - set ${heapOverrideEnv}=<MB> to explicitly attempt the build anyway`, + ] + : [ + `Continuing because ${heapOverrideEnv} explicitly requests ${explicitHeapMb}MB. Existing build output will now be cleaned; the build may stall or fail.`, + ]; + return { + fatal, + message: [ + budget.unresolvedCgroupMemory + ? "[tsdown-build] The process memory limit is not visible through this cgroup mount namespace, so OpenClaw cannot choose a safe default heap." + : `[tsdown-build] The resolved OpenClaw build heap is ${maxOldSpaceMb}MB, ` + + `and a full build needs ${MEASURED_MIN_TSDOWN_HEAP_MB}MB, peaking near 4.7GB once rolldown's ` + + `native allocations are counted; those are not covered by --max-old-space-size.`, + ...outcome, + ].join("\n"), + }; +} + +function parseMaxOldSpaceSizeMb(value: unknown, fallbackMb: number) { + const parsed = Number(value); + if (!Number.isFinite(parsed) || parsed <= 0) { + return fallbackMb; + } + return Math.trunc(parsed); +} + +function normalizeMaxOldSpaceSizeMb(value: unknown, maxOldSpaceMb: number) { + // Build wrappers may inherit smaller runner-level caps; tsdown needs the + // resolved build heap while still respecting cgroup-derived upper bounds. + const parsed = parseMaxOldSpaceSizeMb(value, maxOldSpaceMb); + if (parsed < maxOldSpaceMb) { + return maxOldSpaceMb; + } + return Math.min(parsed, maxOldSpaceMb); +} + +function normalizeTsdownNodeOptions(nodeOptions: string, params: ResolvedMemoryLimitParams = {}) { + const maxOldSpaceMb = resolveTsdownMaxOldSpaceMb(params); + const parts = nodeOptions.trim().split(/\s+/u).filter(Boolean); + const normalized: string[] = []; + let foundMaxOldSpaceSize = false; + + for (let index = 0; index < parts.length; index += 1) { + const part = parts[index]; + if (part === undefined) { + continue; + } + const inlineMatch = part.match(/^--max-old-space-size=(\d+)$/u); + if (inlineMatch) { + foundMaxOldSpaceSize = true; + const value = normalizeMaxOldSpaceSizeMb(inlineMatch[1], maxOldSpaceMb); + normalized.push(`--max-old-space-size=${value}`); + continue; + } + + if (part === "--max-old-space-size") { + foundMaxOldSpaceSize = true; + const next = parts[index + 1]; + const value = normalizeMaxOldSpaceSizeMb(next, maxOldSpaceMb); + normalized.push(`--max-old-space-size=${value}`); + if (next !== undefined) { + index += 1; + } + continue; + } + + normalized.push(part); + } + + if (!foundMaxOldSpaceSize) { + normalized.push(`--max-old-space-size=${maxOldSpaceMb}`); + } + + return normalized.join(" "); +} + +function resolveTsdownEnv( + env: NodeJS.ProcessEnv, + params: ResolvedMemoryLimitParams = {}, +): NodeJS.ProcessEnv { + const nodeOptions = env.NODE_OPTIONS?.trim() ?? ""; + return { + ...env, + NODE_OPTIONS: normalizeTsdownNodeOptions(nodeOptions, params), + }; +} + +function tsdownBuildUsage() { + return [ + "Usage: node --import tsx scripts/tsdown-build.mts [tsdown args...]", + "", + "Builds OpenClaw with tsdown and validates emitted import diagnostics.", + "", + "Options:", + " -h, --help Show this help without starting tsdown.", + "", + "Other arguments are forwarded to tsdown.", + ].join("\n"); +} + +export function parseTsdownBuildArgs(argv: string[]) { + if (argv.includes("--help") || argv.includes("-h")) { + return { + forwardedArgs: [], + help: true, + }; + } + return { + forwardedArgs: argv, + help: false, + }; +} + +export function createTsdownOutputScanner(params: { maxCaptureBytes?: number } = {}) { + const maxCaptureBytes = params.maxCaptureBytes ?? DEFAULT_CAPTURE_BYTES; + let captured = ""; + let pendingLine = ""; + let hasIneffectiveDynamicImport = false; + let fatalUnresolvedImport: string | null = null; + + function scanLines(text: string) { + const combined = pendingLine + text; + const lines = combined.split(/\r?\n/u); + pendingLine = lines.pop() ?? ""; + for (const line of lines) { + fatalUnresolvedImport ??= findFatalUnresolvedImport([line]); + } + } + + return { + append(chunk: unknown) { + const text = Buffer.isBuffer(chunk) ? chunk.toString("utf8") : String(chunk); + if (text.includes(INEFFECTIVE_DYNAMIC_IMPORT_MARKER)) { + hasIneffectiveDynamicImport = true; + } + scanLines(text); + captured += text; + if (captured.length > maxCaptureBytes) { + captured = captured.slice(-maxCaptureBytes); + } + }, + finish() { + if (pendingLine) { + fatalUnresolvedImport ??= findFatalUnresolvedImport([pendingLine]); + pendingLine = ""; + } + return { + captured, + hasIneffectiveDynamicImport, + fatalUnresolvedImport, + }; + }, + }; +} + +export function resolveTsdownBuildInvocation( + params: TsdownBuildParams = {}, +): TsdownBuildInvocation { + const env = resolveTsdownEnv(params.env ?? process.env, params); + const args = params.args ?? []; + const forwardedArgs = wrapperOwnsTsdownCleanup(args) + ? args.filter((arg) => arg !== "--clean" && !arg.startsWith("--clean=")) + : args; + const tsdownArgs = [ + "--config-loader", + "unrun", + "--logLevel", + logLevel, + "--no-clean", + ...forwardedArgs, + ]; + if (env.OPENCLAW_BUILD_ALL_NO_PNPM === "1") { + return { + command: params.nodeExecPath ?? process.execPath, + args: ["node_modules/tsdown/dist/run.mjs", ...tsdownArgs], + options: { + stdio: tsdownStdio(), + shell: false, + windowsVerbatimArguments: undefined, + env, + }, + }; + } + const runner = resolvePnpmRunner({ + env, + pnpmArgs: ["exec", "tsdown", ...tsdownArgs], + nodeExecPath: params.nodeExecPath ?? process.execPath, + npmExecPath: params.npmExecPath ?? env.npm_execpath, + comSpec: params.comSpec, + platform: (params.platform ?? process.platform) === "win32" ? "win32" : "linux", + }); + return { + command: runner.command, + args: runner.args, + options: { + stdio: tsdownStdio(), + shell: runner.shell, + windowsVerbatimArguments: runner.windowsVerbatimArguments, + env, + }, + }; +} + +function selectsMainConfig(args: string[]) { + const config = readForwardedOption(args, ["--config", "-c"]); + if (config === undefined) { + return false; + } + const resolvedConfig = path.resolve(config); + const mainConfig = path.resolve("tsdown.config.ts"); + return resolvedConfig === mainConfig || resolvedConfig === path.dirname(mainConfig); +} + +function resolveSerializedMainConfigGroups(filters: string[]) { + const uniqueFilters = [...new Set(filters)]; + if (filters.length === 0 || filters.includes(".")) { + return SERIALIZED_MAIN_CONFIG_GROUPS; + } + if ( + uniqueFilters.some( + (filter) => filter !== "." && !SERIALIZED_MAIN_CONFIG_GROUPS.includes(filter), + ) + ) { + return null; + } + if (filters.includes(TSDOWN_UNIFIED_CONFIG_GROUP) && !filters.some(isUnifiedDtsGroup)) { + return filters.includes(TSDOWN_PACKAGE_CONFIG_GROUP) + ? SERIALIZED_MAIN_CONFIG_GROUPS + : SERIALIZED_MAIN_CONFIG_GROUPS.slice(1); + } + if (uniqueFilters.length < 2) { + return null; + } + const selectedFilters = new Set(uniqueFilters); + return SERIALIZED_MAIN_CONFIG_GROUPS.filter((group) => selectedFilters.has(group)); +} + +/** Builds declarations in dependency order without overlapping the largest graphs. */ +export function resolveTsdownBuildInvocations(params: TsdownBuildParams = {}) { + const forwardedArgs = params.args ?? []; + readForwardedScalarOption(forwardedArgs, ["--out-dir", "-d"], "--out-dir/-d"); + if (forwardedArgs.filter(isConfigArg).length > 1) { + throw new Error("tsdown build accepts only one --config/-c/--no-config selector"); + } + const env = params.env ?? process.env; + const forwardedFilters = readForwardedOptions(forwardedArgs, ["--filter", "-F"]); + const hasForwardedFilter = forwardedArgs.some(isFilterArg); + const aiArgs = forwardedArgs.filter((arg, index) => { + const previous = forwardedArgs[index - 1]; + return !isFilterArg(arg) && !isFilterFlag(previous); + }); + const dtsArg = aiArgs.findLast((arg) => arg === "--dts" || arg === "--no-dts"); + const declarationsEnabled = dtsArg + ? dtsArg === "--dts" + : env[RUN_NODE_SKIP_DTS_BUILD_ENV] !== "1"; + const hasForwardedConfig = aiArgs.some(isConfigArg); + + const declarationEnv = + declarationsEnabled && env[RUN_NODE_SKIP_DTS_BUILD_ENV] === "1" + ? { ...env, [RUN_NODE_SKIP_DTS_BUILD_ENV]: "0" } + : env; + + if (forwardedArgs.some(isWatchArg)) { + if (!hasForwardedConfig) { + throw new Error( + "tsdown build watch mode requires an explicit --config/-c or --no-config selector. Run separate watchers for tsdown.config.ts and tsdown.ai.config.ts to watch both graphs.", + ); + } + // Watchers are long-lived, so sequential group orchestration would block forever on the + // first child. Keep watch mode inside tsdown's single owning process. + return [resolveTsdownBuildInvocation(params)]; + } + + if (hasForwardedConfig) { + if (declarationsEnabled && selectsMainConfig(forwardedArgs)) { + const serializedGroups = resolveSerializedMainConfigGroups(forwardedFilters); + if (serializedGroups) { + return serializedGroups.map((group) => + resolveTsdownBuildInvocation({ + ...params, + args: ["--filter", group, ...aiArgs], + env: declarationEnv, + }), + ); + } + } + return [resolveTsdownBuildInvocation(params)]; + } + + const invocations = [ + resolveTsdownBuildInvocation({ + ...params, + args: ["--config", "tsdown.ai.config.ts", ...aiArgs], + }), + ]; + + const forwardedFilterSet = new Set(forwardedFilters); + const uniqueForwardedFilters = SERIALIZED_MAIN_CONFIG_GROUPS.filter((group) => + forwardedFilterSet.has(group), + ); + const hasUnknownFilter = forwardedFilters.some( + (filter) => filter !== "." && !SERIALIZED_MAIN_CONFIG_GROUPS.includes(filter), + ); + const serializedGroups = forwardedFilters.includes(".") + ? SERIALIZED_MAIN_CONFIG_GROUPS + : !hasUnknownFilter && uniqueForwardedFilters.length > 1 + ? uniqueForwardedFilters + : null; + if (!declarationsEnabled || (hasForwardedFilter && !serializedGroups)) { + const mainEnv = + !declarationsEnabled && env[RUN_NODE_SKIP_DTS_BUILD_ENV] !== "1" + ? { ...env, [RUN_NODE_SKIP_DTS_BUILD_ENV]: "1" } + : env; + invocations.push(resolveTsdownBuildInvocation({ ...params, env: mainEnv })); + return invocations; + } + + for (const group of serializedGroups ?? SERIALIZED_MAIN_CONFIG_GROUPS) { + invocations.push( + resolveTsdownBuildInvocation({ + ...params, + args: ["--filter", group, ...aiArgs], + env: declarationEnv, + }), + ); + } + return invocations; +} + +function isFullTsdownBuildPlan(args: string[]) { + const filters = readForwardedOptions(args, ["--filter", "-F"]); + const selectsUnifiedRuntime = + filters.length === 0 || filters.includes(TSDOWN_UNIFIED_CONFIG_GROUP) || filters.includes("."); + return selectsUnifiedRuntime && (!args.some(isConfigArg) || selectsMainConfig(args)); +} + +export function resolveTsdownBuildPlan(params: TsdownBuildParams = {}) { + const budget = resolveTsdownMemoryBudget(params); + const maxOldSpaceMb = budget.maxOldSpaceMb; + const preparedParams = { + ...params, + resolvedMaxOldSpaceMb: maxOldSpaceMb, + }; + return { + env: resolveTsdownEnv(params.env ?? process.env, preparedParams), + maxOldSpaceMb, + heapShortfall: + budget.unresolvedCgroupMemory || isFullTsdownBuildPlan(params.args ?? []) + ? describeInsufficientTsdownHeap(preparedParams, budget) + : null, + invocations: resolveTsdownBuildInvocations(preparedParams), + }; +} + +export function prepareTsdownBuildExecution( + params: TsdownBuildParams = {}, + hooks: { + cleanup?: (args: string[]) => void; + reportShortfall?: ( + shortfall: NonNullable<ReturnType<typeof describeInsufficientTsdownHeap>>, + ) => void; + } = {}, +) { + const args = params.args ?? []; + const plan = resolveTsdownBuildPlan(params); + if (plan.heapShortfall) { + hooks.reportShortfall?.(plan.heapShortfall); + if (plan.heapShortfall.fatal) { + return null; + } + } + const cleanup = + hooks.cleanup ?? + ((forwardedArgs: string[]) => { + const roots = resolveTsdownCleanOutputRoots(forwardedArgs); + // Reject unsafe custom output roots before any preparatory mutation. The second + // validation in cleanTsdownOutputRoots closes a symlink race before deletion. + assertTsdownCleanOutputRoots({ roots }); + pruneUntrackedGeneratedSourceDeclarations(); + pruneStaleRuntimeSymlinks(); + cleanTsdownOutputRoots({ roots }); + }); + cleanup(args); + return plan; +} + +type TaskkillRunner = ( + command: string, + args: string[], + options: { killSignal?: NodeJS.Signals; stdio?: StdioOptions; timeout?: number }, +) => { error?: Error; status: number | null }; + +export async function runTsdownBuildInvocation( + invocation: TsdownBuildInvocation, + params: { + stdout?: { write(chunk: unknown): unknown }; + stderr?: { write(chunk: unknown): unknown }; + env?: NodeJS.ProcessEnv; + scanner?: ReturnType<typeof createTsdownOutputScanner>; + platform?: NodeJS.Platform; + runTaskkill?: TaskkillRunner; + } = {}, +): Promise<TsdownBuildResult> { + const stdout = params.stdout ?? process.stdout; + const stderr = params.stderr ?? process.stderr; + const env = params.env ?? process.env; + const scanner = params.scanner ?? createTsdownOutputScanner(); + const timeoutMs = parsePositiveIntegerEnv( + env.OPENCLAW_TSDOWN_TIMEOUT_MS, + "OPENCLAW_TSDOWN_TIMEOUT_MS", + ); + const heartbeatMs = + parseNonNegativeIntegerEnv(env.OPENCLAW_TSDOWN_HEARTBEAT_MS, "OPENCLAW_TSDOWN_HEARTBEAT_MS") ?? + DEFAULT_HEARTBEAT_MS; + let timedOut = false; + let parentSignal: NodeJS.Signals | undefined; + let settled = false; + let lastOutputAt = Date.now(); + let forceKillAt: number | null = null; + + const platform = params.platform ?? process.platform; + const runTaskkill = params.runTaskkill ?? spawnSync; + const useProcessGroup = platform !== "win32"; + const [stdin, stdoutPipe, stderrPipe] = invocation.options.stdio; + if (stdin !== "ignore" || stdoutPipe !== "pipe" || stderrPipe !== "pipe") { + throw new Error("tsdown invocation stdio must be ignore/pipe/pipe"); + } + const child = spawn(invocation.command, invocation.args, { + ...invocation.options, + stdio: tsdownStdio(), + detached: useProcessGroup, + }); + const pidText = child.pid ? ` pid=${child.pid}` : ""; + + function markOutput() { + lastOutputAt = Date.now(); + } + + function signalChild(signal: NodeJS.Signals) { + terminateManagedChild(child, signal, { + platform, + runTaskkill, + useProcessGroup, + }); + } + + const parentSignalHandlers: { signal: NodeJS.Signals; handler: () => void }[] = []; + function cleanupParentSignalHandlers() { + for (const { signal, handler } of parentSignalHandlers) { + process.off(signal, handler); + } + parentSignalHandlers.length = 0; + } + + function relayParentSignal(signal: NodeJS.Signals) { + const handler = () => { + parentSignal ??= signal; + signalChild(signal); + signalChild("SIGKILL"); + }; + parentSignalHandlers.push({ signal, handler }); + process.once(signal, handler); + } + + if (useProcessGroup) { + relayParentSignal("SIGINT"); + relayParentSignal("SIGTERM"); + relayParentSignal("SIGHUP"); + } + + const processTreeAlive = () => + inspectManagedProcessGroup(child, { + errorPolicy: "alive-on-eperm", + inspectLeaderWhenNoGroup: true, + platform, + }) === "live"; + const waitForProcessTreeExit = (timeoutMsToWait: number) => + waitForManagedProcessGroupExit(child, timeoutMsToWait, { + errorPolicy: "alive-on-eperm", + inspectLeaderWhenNoGroup: true, + platform, + }); + + async function finishTimedOutProcessTree() { + const graceRemainingMs = + forceKillAt === null ? TERMINATION_GRACE_MS : Math.max(0, forceKillAt - Date.now()); + if (graceRemainingMs > 0) { + await waitForProcessTreeExit(graceRemainingMs); + } + if (processTreeAlive()) { + signalChild("SIGKILL"); + await waitForProcessTreeExit(POST_FORCE_KILL_WAIT_MS); + } + } + + child.stdout?.on("data", (chunk) => { + markOutput(); + scanner.append(chunk); + stdout.write(chunk); + }); + child.stderr?.on("data", (chunk) => { + markOutput(); + scanner.append(chunk); + stderr.write(chunk); + }); + + const heartbeat = + heartbeatMs > 0 + ? setInterval(() => { + if (settled) { + return; + } + const silentForMs = Date.now() - lastOutputAt; + if (silentForMs < heartbeatMs) { + return; + } + stderr.write( + `[tsdown-build] still running${pidText}; no output for ${Math.round( + silentForMs / 1000, + )}s\n`, + ); + lastOutputAt = Date.now(); + }, heartbeatMs).unref() + : null; + + const timeout = + timeoutMs !== null + ? setTimeout(() => { + timedOut = true; + stderr.write(`[tsdown-build] timeout after ${timeoutMs}ms${pidText}; sending SIGTERM\n`); + signalChild("SIGTERM"); + forceKillAt = Date.now() + TERMINATION_GRACE_MS; + setTimeout(() => { + if (!settled) { + stderr.write(`[tsdown-build] forcing SIGKILL${pidText}\n`); + signalChild("SIGKILL"); + } + }, TERMINATION_GRACE_MS).unref(); + }, timeoutMs).unref() + : null; + + return new Promise<TsdownBuildResult>((resolve) => { + child.once("error", (error) => { + settled = true; + cleanupParentSignalHandlers(); + clearInterval(heartbeat ?? undefined); + clearTimeout(timeout ?? undefined); + stderr.write(`[tsdown-build] failed to start: ${String(error)}\n`); + resolve({ + status: 1, + signal: null, + timedOut, + error, + ...scanner.finish(), + }); + }); + child.once("close", (status, signal) => { + let exitStatus = status; + function finish() { + settled = true; + cleanupParentSignalHandlers(); + clearInterval(heartbeat ?? undefined); + clearTimeout(timeout ?? undefined); + resolve({ + status: parentSignal ? signalExitCode(parentSignal) : exitStatus, + signal: parentSignal ?? signal, + timedOut, + error: null, + ...scanner.finish(), + }); + } + + void (async () => { + if (timedOut || parentSignal) { + await finishTimedOutProcessTree(); + } else if (processTreeAlive()) { + signalChild("SIGKILL"); + await waitForProcessTreeExit(POST_FORCE_KILL_WAIT_MS); + exitStatus = 1; + } + if (processTreeAlive()) { + // Keep ownership when the group could still mutate output after close. + throw Object.assign(new Error("tsdown process group did not exit"), { + code: "EPROCESSGROUP_CLEANUP_FAILED", + processTreeState: "live", + }); + } + finish(); + })().catch((error: unknown) => { + settled = true; + cleanupParentSignalHandlers(); + clearInterval(heartbeat ?? undefined); + clearTimeout(timeout ?? undefined); + resolve({ + status: 1, + signal, + timedOut, + error: toErrorObject(error, "tsdown cleanup failed"), + ...scanner.finish(), + }); + }); + }); + }); +} + +async function executeTsdownInvocation( + invocation: TsdownBuildInvocation, + index: number, + count: number, +): Promise<number> { + const startedAt = performance.now(); + const result = await runTsdownBuildInvocation(invocation); + if (result.error) { + throw result.error; + } + console.log( + `[tsdown-build] invocation ${index + 1}/${count} finished in ${((performance.now() - startedAt) / 1000).toFixed(1)}s`, + ); + if (result.status === 0 && result.hasIneffectiveDynamicImport) { + console.error( + "Build emitted [INEFFECTIVE_DYNAMIC_IMPORT]. Replace transparent runtime re-export facades with real runtime boundaries.", + ); + return 1; + } + + if (result.status === 0 && result.fatalUnresolvedImport) { + console.error( + `Build emitted [UNRESOLVED_IMPORT] outside extensions: ${result.fatalUnresolvedImport}`, + ); + return 1; + } + + if (result.timedOut) { + return 124; + } + + if (typeof result.status === "number") { + return result.status; + } + + return 1; +} + +/** Execute CLI and staged declaration plans with the same diagnostics and deadlines. */ +export async function executeTsdownBuildPlan( + plan: NonNullable<ReturnType<typeof prepareTsdownBuildExecution>>, + concurrency: 1 | 2 = 1, +) { + let next = 0; + let exitCode = plan.invocations.length ? 0 : 1; + const failedExits: { index: number; code: number }[] = []; + const run = async () => { + while (exitCode === 0 && next < plan.invocations.length) { + const index = next++; + try { + const code = await executeTsdownInvocation( + plan.invocations[index]!, + index, + plan.invocations.length, + ); + if (code !== 0) { + failedExits.push({ index, code }); + exitCode ||= code; + } + } catch (error) { + exitCode ||= 1; + throw error; + } + } + }; + // A failed sibling stops admission, not the lifetime of an already admitted compiler. + // Join every result before the writer may seal, publish, or release its private stages. + const results = await Promise.allSettled( + Array.from({ length: Math.min(concurrency, plan.invocations.length) }, run), + ); + const failures = results.flatMap((result) => + result.status === "rejected" ? [result.reason] : [], + ); + if (failures.length || failedExits.length > 1) { + failures.push( + ...failedExits.map(({ index, code }) => + Object.assign(new Error(`tsdown invocation ${index + 1} failed with exit ${code}`), { + exitCode: code, + }), + ), + ); + throw failures.length === 1 + ? failures[0] + : new AggregateError(failures, "tsdown compiler batch failed"); + } + return exitCode; +} + +export async function runTsdownBuild( + argv: string[] = process.argv.slice(2), + options: { + cwd?: string; + executeBuild?: (forwardedArgs: string[]) => Promise<number>; + } = {}, +): Promise<number> { + const args = parseTsdownBuildArgs(argv); + if (args.help) { + console.log(tsdownBuildUsage()); + return 0; + } + let code: number; + if (options.executeBuild) { + code = await options.executeBuild(args.forwardedArgs); + } else { + const plan = prepareTsdownBuildExecution( + { args: args.forwardedArgs }, + { + reportShortfall(shortfall) { + if (shortfall.fatal) { + console.error(shortfall.message); + } else { + console.warn(shortfall.message); + } + }, + }, + ); + if (!plan) { + return 1; + } + code = await executeTsdownBuildPlan(plan); + } + if (code === 0) { + sanitizeTsdownBuildOutputRoots(args.forwardedArgs, options.cwd); + } + return code; +} + +if (isDirectRunUrl(process.argv[1], import.meta.url)) { + const argv = process.argv.slice(2); + process.exitCode = parseTsdownBuildArgs(argv).help + ? await runTsdownBuild(argv) + : await withDistArtifactOwnership(process.cwd(), () => runTsdownBuild(argv)); +} diff --git a/scripts/tsx.mjs b/scripts/tsx.mjs new file mode 100644 index 0000000000000000000000000000000000000000..5a77d5f15a20f084db0c120581ef936970816101 --- /dev/null +++ b/scripts/tsx.mjs @@ -0,0 +1,14 @@ +import { registerToolingTsx } from "./lib/tsx-cli-shim.mjs"; + +// fork() inherits execArgv but can change cwd. Keep our root-script preload +// resolvable there without rewriting unrelated flags or other preloads. +const preload = "./scripts/tsx.mjs"; +for (const [index, arg] of process.execArgv.entries()) { + if (arg === preload && process.execArgv[index - 1] === "--import") { + process.execArgv[index] = import.meta.url; + } else if (arg === `--import=${preload}`) { + process.execArgv[index] = `--import=${import.meta.url}`; + } +} + +await registerToolingTsx(); diff --git a/scripts/update-clawtributors.ts b/scripts/update-clawtributors.ts new file mode 100644 index 0000000000000000000000000000000000000000..550924d04df5454f599f783d3becdee50551b0d4 --- /dev/null +++ b/scripts/update-clawtributors.ts @@ -0,0 +1,1050 @@ +// Update Clawtributors script supports OpenClaw repository automation. +import { execSync } from "node:child_process"; +import { readFileSync, writeFileSync } from "node:fs"; +import { resolve } from "node:path"; +import pMap, { pMapSkip } from "p-map"; +import { expectDefined } from "../packages/normalization-core/src/expect.js"; +import { cancelResponseReaderSoon } from "./lib/bounded-response.mjs"; +import { execPlainGh } from "./lib/plain-gh.mjs"; +import type { ApiContributor, Entry, MapConfig, User } from "./update-clawtributors.types.js"; + +const REPO = "openclaw/openclaw"; +const PER_LINE = 10; +const AVATAR_PROBE_SIZE = 40; +const AVATAR_PROBE_MAX_BYTES = 256 * 1024; +const AVATAR_PROBE_TIMEOUT_MS = 8000; +const AVATAR_SIZE = 48; +// The 5,000-PR history query can take about a minute; preserve healthy pagination +// headroom while bounding a stalled GitHub CLI process. +const GH_COMMAND_TIMEOUT_MS = 120_000; +const CLAWTRIBUTORS_START = "<!-- clawtributors:start -->"; +const CLAWTRIBUTORS_END = "<!-- clawtributors:end -->"; +const CLAWTRIBUTORS_HIDDEN_START = "<!-- clawtributors:hidden:start"; +const CLAWTRIBUTORS_HIDDEN_END = "clawtributors:hidden:end -->"; + +const mapPath = resolve("scripts/clawtributors-map.json"); +const mapConfig = JSON.parse(readFileSync(mapPath, "utf8")) as MapConfig; + +const displayName = mapConfig.displayName ?? {}; +const nameToLogin = normalizeMap(mapConfig.nameToLogin ?? {}); +const emailToLogin = normalizeMap(mapConfig.emailToLogin ?? {}); +const ensureLogins = (mapConfig.ensureLogins ?? []).map((login) => login.toLowerCase()); + +const readmePath = resolve("README.md"); +const seedCommit = mapConfig.seedCommit ?? null; +const seedEntries = seedCommit ? parseReadmeEntries(run(`git show ${seedCommit}:README.md`)) : []; +const currentReadme = readFileSync(readmePath, "utf8"); +const hiddenReadmeLogins = new Set(parseHiddenReadmeLogins(currentReadme)); +const raw = runGh(["api", `repos/${REPO}/contributors?per_page=100&anon=1`, "--paginate"]); +const contributors = parsePaginatedJson(raw) as ApiContributor[]; +const apiByLogin = new Map<string, User>(); +const contributionsByLogin = new Map<string, number>(); +const defaultAvatarByLogin = new Map<string, Promise<boolean>>(); + +for (const item of contributors) { + if (!item?.login || !item?.html_url || !item?.avatar_url) { + continue; + } + if (typeof item.contributions === "number") { + contributionsByLogin.set(item.login.toLowerCase(), item.contributions); + } + apiByLogin.set(item.login.toLowerCase(), { + id: item.id, + login: item.login, + html_url: item.html_url, + avatar_url: normalizeAvatar(item.avatar_url), + }); +} + +for (const login of ensureLogins) { + if (!apiByLogin.has(login)) { + const user = fetchUser(login); + if (user) { + apiByLogin.set(user.login.toLowerCase(), user); + } + } +} + +const linesByLogin = new Map<string, number>(); +const firstCommitByLogin = new Map<string, string>(); + +// %x1f = unit separator to avoid collisions with author names containing "|" +const log = run("git log --reverse --format=%aN%x1f%aE%x1f%aI --numstat"); + +let currentName: string | null = null; +let currentEmail: string | null = null; + +for (const line of log.split("\n")) { + if (!line.trim()) { + continue; + } + + if (line.includes("\x1f") && !/^[0-9-]/.test(line)) { + const [name, email, date] = line.split("\x1f", 3); + currentName = name?.trim() ?? null; + currentEmail = email?.trim().toLowerCase() ?? null; + + // Track first commit date per login (log is --reverse so first seen = earliest) + if (currentName && date) { + const login = resolveLogin(currentName, currentEmail, apiByLogin, nameToLogin, emailToLogin); + if (login) { + const key = login.toLowerCase(); + if (!firstCommitByLogin.has(key)) { + firstCommitByLogin.set(key, date.slice(0, 10)); + } + } + } + continue; + } + + if (!currentName) { + continue; + } + + const parts = line.split("\t"); + if (parts.length < 3) { + continue; + } + + // Skip docs paths so bulk-generated i18n scaffolds don't inflate rankings + const filePath = expectDefined(parts[2], "git numstat file path"); + if (filePath.startsWith("docs/")) { + continue; + } + + const adds = parseCount(expectDefined(parts[0], "git numstat additions")); + const dels = parseCount(expectDefined(parts[1], "git numstat deletions")); + const total = adds + dels; + if (!total) { + continue; + } + + const login = resolveLogin(currentName, currentEmail, apiByLogin, nameToLogin, emailToLogin); + if (!login) { + continue; + } + + const key = login.toLowerCase(); + linesByLogin.set(key, (linesByLogin.get(key) ?? 0) + total); +} + +for (const login of ensureLogins) { + if (!linesByLogin.has(login)) { + linesByLogin.set(login, 0); + } +} + +const prsByLogin = new Map<string, number>(); +const prRaw = runGh([ + "pr", + "list", + "-R", + REPO, + "--state", + "merged", + "--limit", + "5000", + "--json", + "author", + "--jq", + ".[].author.login", +]); +for (const login of prRaw.split("\n")) { + const trimmed = login.trim().toLowerCase(); + if (!trimmed) { + continue; + } + prsByLogin.set(trimmed, (prsByLogin.get(trimmed) ?? 0) + 1); +} + +// Repo epoch for tenure calculation (root commit date) +const rootCommit = run("git rev-list --max-parents=0 HEAD").split("\n")[0]; +const repoEpochStr = run(`git log --format=%aI -1 ${rootCommit}`); +const repoEpoch = new Date(repoEpochStr.slice(0, 10)).getTime(); +const nowDate = new Date().toISOString().slice(0, 10); +const now = new Date(nowDate).getTime(); +const repoAgeDays = Math.max(1, (now - repoEpoch) / 86_400_000); + +// Composite score: +// base = commits*2 + merged_PRs*10 + sqrt(code_LOC) +// tenure = 1.0 + (days_since_first_commit / repo_age)^2 * 0.5 +// score = base * tenure +// Squared curve: only true early contributors get meaningful boost. +// Day-1 = 1.5x, halfway through repo life = 1.125x, recent = ~1.0x. +function computeScore(loc: number, commits: number, prs: number, firstDate: string): number { + const base = commits * 2 + prs * 10 + Math.sqrt(loc); + const daysIn = firstDate + ? Math.max(0, (now - new Date(firstDate.slice(0, 10)).getTime()) / 86_400_000) + : 0; + const tenureRatio = Math.min(1, daysIn / repoAgeDays); + const tenure = 1 + tenureRatio * tenureRatio * 0.5; + return base * tenure; +} + +const entriesByKey = new Map<string, Entry>(); + +for (const seed of seedEntries) { + const login = + (seed.html_url ? loginFromUrl(seed.html_url) : null) ?? + resolveLogin(seed.display, null, apiByLogin, nameToLogin, emailToLogin); + const accountId = accountIdFromAvatarUrl(seed.avatar_url); + if (!login && !accountId) { + continue; + } + const loginKey = login?.toLowerCase(); + const userByLogin = loginKey ? apiByLogin.get(loginKey) : undefined; + // Avatar account IDs survive renames and cannot be claimed by a handle squatter. + const user = accountId + ? userByLogin?.id === accountId + ? userByLogin + : fetchUserByAccountId(accountId) + : (userByLogin ?? (login ? fetchUser(login) : null)); + if (!user) { + const key = accountId ? `github-id:${accountId}` : expectDefined(loginKey, "seed login key"); + entriesByKey.set(key, { + key, + display: seed.display, + html_url: null, + avatar_url: normalizeAvatar(seed.avatar_url), + lines: 0, + commits: 0, + prs: 0, + score: 0, + firstCommitDate: loginKey ? (firstCommitByLogin.get(loginKey) ?? "") : "", + }); + continue; + } + const key = user.login.toLowerCase(); + apiByLogin.set(key, user); + const existing = entriesByKey.get(key); + if (!existing) { + const fd = firstCommitByLogin.get(key) ?? ""; + entriesByKey.set(key, { + key, + login: user.login, + display: seed.display, + html_url: user.html_url, + avatar_url: user.avatar_url, + lines: 0, + commits: 0, + prs: 0, + score: 0, + firstCommitDate: fd, + }); + } else { + existing.display = existing.display || seed.display; + existing.login = user.login; + existing.html_url = user.html_url; + existing.avatar_url = user.avatar_url; + } +} + +for (const item of contributors) { + const baseName = item.name?.trim() || item.email?.trim() || item.login?.trim(); + if (!baseName) { + continue; + } + + const resolvedLogin = item.login + ? item.login + : resolveLogin(baseName, item.email ?? null, apiByLogin, nameToLogin, emailToLogin); + + if (!resolvedLogin) { + continue; + } + + const key = resolvedLogin.toLowerCase(); + const user = apiByLogin.get(key) ?? fetchUser(resolvedLogin); + if (!user) { + continue; + } + apiByLogin.set(key, user); + + const existing = entriesByKey.get(key); + if (!existing) { + const loc = linesByLogin.get(key) ?? 0; + const commits = contributionsByLogin.get(key) ?? 0; + const prs = prsByLogin.get(key) ?? 0; + const fd = firstCommitByLogin.get(key) ?? ""; + entriesByKey.set(key, { + key, + login: user.login, + display: pickDisplay(baseName, user.login), + html_url: user.html_url, + avatar_url: normalizeAvatar(user.avatar_url), + lines: loc > 0 ? loc : commits, + commits, + prs, + score: computeScore(loc, commits, prs, fd), + firstCommitDate: fd, + }); + } else { + existing.login = user.login; + existing.display = pickDisplay(baseName, user.login, existing.display); + existing.html_url = user.html_url; + existing.avatar_url = normalizeAvatar(user.avatar_url); + const loc = linesByLogin.get(key) ?? 0; + const commits = contributionsByLogin.get(key) ?? 0; + const prs = prsByLogin.get(key) ?? 0; + const fd = firstCommitByLogin.get(key) ?? existing.firstCommitDate; + existing.lines = Math.max(existing.lines, loc > 0 ? loc : commits); + existing.commits = Math.max(existing.commits, commits); + existing.prs = Math.max(existing.prs, prs); + existing.firstCommitDate = fd || existing.firstCommitDate; + existing.score = Math.max(existing.score, computeScore(loc, commits, prs, fd)); + } +} + +for (const [login, loc] of linesByLogin.entries()) { + if (entriesByKey.has(login)) { + continue; + } + let user = apiByLogin.get(login); + if (!user) { + user = fetchUser(login) || undefined; + } + if (user) { + const commits = contributionsByLogin.get(login) ?? 0; + const prs = prsByLogin.get(login) ?? 0; + const fd = firstCommitByLogin.get(login) ?? ""; + entriesByKey.set(login, { + key: login, + login: user.login, + display: displayName[user.login.toLowerCase()] ?? user.login, + html_url: user.html_url, + avatar_url: normalizeAvatar(user.avatar_url), + lines: loc > 0 ? loc : commits, + commits, + prs, + score: computeScore(loc, commits, prs, fd), + firstCommitDate: fd, + }); + } +} + +const entries = Array.from(entriesByKey.values()); +const visibleEntries = await filterVisibleEntries(entries, hiddenReadmeLogins); + +visibleEntries.sort((a, b) => { + if (b.score !== a.score) { + return b.score - a.score; + } + return a.display.localeCompare(b.display); +}); + +const markdownLines: string[] = []; +for (let i = 0; i < visibleEntries.length; i += PER_LINE) { + const chunk = visibleEntries.slice(i, i + PER_LINE); + const parts = chunk.map((entry) => { + // Fixed 48px tiles: GitHub's avatar resizer sometimes ignores `s=48` + // (default identicons come back 420px) and never upscales tiny source + // avatars, so markdown images render off-grid without explicit sizing. + const alt = escapeHtmlAttribute(entry.display); + const image = `<img src="${entry.avatar_url}" width="48" height="48" alt="${alt}">`; + return entry.html_url ? `<a href="${entry.html_url}">${image}</a>` : image; + }); + markdownLines.push(parts.join(" ")); +} + +const block = `${CLAWTRIBUTORS_START}\n${markdownLines.join("\n")}\n${CLAWTRIBUTORS_END}`; +const hiddenBlock = buildHiddenReadmeBlock(entries, visibleEntries); +const hiddenRange = findHiddenReadmeRange(currentReadme); +const readmeWithoutMeta = hiddenRange + ? `${currentReadme.slice(0, hiddenRange.start)}${currentReadme.slice(hiddenRange.end)}` + : currentReadme; +const range = findClawtributorsRange(readmeWithoutMeta); + +if (!range) { + throw new Error("README.md missing clawtributors block"); +} + +const next = `${readmeWithoutMeta.slice(0, range.start)}${block}\n${hiddenBlock}${readmeWithoutMeta.slice(range.end)}`; +writeFileSync(readmePath, next); + +console.log( + `Updated README clawtributors: ${visibleEntries.length} visible (${entries.length - visibleEntries.length} default-avatar entries hidden)`, +); +console.log(`\nTop 25 by composite score: (commits*2 + PRs*10 + sqrt(LOC)) * tenure`); +console.log(` tenure = 1.0 + (days_since_first_commit / repo_age)^2 * 0.5`); +console.log( + `${"#".padStart(3)} ${"login".padEnd(24)} ${"score".padStart(8)} ${"tenure".padStart(7)} ${"commits".padStart(8)} ${"PRs".padStart(6)} ${"LOC".padStart(10)} first commit`, +); +console.log("-".repeat(85)); +for (const [index, entry] of visibleEntries.slice(0, 25).entries()) { + const login = (entry.login ?? entry.key).slice(0, 24); + const fd = entry.firstCommitDate || "?"; + const daysIn = + fd !== "?" ? Math.max(0, (now - new Date(fd.slice(0, 10)).getTime()) / 86_400_000) : 0; + const tr = Math.min(1, daysIn / repoAgeDays); + const tenure = 1 + tr * tr * 0.5; + console.log( + `${index + 1}`.padStart(3) + + ` ${login.padEnd(24)} ${entry.score.toFixed(0).padStart(8)} ${tenure.toFixed(2).padStart(6)}x ${String(entry.commits).padStart(8)} ${String(entry.prs).padStart(6)} ${String(entry.lines).padStart(10)} ${fd}`, + ); +} + +function run(cmd: string): string { + return execSync(cmd, { + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + maxBuffer: 1024 * 1024 * 200, + }).trim(); +} + +function runGh(args: string[]): string { + return execPlainGh(args, { + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + maxBuffer: 1024 * 1024 * 200, + timeout: GH_COMMAND_TIMEOUT_MS, + killSignal: "SIGKILL", + }).trim(); +} + +function parsePaginatedJson(rawLocal: string): unknown[] { + const items: unknown[] = []; + for (const line of rawLocal.split("\n")) { + if (!line.trim()) { + continue; + } + const parsed = JSON.parse(line); + if (Array.isArray(parsed)) { + items.push(...parsed); + } else { + items.push(parsed); + } + } + return items; +} + +function normalizeMap(map: Record<string, string>): Record<string, string> { + const out: Record<string, string> = {}; + for (const [key, value] of Object.entries(map)) { + out[normalizeName(key)] = value; + } + return out; +} + +function normalizeName(value: string): string { + return value.trim().toLowerCase().replace(/\s+/g, " "); +} + +function parseCount(value: string): number { + return /^\d+$/.test(value) ? Number(value) : 0; +} + +function isValidLogin(login: string): boolean { + if (!/^[A-Za-z0-9-]{1,39}$/.test(login)) { + return false; + } + if (login.startsWith("-") || login.endsWith("-")) { + return false; + } + if (login.includes("--")) { + return false; + } + return true; +} + +function normalizeLogin(login: string | null): string | null { + if (!login) { + return null; + } + const trimmed = login.trim(); + return isValidLogin(trimmed) ? trimmed : null; +} + +function normalizeAvatar(url: string): string { + if (!/^https?:/i.test(url)) { + return url; + } + try { + const parsed = new URL(url); + parsed.searchParams.delete("s"); + parsed.searchParams.delete("size"); + parsed.searchParams.set("s", String(AVATAR_SIZE)); + return parsed.toString(); + } catch { + return url; + } +} + +function accountIdFromAvatarUrl(url: string): number | null { + try { + const parsed = new URL(url); + if (parsed.hostname !== "avatars.githubusercontent.com") { + return null; + } + const match = /^\/u\/(\d+)\/?$/u.exec(parsed.pathname); + const accountId = match?.[1] ? Number(match[1]) : Number.NaN; + return Number.isSafeInteger(accountId) && accountId > 0 ? accountId : null; + } catch { + return null; + } +} + +function parseUser(responseText: string): User | null { + const parsed = JSON.parse(responseText); + if (!parsed?.login || !parsed?.html_url || !parsed?.avatar_url) { + return null; + } + return { + id: typeof parsed.id === "number" ? parsed.id : undefined, + login: parsed.login, + html_url: parsed.html_url, + avatar_url: normalizeAvatar(parsed.avatar_url), + }; +} + +function fetchUser(login: string): User | null { + const normalized = normalizeLogin(login); + if (!normalized) { + return null; + } + try { + return parseUser(runGh(["api", `users/${normalized}`])); + } catch (error) { + if (isGitHubMissing(error)) { + return null; + } + throw error; + } +} + +function fetchUserByAccountId(accountId: number): User | null { + try { + return parseUser(runGh(["api", `user/${accountId}`])); + } catch (error) { + if (isGitHubMissing(error)) { + return null; + } + throw error; + } +} + +function isGitHubMissing(error: unknown): boolean { + const message = error instanceof Error ? error.message : ""; + const stderr = (error as { stderr?: unknown } | null)?.stderr; + const stderrText = Buffer.isBuffer(stderr) + ? stderr.toString("utf8") + : typeof stderr === "string" + ? stderr + : ""; + return /\bHTTP (?:404|410)\b/u.test(`${message}\n${stderrText}`); +} + +function isDefaultGitHubAvatar(login: string): Promise<boolean> { + const normalized = normalizeLogin(login)?.toLowerCase(); + if (!normalized) { + return Promise.resolve(false); + } + const cached = defaultAvatarByLogin.get(normalized); + if (cached) { + return cached; + } + const pending = probeDefaultGitHubAvatar(normalized); + defaultAvatarByLogin.set(normalized, pending); + return pending; +} + +async function probeDefaultGitHubAvatar(login: string): Promise<boolean> { + try { + return await withAvatarProbeTimeout(login, async ({ signal, timeoutPromise }) => { + const response = await fetch(`https://github.com/${login}.png?size=${AVATAR_PROBE_SIZE}`, { + headers: { "user-agent": "openclaw-clawtributors" }, + signal, + }); + if (!response.ok) { + return false; + } + const buffer = await readAvatarProbeBuffer(response, timeoutPromise); + const dimensions = readImageDimensions(buffer); + return Boolean( + dimensions && + (dimensions.width > AVATAR_PROBE_SIZE || dimensions.height > AVATAR_PROBE_SIZE), + ); + }); + } catch { + return false; + } +} + +type AvatarProbeTimeout = { + signal: AbortSignal; + timeoutPromise: Promise<never>; +}; + +async function withAvatarProbeTimeout<T>( + login: string, + runProbe: (timeout: AvatarProbeTimeout) => Promise<T>, +): Promise<T> { + const controller = new AbortController(); + let timeout: ReturnType<typeof setTimeout> | undefined; + const timeoutPromise = new Promise<never>((_resolve, reject) => { + timeout = setTimeout(() => { + const error = new Error( + `avatar probe for ${login} exceeded timeout of ${AVATAR_PROBE_TIMEOUT_MS}ms`, + ); + reject(error); + controller.abort(error); + }, AVATAR_PROBE_TIMEOUT_MS); + }); + + try { + return await Promise.race([ + runProbe({ signal: controller.signal, timeoutPromise }), + timeoutPromise, + ]); + } finally { + if (timeout) { + clearTimeout(timeout); + } + } +} + +function toAvatarProbeError(value: unknown, fallbackMessage: string): Error { + if (value instanceof Error) { + return value; + } + if (typeof value === "string") { + return new Error(value); + } + return new Error(fallbackMessage, { cause: value }); +} + +async function readAvatarProbeChunkWithTimeout( + reader: ReadableStreamDefaultReader<Uint8Array>, + timeoutPromise: Promise<never> | undefined, + markCanceled: () => void, +): Promise<ReadableStreamReadResult<Uint8Array>> { + const readPromise = reader.read(); + if (!timeoutPromise) { + return await readPromise; + } + + let waitingForRead = true; + const timeoutReadPromise = timeoutPromise.catch((error: unknown) => { + if (waitingForRead) { + markCanceled(); + cancelResponseReaderSoon(reader); + } + throw toAvatarProbeError(error, "avatar probe response body read timed out"); + }); + + try { + return await Promise.race([readPromise, timeoutReadPromise]); + } finally { + waitingForRead = false; + } +} + +async function readAvatarProbeArrayBuffer( + response: Response, + timeoutPromise: Promise<never> | undefined, +): Promise<ArrayBuffer> { + if (!timeoutPromise) { + return await response.arrayBuffer(); + } + return await Promise.race([ + response.arrayBuffer(), + timeoutPromise.catch((error: unknown) => { + void response.body?.cancel().catch(() => undefined); + throw toAvatarProbeError(error, "avatar probe response body read timed out"); + }), + ]); +} + +async function readAvatarProbeBuffer( + response: Response, + timeoutPromise?: Promise<never>, +): Promise<Buffer> { + const contentLengthRaw = response.headers.get("content-length"); + if (contentLengthRaw && /^\d+$/u.test(contentLengthRaw)) { + const contentLength = Number(contentLengthRaw); + if (!Number.isSafeInteger(contentLength) || contentLength > AVATAR_PROBE_MAX_BYTES) { + await response.body?.cancel().catch(() => undefined); + throw new Error(`avatar probe exceeded ${AVATAR_PROBE_MAX_BYTES} bytes`); + } + } + + const reader = response.body?.getReader?.(); + if (!reader) { + const buffer = Buffer.from(await readAvatarProbeArrayBuffer(response, timeoutPromise)); + if (buffer.byteLength > AVATAR_PROBE_MAX_BYTES) { + throw new Error(`avatar probe exceeded ${AVATAR_PROBE_MAX_BYTES} bytes`); + } + return buffer; + } + + const chunks: Buffer[] = []; + let total = 0; + let canceled = false; + try { + for (;;) { + const { done, value } = await readAvatarProbeChunkWithTimeout(reader, timeoutPromise, () => { + canceled = true; + }); + if (done) { + break; + } + if (!value?.byteLength) { + continue; + } + const chunk = Buffer.from(value); + const nextTotal = total + chunk.byteLength; + if (nextTotal > AVATAR_PROBE_MAX_BYTES) { + canceled = true; + await reader.cancel().catch(() => undefined); + throw new Error(`avatar probe exceeded ${AVATAR_PROBE_MAX_BYTES} bytes`); + } + chunks.push(chunk); + total = nextTotal; + } + } finally { + if (!canceled) { + reader.releaseLock(); + } + } + return Buffer.concat(chunks, total); +} + +async function filterVisibleEntries( + entriesResult: Entry[], + hiddenLogins: ReadonlySet<string>, +): Promise<Entry[]> { + return await pMap( + entriesResult, + async (entry) => { + const login = entry.login ?? entry.key; + if (!login) { + return entry; + } + const normalized = normalizeLogin(login)?.toLowerCase(); + if (normalized && hiddenLogins.has(normalized)) { + return pMapSkip; + } + return (await isDefaultGitHubAvatar(login)) ? pMapSkip : entry; + }, + { concurrency: 8, stopOnError: true }, + ); +} + +function readImageDimensions(buffer: Buffer): { width: number; height: number } | null { + if (isPng(buffer)) { + return readPngDimensions(buffer); + } + if (isJpeg(buffer)) { + return readJpegDimensions(buffer); + } + return null; +} + +function isPng(buffer: Buffer): boolean { + return ( + buffer.length >= 24 && + buffer[0] === 0x89 && + buffer[1] === 0x50 && + buffer[2] === 0x4e && + buffer[3] === 0x47 && + buffer[4] === 0x0d && + buffer[5] === 0x0a && + buffer[6] === 0x1a && + buffer[7] === 0x0a + ); +} + +function readPngDimensions(buffer: Buffer): { width: number; height: number } | null { + if (buffer.length < 24) { + return null; + } + return { + width: buffer.readUInt32BE(16), + height: buffer.readUInt32BE(20), + }; +} + +function isJpeg(buffer: Buffer): boolean { + return buffer.length >= 4 && buffer[0] === 0xff && buffer[1] === 0xd8; +} + +function readJpegDimensions(buffer: Buffer): { width: number; height: number } | null { + let offset = 2; + while (offset + 9 < buffer.length) { + if (buffer[offset] !== 0xff) { + offset += 1; + continue; + } + + const marker = expectDefined(buffer[offset + 1], `JPEG marker at byte ${offset + 1}`); + offset += 2; + + if (marker === 0xd8 || marker === 0xd9) { + continue; + } + if (marker === 0x01 || (marker >= 0xd0 && marker <= 0xd7)) { + continue; + } + if (offset + 2 > buffer.length) { + return null; + } + + const length = buffer.readUInt16BE(offset); + if (length < 2 || offset + length > buffer.length) { + return null; + } + + if ( + (marker >= 0xc0 && marker <= 0xc3) || + (marker >= 0xc5 && marker <= 0xc7) || + (marker >= 0xc9 && marker <= 0xcb) || + (marker >= 0xcd && marker <= 0xcf) + ) { + if (length < 7) { + return null; + } + return { + height: buffer.readUInt16BE(offset + 3), + width: buffer.readUInt16BE(offset + 5), + }; + } + + offset += length; + } + return null; +} + +function resolveLogin( + name: string, + email: string | null, + apiByLoginValue: Map<string, User>, + nameToLoginLocal: Record<string, string>, + emailToLoginLocal: Record<string, string>, +): string | null { + if (email && emailToLoginLocal[email]) { + return normalizeLogin(emailToLoginLocal[email]); + } + + if (email && name) { + const guessed = guessLoginFromEmailName(name, email, apiByLoginValue); + if (guessed) { + return normalizeLogin(guessed); + } + } + + if (email && email.endsWith("@users.noreply.github.com")) { + const local = expectDefined(email.split("@", 1)[0], "GitHub noreply email local part"); + const login = local.includes("+") + ? expectDefined(local.split("+")[1], "GitHub noreply email login suffix") + : local; + return normalizeLogin(login); + } + + if (email && email.endsWith("@github.com")) { + const login = expectDefined(email.split("@", 1)[0], "GitHub email local part"); + if (apiByLoginValue.has(login.toLowerCase())) { + return normalizeLogin(login); + } + } + + const normalized = normalizeName(name); + if (nameToLoginLocal[normalized]) { + return normalizeLogin(nameToLoginLocal[normalized]); + } + + const compact = normalized.replace(/\s+/g, ""); + if (nameToLoginLocal[compact]) { + return normalizeLogin(nameToLoginLocal[compact]); + } + + if (apiByLoginValue.has(normalized)) { + return normalizeLogin(normalized); + } + + if (apiByLoginValue.has(compact)) { + return normalizeLogin(compact); + } + + return null; +} + +function guessLoginFromEmailName( + name: string, + email: string, + apiByLoginLocal: Map<string, User>, +): string | null { + const local = email.split("@", 1)[0]?.trim(); + if (!local) { + return null; + } + const normalizedName = normalizeIdentifier(name); + if (!normalizedName) { + return null; + } + const candidates = new Set([local, local.replace(/[._-]/g, "")]); + for (const candidate of candidates) { + if (!candidate) { + continue; + } + if (normalizeIdentifier(candidate) !== normalizedName) { + continue; + } + const key = candidate.toLowerCase(); + if (apiByLoginLocal.has(key)) { + return key; + } + } + return null; +} + +function normalizeIdentifier(value: string): string { + return value.toLowerCase().replace(/[^a-z0-9]/g, ""); +} + +function escapeHtmlAttribute(value: string): string { + return value + .replaceAll("&", "&") + .replaceAll('"', """) + .replaceAll("<", "<") + .replaceAll(">", ">"); +} + +function parseReadmeEntries( + content: string, +): Array<{ display: string; html_url: string | null; avatar_url: string }> { + const rangeValue = findClawtributorsRange(content); + if (!rangeValue) { + return []; + } + const blockValue = content.slice(rangeValue.start, rangeValue.end); + const entriesValue: Array<{ + display: string; + html_url: string | null; + avatar_url: string; + }> = []; + const markdown = /\[!\[([^\]]+)\]\(([^)]+)\)\]\(([^)]+)\)/g; + for (const match of blockValue.matchAll(markdown)) { + const [, alt, src, href] = match; + if (!href || !src || !alt) { + continue; + } + entriesValue.push({ + html_url: href, + avatar_url: src, + display: alt.replace(/\\([\\[\]])/g, "$1"), + }); + } + const linked = /<a href="([^"]+)"><img src="([^"]+)"[^>]*alt="([^"]+)"[^>]*>/g; + for (const match of blockValue.matchAll(linked)) { + const [, href, src, alt] = match; + if (!href || !src || !alt) { + continue; + } + entriesValue.push({ html_url: href, avatar_url: src, display: alt }); + } + const standalone = /<img src="([^"]+)"[^>]*alt="([^"]+)"[^>]*>/g; + for (const match of blockValue.matchAll(standalone)) { + const [, src, alt] = match; + if (!src || !alt) { + continue; + } + if (entriesValue.some((entry) => entry.display === alt && entry.avatar_url === src)) { + continue; + } + entriesValue.push({ html_url: null, avatar_url: src, display: alt }); + } + return entriesValue; +} + +function parseHiddenReadmeLogins(content: string): string[] { + const rangeLocal = findHiddenReadmeRange(content); + if (!rangeLocal) { + return []; + } + const blockLocal = content.slice(rangeLocal.start, rangeLocal.end); + return blockLocal + .split("\n") + .map((line) => normalizeLogin(line.trim())?.toLowerCase() ?? null) + .filter((login): login is string => Boolean(login)); +} + +function buildHiddenReadmeBlock(entriesLocal: Entry[], visibleEntriesLocal: Entry[]): string { + const visibleLogins = new Set( + visibleEntriesLocal + .map((entry) => normalizeLogin(entry.login ?? entry.key)?.toLowerCase() ?? null) + .filter((login): login is string => Boolean(login)), + ); + const hiddenLogins = entriesLocal + .map((entry) => normalizeLogin(entry.login ?? entry.key)?.toLowerCase() ?? null) + .filter((login): login is string => Boolean(login)) + .filter((login) => !visibleLogins.has(login)) + .toSorted((a, b) => a.localeCompare(b)); + const notice = + "default-avatar-cache: hidden from the rendered wall because these users still use GitHub's default avatar"; + if (hiddenLogins.length === 0) { + return `${CLAWTRIBUTORS_HIDDEN_START}\n${notice}\n${CLAWTRIBUTORS_HIDDEN_END}\n`; + } + return `${CLAWTRIBUTORS_HIDDEN_START}\n${notice}\n${hiddenLogins.join("\n")}\n${CLAWTRIBUTORS_HIDDEN_END}\n`; +} + +function findClawtributorsRange(content: string): { start: number; end: number } | null { + const markerStart = content.indexOf(CLAWTRIBUTORS_START); + const markerEnd = content.indexOf(CLAWTRIBUTORS_END, markerStart); + if (markerStart !== -1 && markerEnd !== -1) { + return { + start: markerStart, + end: markerEnd + CLAWTRIBUTORS_END.length, + }; + } + + const legacyStart = content.indexOf('<p align="left">'); + const legacyEnd = content.indexOf("</p>", legacyStart); + if (legacyStart === -1 || legacyEnd === -1) { + return null; + } + return { + start: legacyStart, + end: legacyEnd + "</p>".length, + }; +} + +function findHiddenReadmeRange(content: string): { start: number; end: number } | null { + const markerStart = content.indexOf(CLAWTRIBUTORS_HIDDEN_START); + const markerEnd = content.indexOf(CLAWTRIBUTORS_HIDDEN_END, markerStart); + if (markerStart === -1 || markerEnd === -1) { + return null; + } + return { + start: markerStart, + end: markerEnd + CLAWTRIBUTORS_HIDDEN_END.length, + }; +} + +function loginFromUrl(url: string): string | null { + const match = /^https?:\/\/github\.com\/([^/?#]+)/i.exec(url); + if (!match) { + return null; + } + const login = match[1]; + if (!login || login.toLowerCase() === "search") { + return null; + } + return login; +} + +function pickDisplay( + baseName: string | null | undefined, + login: string, + existing?: string, +): string { + const key = login.toLowerCase(); + if (displayName[key]) { + return displayName[key]; + } + if (existing) { + return existing; + } + if (baseName) { + return baseName; + } + return login; +} diff --git a/scripts/update-clawtributors.types.ts b/scripts/update-clawtributors.types.ts new file mode 100644 index 0000000000000000000000000000000000000000..948c2275e69965bb747458f7a66c05c5c420eb7c --- /dev/null +++ b/scripts/update-clawtributors.types.ts @@ -0,0 +1,38 @@ +// Update Clawtributors.Types script supports OpenClaw repository automation. +export type MapConfig = { + ensureLogins?: string[]; + displayName?: Record<string, string>; + nameToLogin?: Record<string, string>; + emailToLogin?: Record<string, string>; + seedCommit?: string; +}; + +export type ApiContributor = { + id?: number; + login?: string; + html_url?: string; + avatar_url?: string; + name?: string; + email?: string; + contributions?: number; +}; + +export type User = { + id?: number; + login: string; + html_url: string; + avatar_url: string; +}; + +export type Entry = { + key: string; + login?: string; + display: string; + html_url: string | null; + avatar_url: string; + lines: number; + commits: number; + prs: number; + score: number; + firstCommitDate: string; +}; diff --git a/scripts/update-gateway.sh b/scripts/update-gateway.sh new file mode 100644 index 0000000000000000000000000000000000000000..585196b9cf05a887e7f85d4255730f5627fde8dc --- /dev/null +++ b/scripts/update-gateway.sh @@ -0,0 +1,156 @@ +#!/usr/bin/env bash +# Bash 5.3+ can deadlock writing heredoc pipes on macOS before the reader starts. +if [[ ${OSTYPE:-} == darwin* && $BASH != /bin/bash ]] && ((BASH_VERSINFO[0] > 5 || (BASH_VERSINFO[0] == 5 && BASH_VERSINFO[1] >= 3))); then + exec /bin/bash "$0" "$@" +fi +# Updates a self-hosted OpenClaw gateway that runs from this source checkout. +# +# Reference workflow for team-operated servers (see docs/install/updating.md). +# Simple installs should prefer `openclaw update` / `openclaw update --channel +# dev`; this script exists for checkouts that additionally need to: +# - preserve a local branch by rebasing it onto origin/main, +# - refuse all tracked local changes, including build outputs, +# - build clean (incremental builds have shipped stale hashed chunks), +# - restart a custom service unit. +# +# Environment: +# OPENCLAW_UPDATE_RESTART_CMD restart command (default: openclaw gateway restart) +# set to "" to skip the restart step +# OPENCLAW_UPDATE_REMOTE git remote to update from (default: origin) +set -euo pipefail + +pnpm_dir="" +log() { echo "[update-gateway] $*"; } +on_exit() { + local code=$? + if [ -n "$pnpm_dir" ]; then rm -rf "$pnpm_dir"; fi + if [ "$code" -ne 0 ]; then + echo "[update-gateway] FAILED (exit $code)" >&2 + fi +} +trap on_exit EXIT + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$repo_root" + +# Create private shims before fetching; target compatibility needs an invocation +# after fetch, before checkout mutation. Never activate a global version. +if ! command -v corepack >/dev/null 2>&1; then + log "Corepack is required. Install a Corepack version compatible with the target pnpm pin, then retry." + exit 1 +fi +pnpm_dir="$(mktemp -d "${TMPDIR:-/tmp}/openclaw-pnpm.XXXXXX")" +if ! corepack enable --install-directory "$pnpm_dir" pnpm || [ ! -x "$pnpm_dir/pnpm" ]; then + log "Corepack could not create scoped pnpm shims. Repair Corepack before retrying; no Git update was attempted." + exit 1 +fi +run_pnpm() ( + # Source updates own downloads; do not wait for a Corepack terminal prompt. + export COREPACK_ENABLE_DOWNLOAD_PROMPT=0 + export PATH="$pnpm_dir:$PATH" + export NPM_CONFIG_WORKSPACE_DIR="$PWD" npm_config_workspace_dir="$PWD" + export PNPM_CONFIG_LOCKFILE_DIR="$PWD" pnpm_config_lockfile_dir="$PWD" + "$pnpm_dir/pnpm" "$@" +) + +remote="${OPENCLAW_UPDATE_REMOTE:-origin}" + +# Never update over an in-progress git operation: aborting or rebasing on top +# of an operator's paused rebase/merge would discard their progress. +git_dir="$(git rev-parse --git-dir)" +if [ -d "$git_dir/rebase-merge" ] || [ -d "$git_dir/rebase-apply" ] || \ + [ -f "$git_dir/MERGE_HEAD" ] || [ -f "$git_dir/CHERRY_PICK_HEAD" ]; then + log "a git rebase/merge/cherry-pick is in progress; finish or abort it first" + exit 1 +fi + +# Fail closed on any other local changes: an agent or operator may have +# uncommitted work in this checkout, and an update must never eat it. +if ! git diff --quiet || ! git diff --cached --quiet; then + log "working tree has local changes; commit, stash, or restore them first:" + status_lines="$(git status --short)" + head -20 <<<"$status_lines" + exit 1 +fi + +# dist, dist-runtime, and .artifacts/tsgo-cache are wholly disposable build +# outputs: build and check tooling regenerates them, tracked or not — never +# store anything there. Untracked files elsewhere are kept and only warned +# about (servers accumulate harmless scratch files). Accepted tradeoff: an +# untracked file a build tool happens to read stays in effect, same as before +# the update; operators own what they leave in the checkout. +untracked="$(git ls-files --others --exclude-standard)" +if [ -n "$untracked" ]; then + log "warning: untracked files present; they are kept and a build tool that reads them can affect the deployed output:" + head -10 <<<"$untracked" +fi + +log "fetching ${remote}/main" +git fetch "$remote" main + +# Probe only package-manager metadata, not target dependencies or hooks. Freeze +# the fetched commit so the probe and later checkout update cannot drift apart. +probe_dir="$pnpm_dir/probe" +if ! target_sha="$(git rev-parse --verify 'FETCH_HEAD^{commit}')" || + ! target_version="$(git show "$target_sha:package.json" | node -e ' + const fs = require("node:fs"); + const pin = JSON.parse(fs.readFileSync(0, "utf8")).packageManager; + const version = typeof pin === "string" && /^pnpm@(\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?)(?:\+[0-9A-Za-z.-]+)?$/.exec(pin); + if (!version) throw new Error("Fetched package.json must declare an exact pnpm packageManager pin"); + const dir = process.argv[1]; + fs.mkdirSync(dir); + fs.writeFileSync(dir + "/package.json", JSON.stringify({private: true, packageManager: pin})); + fs.writeFileSync(dir + "/pnpm-workspace.yaml", "packages: []\n"); + process.stdout.write(version[1]); + ' "$probe_dir")" || + ! probe_version="$(cd "$probe_dir" && run_pnpm --version)" || + [ "$probe_version" != "$target_version" ]; then + log "Target pnpm preflight failed; no checkout update or restart was attempted. Verify the fetched packageManager pin and install a compatible Corepack, then retry." + exit 1 +fi + +branch="$(git rev-parse --abbrev-ref HEAD)" +if [ "$branch" = "main" ]; then + log "fast-forwarding main" + git merge --ff-only "$target_sha" +else + # A server may carry a local branch (e.g. an agent's in-progress fix) on top + # of main. Rebase preserves that work while still deploying latest main; + # --rebase-merges keeps merge commits (and their conflict resolutions) + # instead of silently flattening them away. + log "rebasing local branch '$branch' onto ${remote}/main" + if ! git rebase --rebase-merges "$target_sha"; then + git rebase --abort + log "rebase of '$branch' conflicts with ${remote}/main; resolve manually" + exit 1 + fi +fi + +log "installing dependencies" +run_pnpm install --frozen-lockfile + +# Incremental builds have left stale hashed chunks and config validators from +# the previous revision in dist; a clean build is the reliable path. +log "clean building" +# These deletes must stay inside the checkout: a symlinked build dir would +# redirect the recursion into its target, so refuse symlinks outright. +for build_path in dist dist-runtime .artifacts; do + if [ -L "$build_path" ]; then + log "$build_path is a symlink; refusing to clean through it" + exit 1 + fi +done +# The build owns cleanup under its checkout-local artifact lock. Deleting here +# would race declaration writers and readers before that ownership is acquired. +# Match CLI updates: build runtime artifacts unless declarations were explicitly requested. +OPENCLAW_UPDATE_IN_PROGRESS=1 run_pnpm build + +restart_cmd="${OPENCLAW_UPDATE_RESTART_CMD-openclaw gateway restart}" +if [ -n "$restart_cmd" ]; then + log "restarting gateway: $restart_cmd" + bash -c "$restart_cmd" +else + log "restart skipped (OPENCLAW_UPDATE_RESTART_CMD is empty)" +fi + +log "OK $(git rev-parse --short HEAD) ($branch)" diff --git a/scripts/upgrade-survivor-diagnostics.mjs b/scripts/upgrade-survivor-diagnostics.mjs new file mode 100644 index 0000000000000000000000000000000000000000..0c0ab14d15890e4828403f4f665a212d7427fbf3 --- /dev/null +++ b/scripts/upgrade-survivor-diagnostics.mjs @@ -0,0 +1,16 @@ +// Host-only entrypoint: this file and app source are never mounted into the +// package-under-test container. Keep the capture module dependency-light. +import { publishDiagnostics } from "./e2e/lib/upgrade-survivor/diagnostics.mjs"; + +try { + const [mode, artifactRoot, destination, outcome = "failed"] = process.argv.slice(2); + if (mode !== "publish") { + throw new Error(); + } + // The wrapper registers this harness's scripts/tsx.mjs before loading source. + const { redactSensitiveText } = await import("../src/logging/redact.ts"); + publishDiagnostics(artifactRoot, destination, redactSensitiveText, outcome); +} catch { + process.stderr.write("Upgrade survivor diagnostics missing: safe host publication failed.\n"); + process.exitCode = 1; +} diff --git a/scripts/validate-qa-runtime-pair-summary.mts b/scripts/validate-qa-runtime-pair-summary.mts new file mode 100644 index 0000000000000000000000000000000000000000..16dd6a7360d3576605829eea419993e6138354ba --- /dev/null +++ b/scripts/validate-qa-runtime-pair-summary.mts @@ -0,0 +1,538 @@ +#!/usr/bin/env node + +// Validates runtime-pair evidence emitted by a frozen release candidate. +import fs from "node:fs"; +import process from "node:process"; +import { pathToFileURL } from "node:url"; +import { isRecord } from "@openclaw/normalization-core/record-coerce"; + +const RUNTIME_IDS = ["openclaw", "codex"]; +const HARD_RUNTIME_ERROR_CLASSES = new Set([ + "missing-api-key", + "failover", + "codex-app-server", + "auth", + "capture-missing", +]); +const EXPLICIT_CODEX_GAP_PREFIXES = ["known-harness-gap ", "codex-native-workspace "]; +const FROZEN_CORE_RUNTIME_PAIR_MANIFEST = { + scenarioIds: [ + "instruction-followthrough-repo-contract", + "subagent-fanout-synthesis", + "subagent-handoff", + "subagent-stale-child-links", + "config-restart-capability-flip", + "image-understanding-attachment", + "memory-recall", + "thread-memory-isolation", + "model-switch-tool-continuity", + "approval-turn-tool-followthrough", + // Preserve the serialized reports from these fixed candidate SHAs. These + // are not live catalog entries or current plugin-compatibility evidence. + "codex-plugin-pinned-new", + "codex-plugin-pinned-old", + "compaction-retry-mutating-tool", + "runtime-first-hour-20-turn", + "runtime-tool-apply-patch", + "runtime-tool-bash", + "runtime-tool-edit", + "runtime-tool-exec", + "runtime-tool-fs-list", + "runtime-tool-fs-read", + "runtime-tool-fs-write", + "runtime-tool-grep", + "runtime-tool-session-status", + "runtime-tool-sessions-spawn", + "runtime-tool-web-fetch", + "runtime-tool-web-search", + "source-docs-discovery-report", + ], + gapScenarioIds: [ + "runtime-tool-apply-patch", + "runtime-tool-bash", + "runtime-tool-edit", + "runtime-tool-exec", + "runtime-tool-fs-list", + "runtime-tool-fs-read", + "runtime-tool-fs-write", + "runtime-tool-grep", + ], +}; +const FROZEN_RUNTIME_PAIR_MANIFESTS = new Map([ + ["311047822ecdde24e824d839ab105ef08f17be00:core", FROZEN_CORE_RUNTIME_PAIR_MANIFEST], + ["c37af96b18776fecc9e24268f27fc89b563481bf:core", FROZEN_CORE_RUNTIME_PAIR_MANIFEST], +]); +const FROZEN_RUNTIME_PAIR_COMPATIBILITY_PROFILES = new Map([ + [ + "ee5ead24b1b46a3560f28f8d57e0afcd911acacb:core", + { + allowMissingRunStatus: true, + scenarioIds: FROZEN_CORE_RUNTIME_PAIR_MANIFEST.scenarioIds.filter( + (scenarioId) => + scenarioId !== "codex-plugin-pinned-new" && scenarioId !== "codex-plugin-pinned-old", + ), + }, + ], +]); + +type RuntimePairValidationOptions = { + candidateSuiteOutcome?: string; + lane?: string; + requireExplicitGap?: boolean; + targetSha?: string; +}; + +function isPassableCell(cell: unknown) { + if (!isRecord(cell) || typeof cell.transportErrorClass === "string") { + return false; + } + const runtimeErrorClass = cell.runtimeErrorClass; + if ( + typeof runtimeErrorClass === "string" && + (HARD_RUNTIME_ERROR_CLASSES.has(runtimeErrorClass) || runtimeErrorClass.startsWith("sentinel:")) + ) { + return false; + } + if (runtimeErrorClass !== undefined && runtimeErrorClass !== "tool-error") { + return false; + } + return ( + Array.isArray(cell.toolCalls) && + !cell.toolCalls.some( + (toolCall) => isRecord(toolCall) && toolCall.errorClass === "tool-result-missing", + ) + ); +} + +function isExplicitCodexGap(cell: unknown) { + if (!isRecord(cell) || cell.status !== "skip" || typeof cell.details !== "string") { + return false; + } + const details = cell.details.trimStart(); + return EXPLICIT_CODEX_GAP_PREFIXES.some((prefix) => details.startsWith(prefix)); +} + +function projectedReportCellStatus(cell: Record<string, unknown>) { + // Frozen parity reports project runtime/transport health, while the raw + // result cell separately preserves an approved scenario-level skip. + return cell.runtimeErrorClass || cell.transportErrorClass ? "fail" : "pass"; +} + +function formatRuntimePairReportValue(value: unknown) { + if (typeof value === "string") { + return value; + } + return value === undefined ? "undefined" : (JSON.stringify(value) ?? "<undefined>"); +} + +function hasPassingRuntimeCellStatus(cell: Record<string, unknown>) { + // Early frozen candidates did not serialize the derived cell status. Accept + // only that absent legacy field when the same runtime evidence projects pass. + return ( + cell.status === "pass" || + (!Object.hasOwn(cell, "status") && projectedReportCellStatus(cell) === "pass") + ); +} + +function requireRuntimePairScenario(scenario: unknown, index: number) { + if (!isRecord(scenario)) { + throw new Error(`scenario ${index + 1} is not an object`); + } + const label = typeof scenario.name === "string" ? scenario.name : `scenario ${index + 1}`; + const parity = scenario.runtimeParity; + if (!isRecord(parity) || !isRecord(parity.cells)) { + throw new Error(`${label} is missing runtimeParity cells`); + } + const openclaw = parity.cells.openclaw; + const codex = parity.cells.codex; + if (!isRecord(openclaw) || !isRecord(codex)) { + throw new Error(`${label} is missing a canonical runtime cell`); + } + if (openclaw.runtime !== "openclaw" || codex.runtime !== "codex") { + throw new Error(`${label} has mismatched canonical runtime cells`); + } + + if (scenario.status === "pass") { + const hasTwoPassingCells = + hasPassingRuntimeCellStatus(openclaw) && hasPassingRuntimeCellStatus(codex); + const hasAdvisoryCodexGap = + parity.drift === "structural" && + hasPassingRuntimeCellStatus(openclaw) && + isExplicitCodexGap(codex); + if ( + parity.drift === "failure-mode" || + (!hasTwoPassingCells && !hasAdvisoryCodexGap) || + !isPassableCell(openclaw) || + !isPassableCell(codex) + ) { + throw new Error(`${label} reports pass without two passing, passable runtime cells`); + } + return false; + } + + // Older candidates emitted this explicit Codex-native limitation as a skip + // before the trusted classifier learned to keep the paired result advisory. + if ( + scenario.status === "skip" && + openclaw.status === "pass" && + codex.status === "skip" && + isExplicitCodexGap(codex) && + isPassableCell(openclaw) && + isPassableCell(codex) + ) { + return true; + } + + throw new Error(`${label} contains a runtime-pair failure or unsupported skip`); +} + +function requireCanonicalRuntimePair(runtimePair: unknown) { + return ( + Array.isArray(runtimePair) && + runtimePair.length === RUNTIME_IDS.length && + runtimePair.every((runtime, index) => runtime === RUNTIME_IDS[index]) + ); +} + +function isCanonicalIsoTimestamp(value: unknown): value is string { + if (typeof value !== "string") { + return false; + } + const timestamp = Date.parse(value); + return Number.isFinite(timestamp) && new Date(timestamp).toISOString() === value; +} + +function isTrustedMissingStatusRun( + run: Record<string, unknown>, + options: RuntimePairValidationOptions, +) { + const profile = FROZEN_RUNTIME_PAIR_COMPATIBILITY_PROFILES.get( + `${options.targetSha}:${options.lane}`, + ); + if ( + Object.hasOwn(run, "status") || + options.candidateSuiteOutcome !== "success" || + profile?.allowMissingRunStatus !== true || + !isCanonicalIsoTimestamp(run.startedAt) || + !isCanonicalIsoTimestamp(run.finishedAt) || + Date.parse(run.finishedAt) < Date.parse(run.startedAt) + ) { + return false; + } + const scenarioIds = run.scenarioIds; + return ( + Array.isArray(scenarioIds) && + scenarioIds.length === profile.scenarioIds.length && + profile.scenarioIds.every((scenarioId, index) => scenarioIds[index] === scenarioId) + ); +} + +export function validateQaRuntimePairSummary( + summary: unknown, + options: RuntimePairValidationOptions = {}, +) { + if (!isRecord(summary) || !isRecord(summary.run) || !Array.isArray(summary.scenarios)) { + throw new Error("runtime-pair summary is missing run or scenario evidence"); + } + // Frozen compatibility profiles may admit a specifically attested legacy + // producer shape. The trusted workflow outcome and exact manifest prevent a + // partial or failed producer from masquerading as terminal evidence. + if (summary.run.status !== "completed" && !isTrustedMissingStatusRun(summary.run, options)) { + throw new Error("runtime-pair summary is not completed"); + } + if (!requireCanonicalRuntimePair(summary.run.runtimePair)) { + throw new Error("runtime-pair summary must compare openclaw and codex in canonical order"); + } + if (summary.scenarios.length === 0) { + throw new Error("runtime-pair summary contains no scenarios"); + } + const run = summary.run; + const scenarioIds = summary.scenarios.map((scenario) => + isRecord(scenario) && isRecord(scenario.runtimeParity) + ? scenario.runtimeParity.scenarioId + : undefined, + ); + const runScenarioIds = Array.isArray(run.scenarioIds) ? run.scenarioIds : null; + if ( + !runScenarioIds || + runScenarioIds.length !== scenarioIds.length || + new Set(runScenarioIds).size !== runScenarioIds.length || + scenarioIds.some( + (scenarioId, index) => typeof scenarioId !== "string" || scenarioId !== runScenarioIds[index], + ) + ) { + throw new Error("runtime-pair results do not match the declared scenario manifest"); + } + + let passed = 0; + let skipped = 0; + const skippedScenarioIds: unknown[] = []; + for (const [index, scenario] of summary.scenarios.entries()) { + if (requireRuntimePairScenario(scenario, index)) { + skipped += 1; + skippedScenarioIds.push(scenarioIds[index]); + } else { + passed += 1; + } + } + + const expectedCounts = { + total: summary.scenarios.length, + passed, + failed: 0, + skipped, + }; + const requiredCountKeys = ["total", "passed", "failed"] as const; + const counts = isRecord(summary.counts) ? summary.counts : undefined; + const skippedCountMatches = + counts?.skipped === skipped || (skipped === 0 && counts?.skipped === undefined); + if ( + !counts || + requiredCountKeys.some((key) => counts[key] !== expectedCounts[key]) || + !skippedCountMatches + ) { + throw new Error("runtime-pair summary counts do not match validated scenario evidence"); + } + if (options.requireExplicitGap === true && skipped === 0) { + throw new Error("nonzero candidate suite exit requires an explicit Codex-native harness gap"); + } + if (options.requireExplicitGap === true) { + const manifest = FROZEN_RUNTIME_PAIR_MANIFESTS.get(`${options.targetSha}:${options.lane}`); + if ( + !manifest || + manifest.scenarioIds.length !== scenarioIds.length || + manifest.scenarioIds.some((scenarioId, index) => scenarioId !== scenarioIds[index]) || + manifest.gapScenarioIds.length !== skippedScenarioIds.length || + manifest.gapScenarioIds.some((scenarioId, index) => scenarioId !== skippedScenarioIds[index]) + ) { + throw new Error("nonzero candidate exit is not covered by a trusted frozen-lane manifest"); + } + } + return expectedCounts; +} + +export function validateQaRuntimePairReport( + summary: unknown, + reportSummary: unknown, + reportMarkdown: string, + options: RuntimePairValidationOptions = {}, +) { + const counts = validateQaRuntimePairSummary(summary, options); + if ( + !isRecord(summary) || + !Array.isArray(summary.scenarios) || + !summary.scenarios.every(isRuntimePairScenario) + ) { + throw new Error("runtime-pair summary is missing canonical scenario evidence"); + } + const scenarios = summary.scenarios; + if ( + !isRecord(reportSummary) || + !requireCanonicalRuntimePair(reportSummary.runtimePair) || + !Array.isArray(reportSummary.scenarios) || + !Array.isArray(reportSummary.failures) || + reportSummary.totalScenarios !== counts.total || + reportSummary.passedScenarios !== counts.passed || + reportSummary.failedScenarios !== counts.skipped || + reportSummary.scenarios.length !== counts.total || + reportSummary.failures.length !== counts.skipped + ) { + throw new Error("runtime-pair report summary does not match validated suite evidence"); + } + const reportScenarios = reportSummary.scenarios.filter(isRecord); + const scenarioNames = scenarios.map((scenario) => scenario.name); + if ( + reportScenarios.length !== reportSummary.scenarios.length || + scenarioNames.some((name) => typeof name !== "string") || + reportScenarios.some((scenario, index) => { + const source = scenarios[index]; + if (!source) { + return true; + } + const expectedStatus = source.status === "skip" ? "fail" : "pass"; + return ( + scenario?.name !== scenarioNames[index] || + scenario.status !== expectedStatus || + scenario.drift !== source.runtimeParity.drift || + scenario.driftDetails !== source.runtimeParity.driftDetails || + scenario.openclawStatus !== + projectedReportCellStatus(source.runtimeParity.cells.openclaw) || + scenario.codexStatus !== projectedReportCellStatus(source.runtimeParity.cells.codex) + ); + }) + ) { + throw new Error("runtime-pair report scenarios do not match validated suite evidence"); + } + const expectedFailures = scenarios + .filter((scenario) => scenario.status === "skip") + .map( + (scenario) => + `${formatRuntimePairReportValue(scenario.name)} drift=${formatRuntimePairReportValue(scenario.runtimeParity.drift)} (${formatRuntimePairReportValue(scenario.runtimeParity.driftDetails)}).`, + ); + const reportFailures = reportSummary.failures; + if ( + reportSummary.pass !== (counts.skipped === 0) || + reportFailures.some((failure, index) => failure !== expectedFailures[index]) + ) { + throw new Error("runtime-pair report failures do not match validated suite evidence"); + } + if ( + typeof reportMarkdown !== "string" || + !reportMarkdown.startsWith("# OpenClaw Runtime Parity Report") || + !reportMarkdown.includes(`- Verdict: ${counts.skipped === 0 ? "pass" : "fail"}`) || + scenarios.some((scenario) => { + const heading = `\n### ${formatRuntimePairReportValue(scenario.name)}\n`; + const start = reportMarkdown.indexOf(heading); + if (start < 0) { + return true; + } + const next = reportMarkdown.indexOf("\n### ", start + heading.length); + const sectionLines = new Set( + reportMarkdown.slice(start, next < 0 ? undefined : next).split("\n"), + ); + const expectedStatus = scenario.status === "skip" ? "fail" : "pass"; + return ( + !sectionLines.has(`- status: ${expectedStatus}`) || + !sectionLines.has( + `- drift: ${formatRuntimePairReportValue(scenario.runtimeParity.drift)}`, + ) || + ![...sectionLines].some((line) => + line.startsWith( + `- openclaw: ${projectedReportCellStatus(scenario.runtimeParity.cells.openclaw)} `, + ), + ) || + ![...sectionLines].some((line) => + line.startsWith( + `- codex: ${projectedReportCellStatus(scenario.runtimeParity.cells.codex)} `, + ), + ) + ); + }) + ) { + throw new Error("runtime-pair Markdown report is incomplete"); + } + return counts; +} + +export function parseArgs(argv: string[]) { + let summaryPath; + let reportSummaryPath; + let reportMarkdownPath; + let requireExplicitGap = false; + let candidateSuiteOutcome; + let targetSha; + let lane; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === "--summary") { + summaryPath = argv[index + 1]; + if (!summaryPath || summaryPath.startsWith("-")) { + throw new Error("--summary requires a path"); + } + index += 1; + } else if (arg === "--report-summary" || arg === "--report-markdown") { + const value = argv[index + 1]; + if (!value || value.startsWith("-")) { + throw new Error(`${arg} requires a path`); + } + if (arg === "--report-summary") { + reportSummaryPath = value; + } else { + reportMarkdownPath = value; + } + index += 1; + } else if (arg === "--require-explicit-gap") { + requireExplicitGap = true; + } else if (arg === "--candidate-suite-outcome") { + candidateSuiteOutcome = argv[index + 1]; + if (!candidateSuiteOutcome || candidateSuiteOutcome.startsWith("-")) { + throw new Error("--candidate-suite-outcome requires a value"); + } + index += 1; + } else if (arg === "--target-sha" || arg === "--lane") { + const value = argv[index + 1]; + if (!value || value.startsWith("-")) { + throw new Error(`${arg} requires a value`); + } + if (arg === "--target-sha") { + targetSha = value; + } else { + lane = value; + } + index += 1; + } else if (arg === "--help" || arg === "-h") { + process.stdout.write( + "Usage: node --import tsx scripts/validate-qa-runtime-pair-summary.mts --summary <qa-suite-summary.json> [--candidate-suite-outcome <outcome>] [--require-explicit-gap] [--report-summary <json> --report-markdown <md>]\n", + ); + return undefined; + } else { + throw new Error(`Unknown argument: ${arg}`); + } + } + if (!summaryPath) { + throw new Error("--summary is required"); + } + if (Boolean(reportSummaryPath) !== Boolean(reportMarkdownPath)) { + throw new Error("--report-summary and --report-markdown must be provided together"); + } + if (requireExplicitGap && (!targetSha || !lane)) { + throw new Error("--require-explicit-gap requires --target-sha and --lane"); + } + return { + summaryPath, + reportSummaryPath, + reportMarkdownPath, + candidateSuiteOutcome, + requireExplicitGap, + targetSha, + lane, + }; +} + +type RuntimePairScenario = { + name: unknown; + status: unknown; + runtimeParity: { + drift: unknown; + driftDetails?: unknown; + cells: { + openclaw: Record<string, unknown>; + codex: Record<string, unknown>; + }; + }; +}; + +function isRuntimePairScenario(value: unknown): value is RuntimePairScenario { + if (!isRecord(value) || !isRecord(value.runtimeParity)) { + return false; + } + const cells = value.runtimeParity.cells; + return isRecord(cells) && isRecord(cells.openclaw) && isRecord(cells.codex); +} + +async function main() { + const options = parseArgs(process.argv.slice(2)); + if (!options) { + return; + } + const summary = JSON.parse(fs.readFileSync(options.summaryPath, "utf8")); + const counts = + options.reportSummaryPath && options.reportMarkdownPath + ? validateQaRuntimePairReport( + summary, + JSON.parse(fs.readFileSync(options.reportSummaryPath, "utf8")), + fs.readFileSync(options.reportMarkdownPath, "utf8"), + options, + ) + : validateQaRuntimePairSummary(summary, options); + process.stdout.write( + `Validated ${counts.total} runtime-pair scenarios (${counts.passed} passed, ${counts.skipped} explicit Codex-native harness gaps).\n`, + ); +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { + main().catch((error: unknown) => { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + process.exitCode = 1; + }); +} diff --git a/scripts/validate-release-publish-approval.mjs b/scripts/validate-release-publish-approval.mjs new file mode 100644 index 0000000000000000000000000000000000000000..5c3254efed8f557866c07f48c6209b30d943a47a --- /dev/null +++ b/scripts/validate-release-publish-approval.mjs @@ -0,0 +1,293 @@ +#!/usr/bin/env node +// Validates that a referenced release-publish workflow run is usable for approval. +import { execFileSync } from "node:child_process"; +import fs from "node:fs"; +import { verifyAndroidNativeCi } from "./android-native-ci.mjs"; +import { validateStablePluginNpmBootstrapApproval } from "./plugin-npm-bootstrap-approval.mjs"; +import { + runReleaseToolingGh, + validateReleasePublishParentRun, + verifyReleaseToolingIdentity, +} from "./release-tooling-identity.mjs"; + +const releasePublishRunId = process.env.RELEASE_PUBLISH_RUN_ID ?? ""; +const expectedBranch = process.env.EXPECTED_WORKFLOW_BRANCH ?? ""; +const directRecovery = process.env.DIRECT_RELEASE_RECOVERY === "true"; +const allowCompletedSuccessfulParent = process.env.ALLOW_COMPLETED_SUCCESSFUL_PARENT === "true"; +const approvalPath = process.env.APPROVAL_PATH ?? ""; +const approvalKind = process.env.RELEASE_APPROVAL_KIND ?? "android"; +const expectedRunAttempt = process.env.EXPECTED_RUN_ATTEMPT ?? ""; +const expectedWorkflowFullRef = process.env.EXPECTED_WORKFLOW_FULL_REF ?? ""; +const expectedWorkflowSha = process.env.EXPECTED_WORKFLOW_SHA ?? ""; +const childWorkflowSha = process.env.CHILD_WORKFLOW_SHA ?? ""; +const run = + approvalKind === "android" && approvalPath ? null : JSON.parse(fs.readFileSync(0, "utf8")); + +function fail(message) { + console.error(message); + process.exit(1); +} + +function canonicalPackages(value) { + const packages = value + .split(",") + .map((entry) => entry.trim()) + .filter(Boolean); + if ( + packages.length === 0 || + new Set(packages).size !== packages.length || + packages.some((entry) => !/^@openclaw\/[a-z0-9][a-z0-9._-]*$/u.test(entry)) + ) { + fail("ClawHub bootstrap approval requires a unique @openclaw/* package set."); + } + return packages.toSorted((a, b) => (a < b ? -1 : a > b ? 1 : 0)); +} + +function positiveRunAttempt(value) { + if (!/^[1-9][0-9]*$/u.test(value)) { + fail("Expected release publish run attempt must be a positive integer."); + } + return Number(value); +} + +if (["clawhub-bootstrap", "npm-stable-bootstrap"].includes(approvalKind) && !approvalPath) { + fail( + `${approvalKind === "clawhub-bootstrap" ? "ClawHub" : "Stable npm"} bootstrap approval requires an attested approval artifact.`, + ); +} + +if (approvalPath) { + const approval = JSON.parse(fs.readFileSync(approvalPath, "utf8")); + let expectedApproval; + let mismatchMessage; + if (approvalKind === "android") { + expectedApproval = { + version: approval.version === 3 ? 3 : 2, + repository: process.env.GITHUB_REPOSITORY, + workflow: "OpenClaw Release Publish", + parentRunId: releasePublishRunId, + parentRunAttempt: positiveRunAttempt(expectedRunAttempt), + workflowBranch: expectedBranch, + workflowFullRef: expectedWorkflowFullRef, + parentWorkflowSha: expectedWorkflowSha, + releaseTag: process.env.RELEASE_TAG, + targetSha: process.env.RELEASE_TARGET_SHA, + ...(approval.version === 3 ? { nativeCi: approval.nativeCi } : {}), + }; + mismatchMessage = "Attested Android release approval does not match this run request."; + } else if (approvalKind === "clawhub-bootstrap") { + if (!/^[a-f0-9]{40}$/u.test(childWorkflowSha)) { + fail("Plugin ClawHub New workflow SHA must be a full lowercase commit SHA."); + } + expectedApproval = { + version: 2, + kind: "clawhub-bootstrap", + repository: process.env.GITHUB_REPOSITORY, + workflow: "OpenClaw Release Publish", + parentRunId: releasePublishRunId, + parentRunAttempt: positiveRunAttempt(expectedRunAttempt), + workflowBranch: expectedBranch, + parentWorkflowSha: run.headSha, + bootstrapWorkflowSha: childWorkflowSha, + releaseTag: process.env.RELEASE_TAG, + targetSha: process.env.RELEASE_TARGET_SHA, + packages: canonicalPackages(process.env.RELEASE_PACKAGES ?? ""), + }; + mismatchMessage = + "Attested ClawHub bootstrap approval does not match this release target and package set."; + } else if (approvalKind === "npm-stable-bootstrap") { + validateStablePluginNpmBootstrapApproval(approval, { + repository: process.env.GITHUB_REPOSITORY, + parentRunId: releasePublishRunId, + parentRunAttempt: positiveRunAttempt(expectedRunAttempt), + workflowBranch: expectedBranch, + workflowFullRef: expectedWorkflowFullRef, + parentWorkflowSha: expectedWorkflowSha, + targetSha: process.env.RELEASE_TARGET_SHA, + packageName: process.env.PACKAGE_NAME, + packageVersion: process.env.PACKAGE_VERSION, + publishTag: process.env.PUBLISH_TAG, + }); + expectedApproval = approval; + mismatchMessage = "Stable npm bootstrap approval mismatch."; + const refs = execFileSync( + "git", + [ + "ls-remote", + "--tags", + "origin", + `refs/tags/${approval.releaseTag}`, + `refs/tags/${approval.releaseTag}^{}`, + ], + { encoding: "utf8", timeout: 60_000, maxBuffer: 1024 * 1024 }, + ) + .trim() + .split("\n") + .map((line) => line.split(/\s+/u)); + const tagSha = + refs.find(([, ref]) => ref === `refs/tags/${approval.releaseTag}^{}`)?.[0] ?? + refs.find(([, ref]) => ref === `refs/tags/${approval.releaseTag}`)?.[0]; + if (tagSha !== approval.targetSha) { + fail("Stable npm bootstrap release tag no longer matches the approved target."); + } + } else { + fail(`Unsupported release approval kind: ${approvalKind}`); + } + if (JSON.stringify(approval) !== JSON.stringify(expectedApproval)) { + fail(mismatchMessage); + } + if (approvalKind === "android") { + const tag = process.env.RELEASE_TAG; + const target = process.env.RELEASE_TARGET_SHA; + // Match the publisher's live direct/peeled tag contract; release tags may + // be signed annotated tags while protected tooling tags must be lightweight. + const refs = execFileSync( + "git", + ["ls-remote", "--tags", "origin", `refs/tags/${tag}`, `refs/tags/${tag}^{}`], + { + encoding: "utf8", + timeout: 60_000, + maxBuffer: 1024 * 1024, + }, + ) + .trim() + .split("\n") + .map((line) => line.split(/\s+/u)); + const targetSha = + refs.find(([, ref]) => ref === `refs/tags/${tag}^{}`)?.[0] ?? + refs.find(([, ref]) => ref === `refs/tags/${tag}`)?.[0]; + if (targetSha !== target) { + fail(`Release tag ${tag} no longer resolves to approved target ${target}.`); + } + const release = JSON.parse( + runReleaseToolingGh([ + "release", + "view", + tag, + "--repo", + process.env.GITHUB_REPOSITORY, + "--json", + "tagName,isPrerelease,createdAt", + ]), + ); + if (release.tagName !== tag || release.isPrerelease !== false) { + fail("Android publication requires the exact approved stable GitHub release."); + } + const buildTimestamp = new Date(release.createdAt).toISOString().replace(/\.000Z$/u, "Z"); + const identity = verifyReleaseToolingIdentity({ + allowPrevalidatedRef: /^release\/[0-9]{4}\.(?:[1-9]|1[0-2])\.[1-9][0-9]*$/u.test( + expectedBranch, + ), + repository: process.env.GITHUB_REPOSITORY, + workflowFullRef: expectedWorkflowFullRef, + workflowRef: expectedBranch, + workflowSha: expectedWorkflowSha, + }); + // Native qualification and parent authority must survive child queue/build + // time and every preceding target, tooling, and release-asset lookup. + verifyAndroidNativeCi(approval); + const currentRun = JSON.parse( + runReleaseToolingGh([ + "api", + `repos/${process.env.GITHUB_REPOSITORY}/actions/runs/${releasePublishRunId}`, + "--method", + "GET", + ]), + ); + // Apps may attach after npm and GitHub publication complete. Keep the exact + // approval binding valid for a successful parent without requiring it to wait. + validateReleasePublishParentRun({ + identity, + releasePublishFullRef: expectedWorkflowFullRef, + releasePublishParentStatePolicy: directRecovery ? "manual-recovery" : "active-or-success", + releasePublishRef: expectedBranch, + releasePublishRunAttempt: expectedRunAttempt, + releasePublishRunId, + repository: process.env.GITHUB_REPOSITORY, + run: currentRun, + }); + process.stdout.write(`${buildTimestamp}\n`); + process.exit(0); + } +} + +const checks = [ + ["workflowName", "OpenClaw Release Publish"], + ["headBranch", expectedBranch], + ["event", "workflow_dispatch"], +]; +if (process.env.GITHUB_REPOSITORY) { + checks.push(["repository", process.env.GITHUB_REPOSITORY]); +} + +for (const [key, expected] of checks) { + if (run[key] !== expected) { + fail( + `Referenced release publish run ${releasePublishRunId} must have ${key}=${expected}, got ${run[key] ?? "<missing>"}.`, + ); + } +} + +if (expectedWorkflowSha && run.headSha !== expectedWorkflowSha) { + fail( + `Referenced release publish run ${releasePublishRunId} must use tooling SHA ${expectedWorkflowSha}, got ${run.headSha ?? "<missing>"}.`, + ); +} +if (expectedWorkflowFullRef) { + const [workflowPath, workflowFullRef] = String(run.path ?? "").split("@", 2); + if (workflowPath !== ".github/workflows/openclaw-release-publish.yml") { + fail(`Referenced release publish run ${releasePublishRunId} has untrusted workflow path.`); + } + if (workflowFullRef && workflowFullRef !== expectedWorkflowFullRef) { + fail(`Referenced release publish run ${releasePublishRunId} has untrusted workflow full ref.`); + } +} + +if (expectedRunAttempt && run.runAttempt !== positiveRunAttempt(expectedRunAttempt)) { + fail( + `Referenced release publish run ${releasePublishRunId} must use attempt ${expectedRunAttempt}, got ${run.runAttempt ?? "<missing>"}.`, + ); +} + +if (!directRecovery) { + if (run.status === "in_progress" && !run.conclusion) { + console.log(`Using release publish approval run ${releasePublishRunId}: ${run.url}`); + process.exit(0); + } + if ( + allowCompletedSuccessfulParent && + run.status === "completed" && + run.conclusion === "success" + ) { + console.log( + `Using successful completed release publish run ${releasePublishRunId}: ${run.url}`, + ); + process.exit(0); + } + if (run.status !== "in_progress") { + fail( + `Referenced release publish run ${releasePublishRunId} must still be in_progress, got ${run.status ?? "<missing>"}.`, + ); + } + if (run.conclusion) { + fail( + `Referenced release publish run ${releasePublishRunId} already concluded ${run.conclusion}.`, + ); + } +} + +if (run.status === "in_progress" && !run.conclusion) { + console.log(`Using active release publish run ${releasePublishRunId}: ${run.url}`); + process.exit(0); +} + +if (run.status === "completed" && ["success", "failure"].includes(run.conclusion)) { + console.log( + `Using completed release publish run ${releasePublishRunId} (${run.conclusion}) for direct recovery: ${run.url}`, + ); + process.exit(0); +} + +fail( + `Direct release recovery run ${releasePublishRunId} must be in_progress or completed with success/failure, got status=${run.status ?? "<missing>"} conclusion=${run.conclusion ?? "<missing>"}.`, +); diff --git a/scripts/verify-full-release-producer-job.mjs b/scripts/verify-full-release-producer-job.mjs new file mode 100644 index 0000000000000000000000000000000000000000..5bf8d4b6462eb61d2b8885023c706951a13ffa73 --- /dev/null +++ b/scripts/verify-full-release-producer-job.mjs @@ -0,0 +1,93 @@ +#!/usr/bin/env node + +import { spawnSync } from "node:child_process"; + +const OPTION_NAMES = ["--repository", "--job-id", "--job-name", "--run-id", "--run-attempt"]; + +function parseOptions(args) { + const options = new Map(); + for (let index = 0; index < args.length; index += 2) { + const name = args[index]; + const value = args[index + 1]; + if (!OPTION_NAMES.includes(name) || !value || options.has(name)) { + throw new Error(`invalid or duplicate option: ${name ?? "<missing>"}`); + } + options.set(name, value); + } + for (const name of OPTION_NAMES) { + if (!options.has(name)) { + throw new Error(`missing ${name}`); + } + } + return options; +} + +function positiveInteger(value, label) { + if (!/^[1-9][0-9]*$/u.test(value)) { + throw new Error(`${label} must be a positive integer`); + } + return value; +} + +function actualIdentifier(value) { + return typeof value === "number" && Number.isSafeInteger(value) ? String(value) : value; +} + +function assertEqual(actual, expected, label) { + if (actualIdentifier(actual) !== expected) { + throw new Error( + `producer ${label} mismatch: expected ${JSON.stringify(expected)}, got ${JSON.stringify(actual)}`, + ); + } +} + +function main() { + const options = parseOptions(process.argv.slice(2)); + const repository = options.get("--repository"); + if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(repository)) { + throw new Error("repository must be an owner/name pair"); + } + const jobId = positiveInteger(options.get("--job-id"), "job ID"); + const jobName = options.get("--job-name"); + const runId = positiveInteger(options.get("--run-id"), "run ID"); + const runAttempt = positiveInteger(options.get("--run-attempt"), "run attempt"); + + // Bind evidence only to the exact producer captured before untrusted code ran. + const result = spawnSync("gh", ["api", `repos/${repository}/actions/jobs/${jobId}`], { + encoding: "utf8", + maxBuffer: 1024 * 1024, + }); + if (result.error) { + throw result.error; + } + if (result.status !== 0) { + if (result.stderr) { + process.stderr.write(result.stderr); + } + throw new Error(`producer job lookup failed with exit code ${result.status ?? "unknown"}`); + } + + let job; + try { + job = JSON.parse(result.stdout); + } catch { + throw new Error("producer job lookup returned invalid JSON"); + } + if (!job || typeof job !== "object" || Array.isArray(job)) { + throw new Error("producer job lookup must return an object"); + } + + assertEqual(job.id, jobId, "job ID"); + assertEqual(job.name, jobName, "job name"); + assertEqual(job.run_id, runId, "run ID"); + assertEqual(job.run_attempt, runAttempt, "run attempt"); + assertEqual(job.status, "completed", "job status"); + assertEqual(job.conclusion, "success", "job conclusion"); +} + +try { + main(); +} catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; +} diff --git a/scripts/verify-mac-node-worker-fs.mjs b/scripts/verify-mac-node-worker-fs.mjs new file mode 100644 index 0000000000000000000000000000000000000000..d2f2285b8d30b8e578e9b9a23fb0e66ed6742b25 --- /dev/null +++ b/scripts/verify-mac-node-worker-fs.mjs @@ -0,0 +1,51 @@ +#!/usr/bin/env node +import assert from "node:assert/strict"; +import fs from "node:fs"; +import { createRequire } from "node:module"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; + +const [packageArg, home] = process.argv.slice(2); +if (!packageArg || !home) { + throw new Error("Usage: verify-mac-node-worker-fs.mjs <package-root> <proof-home>"); +} +assert.equal(process.env.HOME, home); +assert.equal(process.env.TMPDIR, home); +assert.equal(process.env.FS_SAFE_NATIVE_MODE, "require"); +const packageRoot = fs.realpathSync(packageArg); +const { root } = await import( + pathToFileURL(path.join(packageRoot, "dist/plugin-sdk/memory-core-host-engine-fs.js")).href +); +const scoped = await root(home); +const written = "bundled worker write proof\n"; +const created = "bundled worker create proof\n"; +fs.writeFileSync(path.join(home, "native-write-proof"), "before replacement\n"); +await scoped.write("native-write-proof", written); +await scoped.create("native-create-proof", created); +assert.deepEqual(fs.readFileSync(path.join(home, "native-write-proof")), Buffer.from(written)); +assert.deepEqual(fs.readFileSync(path.join(home, "native-create-proof")), Buffer.from(created)); + +// Resolve from fs-safe's dependency scope, including pnpm's nested install layout. +const requireFromPackage = createRequire(path.join(packageRoot, "package.json")); +const requireFromFsSafe = createRequire( + requireFromPackage.resolve("@openclaw/fs-safe/package.json"), +); +const nativeModule = requireFromFsSafe.resolve( + `@openclaw/fs-safe-${process.platform}-${process.arch}`, +); +assert( + nativeModule.startsWith(packageRoot + path.sep), + "fs-safe native package is outside the worker payload", +); +const loaded = Object.keys(createRequire(import.meta.url).cache).filter( + (file) => path.basename(file) === "fs-safe-native.node", +); +assert.deepEqual(loaded, [nativeModule], "Installed fs-safe native module path mismatch"); +console.log( + JSON.stringify({ + architecture: process.arch, + nativeModule, + writeBytes: Buffer.byteLength(written), + createBytes: Buffer.byteLength(created), + }), +); diff --git a/scripts/verify-pr-hosted-gates.mts b/scripts/verify-pr-hosted-gates.mts new file mode 100644 index 0000000000000000000000000000000000000000..a7112db95abbfa55887c12e244d228b2cc3cc895 --- /dev/null +++ b/scripts/verify-pr-hosted-gates.mts @@ -0,0 +1,1161 @@ +#!/usr/bin/env node +import { execFileSync, spawnSync } from "node:child_process"; +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import path from "node:path"; +import { minimatch } from "minimatch"; +import { parse } from "yaml"; +// Materialized PR wrappers must use the verified source, not the caller's tsconfig aliases. +import { isRecord, readStringField } from "../packages/normalization-core/src/record-coerce.ts"; +import { + booleanFlag, + classifyBoundedUnsignedDecimal, + parseFlagArgs, + stringFlag, +} from "./lib/arg-utils.mts"; +import { isDirectRunUrl } from "./lib/direct-run.mjs"; +import { execGhApiRead, plainGhEnv } from "./lib/plain-gh.mjs"; + +const SCHEDULED_HOSTED_WORKFLOW_PATHS = new Map([ + ["Blacksmith Testbox", ".github/workflows/ci-check-testbox.yml"], + ["Blacksmith ARM Testbox", ".github/workflows/ci-check-arm-testbox.yml"], + ["Blacksmith Build Artifacts Testbox", ".github/workflows/ci-build-artifacts-testbox.yml"], + ["Workflow Sanity", ".github/workflows/workflow-sanity.yml"], +]); +export const SCHEDULED_HOSTED_WORKFLOWS = [...SCHEDULED_HOSTED_WORKFLOW_PATHS.keys()]; +const CI_WORKFLOW_PATH = ".github/workflows/ci.yml"; +const BUILD_ARTIFACTS_WORKFLOW = "Blacksmith Build Artifacts Testbox"; +const ARTIFACT_FALLBACK_REQUIRED_WORKFLOWS = [ + "Blacksmith Testbox", + "Blacksmith ARM Testbox", + "Workflow Sanity", +]; +// Full workflow-run objects are large enough for a 100-row response to exceed +// the Octopool relay cap on busy SHAs. Keep each REST page bounded and retain +// the existing 1,000-result search window through pagination. +const WORKFLOW_RUNS_PAGE_SIZE = 30; +const MAX_WORKFLOW_RUN_SEARCH_RESULTS = 1_000; +export const HOSTED_GATE_MAX_AGE_HOURS = 24; +const HOSTED_GATE_MAX_AGE_MS = HOSTED_GATE_MAX_AGE_HOURS * 60 * 60 * 1_000; +const HOSTED_GATE_CLOCK_SKEW_MS = 5 * 60 * 1_000; +const MAX_CI_REUSE_CANDIDATES = 5; +const CI_REUSE_RUN_LIST_LIMIT = 50; +const GIT_MAX_BUFFER_BYTES = 64 * 1024 * 1024; +const PATH_GLOB_OPTIONS = { dot: true, nocomment: true, nonegate: true }; + +type WorkflowRun = Partial<ReturnType<typeof toWorkflowRun>>; +type CiGateJob = ReturnType<typeof toCiGateJob>; + +type ExecGit = (args: string[], options?: { input?: string }) => string; +type PullRequestPathFilter = { paths?: string[]; "paths-ignore"?: string[] }; +type CollectHostedGateEvidenceParams = { + sha: string; + mainSha: string; + pr?: number; + recentSha?: string; + pullRequestCommitShas?: string[]; + pullRequestHeadBranch?: string; + pullRequestHeadRepository?: string; + workflowRuns: WorkflowRun[]; + ciGateJobs?: CiGateJob[]; + loadCiReuseCandidates?: () => WorkflowRun[]; + execGit?: ExecGit; + notApplicableScheduledWorkflows?: string[]; + changelogOnly?: boolean; + nowMs?: number; +}; + +type HostedGateEvidence = { + headSha: string; + evidenceHeadSha?: string; + reusedFromSha?: string; + reusedRunId?: number; + patchIdMatched?: true; + workflows: Array<{ + id: number | undefined; + name: string | undefined; + event: string | undefined; + headSha: string | undefined; + headBranch: string | undefined; + status: string | undefined; + conclusion: string | null | undefined; + createdAt: string | undefined; + updatedAt: string | undefined; + url: string | undefined; + }>; + fallbackCoveredWorkflows?: Array<{ name: string; coveredBy: string; reason: string }>; + notApplicableWorkflows?: string[]; +}; + +function optionalNumber(record: Record<string, unknown>, key: string): number | undefined { + const value = record[key]; + return typeof value === "number" ? value : undefined; +} + +function optionalNullableString( + record: Record<string, unknown>, + key: string, +): string | null | undefined { + const value = record[key]; + return value === null || typeof value === "string" ? value : undefined; +} + +function toWorkflowRun(run: Record<string, unknown>) { + const headRepository = isRecord(run.head_repository) + ? { full_name: readStringField(run.head_repository, "full_name") } + : undefined; + const pullRequests = Array.isArray(run.pull_requests) + ? run.pull_requests.filter(isRecord).map((pullRequest) => ({ + number: optionalNumber(pullRequest, "number"), + })) + : undefined; + return { + id: optionalNumber(run, "id"), + run_number: optionalNumber(run, "run_number"), + run_attempt: optionalNumber(run, "run_attempt"), + name: readStringField(run, "name"), + event: readStringField(run, "event"), + head_sha: readStringField(run, "head_sha"), + head_branch: readStringField(run, "head_branch"), + head_repository: headRepository, + path: readStringField(run, "path"), + display_title: readStringField(run, "display_title"), + status: readStringField(run, "status"), + conclusion: optionalNullableString(run, "conclusion"), + created_at: readStringField(run, "created_at"), + updated_at: readStringField(run, "updated_at"), + html_url: readStringField(run, "html_url"), + pull_requests: pullRequests, + }; +} + +function toCiGateJob(job: Record<string, unknown>) { + return { + name: readStringField(job, "name"), + run_id: optionalNumber(job, "run_id"), + run_attempt: optionalNumber(job, "run_attempt"), + status: readStringField(job, "status"), + conclusion: readStringField(job, "conclusion"), + completed_at: readStringField(job, "completed_at"), + }; +} + +export function parseArgs(argv: readonly string[]) { + const args = { + repo: "", + sha: "", + mainSha: "", + pr: 0, + recentSha: "", + output: "", + changelogOnly: false, + }; + parseFlagArgs( + argv, + args, + [ + ...( + [ + ["--repo", "repo"], + ["--sha", "sha"], + ["--main-sha", "mainSha"], + ["--recent-sha", "recentSha"], + ["--output", "output"], + ] as const + ).map(([flag, key]) => + stringFlag(flag, key, { + allowInline: false, + missingValueMessage: `Expected ${flag} <value>.`, + rejectShortOptions: true, + }), + ), + stringFlag("--pr", "pr", { + allowInline: false, + missingValueMessage: "Expected --pr <value>.", + rejectShortOptions: true, + transform(value: string) { + const result = classifyBoundedUnsignedDecimal(value, 1, Number.MAX_SAFE_INTEGER); + if (result.kind !== "value") { + throw new Error("Expected --pr <positive-integer>."); + } + return result.value; + }, + }), + booleanFlag("--changelog-only", "changelogOnly"), + ], + { + duplicateOptionMessage: (flag: string) => `${flag} was provided more than once.`, + ignoreDoubleDash: false, + onUnhandledArg(arg: string) { + throw new Error(`Unknown option: ${arg}`); + }, + }, + ); + if ( + !args.repo || + !args.sha || + !/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/u.test(args.mainSha) || + !args.pr || + !args.output + ) { + throw new Error( + "Usage: node scripts/verify-pr-hosted-gates.mjs --repo <owner/repo> --sha <sha> --main-sha <sha> --pr <number> [--recent-sha <sha>] --output <path>", + ); + } + return args; +} + +function matchesOrderedPathPatterns(changedPath: string, patterns: string[]) { + let included = false; + for (const declaredPattern of patterns ?? []) { + const excluded = declaredPattern.startsWith("!"); + const pattern = excluded ? declaredPattern.slice(1) : declaredPattern; + if (minimatch(changedPath, pattern, PATH_GLOB_OPTIONS)) { + included = !excluded; + } + } + return included; +} + +function pullRequestPathFilterApplies(pullRequest: PullRequestPathFilter, changedPaths: string[]) { + if (changedPaths.length === 0) { + return false; + } + const includedPatterns = pullRequest.paths ?? []; + if (includedPatterns.length > 0) { + return changedPaths.some((changedPath) => + matchesOrderedPathPatterns(changedPath, includedPatterns), + ); + } + const ignoredPatterns = pullRequest["paths-ignore"] ?? []; + if (ignoredPatterns.length > 0) { + return changedPaths.some((changedPath: string) => + ignoredPatterns.every( + (pattern: string) => !minimatch(changedPath, pattern, PATH_GLOB_OPTIONS), + ), + ); + } + return true; +} + +export function notApplicableScheduledHostedWorkflows(changedPaths: string[]) { + return [...SCHEDULED_HOSTED_WORKFLOW_PATHS] + .filter(([expectedName, workflowPath]) => { + const workflow = parse(readFileSync(workflowPath, "utf8")); + if (workflow?.name !== expectedName) { + throw new Error(`${workflowPath} must declare workflow name ${expectedName}.`); + } + if (!Object.hasOwn(workflow?.on ?? {}, "pull_request")) { + throw new Error(`${workflowPath} must declare a pull_request trigger.`); + } + const pullRequest = workflow.on.pull_request ?? {}; + if (Object.hasOwn(pullRequest, "branches") || Object.hasOwn(pullRequest, "branches-ignore")) { + throw new Error(`${workflowPath} pull_request branch filters are not supported.`); + } + return !pullRequestPathFilterApplies(pullRequest, changedPaths); + }) + .map(([workflowName]) => workflowName); +} + +function formatObservedRuns(runs: WorkflowRun[]) { + if (runs.length === 0) { + return "none"; + } + return runs + .map( + (run) => `${run.id ?? "unknown"}:${run.status ?? "unknown"}/${run.conclusion ?? "unknown"}`, + ) + .join(", "); +} + +function isReleaseGateCiRun(run: WorkflowRun, sha: string) { + return ( + run?.event === "workflow_dispatch" && + run?.head_sha === sha && + (run.path ?? "").split("@", 1)[0] === CI_WORKFLOW_PATH && + run?.display_title === `CI release gate ${sha}` + ); +} + +function matchingAuthoritativeRuns( + runs: WorkflowRun[], + workflowName: string, + sha: string, + allowManual = true, +) { + return runs.filter((run) => { + if (run?.head_sha !== sha) { + return false; + } + if (run?.event === "pull_request") { + return run?.name === workflowName; + } + return allowManual && workflowName === "CI" && isReleaseGateCiRun(run, sha); + }); +} + +function latestRun(runs: WorkflowRun[]) { + // GitHub run_number is creation order; updated_at moves as jobs finish. + return runs.toSorted( + (left, right) => (right.run_number ?? right.id ?? 0) - (left.run_number ?? left.id ?? 0), + )[0]; +} + +function latestNonSkippedScheduledRun(runs: WorkflowRun[]) { + return latestRun( + runs.filter((run) => !(run.status === "completed" && run.conclusion === "skipped")), + ); +} + +function runUpdatedAtMs(run: Pick<WorkflowRun, "updated_at"> | undefined) { + const value = Date.parse(run?.updated_at ?? ""); + return Number.isFinite(value) ? value : null; +} + +function isRecentRun(run: WorkflowRun | undefined, nowMs: number) { + const updatedAtMs = runUpdatedAtMs(run); + return ( + updatedAtMs !== null && + updatedAtMs >= nowMs - HOSTED_GATE_MAX_AGE_MS && + updatedAtMs <= nowMs + HOSTED_GATE_CLOCK_SKEW_MS + ); +} + +function isSuccessfulRecentRun(run: WorkflowRun | undefined, nowMs: number) { + return run?.status === "completed" && run.conclusion === "success" && isRecentRun(run, nowMs); +} + +function runGit(args: string[], { input }: { input?: string } = {}) { + const result = spawnSync("git", args, { + encoding: "utf8", + input, + maxBuffer: GIT_MAX_BUFFER_BYTES, + stdio: [input === undefined ? "ignore" : "pipe", "pipe", "pipe"], + }); + if (result.error) { + throw result.error; + } + if (result.status !== 0) { + throw new Error(`git ${args[0]} failed with exit code ${result.status ?? "unknown"}`); + } + return result.stdout; +} + +function parseSingleObjectId(raw: unknown, label: string) { + const values = String(raw).trim().split(/\s+/u).filter(Boolean); + const value = values[0]; + if (values.length !== 1 || !value || !/^[0-9a-f]{40,64}$/u.test(value)) { + throw new Error(`Expected one ${label} object id.`); + } + return value; +} + +function computePatchId(sha: string, mainRef: string, execGit: ExecGit) { + const mergeBase = parseSingleObjectId( + execGit(["merge-base", mainRef, sha]), + `merge base for ${sha}`, + ); + const diff = execGit(["diff", mergeBase, sha]); + const patchIdLines = execGit(["patch-id", "--stable"], { input: diff }) + .trim() + .split(/\r?\n/u) + .filter(Boolean); + if (patchIdLines.length !== 1) { + throw new Error(`Expected one patch id for ${sha}.`); + } + const patchIdLine = patchIdLines[0]; + if (!patchIdLine) { + throw new Error(`Invalid patch-id output for ${sha}.`); + } + const [patchId, commitId, ...rest] = patchIdLine.trim().split(/\s+/u); + if ( + rest.length > 0 || + !/^[0-9a-f]{40,64}$/u.test(patchId ?? "") || + !/^[0-9a-f]{40,64}$/u.test(commitId ?? "") + ) { + throw new Error(`Invalid patch-id output for ${sha}.`); + } + return patchId; +} + +function ensureCommitAvailable(sha: string, execGit: ExecGit) { + try { + execGit(["cat-file", "-e", `${sha}^{commit}`]); + return true; + } catch { + try { + execGit(["fetch", "origin", sha]); + execGit(["cat-file", "-e", `${sha}^{commit}`]); + return true; + } catch { + return false; + } + } +} + +function isQualifyingCiReuseRun(run: WorkflowRun) { + if (run?.event === "pull_request") { + return run?.name === "CI"; + } + return typeof run.head_sha === "string" && isReleaseGateCiRun(run, run.head_sha); +} + +function findPatchIdenticalCiReuse({ + sha, + candidateRuns, + nowMs, + mainSha, + execGit = runGit, +}: { + sha: string; + candidateRuns: WorkflowRun[]; + nowMs: number; + mainSha: string; + execGit?: ExecGit; +}) { + if (!Array.isArray(candidateRuns)) { + return undefined; + } + const candidates = candidateRuns + .filter( + (run) => + run?.head_sha !== sha && + /^[0-9a-f]{40,64}$/u.test(run.head_sha ?? "") && + isQualifyingCiReuseRun(run) && + isSuccessfulRecentRun(run, nowMs), + ) + .toSorted((left, right) => (right.updated_at ?? "").localeCompare(left.updated_at ?? "")) + .slice(0, MAX_CI_REUSE_CANDIDATES); + if (candidates.length === 0) { + return undefined; + } + + let currentPatchId; + try { + currentPatchId = computePatchId(sha, mainSha, execGit); + } catch { + return undefined; + } + for (const run of candidates) { + if (!run.head_sha || !ensureCommitAvailable(run.head_sha, execGit)) { + continue; + } + try { + if (computePatchId(run.head_sha, mainSha, execGit) === currentPatchId) { + return { + run, + reusedFromSha: run.head_sha, + reusedRunId: run.id, + patchIdMatched: true, + }; + } + } catch { + // A candidate is reusable only when every local git proof step succeeds. + } + } + return undefined; +} + +type PatchIdenticalReuse = NonNullable<ReturnType<typeof findPatchIdenticalCiReuse>>; + +const CI_GATE_CHECK_NAME = "openclaw/ci-gate"; + +/** + * True when this run's own openclaw/ci-gate job already succeeded on the + * run's CURRENT attempt. The gate job needs every selected lane and fails on + * any non-success result, so a successful gate proves the merge-relevant + * outcome minutes before post-gate stragglers (timing summaries, artifact + * uploads) let the run itself reach completed. Check suites survive reruns, + * so binding goes through the attempt-scoped jobs listing: the job must carry + * the run's own run_attempt — a prior attempt's gate success can never vouch + * for a rerun that has not reached its gate yet. + */ +function hasSuccessfulCiGateJob(run: WorkflowRun, ciGateJobs: CiGateJob[], nowMs: number) { + if (!run?.id || !Array.isArray(ciGateJobs)) { + return false; + } + const runAttempt = run.run_attempt ?? 1; + return ciGateJobs.some((job) => { + if (job?.name !== CI_GATE_CHECK_NAME) { + return false; + } + // Workflow attempts share a run id and filter=latest keeps a not-yet-rerun + // job's prior-attempt execution, so bind to the attempt explicitly: the + // REST job payload exposes run_attempt, and jobs are fetched from the + // attempt-specific endpoint. Both must agree with the run's attempt. + if (job?.run_id !== run.id || (job?.run_attempt ?? runAttempt) !== runAttempt) { + return false; + } + if (job?.status !== "completed" || job?.conclusion !== "success") { + return false; + } + const completedAtMs = Date.parse(job?.completed_at ?? ""); + return ( + Number.isFinite(completedAtMs) && + completedAtMs >= nowMs - HOSTED_GATE_MAX_AGE_MS && + completedAtMs <= nowMs + HOSTED_GATE_CLOCK_SKEW_MS + ); + }); +} + +function isGateProvenInProgressRun( + run: WorkflowRun | undefined, + ciGateJobs: CiGateJob[], + nowMs: number, +) { + return ( + (run?.status === "in_progress" || run?.status === "queued") && + isRecentRun(run, nowMs) && + hasSuccessfulCiGateJob(run, ciGateJobs, nowMs) + ); +} + +function preferredCiRun(runs: WorkflowRun[], nowMs: number) { + const scheduledRuns = runs.filter((run) => run.event === "pull_request"); + const latestScheduledRun = latestRun(scheduledRuns); + const latestDecision = latestRun( + scheduledRuns.filter( + (run) => + run.status === "completed" && + typeof run.conclusion === "string" && + !["cancelled", "skipped"].includes(run.conclusion), + ), + ); + const latestManualRun = latestRun(runs.filter((run) => run.event === "workflow_dispatch")); + + // Manual proof may replace stale scheduled success or a pending run, + // never an unresolved terminal non-success. + if (latestDecision && latestDecision.conclusion !== "success") { + return latestDecision; + } + if (latestScheduledRun && latestScheduledRun.status !== "completed") { + return latestRun( + [latestScheduledRun, latestManualRun].filter((run): run is WorkflowRun => run !== undefined), + ); + } + if (latestScheduledRun?.status === "completed" && isSuccessfulRecentRun(latestDecision, nowMs)) { + return latestDecision; + } + return latestManualRun ?? latestScheduledRun; +} + +function successfulRunOrThrow( + runs: WorkflowRun[], + workflowName: string, + sha: string, + { + allowManual = true, + nowMs = Date.now(), + ciGateJobs = [], + }: { allowManual?: boolean; nowMs?: number; ciGateJobs?: CiGateJob[] } = {}, +) { + const matchingRuns = matchingAuthoritativeRuns(runs, workflowName, sha, allowManual); + const run = + workflowName === "CI" + ? preferredCiRun(matchingRuns, nowMs) + : latestNonSkippedScheduledRun(matchingRuns); + if (run && isSuccessfulRecentRun(run, nowMs)) { + return run; + } + if (workflowName === "CI") { + if (run && isGateProvenInProgressRun(run, ciGateJobs, nowMs)) { + return run; + } + // A terminal non-success stays blocking unless a NEWER pending SCHEDULED + // rerun on the same head has already passed its own gate — the gate needs + // every selected lane, so that attempt is authoritative proof the failure + // is re-resolved. The newer-than bound stops a stalled older run's gate + // from masking a later failure, and manual runs can never mask one. + if (run?.status === "completed" && run.conclusion !== "success") { + const failedRunCreatedAtMs = Date.parse(run?.created_at ?? ""); + const gateProvenRerun = matchingRuns.find((candidate) => { + if (candidate === run || candidate.event !== "pull_request") { + return false; + } + const candidateCreatedAtMs = Date.parse(candidate?.created_at ?? ""); + if ( + !Number.isFinite(candidateCreatedAtMs) || + !Number.isFinite(failedRunCreatedAtMs) || + candidateCreatedAtMs <= failedRunCreatedAtMs + ) { + return false; + } + return isGateProvenInProgressRun(candidate, ciGateJobs, nowMs); + }); + if (gateProvenRerun) { + return gateProvenRerun; + } + } + } + throw new Error( + `Missing successful recent ${workflowName} workflow for ${sha}. Observed: ${formatObservedRuns(matchingRuns)}`, + ); +} + +function hasSuccessfulRecentReleaseGate(workflowRuns: WorkflowRun[], sha: string, nowMs: number) { + const releaseGate = latestRun(workflowRuns.filter((run) => isReleaseGateCiRun(run, sha))); + return isSuccessfulRecentRun(releaseGate, nowMs); +} + +function runBelongsToPullRequest( + run: WorkflowRun, + pr: number, + pullRequestCommitShas: Set<string>, + pullRequestHeadBranch: string, + pullRequestHeadRepository: string, +) { + if (run.pull_requests?.some((pullRequest) => pullRequest.number === pr)) { + return true; + } + if (Array.isArray(run?.pull_requests) && run.pull_requests.length > 0) { + return false; + } + // Fork pull_request runs currently arrive with pull_requests: []. Require + // the immutable commit plus its PR head identity; branch identity alone is + // mutable, while ancestry alone can include commits from merged branches. + return ( + typeof run.head_sha === "string" && + pullRequestCommitShas.has(run.head_sha) && + run?.head_branch === pullRequestHeadBranch && + run?.head_repository?.full_name?.toLowerCase() === pullRequestHeadRepository.toLowerCase() + ); +} + +function canCoverQueuedBuildArtifacts( + workflowRuns: WorkflowRun[], + sha: string, + nowMs: number, + notApplicableScheduledWorkflowNames: Set<string> | undefined, +) { + if (!hasSuccessfulRecentReleaseGate(workflowRuns, sha, nowMs)) { + return false; + } + const supportingGatesPassed = ARTIFACT_FALLBACK_REQUIRED_WORKFLOWS.every((workflowName) => { + const matchingRuns = matchingAuthoritativeRuns(workflowRuns, workflowName, sha, false); + if (matchingRuns.length === 0 && notApplicableScheduledWorkflowNames?.has(workflowName)) { + return true; + } + const run = latestNonSkippedScheduledRun(matchingRuns); + return isSuccessfulRecentRun(run, nowMs); + }); + if (!supportingGatesPassed) { + return false; + } + const buildArtifactRuns = matchingAuthoritativeRuns( + workflowRuns, + BUILD_ARTIFACTS_WORKFLOW, + sha, + false, + ); + const latestBuildArtifactRun = latestRun(buildArtifactRuns); + return ( + latestBuildArtifactRun?.status === "queued" && + isRecentRun(latestBuildArtifactRun, nowMs) && + buildArtifactRuns.every( + (run) => + run.status === "queued" || (run.status === "completed" && run.conclusion === "success"), + ) + ); +} + +function stripAnsi(raw: string) { + const escape = String.fromCharCode(27); + return raw.replace(new RegExp(`${escape}\\[[0-?]*[ -/]*[@-~]`, "gu"), ""); +} + +export function parseWorkflowRunPage(raw: string) { + const page = JSON.parse(stripAnsi(raw)) as unknown; + if (!isRecord(page)) { + throw new Error("Expected workflow run page to be an object."); + } + const totalCount = typeof page.total_count === "number" ? page.total_count : 0; + const workflowRuns = Array.isArray(page.workflow_runs) + ? page.workflow_runs.filter(isRecord).map(toWorkflowRun) + : []; + return { + totalCount, + workflowRuns, + }; +} + +export function workflowRunPageCount(totalCount: number) { + return Math.min( + Math.ceil(totalCount / WORKFLOW_RUNS_PAGE_SIZE), + Math.ceil(MAX_WORKFLOW_RUN_SEARCH_RESULTS / WORKFLOW_RUNS_PAGE_SIZE), + ); +} + +export function collectHostedGateEvidence({ + sha, + mainSha, + pr = 0, + recentSha = "", + pullRequestCommitShas = [], + pullRequestHeadBranch = "", + pullRequestHeadRepository = "", + workflowRuns, + ciGateJobs = [], + loadCiReuseCandidates = () => [], + execGit = runGit, + notApplicableScheduledWorkflows, + changelogOnly = false, + nowMs = Date.now(), +}: CollectHostedGateEvidenceParams): HostedGateEvidence { + if (!Array.isArray(workflowRuns)) { + throw new Error("workflowRuns must be an array."); + } + const notApplicableScheduledWorkflowNames = + notApplicableScheduledWorkflows === undefined + ? undefined + : new Set(notApplicableScheduledWorkflows); + const pullRequestCommitShaSet = new Set<string>(pullRequestCommitShas); + + const collectForSha = ( + evidenceSha: string, + { + allowManual, + requiredScheduledWorkflows = new Set<string>(), + ciRun, + }: { + allowManual: boolean; + ciRun?: WorkflowRun; + requiredScheduledWorkflows?: Set<string>; + }, + ) => { + const workflows: WorkflowRun[] = []; + const fallbackCoveredWorkflows: Array<{ name: string; coveredBy: string; reason: string }> = []; + if (!changelogOnly) { + workflows.push( + ciRun ?? + successfulRunOrThrow(workflowRuns, "CI", evidenceSha, { + allowManual, + nowMs, + // Gate proof only vouches for the exact head under verification. + ciGateJobs: evidenceSha === sha ? ciGateJobs : [], + }), + ); + } + for (const workflowName of SCHEDULED_HOSTED_WORKFLOWS) { + const matchingRuns = matchingAuthoritativeRuns( + workflowRuns, + workflowName, + evidenceSha, + allowManual, + ); + if ( + matchingRuns.length === 0 && + !requiredScheduledWorkflows.has(workflowName) && + (notApplicableScheduledWorkflowNames === undefined || + notApplicableScheduledWorkflowNames.has(workflowName)) + ) { + continue; + } + if ( + allowManual && + workflowName === BUILD_ARTIFACTS_WORKFLOW && + canCoverQueuedBuildArtifacts( + workflowRuns, + evidenceSha, + nowMs, + notApplicableScheduledWorkflowNames, + ) + ) { + fallbackCoveredWorkflows.push({ + name: workflowName, + coveredBy: "CI release gate", + reason: "scheduled workflow is queued", + }); + continue; + } + workflows.push( + successfulRunOrThrow(workflowRuns, workflowName, evidenceSha, { + allowManual, + nowMs, + }), + ); + } + return { workflows, fallbackCoveredWorkflows }; + }; + + let ciRun: WorkflowRun | undefined; + let ciReuse: PatchIdenticalReuse | undefined; + if (!changelogOnly) { + try { + ciRun = successfulRunOrThrow(workflowRuns, "CI", sha, { + allowManual: true, + nowMs, + ciGateJobs, + }); + } catch (exactCiError) { + let candidateRuns: WorkflowRun[]; + try { + candidateRuns = loadCiReuseCandidates(); + } catch { + candidateRuns = []; + } + ciReuse = findPatchIdenticalCiReuse({ + sha, + mainSha, + candidateRuns, + nowMs, + execGit, + }); + if (!ciReuse) { + throw exactCiError; + } + ciRun = ciReuse.run; + } + } + + let evidenceSha = sha; + let selected: + | { + workflows: WorkflowRun[]; + fallbackCoveredWorkflows: Array<{ name: string; coveredBy: string; reason: string }>; + } + | undefined; + try { + selected = collectForSha(sha, { allowManual: true, ciRun }); + } catch (exactError) { + // Scheduled hosted workflows retain their existing recent-cohort fallback. + // CI itself is either exact-head proof or the patch-identical run selected + // above; never silently replace it with an unverified prior-head run. + const targetScheduledWorkflows = new Set( + SCHEDULED_HOSTED_WORKFLOWS.filter( + (workflowName) => + (notApplicableScheduledWorkflowNames !== undefined && + !notApplicableScheduledWorkflowNames.has(workflowName)) || + matchingAuthoritativeRuns(workflowRuns, workflowName, sha, false).length > 0, + ), + ); + const fallbackShas = [ + ciReuse?.reusedFromSha, + recentSha, + ...workflowRuns + .filter( + (run) => + run?.event === "pull_request" && + run?.head_sha !== sha && + runBelongsToPullRequest( + run, + pr, + pullRequestCommitShaSet, + pullRequestHeadBranch, + pullRequestHeadRepository, + ) && + isRecentRun(run, nowMs), + ) + .toSorted((left, right) => (right.updated_at ?? "").localeCompare(left.updated_at ?? "")) + .map((run) => run.head_sha), + ].filter((value): value is string => typeof value === "string" && value.length > 0); + let fallbackError; + for (const fallbackSha of new Set(fallbackShas)) { + try { + selected = collectForSha(fallbackSha, { + allowManual: false, + requiredScheduledWorkflows: targetScheduledWorkflows, + ciRun, + }); + evidenceSha = fallbackSha; + break; + } catch (error) { + fallbackError ??= error; + } + } + if (!selected) { + throw fallbackError ?? exactError; + } + } + + const evidence: HostedGateEvidence = { + headSha: sha, + ...(evidenceSha !== sha ? { evidenceHeadSha: evidenceSha } : {}), + ...(ciReuse + ? { + reusedFromSha: ciReuse.reusedFromSha, + reusedRunId: ciReuse.reusedRunId, + patchIdMatched: true satisfies true, + } + : {}), + ...(selected.fallbackCoveredWorkflows.length > 0 + ? { fallbackCoveredWorkflows: selected.fallbackCoveredWorkflows } + : {}), + workflows: selected.workflows.map((run) => ({ + id: run.id, + name: run.name, + event: run.event, + headSha: run.head_sha, + headBranch: run.head_branch, + status: run.status, + conclusion: run.conclusion, + createdAt: run.created_at, + updatedAt: run.updated_at, + url: run.html_url, + })), + }; + const notApplicableWorkflows = (notApplicableScheduledWorkflows ?? []).filter( + (workflowName) => + matchingAuthoritativeRuns(workflowRuns, workflowName, sha, false).length === 0, + ); + return notApplicableWorkflows.length > 0 ? { ...evidence, notApplicableWorkflows } : evidence; +} + +export function workflowRunQueryPaths( + repo: string, + { sha, recentSha, headBranch }: { sha: string; recentSha: string; headBranch?: string }, + page = 1, +) { + const pageSuffix = `per_page=${WORKFLOW_RUNS_PAGE_SIZE}&page=${page}`; + const shas = [...new Set([sha, recentSha].filter(Boolean))]; + const queries = shas.map( + (headSha) => `repos/${repo}/actions/runs?head_sha=${encodeURIComponent(headSha)}&${pageSuffix}`, + ); + if (headBranch) { + queries.push( + `repos/${repo}/actions/runs?branch=${encodeURIComponent(headBranch)}&event=pull_request&${pageSuffix}`, + ); + } + return queries; +} + +function loadWorkflowRunsForQuery(queryForPage: (page: number) => string) { + const loadPage = (page: number) => + parseWorkflowRunPage( + execGhApiRead(queryForPage(page), { + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }), + ); + + // Bound every SHA query to GitHub's documented search window. + const firstPage = loadPage(1); + const workflowRuns = [...firstPage.workflowRuns]; + for (let page = 2; page <= workflowRunPageCount(firstPage.totalCount); page += 1) { + workflowRuns.push(...loadPage(page).workflowRuns); + } + return workflowRuns; +} + +function loadWorkflowRuns(repo: string, sha: string, recentSha: string, headBranch?: string) { + const queries = workflowRunQueryPaths(repo, { sha, recentSha, headBranch }); + const withPage = (query: string, page: number) => query.replace(/page=1$/u, `page=${page}`); + const workflowRuns = queries.flatMap((query) => + loadWorkflowRunsForQuery((page) => withPage(query, page)), + ); + return [...new Map(workflowRuns.map((run) => [run.id, run])).values()]; +} + +function loadCiReuseCandidateRuns(repo: string, headBranch: string) { + const raw = execFileSync( + "gh", + [ + "run", + "list", + "--repo", + repo, + "--workflow", + "ci.yml", + "--branch", + headBranch, + "--limit", + String(CI_REUSE_RUN_LIST_LIMIT), + "--json", + "databaseId,workflowName,headSha,headBranch,event,status,conclusion,createdAt,updatedAt,url,displayTitle", + ], + { + encoding: "utf8", + env: plainGhEnv(), + stdio: ["ignore", "pipe", "pipe"], + }, + ); + const runs = JSON.parse(stripAnsi(raw)) as unknown; + if (!Array.isArray(runs)) { + throw new Error("Expected gh run list to return an array."); + } + if (!runs.every(isRecord)) { + throw new Error("Expected every gh run list entry to be an object."); + } + // The workflow selector above supplies the path identity that the REST + // release-gate matcher normally reads from each full workflow-run object. + return runs.map((run): WorkflowRun => ({ + id: optionalNumber(run, "databaseId"), + name: readStringField(run, "workflowName"), + event: readStringField(run, "event"), + status: readStringField(run, "status"), + conclusion: optionalNullableString(run, "conclusion"), + head_sha: readStringField(run, "headSha"), + head_branch: readStringField(run, "headBranch"), + path: CI_WORKFLOW_PATH, + created_at: readStringField(run, "createdAt"), + updated_at: readStringField(run, "updatedAt"), + html_url: readStringField(run, "url"), + display_title: readStringField(run, "displayTitle"), + })); +} + +export function loadPullRequestCommitShas( + { baseSha, headSha }: { baseSha: string; headSha: string }, + execGit: ExecGit = runGit, +) { + const output = execGit(["rev-list", "--reverse", `${baseSha}..${headSha}`]); + const shas = output.replace(/\r?\n$/u, "").split(/\r?\n/u); + if (shas.length === 0 || shas.some((sha) => !/^[0-9a-f]{40,64}$/u.test(sha))) { + throw new Error("Expected pull request commit object ids from git rev-list."); + } + if (!shas.includes(headSha)) { + throw new Error(`Expected pull request commit list to contain head ${headSha}.`); + } + return shas; +} + +function loadPullRequestChangedPaths(baseSha: string, headSha: string) { + return runGit(["diff", "--name-only", `${baseSha}...${headSha}`]) + .trim() + .split(/\r?\n/u) + .filter(Boolean); +} + +function loadCiGateJobs( + repo: string, + workflowRuns: WorkflowRun[], + sha: string, + nowMs = Date.now(), +) { + // Only an in-progress exact-head CI run can benefit from gate proof. + const candidates = workflowRuns.filter( + (run) => + run?.name === "CI" && + run?.head_sha === sha && + (run?.status === "in_progress" || run?.status === "queued") && + isRecentRun(run, nowMs), + ); + return candidates.flatMap((run) => { + const attempt = run.run_attempt ?? 1; + // The jobs endpoint pages at 100 and full-scope runs already sit near + // that; page until the gate job is visible so growth past one page can + // never silently disable the early-proof path. + const jobs: CiGateJob[] = []; + for (let page = 1; page <= 5; page += 1) { + const payload = JSON.parse( + execGhApiRead( + `repos/${repo}/actions/runs/${run.id}/attempts/${attempt}/jobs?per_page=100&page=${page}`, + { encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] }, + ), + ) as unknown; + const pageJobs = + isRecord(payload) && Array.isArray(payload.jobs) + ? payload.jobs.filter(isRecord).map(toCiGateJob) + : []; + jobs.push(...pageJobs); + const totalCount = Number(isRecord(payload) ? (payload.total_count ?? 0) : 0); + if ( + pageJobs.length === 0 || + jobs.length >= totalCount || + jobs.some((job) => job.name === CI_GATE_CHECK_NAME) + ) { + break; + } + } + // Re-read the run after fetching its attempt jobs and drop the evidence if + // the attempt advanced in between: otherwise a rerun starting in that + // window would let the just-fetched prior-attempt gate vouch for an + // attempt that has not reached its own gate. Same-attempt completion is + // fine — a run that finished successfully still proves this attempt, and + // a non-success completion must not be blessed by its own earlier gate. + const current = JSON.parse( + execGhApiRead(`repos/${repo}/actions/runs/${run.id}`, { + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }), + ) as unknown; + if (!isRecord(current)) { + return []; + } + const sameAttempt = (current.run_attempt ?? attempt) === attempt; + const stillPending = current.status === "in_progress" || current.status === "queued"; + const completedSuccess = current.status === "completed" && current.conclusion === "success"; + if (!sameAttempt || (!stillPending && !completedSuccess)) { + return []; + } + return jobs; + }); +} + +function main(argv = process.argv.slice(2)) { + const args = parseArgs(argv); + const pullRequest = JSON.parse( + execGhApiRead(`repos/${args.repo}/pulls/${args.pr}`, { + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }), + ) as unknown; + const head = isRecord(pullRequest) && isRecord(pullRequest.head) ? pullRequest.head : undefined; + const headRepo = head && isRecord(head.repo) ? head.repo : undefined; + const headBranch = head ? readStringField(head, "ref") : undefined; + const headRepository = headRepo ? readStringField(headRepo, "full_name") : undefined; + const headSha = head ? readStringField(head, "sha") : undefined; + if (!headBranch || !headRepository || !headSha) { + throw new Error(`PR #${args.pr} is missing head metadata.`); + } + if (headSha !== args.sha) { + throw new Error(`PR #${args.pr} head changed from ${args.sha} to ${headSha}.`); + } + // Paths, membership and patch IDs share one snapshot; a newer API base may not exist locally. + const changedPaths = loadPullRequestChangedPaths(args.mainSha, headSha); + const workflowRuns = loadWorkflowRuns(args.repo, args.sha, args.recentSha, headBranch); + const evidence = collectHostedGateEvidence({ + sha: args.sha, + mainSha: args.mainSha, + pr: args.pr, + recentSha: args.recentSha, + pullRequestCommitShas: loadPullRequestCommitShas({ baseSha: args.mainSha, headSha }), + pullRequestHeadBranch: headBranch, + pullRequestHeadRepository: headRepository, + workflowRuns, + ciGateJobs: loadCiGateJobs(args.repo, workflowRuns, args.sha), + loadCiReuseCandidates: () => loadCiReuseCandidateRuns(args.repo, headBranch), + notApplicableScheduledWorkflows: notApplicableScheduledHostedWorkflows(changedPaths), + changelogOnly: args.changelogOnly, + }); + const evidenceHeadSha = evidence.evidenceHeadSha ?? args.sha; + const notApplicableWorkflows = + "notApplicableWorkflows" in evidence ? evidence.notApplicableWorkflows : undefined; + const manifest = { + schemaVersion: 2, + generatedAt: new Date().toISOString(), + repo: args.repo, + pullRequestNumber: args.pr, + selection: { + mode: evidence.patchIdMatched + ? "patch-identical-pre-rebase" + : evidenceHeadSha === args.sha + ? "exact-head" + : "recent-pr-head", + maxAgeHours: HOSTED_GATE_MAX_AGE_HOURS, + }, + ...evidence, + }; + mkdirSync(path.dirname(args.output), { recursive: true }); + writeFileSync(args.output, `${JSON.stringify(manifest, null, 2)}\n`); + if (evidence.patchIdMatched) { + const reusedRun = manifest.workflows.find((workflow) => workflow.id === evidence.reusedRunId); + const updatedAtMs = runUpdatedAtMs({ updated_at: reusedRun?.updatedAt }); + const ageHours = + updatedAtMs === null + ? "unknown" + : `${(Math.max(0, Date.now() - updatedAtMs) / (60 * 60 * 1_000)).toFixed(1)}h`; + console.log( + `hosted CI reused from patch-identical pre-rebase head ${evidence.reusedFromSha} (run ${evidence.reusedRunId}, age ${ageHours})`, + ); + } + console.log( + `Hosted gates passed for PR #${args.pr} at ${args.sha} using ${evidenceHeadSha}: ${manifest.workflows + .map((workflow) => `${workflow.name}#${workflow.id}`) + .join(", ")}${ + notApplicableWorkflows?.length ? `; not applicable: ${notApplicableWorkflows.join(", ")}` : "" + }`, + ); +} + +if (isDirectRunUrl(process.argv[1], import.meta.url)) { + main(); +} diff --git a/scripts/vitest-pair-benchmark-lanes.json b/scripts/vitest-pair-benchmark-lanes.json new file mode 100644 index 0000000000000000000000000000000000000000..2c1cb0b07c072c553d44ee8d5e7fa25656e0b6b4 --- /dev/null +++ b/scripts/vitest-pair-benchmark-lanes.json @@ -0,0 +1,54 @@ +{ + "version": 1, + "rounds": 7, + "thresholds": { + "overallWallRatio": 1.05, + "criticalLaneWallRatio": 1.1, + "criticalLaneWallDeltaMs": 1000, + "improvementRatio": 0.95, + "improvementPairCount": 5 + }, + "lanes": [ + { + "id": "core-unit", + "critical": true, + "config": "test/vitest/vitest.unit-fast.config.ts", + "files": [ + "packages/markdown-core/src/chunk-text.test.ts", + "packages/media-core/src/mime.test.ts", + "src/acp/commands.test.ts", + "src/agents/agent-run-terminal-outcome.test.ts" + ] + }, + { + "id": "gateway-core", + "critical": true, + "config": "test/vitest/vitest.gateway-core.config.ts", + "files": [ + "src/gateway/call.test.ts", + "src/gateway/http-common.test.ts", + "src/gateway/method-scopes.test.ts", + "src/gateway/probe.close-drain.test.ts" + ] + }, + { + "id": "ui-jsdom", + "critical": true, + "config": "test/vitest/vitest.ui.config.ts", + "files": [ + "ui/src/app/connection-bootstrap.test.ts", + "ui/src/app/control-ui-auth.test.ts", + "ui/src/app/gateway-readiness.test.ts", + "ui/src/app/native-route-memory.test.ts" + ] + }, + { + "id": "worker-lifecycle", + "critical": true, + "config": "test/vitest/vitest.tooling-isolated.config.ts", + "files": [ + "test/scripts/vitest-fork-shutdown.test.ts" + ] + } + ] +} diff --git a/scripts/watch-node.mts b/scripts/watch-node.mts new file mode 100644 index 0000000000000000000000000000000000000000..e0be7db25ec7a0ce1a2fdc25457a02384e68e6bc --- /dev/null +++ b/scripts/watch-node.mts @@ -0,0 +1,823 @@ +// Watches dev source paths and restarts scripts/run-node.mjs when relevant +// files change. +import { spawn } from "node:child_process"; +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import process from "node:process"; +import { pathToFileURL } from "node:url"; +import { toErrorObject } from "./lib/error-format.mts"; +import { sleep } from "./lib/sleep.mjs"; +import { + createRunNodePathClassifier, + normalizeRunNodePath as normalizePath, + runNodeWatchedPaths, +} from "./run-node-watch-paths.mts"; + +const WATCH_NODE_RUNNER = "scripts/run-node.mjs"; +const WATCH_RESTART_SIGNAL = "SIGTERM"; +const WATCH_RESTARTABLE_CHILD_EXIT_CODES = new Set([143]); +const WATCH_IGNORED_PATH_SEGMENTS = new Set([".git", "dist", "node_modules"]); +const WATCH_LOCK_WAIT_MS = 5_000; +const WATCH_LOCK_POLL_MS = 100; +const WATCH_SHUTDOWN_KILL_GRACE_MS = 5_000; +const WATCH_LOCK_DIR = path.join(".local", "watch-node"); +const WATCH_DIST_ENTRY_POLL_MS = 1_000; +const WATCH_DIST_ENTRY_TIMEOUT_MS = 5 * 60 * 1_000; +const AUTO_DOCTOR_DISABLE_VALUES = new Set(["0", "false", "no", "off"]); +type ProcessSignal = `SIG${string}`; +type WatchExit = number | ProcessSignal; +type TimerHandle = ReturnType<typeof setTimeout>; + +type WatchChild = { + pid?: number; + kill(signal?: ProcessSignal | number): boolean | void; + on(event: "exit", callback: (code: number | null, signal: ProcessSignal | null) => void): unknown; + on(event: "error", callback: (error: Error) => void): unknown; +}; +type WatchPathStats = { isDirectory(): boolean }; +type WatchOptions = { + ignoreInitial: boolean; + ignored: (watchPath: string, stats?: WatchPathStats) => boolean; +}; +type Watcher = { + on(event: "add" | "change" | "unlink", callback: (path: string) => void): void; + on(event: "error", callback: (error: unknown) => void): void; + close?: () => { catch?(onRejected: () => void): unknown } | void; +}; +type WatcherFactory = (paths: string[], options: WatchOptions) => Watcher; +type WatchPathClassifier = { + refreshGeneratedPluginAssetPaths(): void; + isRestartRelevantRunNodePath(repoPath: unknown): boolean; +}; +type SignalProcess = (pid: number, signal: ProcessSignal | 0) => void; +type WatchMainParams = { + spawn?: ( + command: string, + args: string[], + options: { + cwd: string; + detached: boolean; + env: NodeJS.ProcessEnv; + stdio: "inherit"; + }, + ) => WatchChild; + createWatcher?: WatcherFactory; + loadChokidar?: () => Promise<{ watch: WatcherFactory }>; + watchPaths?: string[]; + pathClassifier?: WatchPathClassifier; + process?: NodeJS.Process; + cwd?: string; + args?: string[]; + env?: NodeJS.ProcessEnv; + fs?: Pick<typeof fs, "existsSync">; + now?: () => number; + sleep?: (milliseconds: number) => Promise<void>; + signalProcess?: SignalProcess; + lockDisabled?: boolean; +}; + +type WatchDeps = Required< + Pick< + WatchMainParams, + | "spawn" + | "loadChokidar" + | "watchPaths" + | "pathClassifier" + | "process" + | "cwd" + | "args" + | "env" + | "fs" + | "now" + | "sleep" + | "signalProcess" + | "lockDisabled" + > +> & { createWatcher?: WatcherFactory }; + +type WatchLock = { + pid: number; + command: string; + createdAt: string; + cwd: string; + watchSession: string; +}; + +const buildRunnerArgs = (args: string[]) => [WATCH_NODE_RUNNER, ...args]; +const buildDoctorRunnerArgs = () => [WATCH_NODE_RUNNER, "doctor", "--fix", "--non-interactive"]; + +const resolveRepoPath = (filePath: unknown, cwd: string) => { + const rawPath = typeof filePath === "string" ? filePath : ""; + if (path.isAbsolute(rawPath)) { + return normalizePath(path.relative(cwd, rawPath)); + } + return normalizePath(rawPath); +}; + +const hasIgnoredPathSegment = (repoPath: string) => + normalizePath(repoPath) + .split("/") + .some((segment) => WATCH_IGNORED_PATH_SEGMENTS.has(segment)); + +const looksLikeDirectoryPath = (repoPath: string) => + path.posix.extname(normalizePath(repoPath)) === ""; + +const isDirectoryLikeWatchedPath = (repoPath: string, watchPaths: string[]) => { + const normalizedRepoPath = normalizePath(repoPath).replace(/\/$/, ""); + return watchPaths.some((watchPath) => { + const normalizedWatchPath = normalizePath(watchPath).replace(/\/$/, ""); + if (!normalizedWatchPath) { + return false; + } + return ( + normalizedRepoPath === normalizedWatchPath || + normalizedRepoPath.startsWith(`${normalizedWatchPath}/`) + ); + }); +}; + +const isIgnoredWatchPath = ( + filePath: unknown, + cwd: string, + watchPaths: string[], + pathClassifier: WatchPathClassifier, + stats?: WatchPathStats, +) => { + const repoPath = resolveRepoPath(filePath, cwd); + if (hasIgnoredPathSegment(repoPath)) { + return true; + } + if (isDirectoryLikeWatchedPath(repoPath, watchPaths)) { + if (stats?.isDirectory?.() || looksLikeDirectoryPath(repoPath)) { + return false; + } + } + return !pathClassifier.isRestartRelevantRunNodePath(repoPath); +}; + +const shouldRestartAfterChildExit = ( + exitCode: number | null, + exitSignal: ProcessSignal | null, + platform: NodeJS.Platform, +) => + (typeof exitCode === "number" && WATCH_RESTARTABLE_CHILD_EXIT_CODES.has(exitCode)) || + (platform === "win32" && exitSignal === "SIGTERM"); + +const isGatewayWatchCommand = (args: string[]) => args[0] === "gateway"; + +const shouldRunAutoDoctor = (deps: WatchDeps, autoDoctorAttempted: boolean) => + !autoDoctorAttempted && + isGatewayWatchCommand(deps.args) && + !AUTO_DOCTOR_DISABLE_VALUES.has( + (deps.env.OPENCLAW_GATEWAY_WATCH_AUTO_DOCTOR ?? "").toLowerCase(), + ); + +const isProcessAlive = (pid: unknown, signalProcess: SignalProcess) => { + if (typeof pid !== "number" || !Number.isInteger(pid) || pid <= 0) { + return false; + } + try { + signalProcess(pid, 0); + } catch { + return false; + } + return true; +}; + +const createWatchLockKey = (cwd: string, args: string[]) => + createHash("sha256").update(cwd).update("\0").update(args.join("\0")).digest("hex").slice(0, 12); + +/** Resolves the lock path that prevents duplicate watch-node loops. */ +const resolveWatchLockPath = (cwd: string, args: string[] = []) => + path.join(cwd, WATCH_LOCK_DIR, `${createWatchLockKey(cwd, args)}.json`); + +const readWatchLock = (lockPath: string): WatchLock | null => { + try { + const value = JSON.parse(fs.readFileSync(lockPath, "utf8")) as unknown; + return isWatchLock(value) ? value : null; + } catch { + return null; + } +}; + +const removeWatchLock = (lockPath: string) => { + try { + fs.unlinkSync(lockPath); + } catch (error) { + if (errorCode(error) !== "ENOENT") { + throw error; + } + } +}; + +const writeWatchLock = (lockPath: string, payload: WatchLock) => { + fs.mkdirSync(path.dirname(lockPath), { recursive: true }); + fs.writeFileSync(lockPath, `${JSON.stringify(payload)}\n`, { + encoding: "utf8", + flag: "wx", + }); +}; + +const logWatcher = (message: string, deps: WatchDeps) => { + deps.process.stderr?.write?.(`[openclaw] ${message}\n`); +}; + +const isInvalidPackageConfigError = (err: unknown) => + errorCode(err) === "ERR_INVALID_PACKAGE_CONFIG"; + +const extractInvalidPackageConfigPath = (err: unknown) => { + const message = errorMessage(err); + const match = message.match(/Invalid package config (.+?) while importing /); + return match?.[1] ?? null; +}; + +const printFriendlyWatchStartupError = (err: unknown) => { + const packageConfigPath = extractInvalidPackageConfigPath(err); + + console.error(""); + console.error( + "[openclaw] gateway:watch could not start because a dependency package config looks corrupted.", + ); + if (packageConfigPath) { + console.error(`[openclaw] Invalid package config: ${packageConfigPath}`); + } + console.error("[openclaw] This usually means a file in node_modules is empty or truncated."); + console.error("[openclaw] Recommended recovery:"); + console.error("[openclaw] rm -rf node_modules"); + console.error("[openclaw] pnpm store prune"); + console.error("[openclaw] pnpm install"); + console.error(""); + console.error("[openclaw] Original error:"); + console.error(err); +}; + +const loadChokidar = async () => { + const mod = await import("chokidar"); + return mod.default ?? mod; +}; + +const waitForWatcherRelease = async (lockPath: string, pid: number, deps: WatchDeps) => { + const deadline = deps.now() + WATCH_LOCK_WAIT_MS; + while (deps.now() < deadline) { + if (!isProcessAlive(pid, deps.signalProcess)) { + return true; + } + if (!fs.existsSync(lockPath)) { + return true; + } + await deps.sleep(WATCH_LOCK_POLL_MS); + } + return !isProcessAlive(pid, deps.signalProcess); +}; + +const acquireWatchLock = async (deps: WatchDeps, watchSession: string) => { + const lockPath = resolveWatchLockPath(deps.cwd, deps.args); + const payload = { + pid: deps.process.pid, + command: deps.args.join(" "), + createdAt: new Date(deps.now()).toISOString(), + cwd: deps.cwd, + watchSession, + }; + + while (true) { + try { + writeWatchLock(lockPath, payload); + return { lockPath, pid: deps.process.pid }; + } catch (error) { + if (errorCode(error) !== "EEXIST") { + throw error; + } + } + + const existing = readWatchLock(lockPath); + const existingPid = existing?.pid; + if (typeof existingPid !== "number" || !isProcessAlive(existingPid, deps.signalProcess)) { + removeWatchLock(lockPath); + continue; + } + + logWatcher(`Replacing existing watcher pid ${existingPid}.`, deps); + try { + deps.signalProcess(existingPid, WATCH_RESTART_SIGNAL); + } catch (error) { + if (isProcessAlive(existingPid, deps.signalProcess)) { + logWatcher( + `Failed to stop existing watcher pid ${existingPid}: ${errorMessage(error) || "unknown error"}`, + deps, + ); + return null; + } + } + + const released = await waitForWatcherRelease(lockPath, existingPid, deps); + if (!released) { + logWatcher(`Timed out waiting for watcher pid ${existingPid} to exit.`, deps); + return null; + } + removeWatchLock(lockPath); + } +}; + +const releaseWatchLock = (lockHandle: { lockPath: string; pid: number } | null) => { + if (!lockHandle) { + return; + } + const current = readWatchLock(lockHandle.lockPath); + if (current?.pid === lockHandle.pid) { + removeWatchLock(lockHandle.lockPath); + } +}; + +/** + * Runs the watch loop and restarts the child process on relevant changes. + */ +export async function runWatchMain(params: WatchMainParams = {}): Promise<WatchExit> { + const cwd = params.cwd ?? process.cwd(); + const deps = { + spawn: params.spawn ?? spawn, + process: params.process ?? process, + cwd, + args: params.args ?? process.argv.slice(2), + env: params.env ? { ...params.env } : { ...process.env }, + fs: params.fs ?? fs, + now: params.now ?? Date.now, + sleep: params.sleep ?? sleep, + signalProcess: + params.signalProcess ?? + ((pid: number, signal: ProcessSignal | 0) => + signal === 0 ? process.kill(pid, signal) : process.kill(pid, signal as NodeJS.Signals)), + lockDisabled: params.lockDisabled === true, + pathClassifier: params.pathClassifier ?? createRunNodePathClassifier({ rootDir: cwd }), + createWatcher: params.createWatcher, + loadChokidar: params.loadChokidar ?? loadChokidar, + watchPaths: params.watchPaths ?? runNodeWatchedPaths, + } satisfies WatchDeps; + + const childEnv = { ...deps.env }; + const watchSession = `${deps.now()}-${deps.process.pid}`; + const platform = deps.process.platform ?? process.platform; + const useChildProcessGroup = platform !== "win32" && !deps.process.stdin?.isTTY; + childEnv.OPENCLAW_WATCH_MODE = "1"; + childEnv.OPENCLAW_WATCH_SESSION = watchSession; + // The watcher owns process restarts; keep SIGUSR1/config reloads in-process + // so inherited launchd/systemd markers do not make the child exit and stall. + childEnv.OPENCLAW_NO_RESPAWN = "1"; + if (deps.args.length > 0) { + childEnv.OPENCLAW_WATCH_COMMAND = deps.args.join(" "); + } + + return await new Promise<WatchExit>((resolve, reject) => { + let settled = false; + let shuttingDown = false; + let restartRequested = false; + let deferredRestartGeneration = 0; + let deferredRestartActive = false; + let watchProcess: WatchChild | null = null; + let watcher: Watcher | null = null; + let lockHandle: { lockPath: string; pid: number } | null = null; + let autoDoctorAttempted = false; + let shutdownExitCode: number | null = null; + let shutdownKillTimer: TimerHandle | null = null; + let watcherStartupError: Error | null = null; + + const signalWatchProcess = (child: WatchChild, signal: ProcessSignal) => { + if (!child || typeof child.kill !== "function") { + return; + } + if (useChildProcessGroup && typeof child.pid === "number") { + try { + deps.signalProcess(-child.pid, signal); + return; + } catch (error) { + if (errorCode(error) === "ESRCH" || errorCode(error) === "EPERM") { + return; + } + } + } + child.kill(signal as NodeJS.Signals); + }; + + const forceKillWatchProcessGroup = (child: WatchChild | null) => { + if (!useChildProcessGroup || typeof child?.pid !== "number") { + return; + } + try { + deps.signalProcess(-child.pid, "SIGKILL"); + } catch (error) { + if (errorCode(error) !== "ESRCH" && errorCode(error) !== "EPERM") { + throw error; + } + } + }; + + const settle = (outcome: WatchExit) => { + if (settled) { + return; + } + settled = true; + if (shutdownKillTimer) { + clearTimeout(shutdownKillTimer); + } + if (onSigInt) { + deps.process.off("SIGINT", onSigInt); + } + if (onSigTerm) { + deps.process.off("SIGTERM", onSigTerm); + } + releaseWatchLock(lockHandle); + watcher?.close?.()?.catch?.(() => {}); + if (watcherStartupError && typeof outcome !== "string") { + reject(watcherStartupError); + } else { + resolve(outcome); + } + }; + + const settleIfSignaled = (child: WatchChild | null, signal: ProcessSignal | null) => { + // Windows emulates SIGTERM with unconditional termination, including + // ordinary requested restarts. It has no Unix completion distinction. + if (!signal || platform === "win32") { + return false; + } + // The native implementation normally acknowledges stop with an exit + // code. Its signal death cannot prove its detached workers have stopped, + // even when we requested a restart or were already shutting down. + try { + forceKillWatchProcessGroup(child); + } catch (error) { + logWatcher( + `Failed to stop the exited runner's process group: ${errorMessage(error)}`, + deps, + ); + } + settle(signal); + return true; + }; + + const requestShutdown = (code: number) => { + shuttingDown = true; + shutdownExitCode = code; + if (!watchProcess || typeof watchProcess.kill !== "function") { + settle(code); + return; + } + const shutdownProcess = watchProcess; + signalWatchProcess(shutdownProcess, WATCH_RESTART_SIGNAL); + shutdownKillTimer ??= setTimeout(() => { + shutdownKillTimer = null; + signalWatchProcess(shutdownProcess, "SIGKILL"); + }, WATCH_SHUTDOWN_KILL_GRACE_MS); + }; + + const settleIfShuttingDown = (exitedProcess: WatchChild | null) => { + if (!shuttingDown || shutdownExitCode === null) { + return false; + } + forceKillWatchProcessGroup(exitedProcess); + settle(shutdownExitCode); + return true; + }; + + const startRunner = () => { + try { + deps.pathClassifier.refreshGeneratedPluginAssetPaths(); + } catch (error) { + logWatcher( + `Failed to refresh generated asset paths: ${errorMessage(error) || "unknown error"}`, + deps, + ); + settle(1); + return; + } + watchProcess = deps.spawn(deps.process.execPath, buildRunnerArgs(deps.args), { + cwd: deps.cwd, + detached: useChildProcessGroup, + env: childEnv, + stdio: "inherit", + }); + watchProcess.on("error", (error) => { + watchProcess = null; + logWatcher( + `Failed to spawn watcher child: ${errorMessage(error) || "unknown error"}`, + deps, + ); + settle(1); + }); + watchProcess.on("exit", (exitCode, exitSignal) => { + const exitedProcess = watchProcess; + watchProcess = null; + if (settled) { + return; + } + if (settleIfSignaled(exitedProcess, exitSignal) || settleIfShuttingDown(exitedProcess)) { + return; + } + if (restartRequested || shouldRestartAfterChildExit(exitCode, exitSignal, platform)) { + forceKillWatchProcessGroup(exitedProcess); + restartRequested = false; + deferredRestartGeneration += 1; + deferredRestartActive = false; + if (!hasDistEntry()) { + deferredRestartActive = true; + const generation = deferredRestartGeneration; + logWatcher("Watcher child exited mid-build; waiting for the build entry.", deps); + deferRestartUntilDistEntryExists({ + generation, + targetProcess: null, + onReady: () => { + if (!watchProcess) { + startRunner(); + } + }, + onTimeout: () => { + logWatcher("Build entry wait timed out; starting run-node recovery.", deps); + if (!watchProcess) { + startRunner(); + } + }, + }); + return; + } + startRunner(); + return; + } + if (shouldRunAutoDoctor(deps, autoDoctorAttempted)) { + runAutoDoctorAndRestart(); + return; + } + settle(exitSignal ? 1 : (exitCode ?? 1)); + }); + }; + + const handleWatcherError = () => { + requestShutdown(1); + }; + + const rejectWatcherStartupError = (err: unknown) => { + if (settled) { + return; + } + // Source already started before the asynchronous watcher was loaded. + // Keep its exit observable until cleanup acknowledges this startup error. + watcherStartupError = toErrorObject(err, "Non-Error rejection"); + requestShutdown(1); + }; + + const resolveCreateWatcher = async () => { + try { + const chokidarModule = await deps.loadChokidar(); + return (watchPaths: string[], options: WatchOptions) => + chokidarModule.watch(watchPaths, options); + } catch (err) { + if (isInvalidPackageConfigError(err)) { + printFriendlyWatchStartupError(err); + } + throw err; + } + }; + + const runAutoDoctorAndRestart = () => { + autoDoctorAttempted = true; + logWatcher( + "Gateway exited early; running `openclaw doctor --fix --non-interactive` once.", + deps, + ); + watchProcess = deps.spawn(deps.process.execPath, buildDoctorRunnerArgs(), { + cwd: deps.cwd, + detached: useChildProcessGroup, + env: { + ...childEnv, + }, + stdio: "inherit", + }); + watchProcess.on("error", (error) => { + watchProcess = null; + logWatcher( + `Failed to spawn doctor repair: ${errorMessage(error) || "unknown error"}`, + deps, + ); + settle(1); + }); + watchProcess.on("exit", (exitCode, exitSignal) => { + const exitedProcess = watchProcess; + watchProcess = null; + if (settled) { + return; + } + if (settleIfSignaled(exitedProcess, exitSignal) || settleIfShuttingDown(exitedProcess)) { + return; + } + if (exitCode === 0 && !exitSignal) { + logWatcher("Doctor repair completed; restarting gateway watch child.", deps); + startRunner(); + return; + } + logWatcher( + `Doctor repair failed; gateway:watch exiting with code ${exitSignal ? 1 : (exitCode ?? 1)}.`, + deps, + ); + settle(exitSignal ? 1 : (exitCode ?? 1)); + }); + }; + + const hasDistEntry = () => deps.fs.existsSync(path.join(deps.cwd, "dist", "entry.js")); + + const deferRestartUntilDistEntryExists = ({ + generation, + targetProcess, + onReady, + onTimeout, + }: { + generation: number; + targetProcess: WatchChild | null; + onReady: () => void; + onTimeout: () => void; + }) => { + void (async () => { + const deadline = deps.now() + WATCH_DIST_ENTRY_TIMEOUT_MS; + while (true) { + if ( + generation !== deferredRestartGeneration || + settled || + shuttingDown || + (targetProcess && (!restartRequested || watchProcess !== targetProcess)) + ) { + return; + } + await deps.sleep(WATCH_DIST_ENTRY_POLL_MS); + if ( + generation !== deferredRestartGeneration || + settled || + shuttingDown || + (targetProcess && (!restartRequested || watchProcess !== targetProcess)) + ) { + return; + } + if (hasDistEntry()) { + deferredRestartActive = false; + onReady(); + return; + } + if (deps.now() >= deadline) { + deferredRestartActive = false; + onTimeout(); + return; + } + } + })(); + }; + + const requestRestart = (changedPath: string) => { + if ( + shuttingDown || + isIgnoredWatchPath(changedPath, deps.cwd, deps.watchPaths, deps.pathClassifier) + ) { + return; + } + if (!watchProcess) { + if (deferredRestartActive) { + return; + } + startRunner(); + return; + } + restartRequested = true; + + // A separate build can temporarily remove dist while this healthy child is + // still serving. Keep it alive until run-node can take ownership safely. + if (!hasDistEntry()) { + if (!deferredRestartActive) { + deferredRestartActive = true; + deferredRestartGeneration += 1; + logWatcher("Build entry missing; keeping watcher child alive until it returns.", deps); + const targetProcess = watchProcess; + deferRestartUntilDistEntryExists({ + generation: deferredRestartGeneration, + targetProcess, + onReady: () => { + logWatcher("Build entry restored; restarting watcher child.", deps); + signalWatchProcess(targetProcess, WATCH_RESTART_SIGNAL); + }, + onTimeout: () => { + restartRequested = false; + logWatcher("Build entry wait timed out; keeping the healthy watcher child.", deps); + }, + }); + } + return; + } + + if (typeof watchProcess.kill === "function") { + signalWatchProcess(watchProcess, WATCH_RESTART_SIGNAL); + } + }; + + const attachWatcher = (createWatcher: WatcherFactory) => { + if (settled) { + return; + } + watcher = createWatcher(deps.watchPaths, { + ignoreInitial: true, + ignored: (watchPath, stats) => + isIgnoredWatchPath(watchPath, deps.cwd, deps.watchPaths, deps.pathClassifier, stats), + }); + watcher.on("add", requestRestart); + watcher.on("change", requestRestart); + watcher.on("unlink", requestRestart); + watcher.on("error", handleWatcherError); + }; + + const startWatcher = () => { + if (deps.createWatcher) { + attachWatcher(deps.createWatcher); + return; + } + void resolveCreateWatcher().then(attachWatcher).catch(rejectWatcherStartupError); + }; + + const onSigInt = () => { + requestShutdown(130); + }; + const onSigTerm = () => { + requestShutdown(143); + }; + + deps.process.on("SIGINT", onSigInt); + deps.process.on("SIGTERM", onSigTerm); + + if (deps.lockDisabled) { + lockHandle = { lockPath: "", pid: deps.process.pid }; + startRunner(); + startWatcher(); + return; + } + + void acquireWatchLock(deps, watchSession) + .then((handle) => { + if (!handle) { + settle(1); + return; + } + lockHandle = handle; + startRunner(); + startWatcher(); + }) + .catch((error: unknown) => { + logWatcher( + `Failed to acquire watcher lock: ${errorMessage(error) || "unknown error"}`, + deps, + ); + settle(1); + }); + }); +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { + void runWatchMain() + .then((outcome) => { + if (typeof outcome === "string") { + process.kill(process.pid, outcome); + return; + } + process.exit(outcome); + }) + .catch((err: unknown) => { + if (!isInvalidPackageConfigError(err)) { + console.error(err); + } + process.exit(1); + }); +} + +function errorCode(error: unknown): unknown { + return typeof error === "object" && error !== null && "code" in error ? error.code : undefined; +} + +function errorMessage(error: unknown): string { + if (error instanceof Error) { + return error.message; + } + if (typeof error !== "object" || error === null || !("message" in error)) { + return ""; + } + return typeof error.message === "string" ? error.message : ""; +} + +function isWatchLock(value: unknown): value is WatchLock { + return ( + typeof value === "object" && + value !== null && + "pid" in value && + typeof value.pid === "number" && + "command" in value && + typeof value.command === "string" && + "createdAt" in value && + typeof value.createdAt === "string" && + "cwd" in value && + typeof value.cwd === "string" && + "watchSession" in value && + typeof value.watchSession === "string" + ); +} diff --git a/scripts/watch-pr-ci.mts b/scripts/watch-pr-ci.mts new file mode 100644 index 0000000000000000000000000000000000000000..e585f86eb7a8589103d763162492aa6ae9364df1 --- /dev/null +++ b/scripts/watch-pr-ci.mts @@ -0,0 +1,952 @@ +#!/usr/bin/env node +import { setTimeout as sleep } from "node:timers/promises"; +import { parseArgs as parseNodeArgs } from "node:util"; +import { z } from "zod"; +import { isDirectRunUrl } from "./lib/direct-run.mjs"; +import { execGhJson, workflowRunsApiArgs } from "./lib/plain-gh.mjs"; + +const USAGE = + "Usage: node scripts/watch-pr-ci.mjs <pr-number> <head-sha> [--repo owner/repo] [--after run-id] [--attach-timeout 900] [--timeout 3600] [--interval 120] [--completion rollup|ci-run]"; + +const optional = <T,>(schema: z.ZodType<T>) => schema.optional().catch(undefined); +const optionalNullable = <T,>(schema: z.ZodType<T>) => optional(schema.nullable()); +const validArray = <T,>(schema: z.ZodType<T>) => + z.array(z.unknown()).transform((values) => + values.flatMap((value) => { + const parsed = schema.safeParse(value); + return parsed.success ? [parsed.data] : []; + }), + ); +const optionalString = optional(z.string()); +const optionalNumber = optional(z.number()); +const RollupCheckSchema = z.object({ + kind: z.enum(["CheckRun", "StatusContext"]), + databaseId: optionalNumber, + name: optionalString, + context: optionalString, + status: optionalString, + conclusion: optionalNullable(z.string()), + state: optionalString, + checkSuite: optionalNullable( + z.object({ + databaseId: optionalNumber, + workflowRun: optionalNullable( + z.object({ + databaseId: optionalNumber, + event: optionalString, + workflow: optional(z.object({ databaseId: optionalNumber })), + }), + ), + }), + ), +}); +const RollupPayloadSchema = z.object({ + state: optionalString, + contexts: optional( + z.object({ + totalCount: optionalNumber, + nodes: optional(validArray(RollupCheckSchema)), + pageInfo: optional( + z.object({ + hasNextPage: optional(z.boolean()), + endCursor: optionalNullable(z.string()), + }), + ), + }), + ), +}); +const RollupPageSchema = z + .object({ + state: optionalString, + mergeable: optional(z.union([z.boolean(), z.string()])), + headRefOid: optionalString, + statusCheckRollup: optionalNullable(RollupPayloadSchema), + }) + .catch({}); +const RollupResponseSchema = z.object({ + data: z.object({ + repository: z.object({ pullRequest: RollupPageSchema.nullish() }).nullish(), + }), +}); +const RunListItemSchema = z.object({ + id: z.number(), + workflow_id: optionalNumber, + check_suite_id: optionalNumber, + event: optionalString, + head_sha: optionalString, + // Do not discard malformed members: that could make an ambiguous association unique. + pull_requests: optional( + z.array(z.object({ number: z.number().int().positive(), head: z.object({ sha: z.string() }) })), + ), + conclusion: optionalNullable(z.string()), +}); +const RunListSchema = z + .object({ workflow_runs: validArray(RunListItemSchema) }) + .transform((response) => response.workflow_runs) + .catch([]); +const RunStatusSchema = z + .object({ status: optionalString, conclusion: optionalNullable(z.string()) }) + .catch({}); +const evidenceId = z.number().int().positive(); +const CompletedRunSchema = z.object({ + id: evidenceId, + workflow_id: evidenceId, + head_sha: z.string(), + run_attempt: evidenceId, + path: z.literal(".github/workflows/ci.yml"), + status: z.literal("completed"), + conclusion: z.literal("success"), +}); +const AttemptJobSchema = z.object({ + id: evidenceId, + run_id: evidenceId, + run_attempt: evidenceId, + head_sha: z.string(), + name: z.string().min(1), + status: z.string(), + conclusion: z.string().nullable(), + runner_id: evidenceId.nullable(), + steps: z.array(z.object({ status: z.string(), conclusion: z.string().nullable() })), +}); +const AttemptJobsPageSchema = z.object({ + total_count: z.number().int().nonnegative().max(1_000), + jobs: z.array(AttemptJobSchema).max(100), +}); + +type RollupCheck = z.infer<typeof RollupCheckSchema>; +type RollupPayload = z.infer<typeof RollupPayloadSchema>; +type RollupPage = z.infer<typeof RollupPageSchema>; +type RunListItem = z.infer<typeof RunListItemSchema>; +type RunStatus = z.infer<typeof RunStatusSchema>; +type JobIdentity = { runId: number; checkId: number }; +type PrRunReplacement = { workflowId: number; checkSuites: ReadonlyMap<number, number> }; +const FAILURE_CONCLUSIONS = new Set([ + "ACTION_REQUIRED", + "CANCELLED", + "FAILURE", + "STARTUP_FAILURE", + "STALE", + "TIMED_OUT", +]); +const ROLLUP_QUERY = `query($owner:String!,$name:String!,$pr:Int!,$cursor:String){repository(owner:$owner,name:$name){pullRequest(number:$pr){state mergeable headRefOid statusCheckRollup{state contexts(first:100,after:$cursor){totalCount pageInfo{hasNextPage endCursor} nodes{kind:__typename ... on CheckRun{name status conclusion databaseId checkSuite{databaseId workflowRun{databaseId event workflow{databaseId}}}} ... on StatusContext{context state}}}}}}}`; +const MAX_EVIDENCE_READS_PER_POLL = 32; +const GH_READ_OPTIONS = { + stdio: ["ignore", "pipe", "pipe"], + timeout: 60_000, +} satisfies Parameters<typeof execGhJson>[1]; +// Adapted from Node's MIT-licensed util.stripVTControlCharacters implementation. +const ANSI_ESCAPE_SEQUENCE = new RegExp( + "[\\u001B\\u009B][[\\]()#;?]*" + + "(?:(?:(?:(?:;[-a-zA-Z\\d\\/#&.:=?%@~_]+)*" + + "|[a-zA-Z\\d]+(?:;[-a-zA-Z\\d\\/#&.:=?%@~_]*)*)?" + + "(?:\\u0007|\\u001B\\u005C|\\u009C))" + + "|(?:(?:\\d{1,4}(?:;\\d{0,4})*)?" + + "[\\dA-PR-TZcf-nq-uy=><~]))", + "g", +); +const UNSAFE_CHECK_NAME_RUN = /[^\u0020-\u007E\p{L}\p{M}\p{N}]+/gu; + +function positiveInteger(value: string, name: string) { + const parsed = Number(value); + if (!Number.isSafeInteger(parsed) || parsed <= 0) { + throw new Error(`${name} must be a positive integer`); + } + return parsed; +} + +export function parseArgs(argv: string[]) { + let parsed; + try { + parsed = parseNodeArgs({ + args: argv, + allowPositionals: true, + options: { + repo: { type: "string", default: "openclaw/openclaw" }, + after: { type: "string" }, + "attach-timeout": { type: "string", default: "900" }, + timeout: { type: "string", default: "3600" }, + interval: { type: "string", default: "120" }, + completion: { type: "string", default: "rollup" }, + }, + }); + } catch { + throw new Error(USAGE); + } + const [prValue, rawSha, ...extra] = parsed.positionals; + if (!prValue || !rawSha || extra.length > 0) { + throw new Error(USAGE); + } + const completion = parsed.values.completion; + if (completion !== "rollup" && completion !== "ci-run") { + throw new Error("--completion must be rollup or ci-run"); + } + const args = { + pr: positiveInteger(prValue, "pr-number"), + headSha: rawSha.toLowerCase(), + repo: parsed.values.repo, + ...(parsed.values.after === undefined + ? {} + : { after: positiveInteger(parsed.values.after, "--after") }), + attachTimeout: positiveInteger(parsed.values["attach-timeout"], "--attach-timeout"), + timeout: positiveInteger(parsed.values.timeout, "--timeout"), + interval: positiveInteger(parsed.values.interval, "--interval"), + completion, + }; + if (!/^[0-9a-f]{40}$/u.test(args.headSha)) { + throw new Error("head-sha must be a full 40-character commit SHA"); + } + if (!/^[^/\s]+\/[^/\s]+$/u.test(args.repo)) { + throw new Error("--repo must be owner/repo"); + } + return args; +} + +const checkName = (check: RollupCheck) => + check.kind === "StatusContext" ? check.context : check.name; +export const sanitizeCheckName = (name: string) => + name.replaceAll(ANSI_ESCAPE_SEQUENCE, "\u0000").replaceAll(UNSAFE_CHECK_NAME_RUN, "?"); +const isAutoResponse = (check: RollupCheck) => + checkName(check) + ?.toLowerCase() + .replaceAll(/[^a-z0-9]+/gu, " ") + .trim() === "auto response"; + +// Run identity for supersession; undefined when any id is missing so ambiguous +// nodes are never dropped (fails toward FAILING, never toward false GREEN). +// These databaseIds are GraphQL Int fields, but GitHub serializes Actions-scale +// 64-bit values in them (live-verified; no fullDatabaseId exists on these types). +// If GitHub ever nulls them, filtering degrades to pre-supersession behavior. +function checkRunIdentity(check: RollupCheck) { + if (check.kind !== "CheckRun") { + return undefined; + } + const runId = check.checkSuite?.workflowRun?.databaseId; + const workflowId = check.checkSuite?.workflowRun?.workflow?.databaseId; + if (typeof runId !== "number" || typeof workflowId !== "number") { + return undefined; + } + return { runId, workflowId, event: check.checkSuite?.workflowRun?.event }; +} +// Strict recency ordering for same-name checks: newest run wins; within one run +// (rerun attempts reuse the run id) the newest check-run id wins. +const newerJob = (a: JobIdentity, b: JobIdentity) => + a.runId !== b.runId ? a.runId > b.runId : a.checkId > b.checkId; + +export function classifyRollup( + rollup: RollupPayload | null | undefined, + reconciledChecks: ReadonlyMap<number, string> = new Map(), + replacement?: PrRunReplacement, + pendingEvidence: ReadonlySet<RollupCheck> = new Set(), +) { + const rawNodes = rollup?.contexts?.nodes ?? []; + const hiddenContextCount = Math.max( + 0, + (rollup?.contexts?.totalCount ?? rawNodes.length) - rawNodes.length, + ); + const isReconciledPlaceholder = (check: RollupCheck) => + check.databaseId !== undefined && + check.name !== undefined && + check.name.length > 0 && + check.status === "QUEUED" && + check.conclusion === null && + reconciledChecks.get(check.databaseId) === check.name; + const bestByJob = new Map<string, JobIdentity & { reconciled: boolean }>(); + for (const check of rawNodes) { + const identity = checkRunIdentity(check); + if (!identity) { + continue; + } + if (check.name && typeof check.databaseId === "number") { + const key = `${identity.workflowId}:${identity.event ?? ""}:${check.name}`; + const reconciled = isReconciledPlaceholder(check); + const candidate = { runId: identity.runId, checkId: check.databaseId, reconciled }; + const best = bestByJob.get(key); + if (!best || newerJob(candidate, best)) { + bestByJob.set(key, candidate); + } + } + } + let supersededCount = 0; + let reconciledCount = 0; + // GitHub CLI deduplicates same-name checks within a workflow/event on the shared SHA. + // Unique jobs, including cancellations, need exact same-PR run/suite replacement proof. + const nodes = rawNodes.filter((check) => { + const identity = checkRunIdentity(check); + if (!identity) { + return true; + } + // A shared head can belong to different PR bases. Only event-bound run/suite + // evidence permits replacing an older graph before new job names appear. + const replacedSuite = replacement?.checkSuites.get(identity.runId); + if ( + replacedSuite !== undefined && + identity.workflowId === replacement?.workflowId && + check.checkSuite?.databaseId === replacedSuite && + check.checkSuite?.workflowRun?.event === "pull_request" + ) { + supersededCount += 1; + return false; + } + // Proof covers the queued observation, not a later name or outcome of the same ID. + if (isReconciledPlaceholder(check)) { + reconciledCount += 1; + supersededCount += 1; + return false; + } + if (check.name && typeof check.databaseId === "number") { + const best = bestByJob.get(`${identity.workflowId}:${identity.event ?? ""}:${check.name}`); + // A removed placeholder cannot hide a verified sibling that changed in this + // attempt. Uncovered earlier attempts and newer runs retain normal supersession. + const changedAlias = reconciledChecks.has(check.databaseId); + if ( + best && + !(best.reconciled && best.runId === identity.runId && changedAlias) && + newerJob(best, { runId: identity.runId, checkId: check.databaseId }) + ) { + supersededCount += 1; + return false; + } + } + return true; + }); + const checks = nodes.filter((check) => !isAutoResponse(check)); + const pendingCount = checks.filter( + (check) => + pendingEvidence.has(check) || + (check.kind === "StatusContext" + ? check.state === "PENDING" || check.state === "EXPECTED" + : check.status !== "COMPLETED"), + ).length; + const failingChecks = checks.filter((check) => { + if (pendingEvidence.has(check)) { + return false; + } + if (check.kind === "StatusContext") { + return check.state === "ERROR" || check.state === "FAILURE"; + } + return typeof check.conclusion === "string" && FAILURE_CONCLUSIONS.has(check.conclusion); + }); + const failingNames = failingChecks + .map(checkName) + .filter((name): name is string => typeof name === "string" && name.length > 0) + .map(sanitizeCheckName) + .toSorted() + .filter((name, index, names) => name !== names[index - 1]); + if (rollup?.state === "SUCCESS") { + return { verdict: "GREEN", pendingCount, failingNames: [], supersededCount }; + } + if ( + rollup?.state === "ERROR" || + rollup?.state === "FAILURE" || + (rollup?.state === "PENDING" && reconciledCount > 0) + ) { + if (failingChecks.length > 0) { + return { + verdict: "FAILING", + pendingCount, + failingNames: [ + ...(failingNames.length > 0 ? failingNames : ["status rollup"]), + ...(hiddenContextCount > 0 ? [`+${hiddenContextCount} more contexts not shown`] : []), + ], + supersededCount, + }; + } + if (hiddenContextCount > 0) { + return { + verdict: "FAILING", + pendingCount, + failingNames: ["status rollup", `+${hiddenContextCount} more contexts not shown`], + supersededCount, + }; + } + // A refresh can invalidate every alias proof. Unknown outcomes still block, + // even when no placeholder was removed from this snapshot. + const hasUnresolvedChecks = checks.some( + (check) => + pendingEvidence.has(check) || + (check.kind === "StatusContext" + ? check.state !== "SUCCESS" + : check.status !== "COMPLETED" || + !["SUCCESS", "SKIPPED", "NEUTRAL"].includes(check.conclusion ?? "")), + ); + if (hasUnresolvedChecks) { + return { verdict: "PENDING", pendingCount, failingNames: [], supersededCount }; + } + // GitHub's aggregate permanently counts superseded cancellations. With full visibility, + // an all-green newest-run set is green; main() also requires the attached run to succeed. + return { verdict: "GREEN", pendingCount, failingNames: [], supersededCount }; + } + return { verdict: "PENDING", pendingCount, failingNames: [], supersededCount }; +} + +function ghReadOptions(deadline?: number) { + if (deadline === undefined) { + return GH_READ_OPTIONS; + } + const remaining = deadline - Date.now(); + if (remaining <= 0) { + throw new Error("watcher evidence deadline elapsed"); + } + return { ...GH_READ_OPTIONS, timeout: Math.min(GH_READ_OPTIONS.timeout, remaining) }; +} + +const readPr = (pr: number, repo: string, deadline?: number) => + RollupPageSchema.parse( + execGhJson( + `pr view ${pr} --repo ${repo} --json state,mergeable,headRefOid`.split(" "), + ghReadOptions(deadline), + ), + ); +export const buildFindRunArgs = (repo: string, sha: string) => + workflowRunsApiArgs(repo, sha, "pull_request", 20); +export const selectRunAfter = (runs: RunListItem[], after?: number) => + runs.find((run) => run.conclusion !== "skipped" && (after === undefined || run.id > after)); +function findRun(repo: string, sha: string, after?: number, pr?: number) { + const runs = RunListSchema.parse(execGhJson(buildFindRunArgs(repo, sha), GH_READ_OPTIONS)); + const candidates = + pr === undefined + ? runs + : runs.filter( + (run) => + !run.pull_requests?.length || + run.pull_requests.some((request) => request.number === pr), + ); + const run = selectRunAfter(candidates, after); + if (!run) { + return undefined; + } + return { run, runs: new Map(runs.map((item) => [item.id, item])) }; +} +const readRun = (repo: string, runId: number, deadline?: number) => + RunStatusSchema.parse( + execGhJson(`run view ${runId} --repo ${repo} --json status,conclusion`.split(" "), { + ...ghReadOptions(deadline), + // Revalidate changing run status instead of reusing a cached snapshot. + env: { ...process.env, OCTOPOOL_FRESH: "1" }, + }), + ); + +function readQueuedPlaceholderEvidence( + repo: string, + headSha: string, + attached: RunListItem, + rollup: RollupPayload, + deadline: number, +) { + const reconciled = new Map<number, string>(); + const contexts = rollup.contexts; + const nodes = contexts?.nodes ?? []; + if ( + !["FAILURE", "ERROR", "PENDING"].includes(rollup.state ?? "") || + contexts?.totalCount !== nodes.length || + contexts.pageInfo?.hasNextPage !== false + ) { + return reconciled; + } + const queued = nodes.flatMap((check) => { + const identity = checkRunIdentity(check); + return check.status === "QUEUED" && + check.conclusion === null && + check.name && + check.databaseId !== undefined && + identity?.runId === attached.id && + identity.workflowId === attached.workflow_id + ? [{ name: check.name, id: check.databaseId }] + : []; + }); + if (queued.length === 0) { + return reconciled; + } + const runPath = `repos/${repo}/actions/runs/${attached.id}`; + // Request current evidence through the shared gh seam; the final run read must + // not deliberately reuse the snapshot from before job collection. + const readEvidence = (endpoint: string) => + execGhJson(["api", endpoint, "-H", "Cache-Control: max-age=0"], ghReadOptions(deadline)); + const readCompletedRun = () => CompletedRunSchema.parse(readEvidence(runPath)); + const run = readCompletedRun(); + if ( + run.id !== attached.id || + run.head_sha !== headSha || + run.workflow_id !== attached.workflow_id + ) { + return reconciled; + } + const jobs: z.infer<typeof AttemptJobSchema>[] = []; + let totalCount: number | undefined; + // Attempt-specific pagination is bounded like the rollup; incomplete or duplicate + // pages cannot establish that no active/failed same-name sibling exists. + for (let page = 1; page <= 10; page += 1) { + const response = AttemptJobsPageSchema.parse( + readEvidence(`${runPath}/attempts/${run.run_attempt}/jobs?per_page=100&page=${page}`), + ); + totalCount ??= response.total_count; + if (response.total_count !== totalCount || response.jobs.length === 0) { + return reconciled; + } + jobs.push(...response.jobs); + if (jobs.length >= totalCount) { + break; + } + } + if (jobs.length !== totalCount || new Set(jobs.map((job) => job.id)).size !== jobs.length) { + return reconciled; + } + const sameAttempt = (job: z.infer<typeof AttemptJobSchema>) => + job.run_id === run.id && job.run_attempt === run.run_attempt && job.head_sha === headSha; + const unassignedQueued = (job: z.infer<typeof AttemptJobSchema>) => + job.status === "queued" && job.conclusion === null && job.runner_id === null; + let remainingAliasReads = MAX_EVIDENCE_READS_PER_POLL; + for (const name of new Set(queued.map((check) => check.name))) { + const checks = queued.filter((check) => check.name === name); + const siblings = jobs.filter((job) => job.name === name); + const aliases = siblings.filter(unassignedQueued); + const executed = siblings.filter((job) => !unassignedQueued(job)); + const replacement = executed[0]; + if ( + executed.length !== 1 || + !replacement || + !siblings.every(sameAttempt) || + !checks.every((check) => aliases.some((job) => job.id === check.id)) || + replacement.status !== "completed" || + replacement.conclusion !== "success" || + replacement.runner_id === null || + replacement.steps.length === 0 || + replacement.steps.some( + (step) => + step.status !== "completed" || + !["success", "skipped", "neutral"].includes(step.conclusion ?? ""), + ) + ) { + continue; + } + // One request per alias can outlive a poll or exhaust API quota. Oversized + // groups stay pending; partial verification never proves supersession. + if (aliases.length > remainingAliasReads) { + console.log("WARN queued-alias evidence exceeds the per-poll request budget"); + continue; + } + // The list can copy executed steps onto queued aliases, including ones absent + // from GraphQL. Prove the entire group unexecuted before dropping any visible ID. + const unexecuted = aliases.every((alias) => { + remainingAliasReads -= 1; + const direct = AttemptJobSchema.parse(readEvidence(`repos/${repo}/actions/jobs/${alias.id}`)); + return ( + direct.id === alias.id && + direct.name === name && + sameAttempt(direct) && + unassignedQueued(direct) && + direct.steps.length === 0 + ); + }); + if (unexecuted) { + // The refreshed rollup can reveal an alias omitted from the first snapshot. + for (const alias of aliases) { + reconciled.set(alias.id, name); + } + } + } + // Reruns reuse a run ID. Evidence from the completed attempt cannot authorize + // completion after a newer attempt starts while the job pages are being read. + const current = readCompletedRun(); + if (JSON.stringify(current) !== JSON.stringify(run)) { + return new Map<number, string>(); + } + return reconciled; +} + +export function classifyRunAttachment(runId: number, run: RunStatus, after?: number) { + if (run.conclusion === "skipped") { + return { attach: false }; + } + return { + attach: true, + warning: + after === undefined && run.status?.toLowerCase() === "completed" + ? `WARN attaching to already-completed run ${runId} (started before watcher); pass --after ${runId} to require a fresh run` + : undefined, + }; +} + +export function classifyAttachedCiRun(run: RunStatus) { + if (run.status !== "completed") { + return { verdict: "PENDING" }; + } + return run.conclusion === "success" + ? { verdict: "GREEN" } + : { verdict: "FAILING", conclusion: run.conclusion ?? "unknown" }; +} + +export function collectRollupContexts( + fetchPage: (cursor: string | null) => RollupPage | null | undefined, +) { + const firstPage = fetchPage(null); + const firstContexts = firstPage?.statusCheckRollup?.contexts; + if (!firstContexts) { + return firstPage; + } + + const nodes = [...(firstContexts.nodes ?? [])]; + let pageInfo = firstContexts.pageInfo; + let pageCount = 1; + // Polling work stays bounded at 1,000 contexts. Any truncation remains visible through + // totalCount and must classify conservatively rather than reading as success. + while (pageInfo?.hasNextPage && pageCount < 10) { + if (typeof pageInfo.endCursor !== "string") { + throw new Error("rollup page advertised a next page without a cursor"); + } + const page = fetchPage(pageInfo.endCursor); + const contexts = page?.statusCheckRollup?.contexts; + pageCount += 1; + // Losing an advertised page (head moved, transient API gap) or reading a changed snapshot + // must not pass off the partial first page as complete; the watch loop catches this error + // and re-reads the rollup on its next bounded poll. + if (!contexts) { + throw new Error("rollup snapshot changed during pagination"); + } + if ( + page.headRefOid !== firstPage.headRefOid || + page.statusCheckRollup?.state !== firstPage.statusCheckRollup?.state || + contexts.totalCount !== firstContexts.totalCount + ) { + throw new Error("rollup snapshot changed during pagination"); + } + nodes.push(...(contexts.nodes ?? [])); + pageInfo = contexts.pageInfo; + } + + return { + ...firstPage, + statusCheckRollup: { + ...firstPage.statusCheckRollup, + contexts: { ...firstContexts, nodes, pageInfo }, + }, + }; +} + +function readRollup(pr: number, repo: string, deadline?: number) { + const [owner, name] = repo.split("/"); + return ( + collectRollupContexts((cursor) => { + const queryArgs = [ + "api", + "graphql", + "-f", + `query=${ROLLUP_QUERY}`, + "-f", + `owner=${owner}`, + "-f", + `name=${name}`, + "-F", + `pr=${pr}`, + ]; + if (cursor !== null) { + queryArgs.push("-f", `cursor=${cursor}`); + } + const response = RollupResponseSchema.safeParse( + execGhJson(queryArgs, ghReadOptions(deadline)), + ); + return response.success ? response.data.data.repository?.pullRequest : undefined; + }) ?? {} + ); +} + +function readPrRollup( + args: ReturnType<typeof parseArgs>, + attachment: NonNullable<ReturnType<typeof findRun>>, + deadline: number, + reads: { remaining: number }, + reconciled?: ReadonlyMap<number, string>, +) { + const { run, runs } = attachment; + const boundToPr = (candidate: RunListItem) => + candidate.event === "pull_request" && + candidate.head_sha === args.headSha && + candidate.pull_requests?.length === 1 && + candidate.pull_requests[0]?.number === args.pr && + candidate.pull_requests[0]?.head.sha === args.headSha; + const checkSuites = new Map<number, number>(); + const pendingEvidence = new Set<RollupCheck>(); + const replacement = + boundToPr(run) && run.workflow_id !== undefined && run.check_suite_id !== undefined + ? { workflowId: run.workflow_id, checkSuites } + : undefined; + while (true) { + const pr = readRollup(args.pr, args.repo, deadline); + const blocked = precheck(pr, args.headSha, true); + if (blocked !== null) { + return { exitCode: blocked }; + } + if (pr.statusCheckRollup?.state === "SUCCESS") { + return { pr, result: classifyRollup(pr.statusCheckRollup, reconciled) }; + } + const knownRuns = runs.size; + // Pending evidence belongs only to these exact observations, never a later + // check state or a different workflow/event sharing its run ID. + pendingEvidence.clear(); + for (const check of pr.statusCheckRollup?.contexts?.nodes ?? []) { + const identity = checkRunIdentity(check); + if ( + !replacement || + !identity || + identity.runId >= run.id || + identity.workflowId !== replacement.workflowId || + identity.event !== "pull_request" || + check.checkSuite?.databaseId === undefined + ) { + continue; + } + // The attachment page is not history. Resolve only rollup-referenced IDs, + // caching even unbound results so repeated jobs/polls do not multiply reads. + let previous = runs.get(identity.runId); + if (!previous) { + if (reads.remaining === 0) { + pendingEvidence.add(check); + continue; + } + reads.remaining -= 1; + previous = RunListItemSchema.parse( + execGhJson( + ["api", `repos/${args.repo}/actions/runs/${identity.runId}`], + ghReadOptions(deadline), + ), + ); + runs.set(identity.runId, previous); + } + if ( + previous.id === identity.runId && + previous.workflow_id === replacement.workflowId && + previous.check_suite_id !== undefined && + boundToPr(previous) + ) { + checkSuites.set(previous.id, previous.check_suite_id); + } + } + // Metadata reads can span a push or new checks. Reobserve before deciding; + // any newly referenced IDs are resolved within the same watcher deadline. + if (runs.size === knownRuns) { + if (pendingEvidence.size > 0) { + console.log(`STATUS evidence=PENDING deferred-checks=${pendingEvidence.size}`); + } + return { + pr, + result: classifyRollup(pr.statusCheckRollup, reconciled, replacement, pendingEvidence), + }; + } + } +} + +const emit = (line: string, code: number) => { + console.log(line); + return code; +}; +export async function pollUntilDeadline<T>({ + deadline, + interval, + poll, + now = Date.now, + wait = sleep, +}: { + deadline: number; + interval: number; + poll: () => T | undefined | Promise<T | undefined>; + now?: () => number; + wait?: (milliseconds: number) => Promise<void>; +}): Promise<T | undefined> { + while (true) { + const result = await poll(); + if (result !== undefined) { + return result; + } + const remaining = deadline - now(); + if (remaining <= 0) { + return undefined; + } + await wait(Math.min(interval * 1000, remaining)); + } +} +function retry(phase: string, error: unknown) { + const message = (error instanceof Error ? error.message : String(error)).replaceAll(/\s+/gu, " "); + // Run-scoped proxy credentials cannot recover while this process keeps polling. + if (/\bProxy Authentication Required\b/iu.test(message)) { + throw new Error( + `PROXY-AUTH-FAILED phase=${phase} status=407 hint="Restart the watcher in an active run with valid proxy authentication; run-scoped credentials expire when their owning run closes."`, + ); + } + console.log(`RETRY phase=${phase} error=${message}`); +} + +function precheck(pr: RollupPage, sha: string, midWait = false) { + const state = (pr.state ?? "MISSING").toUpperCase(); + if (state !== "OPEN") { + return emit(`PR-CLOSED state=${state}`, 10); + } + if (pr.headRefOid !== sha) { + return emit(`HEAD-MOVED expected=${sha} actual=${pr.headRefOid}`, 11); + } + if ( + pr.mergeable === false || + (typeof pr.mergeable === "string" && pr.mergeable.toUpperCase() === "CONFLICTING") + ) { + return emit( + `${midWait ? "CONFLICTING-MID-WAIT" : "CONFLICTING"} mergeable=CONFLICTING`, + midWait ? 14 : 12, + ); + } + return null; +} + +async function main(argv = process.argv.slice(2)) { + const args = parseArgs(argv); + const attachDeadline = Date.now() + args.attachTimeout * 1000; + const attachment = await pollUntilDeadline({ + deadline: attachDeadline, + interval: args.interval, + poll: () => { + try { + const blocked = precheck(readPr(args.pr, args.repo), args.headSha); + if (blocked !== null) { + return { exitCode: blocked }; + } + const candidate = findRun( + args.repo, + args.headSha, + args.after, + args.completion === "rollup" ? args.pr : undefined, + ); + if (candidate) { + const classification = classifyRunAttachment( + candidate.run.id, + readRun(args.repo, candidate.run.id), + args.after, + ); + if (classification.attach) { + if (classification.warning) { + console.log(classification.warning); + } + return candidate; + } + } + } catch (error) { + retry("attach", error); + } + return undefined; + }, + }); + if (attachment === undefined) { + return emit( + 'NO-RUN-ATTACHED hint="close/reopen re-fires CI; pr-ci-sweeper re-fires hourly at :07"', + 13, + ); + } + if ("exitCode" in attachment) { + return attachment.exitCode; + } + const attached = attachment.run; + const runId = attached.id; + console.log(`ATTACHED run=${runId} url=https://github.com/${args.repo}/actions/runs/${runId}`); + + const watchDeadline = Date.now() + args.timeout * 1000; + let lastState = "NONE"; + let lastPending = 0; + const watchResult = await pollUntilDeadline({ + deadline: watchDeadline, + interval: args.interval, + poll: () => { + try { + if (args.completion === "ci-run") { + const blocked = precheck(readPr(args.pr, args.repo, watchDeadline), args.headSha, true); + if (blocked !== null) { + return blocked; + } + const run = readRun(args.repo, runId, watchDeadline); + const result = classifyAttachedCiRun(run); + const runStatus = run.status ?? "undefined"; + const runConclusion = run.conclusion ?? "pending"; + console.log(`STATUS run=${runStatus} conclusion=${runConclusion}`); + if (result.verdict === "FAILING") { + return emit(`FAILING checks=CI workflow (${result.conclusion})`, 15); + } + if (result.verdict === "GREEN") { + return emit("GREEN", 0); + } + return undefined; + } + const reads = { remaining: MAX_EVIDENCE_READS_PER_POLL }; + let observed = readPrRollup(args, attachment, watchDeadline, reads); + if ("exitCode" in observed) { + return observed.exitCode; + } + let { pr, result } = observed; + lastState = pr.statusCheckRollup?.state ?? "NONE"; + lastPending = result.pendingCount; + let run = + result.verdict === "FAILING" ? undefined : readRun(args.repo, runId, watchDeadline); + if ( + result.verdict === "PENDING" && + result.pendingCount > 0 && + run?.status === "completed" && + run.conclusion === "success" && + pr.statusCheckRollup + ) { + const reconciled = readQueuedPlaceholderEvidence( + args.repo, + args.headSha, + attached, + pr.statusCheckRollup, + watchDeadline, + ); + if (reconciled.size > 0) { + // The evidence scan may span pushes or new checks. Reobserve the PR + // before applying proof, then retain the ordinary final CI-run check. + observed = readPrRollup(args, attachment, watchDeadline, reads, reconciled); + if ("exitCode" in observed) { + return observed.exitCode; + } + ({ pr, result } = observed); + run = + result.verdict === "FAILING" ? undefined : readRun(args.repo, runId, watchDeadline); + } + } + lastState = pr.statusCheckRollup?.state ?? "NONE"; + lastPending = result.pendingCount; + console.log( + `STATUS rollup=${result.verdict.toLowerCase()} github_rollup=${lastState} pending=${lastPending} superseded=${result.supersededCount}`, + ); + if (result.verdict === "FAILING") { + return emit(`FAILING checks=${result.failingNames.join(", ")}`, 15); + } + if (run?.status === "completed" && run.conclusion !== "success") { + return emit(`FAILING checks=CI workflow (${run.conclusion ?? "unknown"})`, 15); + } + if ( + result.verdict === "GREEN" && + run?.status === "completed" && + run.conclusion === "success" + ) { + return emit("GREEN", 0); + } + } catch (error) { + retry("watch", error); + } + return undefined; + }, + }); + if (watchResult !== undefined) { + return watchResult; + } + return emit( + args.completion === "rollup" + ? `TIMEOUT github_rollup=${lastState} pending=${lastPending}` + : "TIMEOUT completion=ci-run", + 16, + ); +} + +if (isDirectRunUrl(process.argv[1], import.meta.url)) { + try { + process.exitCode = await main(); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 2; + } +} diff --git a/scripts/windows-cmd-helpers.d.mts b/scripts/windows-cmd-helpers.d.mts new file mode 100644 index 0000000000000000000000000000000000000000..66aa66b33b57f3da681482876990c1237c7f3550 --- /dev/null +++ b/scripts/windows-cmd-helpers.d.mts @@ -0,0 +1,5 @@ +export function resolvePathEnvKey(env: NodeJS.ProcessEnv): string; +export function resolveWindowsSystem32Path(executableName: string, env?: NodeJS.ProcessEnv): string; +export function resolveWindowsCmdExePath(env?: NodeJS.ProcessEnv): string; +export function resolveWindowsPowerShellPath(env?: NodeJS.ProcessEnv): string; +export function buildCmdExeCommandLine(command: string, args: readonly string[]): string; diff --git a/scripts/windows-cmd-helpers.mjs b/scripts/windows-cmd-helpers.mjs new file mode 100644 index 0000000000000000000000000000000000000000..045a4175cf698796fe26e22c8130fbbd482a560c --- /dev/null +++ b/scripts/windows-cmd-helpers.mjs @@ -0,0 +1,98 @@ +// Windows cmd.exe quoting helpers for npm/pnpm command shims. +import path from "node:path"; + +const WINDOWS_UNSAFE_CMD_CHARS_RE = /[&|<>%\r\n]/; +const DEFAULT_WINDOWS_SYSTEM_ROOT = "C:\\Windows"; + +function getEnvValueCaseInsensitive(env, expectedKey) { + const direct = env[expectedKey]; + if (direct !== undefined) { + return direct; + } + const expected = expectedKey.toUpperCase(); + const actualKey = Object.keys(env).find((key) => key.toUpperCase() === expected); + return actualKey ? env[actualKey] : undefined; +} + +function normalizeWindowsSystemRoot(raw) { + const trimmed = raw?.trim(); + if ( + !trimmed || + trimmed.includes("\0") || + trimmed.includes("\r") || + trimmed.includes("\n") || + trimmed.includes(";") + ) { + return null; + } + const normalized = path.win32.normalize(trimmed); + if (!path.win32.isAbsolute(normalized) || normalized.startsWith("\\\\")) { + return null; + } + const parsed = path.win32.parse(normalized); + if (!/^[A-Za-z]:\\$/.test(parsed.root) || normalized.length <= parsed.root.length) { + return null; + } + return normalized.replace(/[\\/]+$/, ""); +} + +/** + * Resolves the correctly cased PATH key in a Windows-style env object. + */ +export function resolvePathEnvKey(env) { + return Object.keys(env).find((key) => key.toLowerCase() === "path") ?? "PATH"; +} + +function resolveWindowsSystemRoot(env = process.env) { + return ( + normalizeWindowsSystemRoot(getEnvValueCaseInsensitive(env, "SystemRoot")) ?? + normalizeWindowsSystemRoot(getEnvValueCaseInsensitive(env, "WINDIR")) ?? + DEFAULT_WINDOWS_SYSTEM_ROOT + ); +} + +/** @internal Shared repository-script contract. */ +export function resolveWindowsSystem32Path(executableName, env = process.env) { + if ( + path.win32.basename(executableName) !== executableName || + !/^[A-Za-z0-9_.-]+\.exe$/u.test(executableName) + ) { + throw new Error(`Invalid Windows System32 executable name: ${executableName}`); + } + return path.win32.join(resolveWindowsSystemRoot(env), "System32", executableName); +} + +export function resolveWindowsCmdExePath(env = process.env) { + return resolveWindowsSystem32Path("cmd.exe", env); +} + +/** @internal Shared repository-script contract. */ +export function resolveWindowsPowerShellPath(env = process.env) { + return path.win32.join( + resolveWindowsSystemRoot(env), + "System32", + "WindowsPowerShell", + "v1.0", + "powershell.exe", + ); +} + +function escapeForCmdExe(arg) { + if (WINDOWS_UNSAFE_CMD_CHARS_RE.test(arg)) { + throw new Error(`unsafe Windows cmd.exe argument detected: ${JSON.stringify(arg)}`); + } + const escaped = arg.replace(/\^/g, "^^"); + if (!escaped.includes(" ") && !escaped.includes('"')) { + return escaped; + } + return `"${escaped.replace(/"/g, '""')}"`; +} + +/** + * Builds a cmd.exe-safe command line or rejects unsafe shell metacharacters. + */ +export function buildCmdExeCommandLine(command, args) { + const escapedCommand = escapeForCmdExe(command); + const commandLine = [escapedCommand, ...args.map(escapeForCmdExe)].join(" "); + return escapedCommand.startsWith('"') ? `"${commandLine}"` : commandLine; +} diff --git a/scripts/write-official-channel-catalog.mjs b/scripts/write-official-channel-catalog.mjs new file mode 100644 index 0000000000000000000000000000000000000000..5f2714415287d594faee6dd17261b630789fb15a --- /dev/null +++ b/scripts/write-official-channel-catalog.mjs @@ -0,0 +1,5 @@ +import { runTsxCliShim } from "./lib/tsx-cli-shim.mjs"; + +await runTsxCliShim(import.meta.url, { + implementation: "./write-official-channel-catalog.mts", +});