Hugging Face's logo

Spaces:
Leon4gr45
/
nova-right-nav
Paused

App Files Community
nova-right-nav / src /cli.ts
Leon4gr45's picture
Leon4gr45
Upload folder using huggingface_hub
a5dea12 verified
Raw History Blame Contribute Delete
95.1 kB
#!/usr/bin/env node
import {
access,
mkdir,
readFile,
readdir,
stat,
writeFile,
} from "node:fs/promises";
import { basename, dirname, join, resolve } from "node:path";
import { homedir } from "node:os";
import { freemem, totalmem } from "node:os";
import { statfs } from "node:fs/promises";
import { pathToFileURL } from "node:url";
import { fileURLToPath } from "node:url";
import { Command } from "commander";
import {
getOAuthApiKey,
getOAuthProvider,
getOAuthProviders,
type OAuthCredentials,
} from "@earendil-works/pi-ai/oauth";
import {
DataLifecycleConfigSchema,
RunResultSchema,
TesterProfileSchema,
UserJourneySchema,
type BrowserDevicePreset,
type BrowserEnvironment,
type RunAttempt,
type RunResult,
type TesterProfile,
type UserJourney,
} from "./core/schemas.js";
import {
DraftJourneyInputSchema,
createDraftUserJourney,
createJsonSchemaDocument,
createTesterProfileTemplate,
createUserJourneyTemplate,
defaultAuthoringDir,
formatLintIssue,
initAuthoringProject,
lintUserJourney,
parseSchemaKind,
stringifyJsonDocument,
writeJsonDocument,
type LintIssue,
type SchemaKind,
} from "./core/authoring.js";
import { readAndParseJsonFile, readJsonFile } from "./core/json.js";
import {
validateAllowedOrigins,
validateJourneyProfileMatch,
type ValidationIssue,
} from "./core/validation.js";
import { emitGitHubAnnotations, writeCiReports } from "./reporters/ci.js";
import {
buildSuiteRunHistory,
runHealth,
snapshotRun,
type RunSnapshot,
type SuiteSelectionSummary,
type SuiteRunHistory,
} from "./reporters/runComparison.js";
import {
renderSuiteDashboard,
type SuiteDashboardRun,
} from "./reporters/suiteDashboard.js";
import { runJourney, type RunJourneyOptions } from "./runner/runJourney.js";
import { isoTimestampForPath, sanitizePathSegment } from "./utils/path.js";
import {
AppLifecycleController,
applyAppTargetOverride,
DataLifecycleController,
} from "./lifecycle/index.js";
import {
createDefaultJourneyTestFactoryRegistry,
type JourneyTestFactoryRegistry,
} from "./factories/index.js";
import type {
AgentDirector,
UiChangeRecordingOptions,
} from "./directors/types.js";
import type { BrowserDriver } from "./drivers/types.js";
import type { BookmarkCurator } from "./curation/types.js";
import { spawn } from "node:child_process";
import { execFile } from "node:child_process";
import { promisify } from "node:util";
import { assertCloudReasoningConfiguration, cognitionConfigStatus, createApiServer, createJourneyRuntime, isHuggingFaceSpace, loadRuntimeCognitionConfig, ManagedLayaService, reasoningModelStatus, resolveLayaMode, assertSafeJourneyNetwork, assertSafeRedirectChain } from "./app/index.js";
export interface RunCliOptions {
factories?: JourneyTestFactoryRegistry;
}
export type ParallelBrowserMode = "shared-tabs" | "isolated-sessions";
const collectRepeatableOption = (value: string, previous: string[] = []): string[] => [
...previous,
value,
];
export async function runCli(
argv = process.argv,
cliOptions: RunCliOptions = {},
): Promise<void> {
const factories =
cliOptions.factories ?? createDefaultJourneyTestFactoryRegistry();
const program = new Command();
program
.name("journeytest")
.description("AI-agent-directed user journey testing for web apps.")
.version("0.1.0");
program
.command("init")
.description("Create starter JourneyTest authoring files.")
.option(
"--dir <dir>",
"Directory for generated authoring files.",
defaultAuthoringDir,
)
.option(
"--app-name <name>",
"App name to use in generated journeys.",
"Acme Admin",
)
.option(
"--base-url <url>",
"Base URL to use in generated journeys.",
"http://127.0.0.1:3000",
)
.option("--profile-id <id>", "Tester profile id.", "admin")
.option("--journey-id <id>", "Starter journey id.", "admin-invite-user")
.option("--risk-level <level>", "Journey risk level.", "writes-test-data")
.option("--force", "Overwrite files if they already exist.")
.action(
async (options: {
dir: string;
appName: string;
baseUrl: string;
profileId: string;
journeyId: string;
riskLevel: UserJourney["riskLevel"];
force?: boolean;
}) => {
const result = await initAuthoringProject({
rootDir: resolve(options.dir),
appName: options.appName,
baseUrl: options.baseUrl,
profileId: options.profileId,
id: options.journeyId,
riskLevel: options.riskLevel,
force: options.force,
});
console.log(`Wrote tester profile: ${result.profilePath}`);
console.log(`Wrote user journey: ${result.journeyPath}`);
},
);
const newCommand = program
.command("new")
.description("Generate JourneyTest authoring JSON.");
newCommand
.command("profile")
.description("Generate a tester profile JSON file.")
.argument("[id]", "Tester profile id.", "admin")
.option("--name <name>", "Tester profile display name.")
.option("--role <role>", "Tester role.")
.option("--perspective <text>", "Tester perspective.")
.option("--out <path>", "Output JSON path.")
.option("--stdout", "Print JSON to stdout instead of writing a file.")
.option("--force", "Overwrite the output file if it already exists.")
.action(
async (
id: string,
options: {
name?: string;
role?: string;
perspective?: string;
out?: string;
stdout?: boolean;
force?: boolean;
},
) => {
const profile = createTesterProfileTemplate({
id,
name: options.name,
role: options.role,
perspective: options.perspective,
});
if (options.stdout) {
process.stdout.write(stringifyJsonDocument(profile));
return;
}
const outPath = resolve(
options.out ??
join(defaultAuthoringDir, "profiles", `${profile.id}.json`),
);
await writeJsonDocument(outPath, profile, { force: options.force });
console.log(`Wrote tester profile: ${outPath}`);
},
);
newCommand
.command("journey")
.description("Generate a user journey JSON file.")
.argument("[id]", "User journey id.", "admin-invite-user")
.option("--title <title>", "Journey title.")
.option("--profile <id>", "Tester profile id used by the journey.", "admin")
.option("--app-name <name>", "App name.", "Acme Admin")
.option("--base-url <url>", "App base URL.", "http://127.0.0.1:3000")
.option("--risk-level <level>", "Journey risk level.", "writes-test-data")
.option("--out <path>", "Output JSON path.")
.option("--stdout", "Print JSON to stdout instead of writing a file.")
.option("--force", "Overwrite the output file if it already exists.")
.action(
async (
id: string,
options: {
title?: string;
profile: string;
appName: string;
baseUrl: string;
riskLevel: UserJourney["riskLevel"];
out?: string;
stdout?: boolean;
force?: boolean;
},
) => {
const journey = createUserJourneyTemplate({
id,
title: options.title,
appName: options.appName,
baseUrl: options.baseUrl,
testerProfile: options.profile,
riskLevel: options.riskLevel,
});
if (options.stdout) {
process.stdout.write(stringifyJsonDocument(journey));
return;
}
const outPath = resolve(
options.out ??
join(defaultAuthoringDir, "journeys", `${journey.id}.json`),
);
await writeJsonDocument(outPath, journey, { force: options.force });
console.log(`Wrote user journey: ${outPath}`);
},
);
const draftCommand = program
.command("draft")
.description(
"Draft richer JourneyTest authoring JSON from structured local inputs.",
);
draftCommand
.command("journey")
.description("Draft a user journey JSON file from flags or a JSON input file.")
.argument("[id]", "User journey id.")
.option("--input <path>", "JSON draft input file.")
.option("--title <title>", "Journey title.")
.option("--profile <id>", "Tester profile id used by the journey.")
.option("--app-name <name>", "App name.")
.option("--base-url <url>", "App base URL.")
.option("--risk-level <level>", "Journey risk level.")
.option("--objective <text>", "Journey objective.")
.option(
"--precondition <text>",
"Precondition for the tester or environment. Repeatable.",
collectRepeatableOption,
[],
)
.option(
"--task <instruction>",
"Task instruction. Repeatable.",
collectRepeatableOption,
[],
)
.option(
"--task-outcome <text>",
"Expected outcome for the task at the same repeat index. Repeatable.",
collectRepeatableOption,
[],
)
.option(
"--pass <statement>",
"Pass criterion statement. Repeatable.",
collectRepeatableOption,
[],
)
.option(
"--fail <statement>",
"Fail criterion statement. Repeatable.",
collectRepeatableOption,
[],
)
.option(
"--blocker <statement>",
"Blocker criterion statement. Repeatable.",
collectRepeatableOption,
[],
)
.option(
"--data <key=value>",
"Journey data value. Repeatable. Values are parsed as JSON when possible.",
collectRepeatableOption,
[],
)
.option("--out <path>", "Output JSON path.")
.option("--stdout", "Print JSON to stdout instead of writing a file.")
.option("--force", "Overwrite the output file if it already exists.")
.action(
async (
id: string | undefined,
options: {
input?: string;
title?: string;
profile?: string;
appName?: string;
baseUrl?: string;
riskLevel?: UserJourney["riskLevel"];
objective?: string;
precondition: string[];
task: string[];
taskOutcome: string[];
pass: string[];
fail: string[];
blocker: string[];
data: string[];
out?: string;
stdout?: boolean;
force?: boolean;
},
) => {
const input = options.input
? DraftJourneyInputSchema.parse(
await readJsonFile(resolve(options.input)),
)
: undefined;
const data = parseDataOptions(options.data);
if (options.taskOutcome.length > options.task.length) {
throw new Error(
"--task-outcome cannot be provided more times than --task.",
);
}
const tasks = options.task.map((instruction, index) => ({
instruction,
...(options.taskOutcome[index]
? { expectedOutcome: options.taskOutcome[index] }
: {}),
}));
const journey = createDraftUserJourney({
input,
id: id ?? input?.id,
title: options.title,
appName: options.appName,
baseUrl: options.baseUrl,
testerProfile: options.profile,
riskLevel: options.riskLevel,
objective: options.objective,
preconditions:
options.precondition.length > 0 ? options.precondition : undefined,
...(Object.keys(data).length > 0 ? { data } : {}),
tasks: tasks.length > 0 ? tasks : undefined,
passCriteria: options.pass.length > 0 ? options.pass : undefined,
failCriteria: options.fail.length > 0 ? options.fail : undefined,
blockerCriteria:
options.blocker.length > 0 ? options.blocker : undefined,
});
if (options.stdout) {
process.stdout.write(stringifyJsonDocument(journey));
return;
}
const outPath = resolve(
options.out ??
join(defaultAuthoringDir, "journeys", `${journey.id}.json`),
);
await writeJsonDocument(outPath, journey, { force: options.force });
console.log(`Wrote drafted user journey: ${outPath}`);
},
);
program
.command("schema")
.description("Print JourneyTest authoring JSON schemas.")
.argument("[kind]", "profile, journey, lifecycle, or all.", "all")
.option("--out <path>", "Write schema JSON to a file.")
.action(async (kindValue: string, options: { out?: string }) => {
const kind: SchemaKind = parseSchemaKind(kindValue);
const document = createJsonSchemaDocument(kind);
if (options.out) {
const outPath = resolve(options.out);
await writeJsonDocument(outPath, document, { force: true });
console.log(`Wrote schema: ${outPath}`);
return;
}
process.stdout.write(stringifyJsonDocument(document));
});
program
.command("lint")
.description(
"Lint JourneyTest authoring files for schema and quality issues.",
)
.requiredOption("--journeys <path>", "Journey JSON file or directory.")
.option("--profiles <path>", "Tester profile JSON file or directory.")
.action(async (options: { journeys: string; profiles?: string }) => {
const result = await lintFiles(options.journeys, options.profiles);
if (result.issueCount > 0) {
process.exitCode = 1;
}
});
program
.command("validate")
.description(
"Validate JourneyTest user journey and tester profile JSON files.",
)
.requiredOption("--journeys <path>", "Journey JSON file or directory.")
.requiredOption(
"--profiles <path>",
"Tester profile JSON file or directory.",
)
.action(async (options: { journeys: string; profiles: string }) => {
const result = await validateFiles(options.journeys, options.profiles);
if (result.issueCount > 0) {
process.exitCode = 1;
}
});
program
.command("auth")
.description("Inspect JourneyTest Pi OAuth authentication without printing secrets.")
.option(
"--auth <path>",
"OAuth auth file. Defaults to JOURNEYTEST_AUTH_PATH, existing ./auth.json, then user config.",
)
.option("--json", "Print machine-readable JSON.")
.action(async (options: { auth?: string; json?: boolean }) => {
const authPath = await resolveAuthPath(options.auth);
const status = await getOAuthAuthStatus(authPath);
if (options.json) {
process.stdout.write(`${JSON.stringify(status, null, 2)}\n`);
return;
}
console.log(`Auth file: ${status.path}`);
console.log(`Source: ${authPathSourceLabel(status.source)}`);
console.log(`Exists: ${status.exists ? "yes" : "no"}`);
console.log(
`Configured providers: ${
status.configuredProviders.length > 0
? status.configuredProviders.join(", ")
: "none"
}`,
);
console.log(
`Pi OAuth providers: ${
status.availableOAuthProviders.length > 0
? status.availableOAuthProviders.join(", ")
: "none"
}`,
);
const unknown = status.configuredProviders.filter(
(provider) => !status.availableOAuthProviders.includes(provider),
);
if (unknown.length > 0) {
console.log(
`Configured but not registered by this Pi version: ${unknown.join(", ")}`,
);
}
});
program
.command("run")
.description(
"Run one JourneyTest user journey file, or all journey JSON files under a directory.",
)
.argument("[journeys]", "Legacy journey JSON file or directory.")
.option("--url <url>", "Start URL for a service journey.")
.option("--goal <goal>", "Goal for a service journey.")
.option("--max-steps <count>", "Maximum Laya steps.", "15")
.option("--timeout <seconds>", "Journey timeout in seconds.", "120")
.option("--no-screenshots", "Disable screenshot capture.")
.option("--trace", "Request trace capture when supported.")
.option("--allowed-domain <domain>", "Allowed destination domain.", collectRepeatableOption, [])
.option("--laya-mode <mode>", "Laya mode: auto, remote, embedded, or mock.", "auto")
.option("--cognition-profile <profile>", "Cognition profile: local-cloud, local-spark-cloud, dual-laya-cloud, or dual-laya-spark-cloud.")
.option("--json", "Print only JourneyResult JSON to stdout.")
.option("--output <dir>", "Service journey output directory.")
.option("--verbose", "Print detailed service progress.")
.option("--profile <path>", "Tester profile JSON file or directory.")
.option("--profiles <path>", "Tester profile JSON file or directory.")
.option("--out <dir>", "Output directory.", "runs")
.option("--director <name>", "Agent director factory to use.", "pi")
.option(
"--browser <name>",
"Browser driver factory to use.",
"agent-browser",
)
.option("--provider <provider>", "Pi model provider.", "anthropic")
.option("--model <model>", "Pi model id or Laya checkpoint.")
.option(
"--auth <path>",
"OAuth auth file. Defaults to JOURNEYTEST_AUTH_PATH, existing ./auth.json, then user config.",
)
.option("--session <name>", "agent-browser session name.")
.option("--state <path>", "agent-browser state file.")
.option("--headed", "Run browser headed.")
.option(
"--viewport <width>x<height>",
"Set browser viewport size, for example 1440x900.",
)
.option(
"--device <preset>",
"Set a curated browser device preset: iphone-14, pixel-7, or ipad-pro-11.",
)
.option(
"--parallel-agents <count>",
"Number of user journeys to run in parallel.",
"1",
)
.option(
"--parallel-browser-mode <mode>",
"Browser strategy for parallel agent-browser runs: shared-tabs or isolated-sessions.",
"shared-tabs",
)
.option(
"--retries <count>",
"Retry each journey up to this many times after a failing attempt.",
"0",
)
.option(
"--journey-timeout-ms <ms>",
"Maximum wall-clock time for one journey director run before it is aborted.",
"1800000",
)
.option("--no-video", "Disable video recording.")
.option(
"--no-ui-change-recording",
"Disable automatic UI change timelines around browser actions.",
)
.option(
"--ui-change-timeout-ms <ms>",
"Maximum time to watch for UI changes after each browser action.",
)
.option(
"--ui-change-quiet-ms <ms>",
"Quiet window before ending UI change observation after changes are seen.",
)
.option(
"--ui-change-max-changes <count>",
"Maximum UI change records to keep per browser action.",
)
.option(
"--ui-change-max-screenshots <count>",
"Maximum change screenshots to keep per browser action.",
)
.option(
"--no-ui-change-screenshots",
"Do not capture before/change/after screenshots for UI change timelines.",
)
.option(
"--no-ui-change-snapshots",
"Do not capture before/after accessibility snapshots for UI change timelines.",
)
.option(
"--no-ui-change-dom-snapshots",
"Do not capture before/after visible DOM summaries for UI change timelines.",
)
.option(
"--no-curate-bookmarks",
"Disable post-run Pi bookmark chapter curation.",
)
.option(
"--bookmark-curator <name>",
"Bookmark curator factory to use.",
"pi",
)
.option(
"--bookmark-system-prompt <path>",
"Path to a custom system prompt for bookmark curation.",
)
.option(
"--data-lifecycle <path>",
"JourneyTest data lifecycle config JSON file.",
)
.option(
"--data-lifecycle-provider <name>",
"Data lifecycle provider factory to use.",
"default",
)
.option("--keep-data", "Skip configured data lifecycle cleanup operations.")
.option("--junit <path>", "Write a JUnit XML report for CI systems.")
.option(
"--github-annotations",
"Emit GitHub Actions workflow-command annotations.",
)
.option("--summary-json <path>", "Write a compact CI summary JSON report.")
.option(
"--compare-to <path>",
"Previous run.json or suite run directory to compare against.",
)
.option(
"--tag <tag>",
"Only run journeys with this tag. Repeat for OR matching.",
collectRepeatableOption,
[],
)
.option(
"--exclude-tag <tag>",
"Skip journeys with this tag. Repeatable.",
collectRepeatableOption,
[],
)
.option(
"--journey-id <id>",
"Only run this journey id. Repeatable.",
collectRepeatableOption,
[],
)
.option(
"--exclude-journey-id <id>",
"Skip this journey id. Repeatable.",
collectRepeatableOption,
[],
)
.option(
"--rerun-failed <path>",
"Prior run.json, history.json, or suite run directory; rerun journeys that were not healthy.",
)
.option(
"--shard <index>/<total>",
"Run one deterministic shard after filtering, using 1-based shard index.",
)
.action(
async (
journeyPath: string | undefined,
options: {
url?: string; goal?: string; maxSteps: string; timeout: string; screenshots: boolean; trace?: boolean; allowedDomain: string[]; layaMode: string; cognitionProfile?: string; json?: boolean; output?: string; verbose?: boolean;
profile?: string;
profiles?: string;
out: string;
director: string;
browser: string;
provider: string;
model: string;
auth?: string;
session?: string;
state?: string;
headed?: boolean;
viewport?: string;
device?: string;
retries: string;
journeyTimeoutMs: string;
parallelAgents: string;
parallelBrowserMode: string;
video?: boolean;
uiChangeRecording?: boolean;
uiChangeTimeoutMs?: string;
uiChangeQuietMs?: string;
uiChangeMaxChanges?: string;
uiChangeMaxScreenshots?: string;
uiChangeScreenshots?: boolean;
uiChangeSnapshots?: boolean;
uiChangeDomSnapshots?: boolean;
curateBookmarks?: boolean;
bookmarkCurator: string;
bookmarkSystemPrompt?: string;
dataLifecycle?: string;
dataLifecycleProvider: string;
keepData?: boolean;
junit?: string;
githubAnnotations?: boolean;
summaryJson?: string;
compareTo?: string;
tag: string[];
excludeTag: string[];
journeyId: string[];
excludeJourneyId: string[];
rerunFailed?: string;
shard?: string;
},
) => {
if (options.url || options.goal) {
if (!options.url || !options.goal) throw new Error("Service runs require both --url and --goal.");
await assertSafeJourneyNetwork(options.url, { allowedDomains: options.allowedDomain }); await assertSafeRedirectChain(options.url, { allowedDomains: options.allowedDomain });
const mode = resolveLayaMode(options.layaMode); if (mode === "mock") throw new Error("CLI does not silently substitute mock inference.");
const managed = mode === "embedded" ? new ManagedLayaService() : undefined; await managed?.start();
const runtime = createJourneyRuntime({ outputDir: options.output ?? options.out, model: options.model, endpoint: managed?.endpoint, cognitionOverrides: options.cognitionProfile ? { COGNITION_PROFILE: options.cognitionProfile } : undefined }); await runtime.backend.start();
const id = await runtime.service.createJourney({ url: options.url, goal: options.goal, maxSteps: Number(options.maxSteps), timeoutMs: Number(options.timeout) * 1000, screenshots: options.screenshots, trace: Boolean(options.trace), cognitionProfile: options.cognitionProfile, context: { allowedDomains: options.allowedDomain } });
const result = await runtime.service.runJourney(id).finally(async () => { await runtime.backend.close(); await managed?.stop(); });
if (options.json) process.stdout.write(`${JSON.stringify(result)}\n`); else { for (const step of result.steps) console.log(`[${String(step.step).padStart(2, "0")}] LAYA ${step.decision?.operation ?? "INVALID"}${step.decision?.elementIndex === undefined ? "" : ` [${step.decision.elementIndex}]`} p=${step.decision?.confidence?.toFixed(3) ?? "-"}`); console.log(`${result.status === "completed" ? "✓" : "✗"} ${result.status}: ${result.termination.reason}`); console.log(`Result: ${result.artifacts.result}`); }
if (result.status !== "completed") process.exitCode = 2; return;
}
if (!journeyPath) throw new Error("Pass a legacy journey path or --url and --goal.");
if (!options.model) throw new Error("Legacy journey runs require --model.");
const profilePath = options.profiles ?? options.profile;
if (!profilePath) {
throw new Error(
"Pass --profile <path> or --profiles <path> so journeys can resolve tester profiles.",
);
}
const bookmarkCuratorName = bookmarkCuratorNameForOptions(options);
factories.directors.assertRegistered(options.director);
factories.browserDrivers.assertRegistered(options.browser);
factories.bookmarkCurators.assertRegistered(bookmarkCuratorName);
if (options.dataLifecycle) {
factories.dataLifecycleProviders.assertRegistered(
options.dataLifecycleProvider,
);
}
const journeyRoot = resolve(journeyPath);
const collectedItems = await collectRunnableJourneys(
journeyRoot,
resolve(profilePath),
);
if (collectedItems.length === 0) {
throw new Error(`No journey JSON files found under ${journeyRoot}.`);
}
const rerunFailed = options.rerunFailed
? await loadRerunFailedSelection(resolve(options.rerunFailed))
: undefined;
const selection = buildJourneySelection({
tags: options.tag,
excludeTags: options.excludeTag,
journeyIds: options.journeyId,
excludeJourneyIds: options.excludeJourneyId,
...(rerunFailed ? { rerunFailed } : {}),
shard: options.shard,
});
const selectionResult = selectRunnableJourneys(
collectedItems,
selection,
);
const items = selectionResult.items;
if (items.length === 0) {
throw new Error(
`No journeys selected after applying run filters. Collected ${collectedItems.length} journey(s).`,
);
}
const parallelAgents = parseParallelAgents(options.parallelAgents);
const retries = parseRetries(options.retries);
const journeyTimeoutMs = parseJourneyTimeoutMs(
options.journeyTimeoutMs,
);
const parallelBrowserMode = parseParallelBrowserMode(
options.parallelBrowserMode,
);
const browserEnvironmentOverrides =
buildBrowserEnvironmentOptions(options);
const uiChangeRecordingOptions = buildUiChangeRecordingOptions(options);
const authPath = await resolveAuthPath(options.auth);
const authLoader = createOAuthAuthLoader(authPath.path);
const bookmarkSystemPrompt = options.bookmarkSystemPrompt
? await readFile(resolve(options.bookmarkSystemPrompt), "utf8")
: undefined;
const directorFactoryContext = {
provider: options.provider,
modelId: options.model,
getApiKey: authLoader,
};
const bookmarkCuratorFactoryContext = {
provider: options.provider,
modelId: options.model,
getApiKey: authLoader,
systemPrompt: bookmarkSystemPrompt,
};
await assertFactoryOAuthCredentialsAvailable(
[
factories.directors.authProvider(
options.director,
directorFactoryContext,
),
factories.bookmarkCurators.authProvider(
bookmarkCuratorName,
bookmarkCuratorFactoryContext,
),
],
authPath,
);
const dataLifecycleConfig = options.dataLifecycle
? await readAndParseJsonFile(
resolve(options.dataLifecycle),
DataLifecycleConfigSchema,
)
: undefined;
const dataLifecycleProvider = dataLifecycleConfig
? factories.dataLifecycleProviders.create(
options.dataLifecycleProvider,
{},
)
: undefined;
const baseOutputDir = resolve(options.out);
const comparisonBaseline = options.compareTo
? await loadComparisonBaseline(resolve(options.compareTo))
: undefined;
const isDirectoryRun = (await stat(journeyRoot)).isDirectory();
const runOutputDir = isDirectoryRun
? join(baseOutputDir, `${isoTimestampForPath()}-run`)
: baseOutputDir;
if (isDirectoryRun || dataLifecycleConfig?.suiteLifecycle) {
await mkdir(runOutputDir, { recursive: true });
}
const suiteRunId = isDirectoryRun
? basename(runOutputDir)
: `${isoTimestampForPath()}-suite`;
console.log(`Collected ${collectedItems.length} journey(s).`);
if (selectionResult.summary) {
console.log(`Selected ${items.length} journey(s) after filters.`);
}
console.log(
`Parallel agents: ${Math.min(parallelAgents, items.length)}`,
);
if (retries > 0) {
console.log(`Retries: ${retries}`);
}
const sharedBrowserTabs = shouldUseSharedBrowserTabs({
browser: options.browser,
parallelAgents,
journeyCount: items.length,
parallelBrowserMode,
});
if (
parallelBrowserMode === "shared-tabs" &&
options.browser !== "agent-browser" &&
parallelAgents > 1 &&
items.length > 1
) {
throw new Error(
'--parallel-browser-mode shared-tabs is only supported by the "agent-browser" browser driver.',
);
}
const sharedBrowserSessionName = sharedBrowserTabs
? parallelBrowserSessionName(options.session, suiteRunId)
: undefined;
if (parallelAgents > 1 && items.length > 1) {
console.log(
`Browser parallel mode: ${
sharedBrowserTabs ? "shared-tabs" : "isolated-sessions"
}`,
);
if (sharedBrowserTabs) {
console.log(`Shared browser session: ${sharedBrowserSessionName}`);
if (options.video ?? true) {
console.log(
"Shared-tab video recording is serialized per journey because agent-browser supports one active recording per session.",
);
}
}
}
if (isDirectoryRun) {
console.log(`Run output: ${runOutputDir}`);
}
if (comparisonBaseline) {
console.log(
`Comparison baseline: ${comparisonBaseline.sourcePath} (${comparisonBaseline.runs.length} run(s))`,
);
}
if (dataLifecycleConfig) {
console.log(
`Data lifecycle config: ${resolve(options.dataLifecycle as string)}`,
);
if (options.keepData) {
console.log("Data lifecycle cleanup: skipped by --keep-data");
}
}
if (browserEnvironmentOverrides) {
console.log(
`Browser environment overrides: ${formatBrowserEnvironment(browserEnvironmentOverrides)}`,
);
}
if (!sharedBrowserTabs && parallelAgents > 1 && options.session) {
console.log(`Browser session prefix: ${options.session}`);
}
let appLifecycleController: AppLifecycleController | undefined;
let removeAppLifecycleSignalHandlers: (() => void) | undefined;
let runtimeItems = items;
let suiteLifecycleController: DataLifecycleController | undefined;
let suiteLifecycleBlocked = false;
let suiteRuns: Array<{
journeyFile: string;
result: Awaited<ReturnType<typeof runJourney>>;
}> = [];
try {
if (dataLifecycleConfig?.appLifecycle) {
appLifecycleController = await AppLifecycleController.create({
config: dataLifecycleConfig.appLifecycle,
suiteRunId,
runDir: join(runOutputDir, "_app-lifecycle"),
});
removeAppLifecycleSignalHandlers =
installAppLifecycleSignalCleanup(appLifecycleController);
console.log("");
console.log("Running app lifecycle start");
await appLifecycleController.runStart();
console.log(
`App lifecycle: ${appLifecycleController.result.status}`,
);
console.log(
`App lifecycle artifacts: ${join(runOutputDir, "_app-lifecycle")}`,
);
if (appLifecycleController.appTarget) {
runtimeItems = items.map((item) => ({
...item,
journey: applyAppTargetOverride(
item.journey,
appLifecycleController?.appTarget,
),
}));
console.log(
`App target: ${appLifecycleController.appTarget.baseUrl}`,
);
}
}
if (dataLifecycleConfig?.suiteLifecycle && dataLifecycleProvider) {
suiteLifecycleController = new DataLifecycleController({
definition: dataLifecycleConfig.suiteLifecycle,
environments: dataLifecycleConfig.dataEnvironments,
provider: dataLifecycleProvider,
scope: "suite",
runId: suiteRunId,
runDir: join(runOutputDir, "_suite-lifecycle"),
suiteRunId,
keepData: options.keepData,
});
console.log("");
console.log("Running suite data lifecycle setup/preflight");
try {
await suiteLifecycleController.runSetupAndPreflight();
console.log(
`Suite data lifecycle: ${suiteLifecycleController.result.status}`,
);
} catch (error) {
suiteLifecycleBlocked = true;
process.exitCode = 1;
const message =
error instanceof Error ? error.message : String(error);
console.error(`Suite data lifecycle blocked the run: ${message}`);
}
}
if (!suiteLifecycleBlocked) {
suiteRuns = await mapConcurrent(
runtimeItems,
parallelAgents,
async (item, index) => {
console.log("");
console.log(
`Running ${item.journey.id} (${index + 1}/${runtimeItems.length})`,
);
console.log(`Source: ${item.journeyFile}`);
const result = await runJourneyWithRetries({
item,
retries,
outputDir: runOutputDir,
createDriver: () =>
factories.browserDrivers.create(options.browser, {}),
createDirector: () =>
factories.directors.create(
options.director,
directorFactoryContext,
),
createBookmarkCurator: () =>
factories.bookmarkCurators.create(
bookmarkCuratorName,
bookmarkCuratorFactoryContext,
),
video: options.video,
directorTimeoutMs: journeyTimeoutMs,
uiChangeRecording: options.uiChangeRecording,
uiChangeRecordingOptions,
sessionName: browserSessionNameForRun({
baseSession: options.session,
parallelAgents,
journeyId: item.journey.id,
index,
suiteRunId,
sharedBrowserTabs,
}),
tabLabel: browserTabLabelForRun({
parallelAgents,
journeyId: item.journey.id,
index,
sharedBrowserTabs,
}),
statePath: options.state ? resolve(options.state) : undefined,
headed: options.headed,
browserEnvironment: mergeBrowserEnvironmentOptions(
item.journey.browserEnvironment,
browserEnvironmentOverrides,
),
...(dataLifecycleConfig && dataLifecycleProvider
? {
dataLifecycle: {
environments: dataLifecycleConfig.dataEnvironments,
provider: dataLifecycleProvider,
keepData: options.keepData,
suiteRunId,
suiteManifest: suiteLifecycleController?.manifest,
},
}
: {}),
});
printRunResult(result);
if (!runPassed(result)) {
process.exitCode = 1;
}
return { journeyFile: item.journeyFile, result };
},
);
}
} finally {
if (suiteLifecycleController) {
if (!suiteLifecycleBlocked) {
await suiteLifecycleController.runPostconditions();
}
await suiteLifecycleController.runCleanup();
suiteLifecycleController.finish();
console.log("");
console.log(
`Suite data lifecycle: ${suiteLifecycleController.result.status}`,
);
console.log(
`Suite lifecycle artifacts: ${join(runOutputDir, "_suite-lifecycle")}`,
);
if (
suiteLifecycleController.result.status === "failed" ||
suiteLifecycleController.result.status === "blocked"
) {
process.exitCode = 1;
}
}
if (appLifecycleController) {
await appLifecycleController.runCleanup();
console.log("");
console.log(
`App lifecycle: ${appLifecycleController.result.status}`,
);
console.log(
`App lifecycle artifacts: ${join(runOutputDir, "_app-lifecycle")}`,
);
if (appLifecycleController.result.status === "failed") {
process.exitCode = 1;
}
}
if (removeAppLifecycleSignalHandlers) {
removeAppLifecycleSignalHandlers();
}
}
if (suiteLifecycleBlocked) {
await writeCliCiOutputs({
suiteRunId,
suiteRuns,
junitPath: options.junit,
summaryJsonPath: options.summaryJson,
githubAnnotations: options.githubAnnotations,
});
return;
}
if (isDirectoryRun) {
const suiteArtifacts = await writeSuiteArtifacts(
runOutputDir,
suiteRuns,
comparisonBaseline,
selectionResult.summary,
);
console.log("");
console.log(`Run dashboard: ${suiteArtifacts.dashboard}`);
console.log(
`Run dashboard URL: ${pathToFileURL(suiteArtifacts.dashboard).href}`,
);
console.log(`Run history: ${suiteArtifacts.history}`);
console.log("");
console.log("Journey dashboards:");
for (const run of suiteRuns) {
console.log(
`- ${run.result.journeyId}: ${run.result.artifacts.dashboard}`,
);
}
}
await writeCliCiOutputs({
suiteRunId,
suiteRuns,
junitPath: options.junit,
summaryJsonPath: options.summaryJson,
githubAnnotations: options.githubAnnotations,
});
},
);
for (const name of ["serve", "ui", "space"] as const) program.command(name).description(`${name} the JourneyTest service.`).option("--host <host>", "Bind host.", "0.0.0.0").option("--port <port>", "Bind port.", "7860").option("--cognition-profile <profile>", "Runtime-only cognition profile override.").action(async (options: { host: string; port: string; cognitionProfile?: string }) => runServerCommand(name, options));
program.command("info").description("Print JourneyTest runtime information without secret values.").option("--json", "Print JSON.").action(async (options: { json?: boolean }) => { const config = loadRuntimeCognitionConfig(); const data = { version: "0.1.2", space: isHuggingFaceSpace(), layaMode: config.localLaya.mode, ...cognitionConfigStatus(config) }; options.json ? process.stdout.write(`${JSON.stringify(data)}\n`) : process.stdout.write(`${JSON.stringify(data, null, 2)}\n`); });
program.command("doctor").description("Check browser, Laya, artifacts, and service dependencies.").option("--smoke-model", "Require a ready SystemOne endpoint without downloading a model.").option("--smoke-reasoning", "Make one explicit live cloud reasoning request.").option("--json", "Print machine-readable diagnostics.").action(async (options: { smokeModel?: boolean; smokeReasoning?: boolean; json?: boolean }) => runDoctor(Boolean(options.smokeModel), Boolean(options.json), Boolean(options.smokeReasoning)));
await program.parseAsync(argv);
}
type StoredOAuthAuth = Record<string, OAuthCredentials & { type?: string }>;
export type AuthPathSource = "cli" | "env" | "local" | "user-config";
export interface AuthPathResolution {
path: string;
source: AuthPathSource;
localPath: string;
userConfigPath: string;
}
export interface AuthPathResolutionOptions {
cwd?: string;
env?: NodeJS.ProcessEnv;
homeDir?: string;
platform?: NodeJS.Platform;
fileExists?: (path: string) => Promise<boolean>;
}
export interface OAuthAuthStatus {
path: string;
source: AuthPathSource;
exists: boolean;
configuredProviders: string[];
availableOAuthProviders: string[];
}
export interface RunnableJourney {
journeyFile: string;
journey: UserJourney;
profile: TesterProfile;
}
interface ComparisonBaseline {
sourcePath: string;
kind: "run" | "suite";
runs: RunResult[];
}
export interface RerunFailedSelection {
sourcePath: string;
unhealthyJourneyIds: string[];
}
export interface JourneySelection {
tags: string[];
excludeTags: string[];
journeyIds: string[];
excludeJourneyIds: string[];
rerunFailed?: RerunFailedSelection;
shard?: ShardSelection;
}
export interface ShardSelection {
index: number;
total: number;
}
export interface JourneySelectionResult {
items: RunnableJourney[];
summary?: SuiteSelectionSummary;
}
export function parseParallelAgents(value: string): number {
const parsed = Number(value);
if (!Number.isInteger(parsed) || parsed < 1) {
throw new Error(
`--parallel-agents must be a positive integer. Received "${value}".`,
);
}
return parsed;
}
export function parseParallelBrowserMode(value: string): ParallelBrowserMode {
if (value === "shared-tabs" || value === "isolated-sessions") {
return value;
}
throw new Error(
`--parallel-browser-mode must be "shared-tabs" or "isolated-sessions". Received "${value}".`,
);
}
export function parseRetries(value: string): number {
const parsed = Number(value);
if (!Number.isInteger(parsed) || parsed < 0) {
throw new Error(`--retries must be a non-negative integer. Received "${value}".`);
}
return parsed;
}
export function parseViewportOption(
value: string,
): BrowserEnvironment["viewport"] {
const match = /^(\d+)x(\d+)(?:@(\d+(?:\.\d+)?))?$/.exec(value.trim());
if (!match) {
throw new Error(
`--viewport must use <width>x<height>, optionally <width>x<height>@<scale>. Received "${value}".`,
);
}
const width = Number(match[1]);
const height = Number(match[2]);
const deviceScaleFactor = match[3] ? Number(match[3]) : undefined;
if (
!Number.isInteger(width) ||
width < 1 ||
!Number.isInteger(height) ||
height < 1 ||
(deviceScaleFactor !== undefined &&
(!Number.isFinite(deviceScaleFactor) || deviceScaleFactor <= 0))
) {
throw new Error(
`--viewport must use positive dimensions and scale. Received "${value}".`,
);
}
return {
width,
height,
...(deviceScaleFactor === undefined ? {} : { deviceScaleFactor }),
};
}
export function parseBrowserDevicePreset(value: string): BrowserDevicePreset {
if (value === "iphone-14" || value === "pixel-7" || value === "ipad-pro-11") {
return value;
}
throw new Error(
`--device must be one of: iphone-14, pixel-7, ipad-pro-11. Received "${value}".`,
);
}
export function buildBrowserEnvironmentOptions(options: {
viewport?: string;
device?: string;
}): BrowserEnvironment | undefined {
const environment: BrowserEnvironment = {
...(options.device ? { device: parseBrowserDevicePreset(options.device) } : {}),
...(options.viewport ? { viewport: parseViewportOption(options.viewport) } : {}),
};
return Object.keys(environment).length > 0 ? environment : undefined;
}
export function mergeBrowserEnvironmentOptions(
defaults: BrowserEnvironment | undefined,
overrides: BrowserEnvironment | undefined,
): BrowserEnvironment | undefined {
const viewport = mergeViewportOptions(defaults?.viewport, overrides?.viewport);
const environment: BrowserEnvironment = {
...(defaults ?? {}),
...(overrides ?? {}),
...(viewport ? { viewport } : {}),
};
return Object.keys(environment).length > 0 ? environment : undefined;
}
function mergeViewportOptions(
defaults: BrowserEnvironment["viewport"] | undefined,
overrides: BrowserEnvironment["viewport"] | undefined,
): BrowserEnvironment["viewport"] | undefined {
if (!defaults && !overrides) {
return undefined;
}
const width = overrides?.width ?? defaults?.width;
const height = overrides?.height ?? defaults?.height;
if (width === undefined || height === undefined) {
return undefined;
}
const deviceScaleFactor = overrides?.deviceScaleFactor ?? defaults?.deviceScaleFactor;
return {
width,
height,
...(deviceScaleFactor === undefined ? {} : { deviceScaleFactor }),
};
}
function formatBrowserEnvironment(environment: BrowserEnvironment): string {
const parts = [
environment.device ? `device=${environment.device}` : undefined,
environment.viewport
? `viewport=${environment.viewport.width}x${environment.viewport.height}${
environment.viewport.deviceScaleFactor === undefined
? ""
: `@${environment.viewport.deviceScaleFactor}`
}`
: undefined,
].filter(Boolean);
return parts.join(", ");
}
export function buildJourneySelection(options: {
tags?: string[];
excludeTags?: string[];
journeyIds?: string[];
excludeJourneyIds?: string[];
rerunFailed?: RerunFailedSelection;
shard?: string;
}): JourneySelection {
const tags = uniqueSelectorValues("--tag", options.tags ?? []);
const excludeTags = uniqueSelectorValues(
"--exclude-tag",
options.excludeTags ?? [],
);
const journeyIds = uniqueSelectorValues(
"--journey-id",
options.journeyIds ?? [],
);
const excludeJourneyIds = uniqueSelectorValues(
"--exclude-journey-id",
options.excludeJourneyIds ?? [],
);
assertDisjointSelectors("--tag", tags, "--exclude-tag", excludeTags);
assertDisjointSelectors(
"--journey-id",
journeyIds,
"--exclude-journey-id",
excludeJourneyIds,
);
return {
tags,
excludeTags,
journeyIds,
excludeJourneyIds,
...(options.rerunFailed ? { rerunFailed: options.rerunFailed } : {}),
...(options.shard ? { shard: parseShardSelection(options.shard) } : {}),
};
}
export function parseShardSelection(value: string): ShardSelection {
const match = /^([1-9]\d*)\/([1-9]\d*)$/.exec(value);
if (!match) {
throw new Error(
`--shard must use the format <index>/<total> with positive integers. Received "${value}".`,
);
}
const index = Number(match[1]);
const total = Number(match[2]);
if (index > total) {
throw new Error(
`--shard index must be less than or equal to total. Received "${value}".`,
);
}
return { index, total };
}
export function selectRunnableJourneys(
items: RunnableJourney[],
selection: JourneySelection,
): JourneySelectionResult {
validateKnownJourneyIds(items, selection.journeyIds, "--journey-id");
validateKnownJourneyIds(
items,
selection.excludeJourneyIds,
"--exclude-journey-id",
);
const rerunIds = selection.rerunFailed
? new Set(selection.rerunFailed.unhealthyJourneyIds)
: undefined;
let selected = items.filter((item) => {
const journeyTags = new Set(item.journey.tags ?? []);
if (
selection.journeyIds.length > 0 &&
!selection.journeyIds.includes(item.journey.id)
) {
return false;
}
if (selection.excludeJourneyIds.includes(item.journey.id)) {
return false;
}
if (
selection.tags.length > 0 &&
!selection.tags.some((tag) => journeyTags.has(tag))
) {
return false;
}
if (selection.excludeTags.some((tag) => journeyTags.has(tag))) {
return false;
}
if (rerunIds && !rerunIds.has(item.journey.id)) {
return false;
}
return true;
});
if (selection.shard) {
selected = selected.filter(
(_item, index) =>
index % selection.shard!.total === selection.shard!.index - 1,
);
}
const summary = buildSelectionSummary(
items.length,
selected.length,
selection,
);
return { items: selected, ...(summary ? { summary } : {}) };
}
function buildSelectionSummary(
collected: number,
selected: number,
selection: JourneySelection,
): SuiteSelectionSummary | undefined {
const hasSelection =
selection.tags.length > 0 ||
selection.excludeTags.length > 0 ||
selection.journeyIds.length > 0 ||
selection.excludeJourneyIds.length > 0 ||
Boolean(selection.rerunFailed) ||
Boolean(selection.shard);
if (!hasSelection) {
return undefined;
}
return {
collected,
selected,
...(selection.tags.length > 0 ? { tags: selection.tags } : {}),
...(selection.excludeTags.length > 0
? { excludeTags: selection.excludeTags }
: {}),
...(selection.journeyIds.length > 0
? { journeyIds: selection.journeyIds }
: {}),
...(selection.excludeJourneyIds.length > 0
? { excludeJourneyIds: selection.excludeJourneyIds }
: {}),
...(selection.rerunFailed
? {
rerunFailed: {
path: selection.rerunFailed.sourcePath,
unhealthyJourneyIds: selection.rerunFailed.unhealthyJourneyIds,
},
}
: {}),
...(selection.shard ? { shard: selection.shard } : {}),
};
}
function uniqueSelectorValues(optionName: string, values: string[]): string[] {
const trimmed = values.map((value) => value.trim());
if (trimmed.some((value) => value.length === 0)) {
throw new Error(`${optionName} values must be non-empty.`);
}
return [...new Set(trimmed)];
}
function assertDisjointSelectors(
includeName: string,
includeValues: string[],
excludeName: string,
excludeValues: string[],
): void {
const excludeSet = new Set(excludeValues);
const overlaps = includeValues.filter((value) => excludeSet.has(value));
if (overlaps.length > 0) {
throw new Error(
`${includeName} and ${excludeName} cannot contain the same value(s): ${overlaps.join(", ")}.`,
);
}
}
function validateKnownJourneyIds(
items: RunnableJourney[],
ids: string[],
optionName: string,
): void {
if (ids.length === 0) {
return;
}
const knownIds = new Set(items.map((item) => item.journey.id));
const unknown = ids.filter((id) => !knownIds.has(id));
if (unknown.length > 0) {
throw new Error(
`${optionName} referenced journey id(s) that were not collected: ${unknown.join(", ")}.`,
);
}
}
export function parseDataOptions(values: string[]): Record<string, unknown> {
const data: Record<string, unknown> = {};
for (const value of values) {
const equalsIndex = value.indexOf("=");
if (equalsIndex <= 0) {
throw new Error(
`--data values must use key=value syntax. Received "${value}".`,
);
}
const key = value.slice(0, equalsIndex).trim();
if (!key) {
throw new Error(
`--data values must include a non-empty key. Received "${value}".`,
);
}
const rawValue = value.slice(equalsIndex + 1);
data[key] = parseJsonLikeValue(rawValue);
}
return data;
}
function parseJsonLikeValue(value: string): unknown {
try {
return JSON.parse(value) as unknown;
} catch {
return value;
}
}
export function buildUiChangeRecordingOptions(options: {
uiChangeTimeoutMs?: string;
uiChangeQuietMs?: string;
uiChangeMaxChanges?: string;
uiChangeMaxScreenshots?: string;
uiChangeScreenshots?: boolean;
uiChangeSnapshots?: boolean;
uiChangeDomSnapshots?: boolean;
}): UiChangeRecordingOptions | undefined {
const recordingOptions: UiChangeRecordingOptions = {
...(options.uiChangeTimeoutMs
? {
timeoutMs: parsePositiveIntegerOption(
"--ui-change-timeout-ms",
options.uiChangeTimeoutMs,
),
}
: {}),
...(options.uiChangeQuietMs
? {
quietMs: parsePositiveIntegerOption(
"--ui-change-quiet-ms",
options.uiChangeQuietMs,
),
}
: {}),
...(options.uiChangeMaxChanges
? {
maxChanges: parsePositiveIntegerOption(
"--ui-change-max-changes",
options.uiChangeMaxChanges,
),
}
: {}),
...(options.uiChangeMaxScreenshots
? {
maxScreenshots: parseNonNegativeIntegerOption(
"--ui-change-max-screenshots",
options.uiChangeMaxScreenshots,
),
}
: {}),
...(options.uiChangeScreenshots === undefined
? {}
: { screenshots: options.uiChangeScreenshots }),
...(options.uiChangeSnapshots === undefined
? {}
: { snapshots: options.uiChangeSnapshots }),
...(options.uiChangeDomSnapshots === undefined
? {}
: { domSnapshots: options.uiChangeDomSnapshots }),
};
return Object.keys(recordingOptions).length > 0
? recordingOptions
: undefined;
}
export function parseJourneyTimeoutMs(value: string): number | undefined {
if (value === "0") {
return undefined;
}
return parsePositiveIntegerOption("--journey-timeout-ms", value);
}
function parsePositiveIntegerOption(name: string, value: string): number {
const parsed = Number(value);
if (!Number.isInteger(parsed) || parsed < 1) {
throw new Error(`${name} must be a positive integer. Received "${value}".`);
}
return parsed;
}
function parseNonNegativeIntegerOption(name: string, value: string): number {
const parsed = Number(value);
if (!Number.isInteger(parsed) || parsed < 0) {
throw new Error(
`${name} must be a non-negative integer. Received "${value}".`,
);
}
return parsed;
}
export function sessionNameForRun(
baseSession: string | undefined,
parallelAgents: number,
journeyId: string,
index: number,
): string | undefined {
if (!baseSession) {
return undefined;
}
if (parallelAgents <= 1) {
return baseSession;
}
return `${baseSession}-${String(index + 1).padStart(2, "0")}-${sanitizePathSegment(journeyId)}`;
}
export function shouldUseSharedBrowserTabs(options: {
browser: string;
parallelAgents: number;
journeyCount: number;
parallelBrowserMode: ParallelBrowserMode;
}): boolean {
return (
options.browser === "agent-browser" &&
options.parallelAgents > 1 &&
options.journeyCount > 1 &&
options.parallelBrowserMode === "shared-tabs"
);
}
export function parallelBrowserSessionName(
baseSession: string | undefined,
suiteRunId: string,
): string {
return baseSession ?? suiteRunId;
}
export function browserSessionNameForRun(options: {
baseSession: string | undefined;
parallelAgents: number;
journeyId: string;
index: number;
suiteRunId: string;
sharedBrowserTabs: boolean;
}): string | undefined {
if (options.sharedBrowserTabs) {
return parallelBrowserSessionName(options.baseSession, options.suiteRunId);
}
return sessionNameForRun(
options.baseSession,
options.parallelAgents,
options.journeyId,
options.index,
);
}
export function browserTabLabelForRun(options: {
parallelAgents: number;
journeyId: string;
index: number;
sharedBrowserTabs: boolean;
}): string | undefined {
if (!options.sharedBrowserTabs) {
return undefined;
}
return `journey-${String(options.index + 1).padStart(2, "0")}-${sanitizePathSegment(options.journeyId)}`;
}
export function bookmarkCuratorNameForOptions(options: {
bookmarkCurator?: string;
curateBookmarks?: boolean;
}): string {
if (options.curateBookmarks === false) {
return "none";
}
return options.bookmarkCurator ?? "pi";
}
interface RunJourneyWithRetriesOptions {
item: RunnableJourney;
retries: number;
outputDir: string;
createDriver: () => BrowserDriver;
createDirector: () => AgentDirector;
createBookmarkCurator: () => BookmarkCurator | undefined;
video?: boolean;
directorTimeoutMs?: number;
uiChangeRecording?: boolean;
uiChangeRecordingOptions?: UiChangeRecordingOptions;
sessionName?: string;
tabLabel?: string;
statePath?: string;
headed?: boolean;
browserEnvironment?: BrowserEnvironment;
dataLifecycle?: RunJourneyOptions["dataLifecycle"];
}
async function runJourneyWithRetries(
options: RunJourneyWithRetriesOptions,
): Promise<RunResult> {
const maxAttempts = options.retries + 1;
const attempts: RunAttempt[] = [];
let finalResult: RunResult | undefined;
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
if (attempt > 1) {
console.log(
`Retrying ${options.item.journey.id} (${attempt}/${maxAttempts})`,
);
}
const result = await runJourney({
journey: options.item.journey,
profile: options.item.profile,
driver: options.createDriver(),
director: options.createDirector(),
outputDir: options.outputDir,
video: options.video,
directorTimeoutMs: options.directorTimeoutMs,
uiChangeRecording: options.uiChangeRecording,
uiChangeRecordingOptions: options.uiChangeRecordingOptions,
sessionName: attemptScopedValue(options.sessionName, attempt),
tabLabel: attemptScopedValue(options.tabLabel, attempt),
statePath: options.statePath,
headed: options.headed,
browserEnvironment: options.browserEnvironment,
bookmarkCurator: options.createBookmarkCurator(),
dataLifecycle: options.dataLifecycle,
});
attempts.push(createRunAttempt(attempt, result));
finalResult = result;
if (runPassed(result)) {
break;
}
}
if (!finalResult) {
throw new Error(`No attempts were run for ${options.item.journey.id}.`);
}
const annotated = annotateFinalRunResult({
result: finalResult,
attempts,
retries: options.retries,
quarantined: options.item.journey.quarantined ?? false,
});
await persistRunResult(annotated);
return annotated;
}
function annotateFinalRunResult(options: {
result: RunResult;
attempts: RunAttempt[];
retries: number;
quarantined: boolean;
}): RunResult {
const passedAttempt = options.attempts.find((attempt) => attempt.passed);
const failedAttemptsBeforePass = passedAttempt
? options.attempts.filter(
(attempt) => attempt.attempt < passedAttempt.attempt && !attempt.passed,
).length
: 0;
return {
...options.result,
attempts: options.attempts,
flake: {
isFlaky: Boolean(passedAttempt && failedAttemptsBeforePass > 0),
retries: options.retries,
attempts: options.attempts.length,
...(passedAttempt ? { passedAttempt: passedAttempt.attempt } : {}),
failedAttemptsBeforePass,
},
...(options.quarantined
? { quarantine: { quarantined: true } }
: options.result.quarantine
? { quarantine: options.result.quarantine }
: {}),
};
}
function createRunAttempt(attempt: number, result: RunResult): RunAttempt {
return {
attempt,
runId: result.runId,
runStatus: result.runStatus,
...(result.verdict?.status ? { verdictStatus: result.verdict.status } : {}),
...(result.dataLifecycle?.status
? { dataLifecycleStatus: result.dataLifecycle.status }
: {}),
passed: runPassed(result),
startedAt: result.startedAt,
endedAt: result.endedAt,
durationMs: result.durationMs,
dashboard: result.artifacts.dashboard,
result: result.artifacts.result,
...(result.verdict?.summary || result.error?.message
? { summary: result.verdict?.summary ?? result.error?.message }
: {}),
};
}
function runPassed(result: RunResult): boolean {
return (
result.runStatus === "completed" &&
result.verdict?.status === "passed" &&
result.dataLifecycle?.status !== "failed" &&
result.dataLifecycle?.status !== "blocked"
);
}
function attemptScopedValue(
value: string | undefined,
attempt: number,
): string | undefined {
if (!value || attempt === 1) {
return value;
}
return `${value}-attempt-${attempt}`;
}
async function persistRunResult(result: RunResult): Promise<void> {
await writeFile(
result.artifacts.result,
`${JSON.stringify(result, null, 2)}\n`,
"utf8",
);
}
async function mapConcurrent<T, TResult>(
items: T[],
concurrency: number,
mapper: (item: T, index: number) => Promise<TResult>,
): Promise<TResult[]> {
const results = new Array<TResult>(items.length);
let nextIndex = 0;
const workerCount = Math.min(concurrency, items.length);
await Promise.all(
Array.from({ length: workerCount }, async () => {
while (nextIndex < items.length) {
const index = nextIndex++;
results[index] = await mapper(items[index] as T, index);
}
}),
);
return results;
}
function createOAuthAuthLoader(
authPath: string,
): (provider: string) => Promise<string | undefined> {
let cachedAuth: StoredOAuthAuth | undefined;
let queue: Promise<void> = Promise.resolve();
return async (provider: string) => {
const operation = queue.then(async () => {
const auth = cachedAuth ?? (cachedAuth = await readOAuthAuth(authPath));
const result = await getOAuthApiKey(provider, auth);
if (!result) {
return undefined;
}
if (auth[provider] !== result.newCredentials) {
auth[provider] = { type: "oauth", ...result.newCredentials };
cachedAuth = auth;
await mkdir(dirname(authPath), { recursive: true });
await writeFile(authPath, `${JSON.stringify(auth, null, 2)}\n`, "utf8");
}
return result.apiKey;
});
queue = operation.then(
() => undefined,
() => undefined,
);
return operation;
};
}
export async function getOAuthAuthStatus(
authPath: AuthPathResolution,
): Promise<OAuthAuthStatus> {
const exists = await pathExists(authPath.path);
const auth = await readOAuthAuth(authPath.path);
return {
path: authPath.path,
source: authPath.source,
exists,
configuredProviders: Object.keys(auth).sort(),
availableOAuthProviders: getOAuthProviders()
.map((provider) => provider.id)
.sort(),
};
}
async function assertFactoryOAuthCredentialsAvailable(
providers: Array<string | undefined>,
authPath: AuthPathResolution,
): Promise<void> {
const authProviders = new Set(
providers.filter((provider): provider is string => Boolean(provider)),
);
await Promise.all(
[...authProviders].map((provider) =>
assertOAuthCredentialsAvailable(provider, authPath),
),
);
}
async function assertOAuthCredentialsAvailable(
provider: string,
authPath: AuthPathResolution,
): Promise<void> {
if (!getOAuthProvider(provider)) {
return;
}
const auth = await readOAuthAuth(authPath.path);
if (auth[provider]) {
return;
}
throw new Error(formatMissingOAuthCredentialsMessage(provider, authPath));
}
async function readOAuthAuth(authPath: string): Promise<StoredOAuthAuth> {
try {
return JSON.parse(await readFile(authPath, "utf8")) as StoredOAuthAuth;
} catch (error) {
const code =
typeof error === "object" && error && "code" in error
? error.code
: undefined;
if (code === "ENOENT") {
return {};
}
const message = error instanceof Error ? error.message : String(error);
if (error instanceof SyntaxError) {
throw new Error(
`Could not parse OAuth auth file at ${authPath}: ${message}`,
);
}
throw new Error(
`Could not read OAuth auth file at ${authPath}: ${message}`,
);
}
}
export async function resolveAuthPath(
authPathOption?: string,
options: AuthPathResolutionOptions = {},
): Promise<AuthPathResolution> {
const cwd = options.cwd ?? process.cwd();
const env = options.env ?? process.env;
const localPath = resolve(cwd, "auth.json");
const userConfigPath = defaultUserAuthPath({
env,
homeDir: options.homeDir,
platform: options.platform,
});
if (authPathOption) {
return {
path: resolve(cwd, authPathOption),
source: "cli",
localPath,
userConfigPath,
};
}
if (env.JOURNEYTEST_AUTH_PATH) {
return {
path: resolve(cwd, env.JOURNEYTEST_AUTH_PATH),
source: "env",
localPath,
userConfigPath,
};
}
const fileExists = options.fileExists ?? pathExists;
if (await fileExists(localPath)) {
return {
path: localPath,
source: "local",
localPath,
userConfigPath,
};
}
return {
path: userConfigPath,
source: "user-config",
localPath,
userConfigPath,
};
}
export function defaultUserAuthPath(
options: {
env?: NodeJS.ProcessEnv;
homeDir?: string;
platform?: NodeJS.Platform;
} = {},
): string {
const env = options.env ?? process.env;
const home = options.homeDir ?? homedir();
const platform = options.platform ?? process.platform;
if (platform === "darwin") {
return join(
home,
"Library",
"Application Support",
"JourneyTest",
"auth.json",
);
}
if (platform === "win32") {
return join(
env.APPDATA ?? join(home, "AppData", "Roaming"),
"JourneyTest",
"auth.json",
);
}
return join(
env.XDG_CONFIG_HOME ?? join(home, ".config"),
"journeytest",
"auth.json",
);
}
export function formatMissingOAuthCredentialsMessage(
provider: string,
authPath: AuthPathResolution,
): string {
const authDir = dirname(authPath.path);
const customFileNote =
basename(authPath.path) === "auth.json"
? ""
: `\nThe Pi login CLI writes auth.json; move it to ${authPath.path} after login if you keep this custom filename.`;
return (
[
`No OAuth credentials found for "${provider}" in ${authPath.path} (${authPathSourceLabel(authPath.source)}).`,
`Create credentials with:`,
` mkdir -p "${authDir}"`,
` (cd "${authDir}" && npx @earendil-works/pi-ai login ${provider})`,
`Override the auth file with --auth <path> or JOURNEYTEST_AUTH_PATH.`,
`Existing local auth files are still supported at ${authPath.localPath}.`,
].join("\n") + customFileNote
);
}
function authPathSourceLabel(source: AuthPathSource): string {
if (source === "cli") {
return "--auth";
}
if (source === "env") {
return "JOURNEYTEST_AUTH_PATH";
}
if (source === "local") {
return "existing local auth.json";
}
return "user config";
}
async function pathExists(path: string): Promise<boolean> {
try {
await access(path);
return true;
} catch (error) {
const code =
typeof error === "object" && error && "code" in error
? error.code
: undefined;
if (code === "ENOENT") {
return false;
}
throw error;
}
}
async function validateFiles(
journeyPath: string,
profilePath: string,
): Promise<{ issueCount: number }> {
const profileFiles = await collectJsonFiles(resolve(profilePath));
const journeyFiles = await collectJsonFiles(resolve(journeyPath));
const profiles = new Map<string, TesterProfile>();
let issueCount = 0;
for (const file of profileFiles) {
try {
const profile = await readAndParseJsonFile(file, TesterProfileSchema);
profiles.set(profile.id, profile);
console.log(`profile ok: ${file}`);
} catch (error) {
issueCount++;
console.error(formatError(file, error));
}
}
for (const file of journeyFiles) {
try {
const journey = await readAndParseJsonFile(file, UserJourneySchema);
const profile = profiles.get(journey.testerProfile);
const issues: ValidationIssue[] = [...validateAllowedOrigins(journey)];
if (!profile) {
issues.push({
path: "testerProfile",
message: `No profile with id "${journey.testerProfile}" was loaded.`,
});
} else {
issues.push(...validateJourneyProfileMatch(journey, profile));
}
if (issues.length > 0) {
issueCount += issues.length;
console.error(`journey invalid: ${file}`);
for (const issue of issues) {
console.error(` ${issue.path}: ${issue.message}`);
}
} else {
console.log(`journey ok: ${file}`);
}
} catch (error) {
issueCount++;
console.error(formatError(file, error));
}
}
console.log(
`Validated ${profileFiles.length} profile(s), ${journeyFiles.length} journey file(s).`,
);
if (issueCount > 0) {
console.error(`${issueCount} validation issue(s).`);
}
return { issueCount };
}
export async function lintFiles(
journeyPath: string,
profilePath?: string,
): Promise<{ issueCount: number }> {
const journeyFiles = await collectJsonFiles(resolve(journeyPath));
const profileFiles = profilePath
? await collectJsonFiles(resolve(profilePath))
: [];
const profiles = new Map<string, TesterProfile>();
let issueCount = 0;
for (const file of profileFiles) {
try {
const profile = await readAndParseJsonFile(file, TesterProfileSchema);
profiles.set(profile.id, profile);
console.log(`profile lint ok: ${file}`);
} catch (error) {
issueCount++;
console.error(formatError(file, error));
}
}
for (const file of journeyFiles) {
try {
const journey = await readAndParseJsonFile(file, UserJourneySchema);
const issues: LintIssue[] = [];
const profile = profilePath
? profiles.get(journey.testerProfile)
: undefined;
if (profilePath && !profile) {
issues.push({
severity: "error",
code: "missing-profile",
path: "testerProfile",
message: `No profile with id "${journey.testerProfile}" was loaded.`,
});
}
issues.push(...lintUserJourney(journey, profile ? { profile } : {}));
if (issues.length > 0) {
issueCount += issues.length;
console.error(`journey lint issues: ${file}`);
for (const issue of issues) {
console.error(` ${formatLintIssue(issue)}`);
}
} else {
console.log(`journey lint ok: ${file}`);
}
} catch (error) {
issueCount++;
console.error(formatError(file, error));
}
}
console.log(
`Linted ${profileFiles.length} profile(s), ${journeyFiles.length} journey file(s).`,
);
if (issueCount > 0) {
console.error(`${issueCount} lint issue(s).`);
}
return { issueCount };
}
async function collectRunnableJourneys(
journeyPath: string,
profilePath: string,
): Promise<RunnableJourney[]> {
const journeyFiles = await collectJsonFiles(journeyPath);
const profiles = await loadProfiles(profilePath);
const items: RunnableJourney[] = [];
const issues: string[] = [];
for (const journeyFile of journeyFiles) {
try {
const journey = await readAndParseJsonFile(
journeyFile,
UserJourneySchema,
);
const profile = profiles.get(journey.testerProfile);
if (!profile) {
issues.push(
`${journeyFile}: no profile with id "${journey.testerProfile}" was loaded from ${profilePath}.`,
);
continue;
}
const validationIssues = [
...validateAllowedOrigins(journey),
...validateJourneyProfileMatch(journey, profile),
];
if (validationIssues.length > 0) {
for (const issue of validationIssues) {
issues.push(`${journeyFile}: ${issue.path}: ${issue.message}`);
}
continue;
}
items.push({ journeyFile, journey, profile });
} catch (error) {
issues.push(formatError(journeyFile, error));
}
}
if (issues.length > 0) {
throw new Error(
`Cannot run journeys until collection issues are fixed:\n${issues.join("\n")}`,
);
}
return items;
}
async function loadProfiles(
profilePath: string,
): Promise<Map<string, TesterProfile>> {
const profileFiles = await collectJsonFiles(profilePath);
const profiles = new Map<string, TesterProfile>();
const issues: string[] = [];
for (const file of profileFiles) {
try {
const profile = await readAndParseJsonFile(file, TesterProfileSchema);
profiles.set(profile.id, profile);
} catch (error) {
issues.push(formatError(file, error));
}
}
if (issues.length > 0) {
throw new Error(`Cannot load tester profiles:\n${issues.join("\n")}`);
}
return profiles;
}
async function loadComparisonBaseline(
compareToPath: string,
): Promise<ComparisonBaseline> {
const info = await stat(compareToPath);
if (info.isFile()) {
return {
sourcePath: compareToPath,
kind: "run",
runs: [await readAndParseJsonFile(compareToPath, RunResultSchema)],
};
}
const runFiles = await collectRunJsonFiles(compareToPath);
if (runFiles.length === 0) {
throw new Error(
`No run.json files found under --compare-to directory ${compareToPath}.`,
);
}
return {
sourcePath: compareToPath,
kind: "suite",
runs: await Promise.all(
runFiles.map((runFile) => readAndParseJsonFile(runFile, RunResultSchema)),
),
};
}
export async function loadRerunFailedSelection(
rerunFailedPath: string,
): Promise<RerunFailedSelection> {
const snapshots = await loadRunSnapshotsForSelection(rerunFailedPath);
const unhealthyJourneyIds = [
...new Set(
snapshots
.filter((snapshot) => runHealth(snapshot) !== "passed")
.map((snapshot) => snapshot.journeyId),
),
];
if (unhealthyJourneyIds.length === 0) {
throw new Error(
`--rerun-failed found no unhealthy journeys in ${rerunFailedPath}.`,
);
}
return {
sourcePath: rerunFailedPath,
unhealthyJourneyIds,
};
}
async function loadRunSnapshotsForSelection(path: string): Promise<RunSnapshot[]> {
const info = await stat(path);
if (info.isFile()) {
if (basename(path) === "history.json") {
return readHistorySnapshots(path);
}
return [snapshotRun(await readAndParseJsonFile(path, RunResultSchema))];
}
const historyPath = join(path, "history.json");
if (await pathExists(historyPath)) {
return readHistorySnapshots(historyPath);
}
const runFiles = await collectRunJsonFiles(path);
if (runFiles.length === 0) {
throw new Error(
`No run.json or history.json files found under --rerun-failed directory ${path}.`,
);
}
return Promise.all(
runFiles.map(async (runFile) =>
snapshotRun(await readAndParseJsonFile(runFile, RunResultSchema)),
),
);
}
async function readHistorySnapshots(path: string): Promise<RunSnapshot[]> {
let parsed: unknown;
try {
parsed = JSON.parse(await readFile(path, "utf8"));
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
throw new Error(`Could not parse history.json at ${path}: ${message}`);
}
if (!isRecord(parsed) || !Array.isArray(parsed.runs)) {
throw new Error(`history.json at ${path} must contain a runs array.`);
}
return parsed.runs.map((run, index) =>
parseHistoryRunSnapshot(run, `${path}: runs[${index}]`),
);
}
function parseHistoryRunSnapshot(value: unknown, label: string): RunSnapshot {
if (!isRecord(value)) {
throw new Error(`${label} must be an object.`);
}
const snapshot = {
journeyId: readRequiredString(value, "journeyId", label),
runId: readRequiredString(value, "runId", label),
runStatus: readRequiredString(value, "runStatus", label),
verdictStatus: readRequiredString(value, "verdictStatus", label),
dataLifecycleStatus: readRequiredString(
value,
"dataLifecycleStatus",
label,
),
startedAt: readRequiredString(value, "startedAt", label),
endedAt: readRequiredString(value, "endedAt", label),
durationMs:
typeof value.durationMs === "number" && Number.isFinite(value.durationMs)
? value.durationMs
: 0,
...(typeof value.summary === "string" ? { summary: value.summary } : {}),
...(typeof value.dashboard === "string"
? { dashboard: value.dashboard }
: {}),
...(typeof value.result === "string" ? { result: value.result } : {}),
} as RunSnapshot;
assertRunSnapshotEnums(snapshot, label);
return snapshot;
}
function readRequiredString(
value: Record<string, unknown>,
key: string,
label: string,
): string {
const field = value[key];
if (typeof field !== "string" || field.length === 0) {
throw new Error(`${label}.${key} must be a non-empty string.`);
}
return field;
}
function assertRunSnapshotEnums(snapshot: RunSnapshot, label: string): void {
if (
!["completed", "error", "cancelled", "blocked"].includes(
snapshot.runStatus,
)
) {
throw new Error(`${label}.runStatus is not a valid run status.`);
}
if (
!["passed", "failed", "blocked", "inconclusive", "none"].includes(
snapshot.verdictStatus,
)
) {
throw new Error(`${label}.verdictStatus is not a valid verdict status.`);
}
if (
!["passed", "failed", "blocked", "skipped", "none"].includes(
snapshot.dataLifecycleStatus,
)
) {
throw new Error(
`${label}.dataLifecycleStatus is not a valid data lifecycle status.`,
);
}
}
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value);
}
async function writeSuiteArtifacts(
outputDir: string,
suiteRuns: SuiteDashboardRun[],
comparisonBaseline?: ComparisonBaseline,
selection?: SuiteSelectionSummary,
): Promise<{ dashboard: string; history: string }> {
const suiteId =
outputDir.split(/[\\/]/).at(-1) ?? `${isoTimestampForPath()}-run`;
const generatedAt = new Date().toISOString();
const suitePath = join(outputDir, "dashboard.html");
const historyPath = join(outputDir, "history.json");
const history = buildSuiteRunHistory({
suiteId,
generatedAt,
runs: suiteRuns,
...(selection ? { selection } : {}),
...(comparisonBaseline
? {
compareTo: {
path: comparisonBaseline.sourcePath,
kind: comparisonBaseline.kind,
runs: comparisonBaseline.runs,
},
}
: {}),
});
await writeHistoryArtifact(historyPath, history);
await writeFile(
suitePath,
renderSuiteDashboard({
suiteId,
generatedAt,
baseDir: outputDir,
runs: suiteRuns,
history,
...(selection ? { selection } : {}),
}),
"utf8",
);
return { dashboard: suitePath, history: historyPath };
}
async function writeHistoryArtifact(
path: string,
history: SuiteRunHistory,
): Promise<void> {
await writeFile(path, `${JSON.stringify(history, null, 2)}\n`, "utf8");
}
async function writeCliCiOutputs(options: {
suiteRunId: string;
suiteRuns: SuiteDashboardRun[];
junitPath?: string;
summaryJsonPath?: string;
githubAnnotations?: boolean;
}): Promise<void> {
if (
!options.junitPath &&
!options.summaryJsonPath &&
!options.githubAnnotations
) {
return;
}
const generatedAt = new Date().toISOString();
const written = await writeCiReports({
suiteName: options.suiteRunId,
generatedAt,
runs: options.suiteRuns,
...(options.junitPath ? { junitPath: resolve(options.junitPath) } : {}),
...(options.summaryJsonPath
? { summaryJsonPath: resolve(options.summaryJsonPath) }
: {}),
});
if (written.junitPath) {
console.log(`JUnit report: ${written.junitPath}`);
}
if (written.summaryJsonPath) {
console.log(`Summary JSON: ${written.summaryJsonPath}`);
}
if (options.githubAnnotations) {
emitGitHubAnnotations(options.suiteRuns, process.stderr);
}
}
function installAppLifecycleSignalCleanup(
controller: AppLifecycleController,
): () => void {
const signals: NodeJS.Signals[] = ["SIGINT", "SIGTERM"];
let handling = false;
const handler = async (signal: NodeJS.Signals) => {
if (handling) {
return;
}
handling = true;
console.error(`Received ${signal}; running app lifecycle cleanup.`);
await controller.runCleanup().catch((error: unknown) => {
const message = error instanceof Error ? error.message : String(error);
console.error(`App lifecycle cleanup failed: ${message}`);
});
process.exit(signal === "SIGINT" ? 130 : 143);
};
for (const signal of signals) {
process.once(signal, handler);
}
return () => {
for (const signal of signals) {
process.off(signal, handler);
}
};
}
function printRunResult(result: Awaited<ReturnType<typeof runJourney>>): void {
console.log(`Run: ${result.runId}`);
console.log(`Run status: ${result.runStatus}`);
if (result.verdict) {
console.log(`Verdict: ${result.verdict.status}`);
}
if (result.dataLifecycle) {
console.log(`Data lifecycle: ${result.dataLifecycle.status}`);
}
console.log(`Dashboard: ${result.artifacts.dashboard}`);
console.log(
`Dashboard URL: ${pathToFileURL(result.artifacts.dashboard).href}`,
);
console.log(`Report: ${result.artifacts.report}`);
console.log(`Result: ${result.artifacts.result}`);
}
async function collectJsonFiles(path: string): Promise<string[]> {
const info = await stat(path);
if (info.isFile()) {
return path.endsWith(".json") ? [path] : [];
}
const entries = await readdir(path, { withFileTypes: true });
const files = await Promise.all(
entries.map(async (entry) => {
const child = resolve(path, entry.name);
if (entry.isDirectory()) {
return collectJsonFiles(child);
}
return child.endsWith(".json") ? [child] : [];
}),
);
return files.flat().sort();
}
async function collectRunJsonFiles(path: string): Promise<string[]> {
const info = await stat(path);
if (info.isFile()) {
return basename(path) === "run.json" ? [path] : [];
}
const entries = await readdir(path, { withFileTypes: true });
const files = await Promise.all(
entries.map(async (entry) => {
const child = resolve(path, entry.name);
if (entry.isDirectory()) {
return collectRunJsonFiles(child);
}
return entry.name === "run.json" ? [child] : [];
}),
);
return files.flat().sort();
}
function formatError(file: string, error: unknown): string {
const message = error instanceof Error ? error.message : String(error);
return `${file} is invalid:\n${message}`;
}
if (import.meta.url === `file://${process.argv[1]}`) {
runCli().catch((error) => {
console.error(error instanceof Error ? error.message : String(error));
process.exitCode = 1;
});
}
async function runServerCommand(name: "serve" | "ui" | "space", options: { host: string; port: string; cognitionProfile?: string }) {
const port = Number(options.port); if (!Number.isInteger(port) || port <= 0) throw new Error("--port must be a positive integer");
const loadedConfig = loadRuntimeCognitionConfig({ overrides: options.cognitionProfile ? { COGNITION_PROFILE: options.cognitionProfile } : undefined });
assertCloudReasoningConfiguration(loadedConfig);
const mode = resolveLayaMode(loadedConfig.localLaya.mode); let managed: ManagedLayaService | undefined;
console.error("JourneyTest starting"); console.error(`Environment: ${isHuggingFaceSpace() ? "Hugging Face Space" : "local"}`); console.error(`Laya mode: ${mode}`); if (loadedConfig.cloud.enabled) { const resolution = reasoningModelStatus(loadedConfig); console.error(`Reasoning transport: ${loadedConfig.cloud.transport}`); if (loadedConfig.cloud.openAiCompatibleUrl) console.error(`Reasoning endpoint: ${new URL(loadedConfig.cloud.openAiCompatibleUrl).host}`); else console.error(`Reasoning provider: ${loadedConfig.cloud.provider}`); console.error(`Reasoning model: ${loadedConfig.cloud.model}`); console.error(`Credential configured: ${cognitionConfigStatus(loadedConfig).reasoning.credential_configured ? "yes" : "no"}`); console.error(`Reasoning model resolved: ${resolution.provider}/${resolution.model}`); }
if (name === "space" && mode === "embedded") { console.error("Starting localdecide"); managed = new ManagedLayaService(); await managed.start(); console.error("Laya ready"); }
if (name === "space") { await checkBrowserLaunch(); console.error("Browser ready"); }
const runtime = createJourneyRuntime({ endpoint: managed?.endpoint, cognitionOverrides: options.cognitionProfile ? { COGNITION_PROFILE: options.cognitionProfile } : undefined }); await runtime.backend.start();
const useGradio = name !== "serve"; const apiPort = useGradio ? port + 1 : port; const server = createApiServer({ service: runtime.service, backend: runtime.backend, model: runtime.config.localLaya.model, cognitionStatus: cognitionConfigStatus(runtime.config) });
await new Promise<void>((resolveReady, reject) => { server.once("error", reject); server.listen(apiPort, useGradio ? "127.0.0.1" : options.host, resolveReady); });
console.error("API mounted");
let child: ReturnType<typeof spawn> | undefined;
if (useGradio) { const script = resolve(dirname(fileURLToPath(import.meta.url)), "../python/journeytest_web.py"); child = spawn(process.env.JOURNEYTEST_PYTHON ?? "python3", [script], { stdio: "inherit", env: { ...process.env, JOURNEYTEST_API_BASE: `http://127.0.0.1:${apiPort}`, JOURNEYTEST_HOST: options.host, JOURNEYTEST_PORT: String(port) } }); }
try { if (useGradio && child) { await waitForGradioReadiness(`http://127.0.0.1:${port}/api/v1/health`, 30_000, child); console.error("Gradio ready"); } }
catch (error) { child?.kill("SIGTERM"); await new Promise<void>(resolveClose => server.close(() => resolveClose())); await runtime.backend.close(); await managed?.stop(); throw error; }
console.error(`JourneyTest ready on :${port}`);
await new Promise<void>(resolveStop => { const stop = () => resolveStop(); process.once("SIGINT", stop); process.once("SIGTERM", stop); child?.once("exit", () => resolveStop()); });
child?.kill("SIGTERM"); await new Promise<void>(resolveClose => server.close(() => resolveClose())); await runtime.backend.close(); await managed?.stop();
}
async function runDoctor(smokeModel = false, json = false, smokeReasoning = false) {
const checks: Array<{ name: string; ok: boolean; detail: string }> = [];
checks.push({ name: "JourneyTest version", ok: true, detail: "0.1.2" }); checks.push({ name: "Node version", ok: true, detail: process.version }); checks.push({ name: "environment", ok: true, detail: isHuggingFaceSpace() ? "Hugging Face Space" : "local" }); checks.push({ name: "Space detection", ok: true, detail: isHuggingFaceSpace() ? "yes" : "no" });
try { const { stdout } = await promisify(execFile)(process.env.JOURNEYTEST_PYTHON ?? "python3", ["--version"]); checks.push({ name: "Python", ok: true, detail: stdout.trim() }); } catch (error) { checks.push({ name: "Python", ok: false, detail: String(error) }); }
try { const { stdout } = await promisify(execFile)("agent-browser", ["--version"]); checks.push({ name: "browser executable", ok: true, detail: stdout.trim() }); } catch (error) { checks.push({ name: "browser executable", ok: false, detail: String(error) }); }
try { await checkBrowserLaunch(); checks.push({ name: "browser launch", ok: true, detail: "about:blank" }); } catch (error) { checks.push({ name: "browser launch", ok: false, detail: String(error) }); }
try { await promisify(execFile)("localdecide", ["--help"]); checks.push({ name: "localdecide availability", ok: true, detail: "installed" }); } catch (error) { checks.push({ name: "localdecide availability", ok: false, detail: String(error) }); }
try { await mkdir(resolve(process.env.JOURNEYTEST_OUTPUT ?? "runs"), { recursive: true }); await access(resolve(process.env.JOURNEYTEST_OUTPUT ?? "runs")); checks.push({ name: "artifact directory", ok: true, detail: resolve(process.env.JOURNEYTEST_OUTPUT ?? "runs") }); } catch (error) { checks.push({ name: "artifact directory", ok: false, detail: String(error) }); }
checks.push({ name: "Laya backend", ok: true, detail: process.env.LAYA_BACKEND ?? "torch" }); checks.push({ name: "Laya checkpoint", ok: true, detail: process.env.LAYA_MODEL_REPO ?? "ichenney/laya-browser-v32b" }); checks.push({ name: "checkpoint revision", ok: true, detail: process.env.LAYA_MODEL_REVISION ?? "unpinned at runtime" });
const cognition = loadRuntimeCognitionConfig(); const modelStatus = reasoningModelStatus(cognition); const cognitionStatus = cognitionConfigStatus(cognition); checks.push({ name: "cloud reasoning configuration", ok: !cognition.cloud.enabled || (modelStatus.modelResolved && (cognitionStatus.reasoning.credential_configured || (cognition.cloud.transport === "openai-compatible" && !cognition.cloud.authRequired))), detail: cognition.cloud.enabled ? `${cognition.cloud.provider}/${cognition.cloud.model}; resolved=${modelStatus.modelResolved}; credential=${cognitionStatus.reasoning.credential_configured ? "configured" : "missing"}` : "disabled" });
let runtime: ReturnType<typeof createJourneyRuntime> | undefined; try { runtime = createJourneyRuntime({ config: cognition }); } catch (error) { checks.push({ name: "cloud reasoning startup validation", ok: false, detail: error instanceof Error ? error.message : String(error) }); }
const health = runtime ? await runtime.backend.health() : { ok: false, detail: "runtime configuration invalid" }; checks.push({ name: "SystemOne endpoint", ok: health.ok, detail: `${runtime?.endpoint ?? cognition.localLaya.endpoint} (${health.backend ?? health.detail ?? "unavailable"})` });
if (smokeModel && health.ok && runtime) { try { const smoke = await runtime.backend.infer({ state: { page: { url: "about:blank", title: "Doctor", text: "Ready" }, recent_actions: [] }, questions: { operation: { type: "choice", instructions: "Readiness smoke test", criteria: { DONE: "Ready", BLOCKED: "Not ready" } } } }); checks.push({ name: "model readiness", ok: true, detail: `${smoke.backend}, ${smoke.latencyMs} ms` }); } catch (error) { checks.push({ name: "model readiness", ok: false, detail: error instanceof Error ? error.message : String(error) }); } } else checks.push({ name: "model readiness", ok: !smokeModel, detail: smokeModel ? "SystemOne endpoint unavailable" : "not loaded (no unexpected download)" });
if (smokeReasoning && runtime && cognition.cloud.enabled) { const started = Date.now(); try { const smoke = await runtime.cognitionRouter.initializeJourney({ goal: "Confirm reasoning readiness", successCriteria: [], context: {}, signal: new AbortController().signal }); checks.push({ name: "cloud reasoning live smoke", ok: true, detail: `${smoke.metadata.provider}/${smoke.metadata.model}, ${Date.now() - started} ms` }); } catch (error) { checks.push({ name: "cloud reasoning live smoke", ok: false, detail: error instanceof Error ? error.message : String(error) }); } } else checks.push({ name: "cloud reasoning live smoke", ok: true, detail: smokeReasoning ? "cloud reasoning disabled or invalid" : "not requested" }); checks.push({ name: "API dependencies", ok: true, detail: "node:http; Gradio/FastAPI checked by ui/space startup" });
const disk = await statfs(resolve(process.env.JOURNEYTEST_OUTPUT ?? "runs")); checks.push({ name: "available disk", ok: disk.bavail > 0, detail: `${Math.round(Number(disk.bavail * disk.bsize) / 1024 ** 3)} GiB` }); checks.push({ name: "available memory", ok: freemem() > 0, detail: `${Math.round(freemem() / 1024 ** 3)} / ${Math.round(totalmem() / 1024 ** 3)} GiB` });
if (json) process.stdout.write(`${JSON.stringify({ ok: checks.every(check => check.ok), checks })}\n`); else for (const check of checks) console.log(`${check.ok ? "OK" : "FAIL"} ${check.name}: ${check.detail}`); if (checks.some(check => !check.ok)) process.exitCode = 1;
}
async function checkBrowserLaunch() { const run = promisify(execFile); await run("agent-browser", ["--session", `journeytest-health-${process.pid}`, "open", "about:blank"], { timeout: 30_000 }); await run("agent-browser", ["--session", `journeytest-health-${process.pid}`, "close"], { timeout: 30_000 }).catch(() => undefined); }
async function waitForHttp(url: string, timeoutMs: number) { const deadline = Date.now() + timeoutMs; while (Date.now() < deadline) { try { if ((await fetch(url)).ok) return; } catch {} await new Promise(resolveWait => setTimeout(resolveWait, 250)); } throw new Error(`Timed out waiting for ${url}`); }
export async function waitForGradioReadiness(url: string, timeoutMs: number, child: Pick<ReturnType<typeof spawn>, "once">, wait = waitForHttp) { const exited = new Promise<never>((_, reject) => child.once("exit", (code, signal) => reject(new Error(`Gradio process exited before readiness (exit code ${code ?? "unknown"}${signal ? `, signal ${signal}` : ""})`)))); await Promise.race([wait(url, timeoutMs), exited]); }