import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import type { ProjectCloneFailureCause } from "../../packages/gateway-protocol/src/index.js"; import { executeGitCommand, gitNullConfigPath, requireGitCommandOutput, } from "../infra/git-exec.js"; import { withGitNetworkRetry } from "../infra/git-network-retry.js"; import { runCommandWithTimeout } from "../process/exec.js"; const PROJECT_CLONE_TIMEOUT_MS = 10 * 60_000; type ProjectCloneOptions = { env?: NodeJS.ProcessEnv; objectDirectory?: string; signal?: AbortSignal; timeoutMs?: number; token?: string; }; const PROJECT_FETCH_TIMEOUT_MS = 60_000; export class ProjectCloneError extends Error { constructor( readonly failure: ProjectCloneFailureCause, message: string, ) { super(message); this.name = "ProjectCloneError"; } } function cloneCommandEnv(token: string | undefined, env: NodeJS.ProcessEnv): NodeJS.ProcessEnv { const gitEnv: NodeJS.ProcessEnv = { ...env, GIT_TERMINAL_PROMPT: "0", GIT_CONFIG_NOSYSTEM: "1", GIT_CONFIG_GLOBAL: gitNullConfigPath(), GIT_TEMPLATE_DIR: "", GIT_EDITOR: "", GIT_SEQUENCE_EDITOR: "", GIT_EXTERNAL_DIFF: "", GIT_ASKPASS: undefined, SSH_ASKPASS: undefined, GIT_DIR: undefined, GIT_WORK_TREE: undefined, GIT_COMMON_DIR: undefined, GIT_INDEX_FILE: undefined, GIT_OBJECT_DIRECTORY: undefined, GIT_ALTERNATE_OBJECT_DIRECTORIES: undefined, GIT_NAMESPACE: undefined, GIT_EXEC_PATH: undefined, GIT_SSH: undefined, GIT_SSH_COMMAND: undefined, GIT_SSL_NO_VERIFY: undefined, }; if (token) { gitEnv.GIT_CONFIG_COUNT = "1"; gitEnv.GIT_CONFIG_KEY_0 = "http.https://github.com/.extraHeader"; gitEnv.GIT_CONFIG_VALUE_0 = `Authorization: Basic ${Buffer.from(`x-access-token:${token}`).toString("base64")}`; } return gitEnv; } function classifyProjectGitFailure(params: { output: string; operation: "clone" | "fetch"; tokenConfigured: boolean; timedOut?: boolean; }): ProjectCloneError { const detail = params.output.toLowerCase(); if ( params.timedOut || /could not resolve host|connection timed out|failed to connect/u.test(detail) ) { return new ProjectCloneError( "network", `Git ${params.operation} could not reach GitHub. Check the Gateway network connection and retry.`, ); } if ( /authentication failed|permission denied|could not read username|access denied/u.test(detail) ) { return new ProjectCloneError( "auth_required", params.tokenConfigured ? "GitHub rejected the active Control UI credential. Update gateway.controlUi.github.token when set; otherwise update the shared Gateway process environment, then retry." : "GitHub authentication is required. Configure gateway.controlUi.github.token or set GH_TOKEN/GITHUB_TOKEN in the shared Gateway process environment to clone private repositories.", ); } if (/repository not found|not found/u.test(detail)) { return params.tokenConfigured ? new ProjectCloneError( "not_found", "GitHub could not find that repository. Check the URL and repository access.", ) : new ProjectCloneError( "auth_required", "The repository was not found or is private. Check the URL, or configure gateway.controlUi.github.token (or the shared Gateway process environment) for private repositories.", ); } return new ProjectCloneError( "clone_failed", `Git could not ${params.operation === "clone" ? "clone" : "refresh"} that repository. Check the URL and Gateway Git configuration, then retry.`, ); } /** Clones one already-validated source into an unoccupied managed target. */ export async function cloneProjectCheckout( input: { url: string; target: string; requiredCommit?: string }, options: ProjectCloneOptions = {}, ): Promise { const env = options.env ?? process.env; const existed = await fs.lstat(input.target).then( () => true, () => false, ); if (existed) { throw new ProjectCloneError( "target_exists", "A managed checkout already exists for this repository. Register or remove it before retrying.", ); } await fs.mkdir(path.dirname(input.target), { recursive: true }); const commandEnv = cloneCommandEnv(options.token, env); const result = await withGitNetworkRetry( "clone", { timeoutMs: options.timeoutMs ?? PROJECT_CLONE_TIMEOUT_MS, signal: options.signal, }, async (timeoutMs) => { const attempt = await runCommandWithTimeout( ["git", "clone", "--no-recurse-submodules", "--", input.url, input.target], { env: commandEnv, timeoutMs, signal: options.signal, killProcessTree: true, maxOutputBytes: 256 * 1024, }, ); if (attempt.code !== 0 || attempt.termination !== "exit") { await fs.rm(input.target, { recursive: true, force: true }).catch(() => {}); } return attempt; }, ); if (result.code === 0 && result.termination === "exit") { if (input.requiredCommit) { try { await ensureProjectCheckoutCommit({ ...input, commit: input.requiredCommit }, options); } catch (error) { await fs.rm(input.target, { recursive: true, force: true }); throw error; } } return; } throw classifyProjectGitFailure({ output: `${result.stderr}\n${result.stdout}`, operation: "clone", tokenConfigured: Boolean(options.token), timedOut: result.termination === "timeout" || result.termination === "no-output-timeout", }); } /** Refreshes refs in an existing Gateway-managed project checkout. */ export async function refreshProjectCheckout( input: { target: string; url: string }, options: ProjectCloneOptions = {}, ): Promise { const [objectPath, objectFormatResult, currentRefs] = await Promise.all([ runProjectCheckoutGit(input, options, [ "rev-parse", "--path-format=absolute", "--git-path", "objects", ]), runProjectCheckoutGit(input, options, ["rev-parse", "--show-object-format"]), readProjectRemoteRefs(input, options), ]); if (objectPath.code !== 0 || objectPath.termination !== "exit") { throw new ProjectCloneError( "clone_failed", "The managed repository object store could not be verified. Remove the repository and retry.", ); } const objectFormat = objectFormatResult.stdout.trim(); if ( objectFormatResult.code !== 0 || objectFormatResult.termination !== "exit" || (objectFormat !== "sha1" && objectFormat !== "sha256") ) { throw new ProjectCloneError( "clone_failed", "The managed repository object format could not be verified. Remove the repository and retry.", ); } const objects = await fs.realpath(objectPath.stdout.trim()); const staging = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-project-fetch-")); try { const initialized = await runProjectCheckoutGit({ target: staging }, options, [ "init", "--bare", `--object-format=${objectFormat}`, ]); if (initialized.code !== 0 || initialized.termination !== "exit") { throw new ProjectCloneError( "clone_failed", "Git could not prepare a safe repository refresh. Retry the session.", ); } const stagingOptions = { ...options, objectDirectory: objects }; if (currentRefs.size > 0) { // Seed only refs already owned by the managed checkout. Besides enabling // incremental negotiation, this keeps transport isolated from local branches. const seeded = await runProjectCheckoutGit( { target: staging }, stagingOptions, ["update-ref", "--stdin"], { input: `${Array.from(currentRefs, ([ref, commit]) => `update ${ref} ${commit}`).join("\n")}\n`, }, ); if (seeded.code !== 0 || seeded.termination !== "exit") { throw new ProjectCloneError( "clone_failed", "Git could not prepare the managed repository refs for refresh. Retry the session.", ); } } // The isolated repository owns transport, but it borrows the managed object store. // It must not run maintenance using its incomplete temporary ref inventory. const result = await runProjectCheckoutGit( { target: staging }, { ...stagingOptions, timeoutMs: options.timeoutMs ?? PROJECT_FETCH_TIMEOUT_MS, }, [ "fetch", "--no-auto-maintenance", "--no-recurse-submodules", "--prune", "--", input.url, "+refs/heads/*:refs/remotes/origin/*", ], ); if (result.code !== 0 || result.termination !== "exit") { throw classifyProjectGitFailure({ output: `${result.stderr}\n${result.stdout}`, operation: "fetch", tokenConfigured: Boolean(options.token), timedOut: result.termination === "timeout" || result.termination === "no-output-timeout", }); } const fetchedRefs = await readProjectRemoteRefs({ target: staging }, stagingOptions); const updates = [ ...Array.from(fetchedRefs, ([ref, commit]) => `update ${ref} ${commit}`), ...Array.from(currentRefs.keys()) .filter((ref) => !fetchedRefs.has(ref)) .map((ref) => `delete ${ref}`), ]; if (updates.length > 0) { const updated = await runProjectCheckoutGit(input, options, ["update-ref", "--stdin"], { input: `${updates.join("\n")}\n`, }); if (updated.code !== 0 || updated.termination !== "exit") { throw new ProjectCloneError( "clone_failed", "The managed repository changed while its branches were refreshed. Retry the session.", ); } } } finally { await fs.rm(staging, { recursive: true, force: true }); } } async function readProjectRemoteRefs( input: { target: string }, options: ProjectCloneOptions, ): Promise> { const result = await runProjectCheckoutGit(input, options, [ "for-each-ref", "--format=%(refname) %(objectname) %(symref)", "refs/remotes/origin", ]); const stdout = requireGitCommandOutput("git for-each-ref", result, (_command, failed) => failed.outputLimitExceeded ? new ProjectCloneError( "clone_failed", "Git returned too many managed repository refs to refresh safely. Remove obsolete remote branches, then retry.", ) : new ProjectCloneError("clone_failed", "Git could not read the managed repository refs."), ); const refs = new Map(); for (const line of stdout.trim().split("\n").filter(Boolean)) { const match = /^(refs\/remotes\/origin\/\S+) ([a-f0-9]{40}|[a-f0-9]{64})(?: (\S+))?$/u.exec( line.trimEnd(), ); if (!match) { throw new ProjectCloneError( "clone_failed", "Git returned an invalid managed repository ref.", ); } const [, ref, commit, symbolicTarget] = match; if (!ref || !commit) { throw new ProjectCloneError( "clone_failed", "Git returned an incomplete managed repository ref.", ); } if (symbolicTarget) { continue; } refs.set(ref, commit); } return refs; } function runProjectCheckoutGit( input: { target: string }, options: ProjectCloneOptions, args: string[], commandOptions: { input?: string } = {}, ) { return executeGitCommand( input.target, ["-c", `core.hooksPath=${os.devNull}`, "-c", "core.fsmonitor=false", ...args], { env: { ...cloneCommandEnv(options.token, options.env ?? process.env), ...(options.objectDirectory ? { GIT_OBJECT_DIRECTORY: options.objectDirectory } : {}), }, timeoutMs: options.timeoutMs ?? PROJECT_CLONE_TIMEOUT_MS, signal: options.signal, killProcessTree: true, maxOutputBytes: 256 * 1024, ...commandOptions, }, ); } /** Fetch the pinned source when a reused project clone predates the remote session. */ export async function ensureProjectCheckoutCommit( input: { url: string; target: string; commit: string }, options: ProjectCloneOptions = {}, ): Promise { if (!/^[a-f0-9]{40}(?:[a-f0-9]{24})?$/u.test(input.commit)) { throw new ProjectCloneError("clone_failed", "The repository commit is invalid."); } const command = (args: string[]) => runProjectCheckoutGit(input, options, args); const present = await command(["cat-file", "-e", `${input.commit}^{commit}`]); if (present.code === 0 && present.termination === "exit") { return; } const fetched = await command([ "fetch", "--no-tags", "--no-recurse-submodules", "--", input.url, input.commit, ]); if (fetched.code !== 0 || fetched.termination !== "exit") { throw classifyProjectGitFailure({ operation: "fetch", output: `${fetched.stderr}\n${fetched.stdout}`, tokenConfigured: Boolean(options.token), timedOut: fetched.termination === "timeout" || fetched.termination === "no-output-timeout", }); } } /** Observe only the named source branch using the same shared fetch identity as cloning. */ export async function readProjectCheckoutRemoteHead( input: { url: string; target: string; branch: string }, options: ProjectCloneOptions = {}, ): Promise { const ref = `refs/heads/${input.branch}`; const result = await runProjectCheckoutGit(input, options, [ "ls-remote", "--refs", "--", input.url, ref, ]); if (result.code !== 0 || result.termination !== "exit") { throw new ProjectCloneError( "network", "The repository branch could not be verified; retry the Gateway move.", ); } const raw = result.stdout.trim(); if (!raw) { return undefined; } const [sha, observedRef, ...extra] = raw.split(/\s+/u); if ( !sha || !/^[a-f0-9]{40}(?:[a-f0-9]{24})?$/u.test(sha) || observedRef !== ref || extra.length ) { throw new ProjectCloneError("clone_failed", "The repository branch observation is invalid."); } return sha; }