src/projects/project-clone-runtime.ts

import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import type { ProjectCloneFailureCause } from "../../packages/gateway-protocol/src/index.js";
import {
  executeGitCommand,
  gitNullConfigPath,
  requireGitCommandOutput,
} from "../infra/git-exec.js";
import { withGitNetworkRetry } from "../infra/git-network-retry.js";
import { runCommandWithTimeout } from "../process/exec.js";

const PROJECT_CLONE_TIMEOUT_MS = 10 * 60_000;
type ProjectCloneOptions = {
  env?: NodeJS.ProcessEnv;
  objectDirectory?: string;
  signal?: AbortSignal;
  timeoutMs?: number;
  token?: string;
};
const PROJECT_FETCH_TIMEOUT_MS = 60_000;

export class ProjectCloneError extends Error {
  constructor(
    readonly failure: ProjectCloneFailureCause,
    message: string,
  ) {
    super(message);
    this.name = "ProjectCloneError";
  }
}

function cloneCommandEnv(token: string | undefined, env: NodeJS.ProcessEnv): NodeJS.ProcessEnv {
  const gitEnv: NodeJS.ProcessEnv = {
    ...env,
    GIT_TERMINAL_PROMPT: "0",
    GIT_CONFIG_NOSYSTEM: "1",
    GIT_CONFIG_GLOBAL: gitNullConfigPath(),
    GIT_TEMPLATE_DIR: "",
    GIT_EDITOR: "",
    GIT_SEQUENCE_EDITOR: "",
    GIT_EXTERNAL_DIFF: "",
    GIT_ASKPASS: undefined,
    SSH_ASKPASS: undefined,
    GIT_DIR: undefined,
    GIT_WORK_TREE: undefined,
    GIT_COMMON_DIR: undefined,
    GIT_INDEX_FILE: undefined,
    GIT_OBJECT_DIRECTORY: undefined,
    GIT_ALTERNATE_OBJECT_DIRECTORIES: undefined,
    GIT_NAMESPACE: undefined,
    GIT_EXEC_PATH: undefined,
    GIT_SSH: undefined,
    GIT_SSH_COMMAND: undefined,
    GIT_SSL_NO_VERIFY: undefined,
  };
  if (token) {
    gitEnv.GIT_CONFIG_COUNT = "1";
    gitEnv.GIT_CONFIG_KEY_0 = "http.https://github.com/.extraHeader";
    gitEnv.GIT_CONFIG_VALUE_0 = `Authorization: Basic ${Buffer.from(`x-access-token:${token}`).toString("base64")}`;
  }
  return gitEnv;
}

function classifyProjectGitFailure(params: {
  output: string;
  operation: "clone" | "fetch";
  tokenConfigured: boolean;
  timedOut?: boolean;
}): ProjectCloneError {
  const detail = params.output.toLowerCase();
  if (
    params.timedOut ||
    /could not resolve host|connection timed out|failed to connect/u.test(detail)
  ) {
    return new ProjectCloneError(
      "network",
      `Git ${params.operation} could not reach GitHub. Check the Gateway network connection and retry.`,
    );
  }
  if (
    /authentication failed|permission denied|could not read username|access denied/u.test(detail)
  ) {
    return new ProjectCloneError(
      "auth_required",
      params.tokenConfigured
        ? "GitHub rejected the active Control UI credential. Update gateway.controlUi.github.token when set; otherwise update the shared Gateway process environment, then retry."
        : "GitHub authentication is required. Configure gateway.controlUi.github.token or set GH_TOKEN/GITHUB_TOKEN in the shared Gateway process environment to clone private repositories.",
    );
  }
  if (/repository not found|not found/u.test(detail)) {
    return params.tokenConfigured
      ? new ProjectCloneError(
          "not_found",
          "GitHub could not find that repository. Check the URL and repository access.",
        )
      : new ProjectCloneError(
          "auth_required",
          "The repository was not found or is private. Check the URL, or configure gateway.controlUi.github.token (or the shared Gateway process environment) for private repositories.",
        );
  }
  return new ProjectCloneError(
    "clone_failed",
    `Git could not ${params.operation === "clone" ? "clone" : "refresh"} that repository. Check the URL and Gateway Git configuration, then retry.`,
  );
}

/** Clones one already-validated source into an unoccupied managed target. */
export async function cloneProjectCheckout(
  input: { url: string; target: string; requiredCommit?: string },
  options: ProjectCloneOptions = {},
): Promise<void> {
  const env = options.env ?? process.env;
  const existed = await fs.lstat(input.target).then(
    () => true,
    () => false,
  );
  if (existed) {
    throw new ProjectCloneError(
      "target_exists",
      "A managed checkout already exists for this repository. Register or remove it before retrying.",
    );
  }
  await fs.mkdir(path.dirname(input.target), { recursive: true });
  const commandEnv = cloneCommandEnv(options.token, env);
  const result = await withGitNetworkRetry(
    "clone",
    {
      timeoutMs: options.timeoutMs ?? PROJECT_CLONE_TIMEOUT_MS,
      signal: options.signal,
    },
    async (timeoutMs) => {
      const attempt = await runCommandWithTimeout(
        ["git", "clone", "--no-recurse-submodules", "--", input.url, input.target],
        {
          env: commandEnv,
          timeoutMs,
          signal: options.signal,
          killProcessTree: true,
          maxOutputBytes: 256 * 1024,
        },
      );
      if (attempt.code !== 0 || attempt.termination !== "exit") {
        await fs.rm(input.target, { recursive: true, force: true }).catch(() => {});
      }
      return attempt;
    },
  );
  if (result.code === 0 && result.termination === "exit") {
    if (input.requiredCommit) {
      try {
        await ensureProjectCheckoutCommit({ ...input, commit: input.requiredCommit }, options);
      } catch (error) {
        await fs.rm(input.target, { recursive: true, force: true });
        throw error;
      }
    }
    return;
  }
  throw classifyProjectGitFailure({
    output: `${result.stderr}\n${result.stdout}`,
    operation: "clone",
    tokenConfigured: Boolean(options.token),
    timedOut: result.termination === "timeout" || result.termination === "no-output-timeout",
  });
}

/** Refreshes refs in an existing Gateway-managed project checkout. */
export async function refreshProjectCheckout(
  input: { target: string; url: string },
  options: ProjectCloneOptions = {},
): Promise<void> {
  const [objectPath, objectFormatResult, currentRefs] = await Promise.all([
    runProjectCheckoutGit(input, options, [
      "rev-parse",
      "--path-format=absolute",
      "--git-path",
      "objects",
    ]),
    runProjectCheckoutGit(input, options, ["rev-parse", "--show-object-format"]),
    readProjectRemoteRefs(input, options),
  ]);
  if (objectPath.code !== 0 || objectPath.termination !== "exit") {
    throw new ProjectCloneError(
      "clone_failed",
      "The managed repository object store could not be verified. Remove the repository and retry.",
    );
  }
  const objectFormat = objectFormatResult.stdout.trim();
  if (
    objectFormatResult.code !== 0 ||
    objectFormatResult.termination !== "exit" ||
    (objectFormat !== "sha1" && objectFormat !== "sha256")
  ) {
    throw new ProjectCloneError(
      "clone_failed",
      "The managed repository object format could not be verified. Remove the repository and retry.",
    );
  }
  const objects = await fs.realpath(objectPath.stdout.trim());
  const staging = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-project-fetch-"));
  try {
    const initialized = await runProjectCheckoutGit({ target: staging }, options, [
      "init",
      "--bare",
      `--object-format=${objectFormat}`,
    ]);
    if (initialized.code !== 0 || initialized.termination !== "exit") {
      throw new ProjectCloneError(
        "clone_failed",
        "Git could not prepare a safe repository refresh. Retry the session.",
      );
    }
    const stagingOptions = { ...options, objectDirectory: objects };
    if (currentRefs.size > 0) {
      // Seed only refs already owned by the managed checkout. Besides enabling
      // incremental negotiation, this keeps transport isolated from local branches.
      const seeded = await runProjectCheckoutGit(
        { target: staging },
        stagingOptions,
        ["update-ref", "--stdin"],
        {
          input: `${Array.from(currentRefs, ([ref, commit]) => `update ${ref} ${commit}`).join("\n")}\n`,
        },
      );
      if (seeded.code !== 0 || seeded.termination !== "exit") {
        throw new ProjectCloneError(
          "clone_failed",
          "Git could not prepare the managed repository refs for refresh. Retry the session.",
        );
      }
    }
    // The isolated repository owns transport, but it borrows the managed object store.
    // It must not run maintenance using its incomplete temporary ref inventory.
    const result = await runProjectCheckoutGit(
      { target: staging },
      {
        ...stagingOptions,
        timeoutMs: options.timeoutMs ?? PROJECT_FETCH_TIMEOUT_MS,
      },
      [
        "fetch",
        "--no-auto-maintenance",
        "--no-recurse-submodules",
        "--prune",
        "--",
        input.url,
        "+refs/heads/*:refs/remotes/origin/*",
      ],
    );
    if (result.code !== 0 || result.termination !== "exit") {
      throw classifyProjectGitFailure({
        output: `${result.stderr}\n${result.stdout}`,
        operation: "fetch",
        tokenConfigured: Boolean(options.token),
        timedOut: result.termination === "timeout" || result.termination === "no-output-timeout",
      });
    }
    const fetchedRefs = await readProjectRemoteRefs({ target: staging }, stagingOptions);
    const updates = [
      ...Array.from(fetchedRefs, ([ref, commit]) => `update ${ref} ${commit}`),
      ...Array.from(currentRefs.keys())
        .filter((ref) => !fetchedRefs.has(ref))
        .map((ref) => `delete ${ref}`),
    ];
    if (updates.length > 0) {
      const updated = await runProjectCheckoutGit(input, options, ["update-ref", "--stdin"], {
        input: `${updates.join("\n")}\n`,
      });
      if (updated.code !== 0 || updated.termination !== "exit") {
        throw new ProjectCloneError(
          "clone_failed",
          "The managed repository changed while its branches were refreshed. Retry the session.",
        );
      }
    }
  } finally {
    await fs.rm(staging, { recursive: true, force: true });
  }
}

async function readProjectRemoteRefs(
  input: { target: string },
  options: ProjectCloneOptions,
): Promise<Map<string, string>> {
  const result = await runProjectCheckoutGit(input, options, [
    "for-each-ref",
    "--format=%(refname) %(objectname) %(symref)",
    "refs/remotes/origin",
  ]);
  const stdout = requireGitCommandOutput("git for-each-ref", result, (_command, failed) =>
    failed.outputLimitExceeded
      ? new ProjectCloneError(
          "clone_failed",
          "Git returned too many managed repository refs to refresh safely. Remove obsolete remote branches, then retry.",
        )
      : new ProjectCloneError("clone_failed", "Git could not read the managed repository refs."),
  );
  const refs = new Map<string, string>();
  for (const line of stdout.trim().split("\n").filter(Boolean)) {
    const match = /^(refs\/remotes\/origin\/\S+) ([a-f0-9]{40}|[a-f0-9]{64})(?: (\S+))?$/u.exec(
      line.trimEnd(),
    );
    if (!match) {
      throw new ProjectCloneError(
        "clone_failed",
        "Git returned an invalid managed repository ref.",
      );
    }
    const [, ref, commit, symbolicTarget] = match;
    if (!ref || !commit) {
      throw new ProjectCloneError(
        "clone_failed",
        "Git returned an incomplete managed repository ref.",
      );
    }
    if (symbolicTarget) {
      continue;
    }
    refs.set(ref, commit);
  }
  return refs;
}

function runProjectCheckoutGit(
  input: { target: string },
  options: ProjectCloneOptions,
  args: string[],
  commandOptions: { input?: string } = {},
) {
  return executeGitCommand(
    input.target,
    ["-c", `core.hooksPath=${os.devNull}`, "-c", "core.fsmonitor=false", ...args],
    {
      env: {
        ...cloneCommandEnv(options.token, options.env ?? process.env),
        ...(options.objectDirectory ? { GIT_OBJECT_DIRECTORY: options.objectDirectory } : {}),
      },
      timeoutMs: options.timeoutMs ?? PROJECT_CLONE_TIMEOUT_MS,
      signal: options.signal,
      killProcessTree: true,
      maxOutputBytes: 256 * 1024,
      ...commandOptions,
    },
  );
}

/** Fetch the pinned source when a reused project clone predates the remote session. */
export async function ensureProjectCheckoutCommit(
  input: { url: string; target: string; commit: string },
  options: ProjectCloneOptions = {},
): Promise<void> {
  if (!/^[a-f0-9]{40}(?:[a-f0-9]{24})?$/u.test(input.commit)) {
    throw new ProjectCloneError("clone_failed", "The repository commit is invalid.");
  }
  const command = (args: string[]) => runProjectCheckoutGit(input, options, args);
  const present = await command(["cat-file", "-e", `${input.commit}^{commit}`]);
  if (present.code === 0 && present.termination === "exit") {
    return;
  }
  const fetched = await command([
    "fetch",
    "--no-tags",
    "--no-recurse-submodules",
    "--",
    input.url,
    input.commit,
  ]);
  if (fetched.code !== 0 || fetched.termination !== "exit") {
    throw classifyProjectGitFailure({
      operation: "fetch",
      output: `${fetched.stderr}\n${fetched.stdout}`,
      tokenConfigured: Boolean(options.token),
      timedOut: fetched.termination === "timeout" || fetched.termination === "no-output-timeout",
    });
  }
}

/** Observe only the named source branch using the same shared fetch identity as cloning. */
export async function readProjectCheckoutRemoteHead(
  input: { url: string; target: string; branch: string },
  options: ProjectCloneOptions = {},
): Promise<string | undefined> {
  const ref = `refs/heads/${input.branch}`;
  const result = await runProjectCheckoutGit(input, options, [
    "ls-remote",
    "--refs",
    "--",
    input.url,
    ref,
  ]);
  if (result.code !== 0 || result.termination !== "exit") {
    throw new ProjectCloneError(
      "network",
      "The repository branch could not be verified; retry the Gateway move.",
    );
  }
  const raw = result.stdout.trim();
  if (!raw) {
    return undefined;
  }
  const [sha, observedRef, ...extra] = raw.split(/\s+/u);
  if (
    !sha ||
    !/^[a-f0-9]{40}(?:[a-f0-9]{24})?$/u.test(sha) ||
    observedRef !== ref ||
    extra.length
  ) {
    throw new ProjectCloneError("clone_failed", "The repository branch observation is invalid.");
  }
  return sha;
}