src/fleet/backup.runtime.ts

import { randomUUID } from "node:crypto";
import { createWriteStream, type Stats } from "node:fs";
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { pipeline } from "node:stream/promises";
import * as tar from "tar";
import {
  ARCHIVE_LIMIT_ERROR_CODE,
  ArchiveFormatError,
  ArchiveLimitError,
  ArchiveSecurityError,
  extractArchive,
} from "../infra/archive.js";
import { createBackupVolatileStatCache } from "../infra/backup-volatile-stat-cache.js";
import {
  getPublishFileExclusiveFailureDetails,
  publishFileNoClobber,
} from "../infra/directory-durability.js";
import { formatErrorMessage as errorMessage } from "../infra/errors.js";
import { root as fsSafeRoot } from "../infra/fs-safe.js";
import { isPathInside } from "../infra/path-guards.js";
import {
  cellAuthSecretDir,
  cellNetworkName,
  FLEET_ATTEMPT_LABEL,
  validateCellContainerProfile,
  type CellContainerProfile,
} from "./cell-profile.js";
import type { FleetContainerRuntime } from "./containers.runtime.js";
import type { FleetCellRecord } from "./registry.js";
import {
  assertManagedInspection,
  assertManagedNetwork,
  buildProfileBaseFromInspection,
  prepareCellConfig,
  prepareCellDirectories,
  requireInspectedAttemptId,
  requireInspectedGatewayToken,
  resolveContainerUser,
  resolvePurgeTarget,
  verifyReplacementHealthy,
  type HostIdentity,
} from "./service-support.runtime.js";

const DEFAULT_FLEET_BACKUP_MAX_BYTES = 16 * 1024 ** 3;
// Byte caps alone do not stop metadata-only archive bombs: millions of empty
// files stay under --max-bytes but can exhaust host inodes during extraction.
const FLEET_BACKUP_MAX_ENTRIES = 1_000_000;
const MANIFEST_MAX_BYTES = 4 * 1024 * 1024;
// Well under the 5-minute lease TTL so a stalled archive stream cannot outlive
// the lease by more than one probe interval before the backup aborts.
const BACKUP_LEASE_PROBE_INTERVAL_MS = 30_000;
const RESTORE_VERIFY_TIMEOUT_MS = 60_000;
const RESTORE_VERIFY_POLL_MS = 1_000;
const RESTORE_EXTRACT_TIMEOUT_MS = 30 * 60_000;

type FleetBackupManifest = {
  schemaVersion: 1;
  kind: "openclaw-fleet-cell-backup";
  tenant: string;
  createdAt: string;
  hostPort: number;
  image: string;
  runtime: "docker" | "podman";
};

type FleetBackupResult = {
  tenant: string;
  archivePath: string;
  fileCount: number;
  skippedSymlinks: number;
  skippedSpecial: number;
  note: string;
};

type FleetRestoreResult = {
  tenant: string;
  archivePath: string;
  token: string;
  tokenNote: string;
  started: boolean;
  url: string;
};

function timestampBasename(tenant: string, nowMs: number): string {
  const stamp = new Date(nowMs)
    .toISOString()
    .replaceAll(":", "-")
    .replace(/\.\d{3}Z$/u, "Z");
  return `openclaw-fleet-backup-${tenant}-${stamp}.tgz`;
}

async function resolveOutputPath(out: string | undefined, basename: string): Promise<string> {
  if (!out) {
    return path.resolve(process.cwd(), basename);
  }
  const resolved = path.resolve(out);
  if (out.endsWith(path.sep) || out.endsWith("/") || out.endsWith("\\")) {
    return path.join(resolved, basename);
  }
  try {
    return (await fs.stat(resolved)).isDirectory() ? path.join(resolved, basename) : resolved;
  } catch {
    return resolved;
  }
}

async function canonicalizeForContainment(targetPath: string): Promise<string> {
  const resolved = path.resolve(targetPath);
  const suffix: string[] = [];
  let probe = resolved;
  for (;;) {
    try {
      const real = await fs.realpath(probe);
      return path.join(real, ...suffix.toReversed());
    } catch {
      const parent = path.dirname(probe);
      if (parent === probe) {
        return resolved;
      }
      suffix.push(path.basename(probe));
      probe = parent;
    }
  }
}

function remapArchivePath(
  entryPath: string,
  manifestPath: string,
  dataTarget: string,
  authTarget: string,
): string {
  const resolved = path.resolve(entryPath);
  if (resolved === manifestPath) {
    return "manifest.json";
  }
  if (isPathInside(dataTarget, resolved)) {
    const relative = path.relative(dataTarget, resolved).split(path.sep).join(path.posix.sep);
    return relative ? path.posix.join("data", relative) : "data";
  }
  if (isPathInside(authTarget, resolved)) {
    const relative = path.relative(authTarget, resolved).split(path.sep).join(path.posix.sep);
    return relative ? path.posix.join("auth", relative) : "auth";
  }
  throw new Error(`Fleet backup encountered a path outside the cell roots: ${entryPath}`);
}

export async function backupFleetCell(params: {
  record: FleetCellRecord;
  stateDir: string;
  containers: FleetContainerRuntime;
  now: () => number;
  checkpoint: () => void;
  out?: string;
  maxBytes?: number;
  maxEntries?: number;
}): Promise<FleetBackupResult> {
  await params.containers.assertLocal(params.record.runtime);
  const inspection = await params.containers.inspect(
    params.record.runtime,
    params.record.containerName,
  );
  if (inspection.kind === "unavailable") {
    throw new Error(
      `Cannot inspect ${params.record.runtime} container for tenant ${params.record.tenantId}: ${inspection.error}`,
    );
  }
  if (inspection.kind === "ok") {
    assertManagedInspection(params.record, inspection);
    if (inspection.running) {
      throw new Error(
        `Fleet cell ${params.record.tenantId} is running; stop it first (openclaw fleet stop ${params.record.tenantId}) so SQLite state is captured consistently.`,
      );
    }
  }

  const dataTarget = await resolvePurgeTarget(
    path.join(params.stateDir, "fleet", "cells"),
    params.record.dataDir,
    params.record.tenantId,
  );
  if (!dataTarget) {
    throw new Error(`Fleet cell ${params.record.tenantId} has no cell data to back up.`);
  }
  const authTarget = await resolvePurgeTarget(
    path.join(params.stateDir, "fleet", "auth-profile-secrets"),
    cellAuthSecretDir(params.stateDir, params.record.tenantId),
    params.record.tenantId,
  );
  // Refuse auth-less backups: restore requires the auth tree, and an archive
  // without it could later replace a cell's auth-profile encryption keys with
  // an empty directory.
  if (!authTarget) {
    throw new Error(
      `Fleet cell ${params.record.tenantId} has no auth-secret directory to back up.`,
    );
  }
  const nowMs = params.now();
  const archivePath = await resolveOutputPath(
    params.out,
    timestampBasename(params.record.tenantId, nowMs),
  );
  const canonicalOutput = await canonicalizeForContainment(archivePath);
  const roots = [dataTarget, authTarget];
  if (roots.some((root) => isPathInside(root, canonicalOutput))) {
    throw new Error(
      "Fleet backup output must not be written inside the cell data or auth directory.",
    );
  }
  try {
    await fs.lstat(archivePath);
    throw new Error(`Refusing to overwrite existing fleet backup archive: ${archivePath}`);
  } catch (error) {
    if ((error as NodeJS.ErrnoException).code !== "ENOENT") {
      throw error;
    }
  }
  await fs.mkdir(path.dirname(archivePath), { recursive: true });
  const tempArchivePath = `${archivePath}.${randomUUID()}.tmp`;
  const tempRoot = await fs.realpath(os.tmpdir());
  const tempDir = await fs.mkdtemp(path.join(tempRoot, "openclaw-fleet-backup-"));
  const manifestPath = path.join(tempDir, "manifest.json");
  const manifest: FleetBackupManifest = {
    schemaVersion: 1,
    kind: "openclaw-fleet-cell-backup",
    tenant: params.record.tenantId,
    createdAt: new Date(nowMs).toISOString(),
    hostPort: params.record.hostPort,
    image: params.record.image,
    runtime: params.record.runtime,
  };
  let fileCount = 0;
  let skippedSymlinks = 0;
  let skippedSpecial = 0;
  let totalBytes = 0;
  let totalEntries = 0;
  let exceeded = false;
  let tooManyEntries = false;
  let leaseLost = false;
  let unrestorablePath: string | undefined;
  let lastLeaseProbeMs = params.now();
  const maxBytes = params.maxBytes ?? DEFAULT_FLEET_BACKUP_MAX_BYTES;
  const maxEntries = params.maxEntries ?? FLEET_BACKUP_MAX_ENTRIES;
  try {
    await fs.writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, { mode: 0o600 });
    const filter = (entryPath: string, stat: Stats | tar.ReadEntry): boolean => {
      if (exceeded || tooManyEntries || leaseLost) {
        return false;
      }
      // Probe the mutation lease during long archive streams so a lost lease
      // (another operation could start the cell mid-read) aborts the backup
      // instead of publishing a possibly-torn archive. node-tar filters run
      // from async callbacks, so record the loss and throw after tar settles.
      if (params.now() - lastLeaseProbeMs >= BACKUP_LEASE_PROBE_INTERVAL_MS) {
        lastLeaseProbeMs = params.now();
        try {
          params.checkpoint();
        } catch {
          leaseLost = true;
          return false;
        }
      }
      const type = "type" in stat ? stat.type : undefined;
      const isSymlink = "isSymbolicLink" in stat ? stat.isSymbolicLink() : type === "SymbolicLink";
      if (isSymlink) {
        skippedSymlinks += 1;
        return false;
      }
      const isFile = "isFile" in stat ? stat.isFile() : type === "File";
      const isDirectory = "isDirectory" in stat ? stat.isDirectory() : type === "Directory";
      if (!isFile && !isDirectory) {
        skippedSpecial += 1;
        return false;
      }
      // Validate and budget with the same path rules restore applies, so every
      // archive backup accepts stays restorable: same segment accounting, and
      // no names (e.g. literal backslashes) restore would reject as traversal.
      const archivedPath = remapArchivePath(entryPath, manifestPath, dataTarget, authTarget);
      if (!isAllowedRestorePath(archivedPath)) {
        unrestorablePath = unrestorablePath ?? entryPath;
        return false;
      }
      totalEntries += archivedPath.split("/").filter(Boolean).length;
      if (totalEntries > maxEntries) {
        tooManyEntries = true;
        return false;
      }
      if (isFile) {
        totalBytes += stat.size;
        fileCount += 1;
        if (totalBytes > maxBytes) {
          exceeded = true;
          return false;
        }
      }
      return true;
    };
    await pipeline(
      tar.c(
        {
          gzip: true,
          portable: true,
          preservePaths: true,
          statCache: createBackupVolatileStatCache(() => false),
          filter,
          onWriteEntry: (entry) => {
            entry.path = remapArchivePath(entry.path, manifestPath, dataTarget, authTarget);
          },
        },
        [manifestPath, dataTarget, authTarget],
      ),
      // Finish streaming into a same-directory temp path before publication;
      // filesystems without hard-link support still need a non-atomic copy.
      createWriteStream(tempArchivePath, { flags: "wx", mode: 0o600 }),
    );
    // A single large file can stream past the lease TTL without a filter
    // callback, so validate lease ownership once more before the archive is
    // declared good; a lost lease means the cell may have run mid-read.
    try {
      params.checkpoint();
    } catch {
      leaseLost = true;
    }
    if (leaseLost) {
      throw new Error(
        `Fleet backup for ${params.record.tenantId} lost its operation lease; the partial archive was discarded. Retry the backup.`,
      );
    }
    if (exceeded) {
      throw new Error(
        `Fleet backup exceeds the ${maxBytes}-byte limit; raise --max-bytes or reduce the cell data.`,
      );
    }
    if (tooManyEntries) {
      throw new Error(
        `Fleet backup exceeds the ${maxEntries}-entry limit; reduce the number of files in the cell data.`,
      );
    }
    if (unrestorablePath !== undefined) {
      throw new Error(
        `Fleet backup refuses a file name its restore path rules would reject: ${unrestorablePath}. Rename the file inside the cell and retry.`,
      );
    }
    try {
      await publishFileNoClobber(tempArchivePath, archivePath, {
        strategy: "link-or-copy",
        durability: "degrade",
      });
    } catch (error) {
      if ((error as NodeJS.ErrnoException).code === "EEXIST") {
        throw new Error(`Refusing to overwrite existing fleet backup archive: ${archivePath}`, {
          cause: error,
        });
      }
      if (getPublishFileExclusiveFailureDetails(error)?.cleanup === "unknown") {
        throw new Error(
          `Fleet backup publication failed: ${errorMessage(error)}. A partial archive may remain at ${archivePath}; inspect or remove it before retrying.`,
          { cause: error },
        );
      }
      throw error;
    }
    return {
      tenant: params.record.tenantId,
      archivePath,
      fileCount,
      skippedSymlinks,
      skippedSpecial,
      note: "Archive contains tenant state and auth secrets; store it like a credential.",
    };
  } finally {
    await fs.rm(tempArchivePath, { force: true }).catch(() => undefined);
    await fs.rm(tempDir, { recursive: true, force: true }).catch(() => undefined);
  }
}

function isAllowedRestorePath(rawPath: string): boolean {
  // Backup writes canonical POSIX names; restore receives fs-safe's canonical
  // paths. Reject raw aliases here when validating backup source names.
  if (rawPath.includes("\\")) {
    return false;
  }
  const normalized = path.posix.normalize(rawPath);
  if (normalized !== rawPath || normalized.startsWith("/") || normalized.startsWith("../")) {
    return false;
  }
  return (
    normalized === "manifest.json" ||
    normalized === "data" ||
    normalized.startsWith("data/") ||
    normalized === "auth" ||
    normalized.startsWith("auth/")
  );
}

// Extraction runs as the invoking user, so a root-invoked restore must repair
// ownership for whichever identity the cell container actually runs as: the
// explicit non-root user mapping when one exists, else the image default
// (uid 1000). Rootless mappings (uid 0) keep root ownership, which the user
// namespace translates to the daemon user.
function resolveRestoreOwner(
  hostIdentity: HostIdentity | undefined,
  containerUser: CellContainerProfile["containerUser"],
): { uid: number; gid: number } | undefined {
  if (hostIdentity?.uid !== 0) {
    return undefined;
  }
  if (!containerUser) {
    return { uid: 1000, gid: 1000 };
  }
  return containerUser.uid > 0 ? { uid: containerUser.uid, gid: containerUser.gid } : undefined;
}

// Lease liveness during this walk comes from withFleetCellOperation's interval
// heartbeat (60s, well under the 5-minute TTL); per-entry checkpoints would add
// a SQLite write per file for a stall-only risk the v1 lease design accepts.
async function chownTree(root: string, owner: { uid: number; gid: number }): Promise<void> {
  await fs.chown(root, owner.uid, owner.gid);
  for (const entry of await fs.readdir(root, { withFileTypes: true })) {
    const entryPath = path.join(root, entry.name);
    if (entry.isSymbolicLink()) {
      throw new Error(`Refusing to chown symlink in restored fleet data: ${entryPath}`);
    }
    if (entry.isDirectory()) {
      await chownTree(entryPath, owner);
    } else {
      await fs.chown(entryPath, owner.uid, owner.gid);
    }
  }
}

export async function restoreFleetCell(params: {
  record: FleetCellRecord;
  stateDir: string;
  containers: FleetContainerRuntime;
  fetchImpl: typeof fetch;
  now: () => number;
  sleep: (ms: number) => Promise<void>;
  checkpoint: () => void;
  generateToken: () => string;
  generateAttemptId: () => string;
  hostIdentity: HostIdentity | undefined;
  selinuxRelabel: boolean;
  from: string;
  force?: boolean;
  maxBytes?: number;
  maxEntries?: number;
}): Promise<FleetRestoreResult> {
  await params.containers.assertLocal(params.record.runtime);
  const archivePath = path.resolve(params.from);
  const archiveStat = await fs.lstat(archivePath);
  if (!archiveStat.isFile() || archiveStat.isSymbolicLink()) {
    throw new Error(`Fleet restore archive must be a regular file: ${archivePath}`);
  }
  const canonicalArchive = await fs.realpath(archivePath);
  const restoreRoots = await Promise.all([
    canonicalizeForContainment(params.record.dataDir),
    canonicalizeForContainment(cellAuthSecretDir(params.stateDir, params.record.tenantId)),
  ]);
  if (restoreRoots.some((root) => isPathInside(root, canonicalArchive))) {
    throw new Error(
      "Fleet restore archive must not be stored inside the cell data or auth directory.",
    );
  }
  const inspectionResult = await params.containers.inspect(
    params.record.runtime,
    params.record.containerName,
  );
  if (inspectionResult.kind === "missing") {
    throw new Error(
      `Fleet cell container is missing for ${params.record.tenantId}; remove the stale registration without purging data (openclaw fleet rm ${params.record.tenantId} --force), recreate a stopped cell with the intended image (openclaw fleet create ${params.record.tenantId} --no-start --image <image>), then retry fleet restore.`,
    );
  }
  const inspection = assertManagedInspection(params.record, inspectionResult);
  if (inspection.running && !params.force) {
    throw new Error(
      `Fleet cell ${params.record.tenantId} is running; pass --force to stop it and replace its state.`,
    );
  }
  const wasRunning = inspection.running;
  requireInspectedGatewayToken(inspection, "restore");
  requireInspectedAttemptId(inspection, "restore");

  const tempRoot = path.join(params.stateDir, "fleet", "restore-tmp");
  await fs.mkdir(tempRoot, { recursive: true, mode: 0o700 });
  const tempDir = await fs.mkdtemp(path.join(tempRoot, `${params.record.tenantId}-`));
  await fs.chmod(tempDir, 0o700);
  let preserveTemp = false;
  let stoppedForRestore = false;
  let containerRemoved = false;
  let previousDisplaced = false;
  let stateSwapped = false;
  // Captured outside the try so the failure path can recognize the replacement
  // generation by attempt label.
  let replacementAttemptId = "";
  try {
    let invalidArchive = false;
    let totalPathComponents = 0;
    let tooManyPathComponents = false;
    const maxBytes = params.maxBytes ?? DEFAULT_FLEET_BACKUP_MAX_BYTES;
    const maxEntries = params.maxEntries ?? FLEET_BACKUP_MAX_ENTRIES;
    try {
      await extractArchive({
        archivePath,
        destDir: tempDir,
        kind: "tar",
        tarGzip: true,
        timeoutMs: RESTORE_EXTRACT_TIMEOUT_MS,
        entryModes: "clamp",
        entryFilter: (entry) => {
          // Preserve Fleet's aggregate component budget in addition to the
          // per-entry preflight enforced by maxEntryPathComponents.
          totalPathComponents += entry.path.split("/").filter(Boolean).length;
          if (totalPathComponents > maxEntries) {
            tooManyPathComponents = true;
            return "skip";
          }
          if (
            (entry.kind !== "file" && entry.kind !== "directory") ||
            !isAllowedRestorePath(entry.path)
          ) {
            invalidArchive = true;
            return "skip";
          }
          return "extract";
        },
        onFiltered: "reject-archive",
        limits: {
          // The caller already pinned this exact archive path and size; setting
          // the observed size avoids fs-safe's smaller generic upload default.
          maxArchiveBytes: Math.max(1, archiveStat.size),
          maxEntries,
          maxExtractedBytes: maxBytes,
          maxEntryBytes: maxBytes,
          maxEntryPathComponents: maxEntries,
        },
      });
    } catch (error) {
      if (
        tooManyPathComponents ||
        (error instanceof ArchiveLimitError &&
          (error.code === ARCHIVE_LIMIT_ERROR_CODE.ENTRY_COUNT_EXCEEDS_LIMIT ||
            error.code === ARCHIVE_LIMIT_ERROR_CODE.ENTRY_PATH_COMPONENTS_EXCEEDS_LIMIT))
      ) {
        throw new Error(
          `Fleet restore exceeds the ${maxEntries}-entry limit; the archive is not a usable fleet cell backup.`,
          { cause: error },
        );
      }
      if (
        error instanceof ArchiveLimitError &&
        (error.code === ARCHIVE_LIMIT_ERROR_CODE.EXTRACTED_SIZE_EXCEEDS_LIMIT ||
          error.code === ARCHIVE_LIMIT_ERROR_CODE.ENTRY_EXTRACTED_SIZE_EXCEEDS_LIMIT)
      ) {
        throw new Error(
          `Fleet restore exceeds the ${maxBytes}-byte limit; raise --max-bytes or use a smaller archive.`,
          { cause: error },
        );
      }
      if (
        invalidArchive ||
        error instanceof ArchiveSecurityError ||
        error instanceof ArchiveFormatError
      ) {
        throw new Error("Archive is not a fleet cell backup or was tampered with.", {
          cause: error,
        });
      }
      throw error;
    }
    const safeRoot = await fsSafeRoot(tempDir, {
      symlinks: "reject",
      hardlinks: "reject",
      maxBytes: MANIFEST_MAX_BYTES,
      nonBlockingRead: true,
    });
    let manifest: unknown;
    try {
      manifest = JSON.parse((await safeRoot.read("manifest.json")).buffer.toString("utf8"));
    } catch (error) {
      throw new Error("Archive is not a fleet cell backup or was tampered with.", {
        cause: error,
      });
    }
    if (
      typeof manifest !== "object" ||
      manifest === null ||
      !("kind" in manifest) ||
      manifest.kind !== "openclaw-fleet-cell-backup" ||
      !("schemaVersion" in manifest) ||
      manifest.schemaVersion !== 1 ||
      !("tenant" in manifest) ||
      typeof manifest.tenant !== "string"
    ) {
      throw new Error("Archive is not a fleet cell backup or was tampered with.");
    }
    if (manifest.tenant !== params.record.tenantId) {
      throw new Error(
        `Backup archive belongs to tenant ${manifest.tenant}; refusing to restore it into ${params.record.tenantId}.`,
      );
    }
    const extractedData = path.join(tempDir, "data");
    const dataStat = await fs.lstat(extractedData);
    if (!dataStat.isDirectory() || dataStat.isSymbolicLink()) {
      throw new Error("Archive is not a fleet cell backup or was tampered with.");
    }
    // The auth tree is mandatory: accepting an auth-less archive would swap the
    // cell's auth-profile encryption keys for an empty directory and then delete
    // the displaced originals on successful cleanup.
    const extractedAuth = path.join(tempDir, "auth");
    let authStat: Awaited<ReturnType<typeof fs.lstat>>;
    try {
      authStat = await fs.lstat(extractedAuth);
    } catch (error) {
      if ((error as NodeJS.ErrnoException).code === "ENOENT") {
        throw new Error("Archive is not a fleet cell backup or was tampered with.", {
          cause: error,
        });
      }
      throw error;
    }
    if (!authStat.isDirectory() || authStat.isSymbolicLink()) {
      throw new Error("Archive is not a fleet cell backup or was tampered with.");
    }

    const containerUser = await resolveContainerUser({
      runtime: params.record.runtime,
      containers: params.containers,
      hostIdentity: params.hostIdentity,
      user: inspection.user,
    });
    const imageOwner = resolveRestoreOwner(params.hostIdentity, containerUser);
    // Build and validate the replacement profile before any destructive step so a
    // drifted-but-managed container (bad provenance label, invalid inspected limits)
    // fails preflight instead of after the old container and state are gone.
    const token = params.generateToken();
    const attemptId = params.generateAttemptId();
    replacementAttemptId = attemptId;
    const profile: CellContainerProfile = {
      ...buildProfileBaseFromInspection({
        record: params.record,
        stateDir: params.stateDir,
        inspection,
        containerUser,
        selinuxRelabel: params.selinuxRelabel,
        token,
        context: "restore",
      }),
      image: inspection.imageId,
      attemptId,
    };
    validateCellContainerProfile(profile);
    const authSecretDir = cellAuthSecretDir(params.stateDir, params.record.tenantId);
    const dataTarget = await resolvePurgeTarget(
      path.join(params.stateDir, "fleet", "cells"),
      params.record.dataDir,
      params.record.tenantId,
    );
    const authTarget = await resolvePurgeTarget(
      path.join(params.stateDir, "fleet", "auth-profile-secrets"),
      authSecretDir,
      params.record.tenantId,
    );
    const replacedRoot = path.join(tempDir, "replaced");
    await fs.mkdir(replacedRoot, { mode: 0o700 });

    assertManagedNetwork(
      params.record,
      await params.containers.inspectNetwork(
        params.record.runtime,
        cellNetworkName(params.record.tenantId),
      ),
    );

    // Restore decided to displace the generation inspected above, so every
    // re-validation below re-inspects that identity rather than the cell name.
    // Re-inspecting the name would let a container that claimed it in the
    // meantime pass the ownership guard and be stopped or removed instead.
    if (wasRunning) {
      params.checkpoint();
      const running = assertManagedInspection(
        params.record,
        await params.containers.inspect(params.record.runtime, inspection.containerId),
      );
      await params.containers.stop(params.record.runtime, running.containerId);
      stoppedForRestore = true;
    }
    params.checkpoint();
    const removable = assertManagedInspection(
      params.record,
      await params.containers.inspect(params.record.runtime, inspection.containerId),
    );
    await params.containers.remove(params.record.runtime, removable.containerId, false);
    containerRemoved = true;
    params.checkpoint();
    previousDisplaced = true;
    if (dataTarget) {
      await fs.rename(dataTarget, path.join(replacedRoot, "data"));
    }
    if (authTarget) {
      await fs.rename(authTarget, path.join(replacedRoot, "auth"));
    }
    await fs.rename(extractedData, params.record.dataDir);
    await fs.rename(extractedAuth, authSecretDir);
    stateSwapped = true;
    await prepareCellDirectories(params.record, authSecretDir, imageOwner);
    if (imageOwner) {
      await Promise.all([
        chownTree(params.record.dataDir, imageOwner),
        chownTree(authSecretDir, imageOwner),
      ]);
    }
    await prepareCellConfig(params.record, imageOwner);

    params.checkpoint();
    await params.containers.run(profile, wasRunning);
    if (wasRunning) {
      await verifyReplacementHealthy({
        containers: params.containers,
        record: params.record,
        attemptId,
        fetchImpl: params.fetchImpl,
        now: params.now,
        sleep: params.sleep,
        checkpoint: params.checkpoint,
        timeoutMs: RESTORE_VERIFY_TIMEOUT_MS,
        pollMs: RESTORE_VERIFY_POLL_MS,
        context: "restore",
      });
    }
    return {
      tenant: params.record.tenantId,
      archivePath,
      token,
      tokenNote: "Shown once. The previous Gateway token was rotated by this restore.",
      started: wasRunning,
      url: `http://127.0.0.1:${params.record.hostPort}`,
    };
  } catch (error) {
    if (containerRemoved) {
      preserveTemp = true;
      // The rotated token was never delivered, so a live replacement must not
      // keep serving. Detect it by attempt label (containers.run can fail after
      // partially starting), and never claim it was stopped when it was not.
      let replacementNote = "";
      try {
        const current = await params.containers.inspect(
          params.record.runtime,
          params.record.containerName,
        );
        if (
          current.kind === "ok" &&
          current.labels[FLEET_ATTEMPT_LABEL] === replacementAttemptId &&
          current.running
        ) {
          await params.containers.stop(params.record.runtime, current.containerId);
          replacementNote =
            " The interrupted replacement container was stopped; retry fleet restore to rotate a fresh Gateway token.";
        } else if (current.kind === "unavailable") {
          // An unavailable runtime cannot prove the replacement is down.
          replacementNote =
            " The replacement container state could not be verified; stop it manually before retrying fleet restore.";
        }
      } catch {
        replacementNote =
          " The interrupted replacement container could not be stopped; stop it manually before retrying fleet restore.";
      }
      // Report where each tree actually is for the failure phase: before the
      // swap the previous data never moved; after it, the restored trees are
      // already live and only the displaced originals sit under replaced/.
      const recoveryNote = stateSwapped
        ? ` Restored data is already in place under the cell directories; displaced previous data is preserved at ${tempDir}/replaced.`
        : previousDisplaced
          ? ` Previous data is preserved at ${tempDir}/replaced and the extracted archive remains at ${tempDir}.`
          : ` Previous cell data remains in place; the extracted archive remains at ${tempDir}.`;
      throw new Error(
        `Fleet restore for ${params.record.tenantId} was interrupted after the cell container was removed: ${errorMessage(error)}.${replacementNote}${recoveryNote}`,
        { cause: error },
      );
    }
    if (stoppedForRestore) {
      // A --force restore stopped a running cell but failed before removal.
      // Restart the same managed generation so an aborted restore does not
      // strand a healthy tenant stopped; the original error stays primary.
      try {
        // Same generation by identity, so no attempt-label comparison is needed:
        // the cell name may already point at something this must not start.
        const current = assertManagedInspection(
          params.record,
          await params.containers.inspect(params.record.runtime, inspection.containerId),
        );
        if (!current.running) {
          await params.containers.start(params.record.runtime, current.containerId);
        }
      } catch {
        // Best-effort recovery; the container remains stopped but intact.
      }
    }
    throw error;
  } finally {
    if (!preserveTemp) {
      await fs.rm(tempDir, { recursive: true, force: true }).catch(() => undefined);
    }
  }
}
/* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */