Add files using upload-large-folder tool

253e783 verified about 1 month ago

8.49 kB

import type { MsgContext } from "../auto-reply/templating.js";
import { applyTemplate } from "../auto-reply/templating.js";
import type { OpenClawConfig } from "../config/types.openclaw.js";
import type { LinkModelConfig, LinkToolsConfig } from "../config/types.tools.js";
import { logVerbose, shouldLogVerbose } from "../globals.js";
// Link-understanding runner fetches allowed URLs and invokes configured commands with bounded content.
import { createAbortError, isAbortError } from "../infra/abort-signal.js";
import { cancelUnreadResponseBody, readResponseWithLimit } from "../infra/http-body.js";
import { fetchWithSsrFGuard, GUARDED_FETCH_MODE } from "../infra/net/fetch-guard.js";
import { CLI_OUTPUT_MAX_BUFFER } from "../media-understanding/defaults.js";
import { resolveScopeDecision, resolveTimeoutMs } from "../media-understanding/resolve.js";
import { runCommandWithTimeout } from "../process/exec.js";
import { DEFAULT_LINK_TIMEOUT_SECONDS } from "./defaults.js";
import { extractLinksFromMessage } from "./detect.js";

function resolveTimeoutMsFromConfig(params: {
  config?: LinkToolsConfig;
  entry: LinkModelConfig;
}): number {
  const configured = params.entry.timeoutSeconds ?? params.config?.timeoutSeconds;
  return resolveTimeoutMs(configured, DEFAULT_LINK_TIMEOUT_SECONDS);
}

function resolveFetchTimeoutMsFromConfig(params: {
  config?: LinkToolsConfig;
  entries: LinkModelConfig[];
}): number {
  // The HTTP fetch phase is independent of any single CLI execution, so honor
  // an explicit global link-tools timeout first. Otherwise use the largest
  // per-entry timeout so slower entries are not capped by the first entry.
  if (params.config?.timeoutSeconds != null) {
    return resolveTimeoutMs(params.config.timeoutSeconds, DEFAULT_LINK_TIMEOUT_SECONDS);
  }
  return Math.max(
    ...params.entries.map((entry) => resolveTimeoutMsFromConfig({ config: params.config, entry })),
  );
}

function isLinkUrlTemplate(value: string): boolean {
  return value.includes("LinkUrl") || value.includes("LinkFinalUrl");
}

function commandName(command: string): string {
  return (command.split(/[\\/]/).pop() ?? command).toLowerCase();
}

function isUrlFetcherCommand(command: string): boolean {
  return commandName(command) === "curl" || commandName(command) === "wget";
}

function buildLinkCliArgs(params: {
  args: string[];
  ctx: MsgContext;
  finalUrl: string;
  url: string;
}): string[] {
  const templCtx = {
    ...params.ctx,
    LinkFinalUrl: params.finalUrl,
    LinkUrl: params.url,
  };
  return params.args
    .filter((arg) => !isLinkUrlTemplate(arg))
    .map((arg) => applyTemplate(arg, templCtx));
}

async function fetchLinkContent(params: {
  timeoutMs: number;
  url: string;
  signal?: AbortSignal;
}): Promise<{ content: string; finalUrl: string } | null> {
  const { response, finalUrl, release } = await fetchWithSsrFGuard({
    url: params.url,
    timeoutMs: params.timeoutMs,
    mode: GUARDED_FETCH_MODE.STRICT,
    auditContext: "link-understanding",
    signal: params.signal,
    init: {
      headers: {
        Accept: "text/*,application/json,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
        "User-Agent": "OpenClaw-LinkUnderstanding/1.0",
      },
    },
  });
  try {
    if (!response.ok) {
      // Do not await: a debug-capture tee settles only after its sibling branch cancels.
      void cancelUnreadResponseBody(response);
      throw new Error(`Link fetch failed with HTTP ${response.status}`);
    }
    const buffer = await readResponseWithLimit(response, CLI_OUTPUT_MAX_BUFFER);
    const content = new TextDecoder().decode(buffer).trim();
    if (!content) {
      return null;
    }
    return { content, finalUrl };
  } finally {
    await release();
  }
}

async function runCliEntry(params: {
  content: string;
  entry: LinkModelConfig;
  finalUrl: string;
  ctx: MsgContext;
  url: string;
  config?: LinkToolsConfig;
  signal?: AbortSignal;
}): Promise<string | null> {
  if ((params.entry.type ?? "cli") !== "cli") {
    return null;
  }
  const command = params.entry.command.trim();
  if (!command) {
    return null;
  }
  const args = params.entry.args ?? [];
  const timeoutMs = resolveTimeoutMsFromConfig({ config: params.config, entry: params.entry });
  if (isUrlFetcherCommand(command) && args.some(isLinkUrlTemplate)) {
    // curl/wget URL templates mark the entry as a fetcher; guarded fetch already supplied content.
    return params.content;
  }

  const argv = [
    command,
    ...buildLinkCliArgs({
      args,
      ctx: params.ctx,
      finalUrl: params.finalUrl,
      url: params.url,
    }),
  ];

  if (shouldLogVerbose()) {
    logVerbose(`Link understanding via CLI: ${argv.join(" ")}`);
  }

  const result = await runCommandWithTimeout(argv, {
    timeoutMs,
    input: params.content,
    signal: params.signal,
    // Processor wrappers and their children share the reply's cancellation lifetime.
    killProcessTree: true,
    env: {
      OPENCLAW_LINK_FINAL_URL: params.finalUrl,
      OPENCLAW_LINK_URL: params.url,
    },
  });
  if (params.signal?.aborted) {
    throw createAbortError("Link understanding command aborted", { cause: params.signal.reason });
  }
  if (result.code !== 0) {
    throw new Error(`Link understanding command exited with code ${result.code ?? "unknown"}`);
  }
  const trimmed = result.stdout.trim();
  return trimmed || null;
}

async function runLinkEntries(params: {
  content: string;
  entries: LinkModelConfig[];
  finalUrl: string;
  ctx: MsgContext;
  url: string;
  config?: LinkToolsConfig;
  signal?: AbortSignal;
}): Promise<string | null> {
  let lastError: unknown;
  for (const entry of params.entries) {
    if (params.signal?.aborted) {
      break;
    }
    try {
      const output = await runCliEntry({
        content: params.content,
        entry,
        finalUrl: params.finalUrl,
        ctx: params.ctx,
        url: params.url,
        config: params.config,
        signal: params.signal,
      });
      if (output) {
        return output;
      }
    } catch (err) {
      if (isAbortError(err)) {
        throw err;
      }
      lastError = err;
      if (shouldLogVerbose()) {
        logVerbose(`Link understanding failed for ${params.url}: ${String(err)}`);
      }
    }
  }
  if (lastError && shouldLogVerbose()) {
    logVerbose(`Link understanding exhausted for ${params.url}`);
  }
  return null;
}

/**
 * Fetches detected links through the SSRF guard and runs configured CLI processors.
 */
export async function runLinkUnderstanding(params: {
  cfg: OpenClawConfig;
  ctx: MsgContext;
  message?: string;
  signal?: AbortSignal;
}): Promise<string[]> {
  const config = params.cfg.tools?.links;
  if (!config || config.enabled === false) {
    return [];
  }

  const scopeDecision = resolveScopeDecision({ scope: config.scope, ctx: params.ctx });
  if (scopeDecision === "deny") {
    if (shouldLogVerbose()) {
      logVerbose("Link understanding disabled by scope policy.");
    }
    return [];
  }

  const message = params.message ?? params.ctx.CommandBody ?? params.ctx.RawBody ?? params.ctx.Body;
  const links = extractLinksFromMessage(message ?? "", { maxLinks: config?.maxLinks });
  if (links.length === 0) {
    return [];
  }

  const entries = config?.models ?? [];
  if (entries.length === 0) {
    return [];
  }

  const outputs: string[] = [];
  const timeoutMs = resolveFetchTimeoutMsFromConfig({ config, entries });
  for (const url of links) {
    if (params.signal?.aborted) {
      break;
    }
    let fetched: Awaited<ReturnType<typeof fetchLinkContent>>;
    try {
      fetched = await fetchLinkContent({ url, timeoutMs, signal: params.signal });
    } catch (err) {
      if (params.signal?.aborted) {
        throw createAbortError("Link understanding fetch aborted", { cause: params.signal.reason });
      }
      if (isAbortError(err)) {
        throw err;
      }
      if (shouldLogVerbose()) {
        logVerbose(`Link understanding fetch blocked or failed for ${url}: ${String(err)}`);
      }
      continue;
    }
    if (!fetched) {
      continue;
    }
    const output =
      (await runLinkEntries({
        content: fetched.content,
        entries,
        finalUrl: fetched.finalUrl,
        ctx: params.ctx,
        url,
        config,
        signal: params.signal,
      })) ?? fetched.content;
    if (params.signal?.aborted) {
      break;
    }
    if (output) {
      outputs.push(output);
    }
  }

  return outputs;
}

	import type { MsgContext } from "../auto-reply/templating.js";
	import { applyTemplate } from "../auto-reply/templating.js";
	import type { OpenClawConfig } from "../config/types.openclaw.js";
	import type { LinkModelConfig, LinkToolsConfig } from "../config/types.tools.js";
	import { logVerbose, shouldLogVerbose } from "../globals.js";
	// Link-understanding runner fetches allowed URLs and invokes configured commands with bounded content.
	import { createAbortError, isAbortError } from "../infra/abort-signal.js";
	import { cancelUnreadResponseBody, readResponseWithLimit } from "../infra/http-body.js";
	import { fetchWithSsrFGuard, GUARDED_FETCH_MODE } from "../infra/net/fetch-guard.js";
	import { CLI_OUTPUT_MAX_BUFFER } from "../media-understanding/defaults.js";
	import { resolveScopeDecision, resolveTimeoutMs } from "../media-understanding/resolve.js";
	import { runCommandWithTimeout } from "../process/exec.js";
	import { DEFAULT_LINK_TIMEOUT_SECONDS } from "./defaults.js";
	import { extractLinksFromMessage } from "./detect.js";

	function resolveTimeoutMsFromConfig(params: {
	config?: LinkToolsConfig;
	entry: LinkModelConfig;
	}): number {
	const configured = params.entry.timeoutSeconds ?? params.config?.timeoutSeconds;
	return resolveTimeoutMs(configured, DEFAULT_LINK_TIMEOUT_SECONDS);
	}

	function resolveFetchTimeoutMsFromConfig(params: {
	config?: LinkToolsConfig;
	entries: LinkModelConfig[];
	}): number {
	// The HTTP fetch phase is independent of any single CLI execution, so honor
	// an explicit global link-tools timeout first. Otherwise use the largest
	// per-entry timeout so slower entries are not capped by the first entry.
	if (params.config?.timeoutSeconds != null) {
	return resolveTimeoutMs(params.config.timeoutSeconds, DEFAULT_LINK_TIMEOUT_SECONDS);
	}
	return Math.max(
	...params.entries.map((entry) => resolveTimeoutMsFromConfig({ config: params.config, entry })),
	);
	}

	function isLinkUrlTemplate(value: string): boolean {
	return value.includes("LinkUrl") \|\| value.includes("LinkFinalUrl");
	}

	function commandName(command: string): string {
	return (command.split(/[\\/]/).pop() ?? command).toLowerCase();
	}

	function isUrlFetcherCommand(command: string): boolean {
	return commandName(command) === "curl" \|\| commandName(command) === "wget";
	}

	function buildLinkCliArgs(params: {
	args: string[];
	ctx: MsgContext;
	finalUrl: string;
	url: string;
	}): string[] {
	const templCtx = {
	...params.ctx,
	LinkFinalUrl: params.finalUrl,
	LinkUrl: params.url,
	};
	return params.args
	.filter((arg) => !isLinkUrlTemplate(arg))
	.map((arg) => applyTemplate(arg, templCtx));
	}

	async function fetchLinkContent(params: {
	timeoutMs: number;
	url: string;
	signal?: AbortSignal;
	}): Promise<{ content: string; finalUrl: string } \| null> {
	const { response, finalUrl, release } = await fetchWithSsrFGuard({
	url: params.url,
	timeoutMs: params.timeoutMs,
	mode: GUARDED_FETCH_MODE.STRICT,
	auditContext: "link-understanding",
	signal: params.signal,
	init: {
	headers: {
	Accept: "text/,application/json,application/xhtml+xml,application/xml;q=0.9,/*;q=0.8",
	"User-Agent": "OpenClaw-LinkUnderstanding/1.0",
	},
	},
	});
	try {
	if (!response.ok) {
	// Do not await: a debug-capture tee settles only after its sibling branch cancels.
	void cancelUnreadResponseBody(response);
	throw new Error(`Link fetch failed with HTTP ${response.status}`);
	}
	const buffer = await readResponseWithLimit(response, CLI_OUTPUT_MAX_BUFFER);
	const content = new TextDecoder().decode(buffer).trim();
	if (!content) {
	return null;
	}
	return { content, finalUrl };
	} finally {
	await release();
	}
	}

	async function runCliEntry(params: {
	content: string;
	entry: LinkModelConfig;
	finalUrl: string;
	ctx: MsgContext;
	url: string;
	config?: LinkToolsConfig;
	signal?: AbortSignal;
	}): Promise<string \| null> {
	if ((params.entry.type ?? "cli") !== "cli") {
	return null;
	}
	const command = params.entry.command.trim();
	if (!command) {
	return null;
	}
	const args = params.entry.args ?? [];
	const timeoutMs = resolveTimeoutMsFromConfig({ config: params.config, entry: params.entry });
	if (isUrlFetcherCommand(command) && args.some(isLinkUrlTemplate)) {
	// curl/wget URL templates mark the entry as a fetcher; guarded fetch already supplied content.
	return params.content;
	}

	const argv = [
	command,
	...buildLinkCliArgs({
	args,
	ctx: params.ctx,
	finalUrl: params.finalUrl,
	url: params.url,
	}),
	];

	if (shouldLogVerbose()) {
	logVerbose(`Link understanding via CLI: ${argv.join(" ")}`);
	}

	const result = await runCommandWithTimeout(argv, {
	timeoutMs,
	input: params.content,
	signal: params.signal,
	// Processor wrappers and their children share the reply's cancellation lifetime.
	killProcessTree: true,
	env: {
	OPENCLAW_LINK_FINAL_URL: params.finalUrl,
	OPENCLAW_LINK_URL: params.url,
	},
	});
	if (params.signal?.aborted) {
	throw createAbortError("Link understanding command aborted", { cause: params.signal.reason });
	}
	if (result.code !== 0) {
	throw new Error(`Link understanding command exited with code ${result.code ?? "unknown"}`);
	}
	const trimmed = result.stdout.trim();
	return trimmed \|\| null;
	}

	async function runLinkEntries(params: {
	content: string;
	entries: LinkModelConfig[];
	finalUrl: string;
	ctx: MsgContext;
	url: string;
	config?: LinkToolsConfig;
	signal?: AbortSignal;
	}): Promise<string \| null> {
	let lastError: unknown;
	for (const entry of params.entries) {
	if (params.signal?.aborted) {
	break;
	}
	try {
	const output = await runCliEntry({
	content: params.content,
	entry,
	finalUrl: params.finalUrl,
	ctx: params.ctx,
	url: params.url,
	config: params.config,
	signal: params.signal,
	});
	if (output) {
	return output;
	}
	} catch (err) {
	if (isAbortError(err)) {
	throw err;
	}
	lastError = err;
	if (shouldLogVerbose()) {
	logVerbose(`Link understanding failed for ${params.url}: ${String(err)}`);
	}
	}
	}
	if (lastError && shouldLogVerbose()) {
	logVerbose(`Link understanding exhausted for ${params.url}`);
	}
	return null;
	}

	/**
	* Fetches detected links through the SSRF guard and runs configured CLI processors.
	*/
	export async function runLinkUnderstanding(params: {
	cfg: OpenClawConfig;
	ctx: MsgContext;
	message?: string;
	signal?: AbortSignal;
	}): Promise<string[]> {
	const config = params.cfg.tools?.links;
	if (!config \|\| config.enabled === false) {
	return [];
	}

	const scopeDecision = resolveScopeDecision({ scope: config.scope, ctx: params.ctx });
	if (scopeDecision === "deny") {
	if (shouldLogVerbose()) {
	logVerbose("Link understanding disabled by scope policy.");
	}
	return [];
	}

	const message = params.message ?? params.ctx.CommandBody ?? params.ctx.RawBody ?? params.ctx.Body;
	const links = extractLinksFromMessage(message ?? "", { maxLinks: config?.maxLinks });
	if (links.length === 0) {
	return [];
	}

	const entries = config?.models ?? [];
	if (entries.length === 0) {
	return [];
	}

	const outputs: string[] = [];
	const timeoutMs = resolveFetchTimeoutMsFromConfig({ config, entries });
	for (const url of links) {
	if (params.signal?.aborted) {
	break;
	}
	let fetched: Awaited<ReturnType<typeof fetchLinkContent>>;
	try {
	fetched = await fetchLinkContent({ url, timeoutMs, signal: params.signal });
	} catch (err) {
	if (params.signal?.aborted) {
	throw createAbortError("Link understanding fetch aborted", { cause: params.signal.reason });
	}
	if (isAbortError(err)) {
	throw err;
	}
	if (shouldLogVerbose()) {
	logVerbose(`Link understanding fetch blocked or failed for ${url}: ${String(err)}`);
	}
	continue;
	}
	if (!fetched) {
	continue;
	}
	const output =
	(await runLinkEntries({
	content: fetched.content,
	entries,
	finalUrl: fetched.finalUrl,
	ctx: params.ctx,
	url,
	config,
	signal: params.signal,
	})) ?? fetched.content;
	if (params.signal?.aborted) {
	break;
	}
	if (output) {
	outputs.push(output);
	}
	}

	return outputs;
	}