server/pageContentEndpointServerHook.ts

import type { PreviewServer, ViteDevServer } from "vite";
import { z } from "zod";
import { handleTokenVerification } from "./handleTokenVerification.ts";
import { fetchPageContents } from "./pageContentService.ts";
import { recordGroundingOutcome } from "./pageReadsSinceLastRestart.ts";
import { sendValidationError } from "./utils/httpResponse.ts";

const MAX_QUERY_LENGTH = 2000;
const MAX_URL_LENGTH = 2048;
/** Matches `searchResultsToConsider` on the client: the pages that reach the prompt. */
const MAX_URLS = 6;

const pageContentParamsSchema = z.object({
  query: z
    .string({ error: "Missing query parameter" })
    .trim()
    .min(1, "Missing query parameter")
    .max(
      MAX_QUERY_LENGTH,
      `Query parameter must not exceed ${MAX_QUERY_LENGTH} characters`,
    ),
  urls: z
    .array(
      z
        .url({ protocol: /^https?$/, error: "Invalid URL parameter" })
        .max(MAX_URL_LENGTH),
    )
    .min(1, "Missing url parameter")
    .max(MAX_URLS, `No more than ${MAX_URLS} URLs can be read per request`)
    // Repeats would otherwise multiply the requests one caller can aim at a
    // single site, and a page read twice adds nothing to the excerpt.
    .transform((urls) => [...new Set(urls)]),
});

/**
 * Sets up the `/page-content` endpoint, which reads the pages behind a handful
 * of search results and answers with the passages most relevant to the query.
 *
 * Kept off the `/search/` prefix on purpose: `searchEndpointServerHook` claims
 * every path under it, so a sibling route there would depend on middleware
 * ordering to ever be reached.
 *
 * Unlike `/search/`, which only ever reaches the configured SearXNG instance,
 * this endpoint fetches URLs the caller chose. It is always available; each
 * browser decides whether to call it, under AI Settings > Read Page Content.
 */
export function pageContentEndpointServerHook<
  T extends ViteDevServer | PreviewServer,
>(server: T) {
  server.middlewares.use(async (request, response, next) => {
    if (!request.url?.startsWith("/page-content")) return next();

    const url = new URL(request.url, `http://${request.headers.host}`);
    if (url.pathname !== "/page-content") return next();

    // Verified before the parameters are read, so malformed requests from an
    // unauthenticated caller still count against the rate limiter.
    const { shouldContinue } = await handleTokenVerification(
      url.searchParams.get("token"),
      response,
      request,
    );

    if (!shouldContinue) return;

    const parsedParams = pageContentParamsSchema.safeParse({
      query: url.searchParams.get("q") ?? undefined,
      urls: url.searchParams.getAll("url"),
    });

    if (!parsedParams.success) {
      sendValidationError(response, parsedParams.error);
      return;
    }

    try {
      const { query, urls } = parsedParams.data;
      const contents = await fetchPageContents(query, urls);
      recordGroundingOutcome(contents.length > 0);

      response.setHeader("Content-Type", "application/json");
      response.end(
        JSON.stringify(
          Object.fromEntries(
            contents.map(({ url: page, content }) => [page, content]),
          ),
        ),
      );
    } catch (error) {
      console.error(
        "Error reading page content:",
        error instanceof Error ? error.message : error,
      );
      response.statusCode = 500;
      response.setHeader("Content-Type", "application/json");
      response.end(JSON.stringify({ error: "Internal server error" }));
    }
  });
}