Spaces:
Sleeping
Sleeping
File size: 3,775 Bytes
097eaf0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 | import { Response } from "express";
import { z } from "zod";
import { ErrorResponse, RequestWithAuth } from "./types";
import { logger as _logger } from "../../lib/logger";
import { generateCrawlerOptionsFromPrompt } from "../../scraper/scrapeURL/transformers/llmExtract";
import { CostTracking } from "../../lib/cost-tracking";
import { buildPromptWithWebsiteStructure } from "../../lib/map-utils";
// Define the request schema for params preview
// Only url and prompt are required/relevant for preview
const crawlParamsPreviewRequestSchema = z.object({
url: z.url(),
prompt: z.string().max(10000),
});
type CrawlParamsPreviewRequest = z.infer<
typeof crawlParamsPreviewRequestSchema
>;
type CrawlParamsPreviewResponse =
| {
success: true;
data?: {
url: string;
includePaths?: string[];
excludePaths?: string[];
maxDepth?: number;
maxDiscoveryDepth?: number;
crawlEntireDomain?: boolean;
allowExternalLinks?: boolean;
allowSubdomains?: boolean;
sitemap?: "skip" | "include" | "only";
ignoreQueryParameters?: boolean;
deduplicateSimilarURLs?: boolean;
delay?: number;
limit?: number;
};
}
| ErrorResponse;
export async function crawlParamsPreviewController(
req: RequestWithAuth<
{},
CrawlParamsPreviewResponse,
CrawlParamsPreviewRequest
>,
res: Response<CrawlParamsPreviewResponse>,
) {
const logger = _logger.child({
module: "api/v2",
method: "crawlParamsPreviewController",
teamId: req.auth.team_id,
});
try {
// Parse and validate request body
const parsedBody = crawlParamsPreviewRequestSchema.parse(req.body);
logger.debug("Crawl params preview request", {
url: parsedBody.url,
prompt: parsedBody.prompt,
});
// Build enhanced prompt with website structure
const { prompt: enhancedPrompt, websiteUrls } =
await buildPromptWithWebsiteStructure({
basePrompt: parsedBody.prompt,
url: parsedBody.url,
teamId: req.auth.team_id,
flags: req.acuc?.flags ?? null,
logger,
limit: 50,
includeSubdomains: true,
allowExternalLinks: false,
useIndex: true,
maxFireEngineResults: 500,
});
// Generate crawler options from enhanced prompt
const costTracking = new CostTracking();
const { extract } = await generateCrawlerOptionsFromPrompt(
enhancedPrompt,
logger,
costTracking,
{ teamId: req.auth.team_id },
);
const generatedOptions = extract || {};
logger.debug("Generated crawler options from enhanced prompt", {
originalPrompt: parsedBody.prompt,
websiteUrlCount: websiteUrls.length,
generatedOptions: generatedOptions,
});
// Prepare response data
const responseData = {
url: parsedBody.url,
...generatedOptions,
};
// Remove any undefined values for cleaner response
Object.keys(responseData).forEach(key => {
if (responseData[key] === undefined) {
delete responseData[key];
}
});
return res.status(200).json({
success: true,
data: responseData,
});
} catch (error) {
if (error instanceof z.ZodError) {
return res.status(400).json({
success: false,
code: "BAD_REQUEST",
error:
"Invalid request parameters: " +
error.issues.map(e => e.message).join(", "),
});
}
logger.error("Failed to generate crawler params preview", {
error: error.message,
prompt: req.body.prompt,
});
return res.status(400).json({
success: false,
error:
"Failed to process natural language prompt. Please try rephrasing.",
});
}
}
|