Spaces:
Sleeping
Sleeping
File size: 3,775 Bytes
097eaf0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 | import { v7 as uuidv7 } from "uuid";
import { Response } from "express";
import {
RequestWithAuth,
ExtractRequest,
extractRequestSchema,
ExtractResponse,
} from "./types";
import { addExtractJobToQueue } from "../../services/queue-service";
import { saveExtract } from "../../lib/extract/extract-redis";
import { UNSUPPORTED_SITE_MESSAGE } from "../../lib/strings";
import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
import { logger as _logger } from "../../lib/logger";
import { logRequest } from "../../services/logging/log_job";
import { config } from "../../config";
import { getScrapeZDR } from "../../lib/zdr-helpers";
/**
* Extracts data from the provided URLs based on the request parameters.
* Currently in beta.
* @param req - The request object containing authentication and extraction details.
* @param res - The response object to send the extraction results.
* @returns A promise that resolves when the extraction process is complete.
*/
export async function extractController(
req: RequestWithAuth<{}, ExtractResponse, ExtractRequest>,
res: Response<ExtractResponse>,
) {
const originalRequest = { ...req.body };
req.body = extractRequestSchema.parse(req.body);
if (getScrapeZDR(req.acuc?.flags) === "forced") {
return res.status(400).json({
success: false,
error:
"Your team has zero data retention enabled. This is not supported on extract. Please contact support@firecrawl.com to unblock this feature.",
});
}
const extractId = uuidv7();
const createdAt = Date.now();
_logger.info("Extract starting...", {
request: req.body,
originalRequest,
teamId: req.auth.team_id,
team_id: req.auth.team_id,
subId: req.acuc?.sub_id,
extractId,
zeroDataRetention: getScrapeZDR(req.acuc?.flags) === "forced",
});
if (req.body.agent?.model === "v3-beta") {
return res.status(400).json({
success: false,
error:
"Use the new /agent endpoint instead of passing agent.model=v3-beta into /extract.",
});
}
const invalidURLs: string[] =
req.body.urls?.filter((url: string) =>
isUrlBlocked(url, req.acuc?.flags ?? null, {
team_id: req.auth.team_id,
origin: req.body.origin ?? null,
}),
) ?? [];
if (invalidURLs.length > 0 && !req.body.ignoreInvalidURLs) {
if (!res.headersSent) {
return res.status(403).json({
success: false,
error: UNSUPPORTED_SITE_MESSAGE,
});
}
}
await logRequest({
id: extractId,
kind: "extract",
api_version: "v2",
team_id: req.auth.team_id,
origin: req.body.origin ?? "api",
integration: req.body.integration,
target_hint: req.body.urls?.[0] ?? "",
zeroDataRetention: false, // not supported for extract
api_key_id: req.acuc?.api_key_id ?? null,
});
const jobData = {
request: req.body,
teamId: req.auth.team_id,
subId: req.acuc?.sub_id,
extractId,
agent: req.body.agent,
createdAt,
};
await saveExtract(extractId, {
id: extractId,
team_id: req.auth.team_id,
createdAt,
status: "processing",
showSteps: req.body.__experimental_streamSteps,
showLLMUsage: req.body.__experimental_llmUsage,
showSources: req.body.__experimental_showSources || req.body.showSources,
showCostTracking: req.body.__experimental_showCostTracking,
zeroDataRetention: getScrapeZDR(req.acuc?.flags) === "forced",
});
await addExtractJobToQueue(extractId, {
...jobData,
apiKeyId: req.acuc?.api_key_id ?? undefined,
});
return res.status(200).json({
success: true,
id: extractId,
urlTrace: [],
...(invalidURLs.length > 0 && req.body.ignoreInvalidURLs
? {
invalidURLs,
}
: {}),
});
}
|