Spaces:
Sleeping
Sleeping
Download apps/api/src/controllers/auth.ts from s-eto-hub/firecrawl-api: direct link, hf CLI and curl.
- Browser
- Download file 19.7 kB
-
https://huggingface.co/spaces/s-eto-hub/firecrawl-api/resolve/main/apps/api/src/controllers/auth.ts
- Command line
-
hf download hf://spaces/s-eto-hub/firecrawl-api/apps/api/src/controllers/auth.ts
-
curl -L -o auth.ts https://huggingface.co/spaces/s-eto-hub/firecrawl-api/resolve/main/apps/api/src/controllers/auth.ts
19.7 kB
| import * as crypto from "crypto"; | |
| import { RateLimiterRedis } from "rate-limiter-flexible"; | |
| import { validate } from "uuid"; | |
| import { config } from "../config"; | |
| import { logger } from "../lib/logger"; | |
| import { parseApi } from "../lib/parseApi"; | |
| import { withAuth } from "../lib/withAuth"; | |
| import { getAgentSponsorStatus } from "../services/agent-sponsor"; | |
| import { getRedisConnection } from "../services/queue-service"; | |
| import { getRateLimiter } from "../services/rate-limiter"; | |
| import { deleteKey, getValue, setValue } from "../services/redis"; | |
| import { redlock } from "../services/redlock"; | |
| import { supabase_rr_service, supabase_service } from "../services/supabase"; | |
| import { AuthResponse, RateLimiterMode } from "../types"; | |
| import { AuthCreditUsageChunk, AuthCreditUsageChunkFromTeam } from "./v1/types"; | |
| function normalizedApiIsUuid(potentialUuid: string): boolean { | |
| // Check if the string is a valid UUID | |
| return validate(potentialUuid); | |
| } | |
| export async function setCachedACUC( | |
| api_key: string, | |
| is_extract: boolean, | |
| acuc: | |
| | AuthCreditUsageChunk | |
| | null | |
| | ((acuc: AuthCreditUsageChunk) => AuthCreditUsageChunk | null), | |
| ) { | |
| const cacheKeyACUC = `acuc_${api_key}_${is_extract ? "extract" : "scrape"}`; | |
| const redLockKey = `lock_${cacheKeyACUC}`; | |
| try { | |
| await redlock.using([redLockKey], 10000, {}, async signal => { | |
| if (typeof acuc === "function") { | |
| acuc = acuc(JSON.parse((await getValue(cacheKeyACUC)) ?? "null")); | |
| if (acuc === null) { | |
| if (signal.aborted) { | |
| throw signal.error; | |
| } | |
| return; | |
| } | |
| } | |
| if (signal.aborted) { | |
| throw signal.error; | |
| } | |
| // Cache for 10 minutes. - mogery | |
| await setValue(cacheKeyACUC, JSON.stringify(acuc), 600, true); | |
| }); | |
| } catch (error) { | |
| logger.error(`Error updating cached ACUC ${cacheKeyACUC}: ${error}`); | |
| } | |
| } | |
| const mockPreviewACUC: ( | |
| team_id: string, | |
| is_extract: boolean, | |
| ) => AuthCreditUsageChunk = (team_id, is_extract) => ({ | |
| api_key: "preview", | |
| api_key_id: 0, | |
| team_id, | |
| sub_id: null, | |
| sub_current_period_start: null, | |
| sub_current_period_end: null, | |
| sub_user_id: null, | |
| price_id: null, | |
| rate_limits: { | |
| crawl: 2, | |
| scrape: 10, | |
| extract: 10, | |
| search: 5, | |
| map: 5, | |
| preview: 5, | |
| crawlStatus: 500, | |
| extractStatus: 500, | |
| extractAgentPreview: 1, | |
| scrapeAgentPreview: 5, | |
| }, | |
| price_credits: 99999999, | |
| price_should_be_graceful: false, | |
| price_associated_auto_recharge_price_id: null, | |
| credits_used: 0, | |
| coupon_credits: 99999999, | |
| adjusted_credits_used: 0, | |
| remaining_credits: 99999999, | |
| total_credits_sum: 99999999, | |
| plan_priority: { | |
| bucketLimit: 25, | |
| planModifier: 0.1, | |
| }, | |
| concurrency: is_extract ? 200 : 2, | |
| flags: null, | |
| is_extract, | |
| }); | |
| const mockACUC: () => AuthCreditUsageChunk = () => ({ | |
| api_key: "bypass", | |
| api_key_id: 0, | |
| team_id: "bypass", | |
| sub_id: "bypass", | |
| sub_current_period_start: new Date().toISOString(), | |
| sub_current_period_end: new Date( | |
| new Date().getTime() + 30 * 24 * 60 * 60 * 1000, | |
| ).toISOString(), | |
| sub_user_id: "bypass", | |
| price_id: "bypass", | |
| rate_limits: { | |
| crawl: 99999999, | |
| scrape: 99999999, | |
| extract: 99999999, | |
| search: 99999999, | |
| map: 99999999, | |
| preview: 99999999, | |
| crawlStatus: 99999999, | |
| extractStatus: 99999999, | |
| extractAgentPreview: 99999999, | |
| scrapeAgentPreview: 99999999, | |
| }, | |
| price_credits: 99999999, | |
| price_should_be_graceful: false, | |
| price_associated_auto_recharge_price_id: null, | |
| credits_used: 0, | |
| coupon_credits: 99999999, | |
| adjusted_credits_used: 0, | |
| remaining_credits: 99999999, | |
| total_credits_sum: 99999999, | |
| plan_priority: { | |
| bucketLimit: 25, | |
| planModifier: 0.1, | |
| }, | |
| concurrency: 99999999, | |
| flags: null, | |
| is_extract: false, | |
| }); | |
| /** | |
| * Introspection response from the OAuth token endpoint. | |
| */ | |
| interface OAuthIntrospectionResponse { | |
| active: boolean; | |
| api_key: string; | |
| scope: string; | |
| client_id: string; | |
| team_id: string; | |
| exp: number; | |
| } | |
| /** | |
| * Resolve an OAuth access token (fco_…) to the underlying API key via | |
| * the introspection endpoint. Results are cached in Redis for the | |
| * remaining token TTL (up to 5 minutes). | |
| */ | |
| async function resolveOAuthToken( | |
| token: string, | |
| ): Promise<OAuthIntrospectionResponse | null> { | |
| const introspectUrl = config.OAUTH_INTROSPECT_URL; | |
| const introspectSecret = config.OAUTH_INTROSPECT_SECRET; | |
| if (!introspectUrl || !introspectSecret) { | |
| logger.warn( | |
| "OAuth introspection not configured (OAUTH_INTROSPECT_URL / OAUTH_INTROSPECT_SECRET)", | |
| ); | |
| return null; | |
| } | |
| // Check Redis cache first (hash the token to avoid leaking material in key names) | |
| const tokenHash = crypto | |
| .createHash("sha256") | |
| .update(token) | |
| .digest("hex") | |
| .substring(0, 32); | |
| const cacheKey = `oauth_token:${tokenHash}`; | |
| const cached = await getValue(cacheKey); | |
| if (cached !== null) { | |
| try { | |
| const parsed = JSON.parse(cached); | |
| if (!parsed.active) return null; | |
| return parsed; | |
| } catch { | |
| // Corrupt cache entry — treat as a miss | |
| } | |
| } | |
| try { | |
| const response = await fetch(introspectUrl, { | |
| method: "POST", | |
| headers: { | |
| "Content-Type": "application/json", | |
| Authorization: `Bearer ${introspectSecret}`, | |
| }, | |
| body: JSON.stringify({ token }), | |
| }); | |
| if (!response.ok) { | |
| logger.error("OAuth introspection request failed", { | |
| status: response.status, | |
| }); | |
| return null; | |
| } | |
| const data = (await response.json()) as OAuthIntrospectionResponse; | |
| // Cache the result — use remaining TTL (max 5 minutes) or 60s for inactive | |
| if (data.active) { | |
| const remainingSeconds = Math.max( | |
| 0, | |
| data.exp - Math.floor(Date.now() / 1000), | |
| ); | |
| const cacheTtl = Math.min(remainingSeconds, 300); // Cap at 5 minutes | |
| if (cacheTtl > 0) { | |
| await setValue(cacheKey, JSON.stringify(data), cacheTtl); | |
| } | |
| } else { | |
| // Cache negative results briefly to avoid hammering introspection | |
| await setValue(cacheKey, JSON.stringify({ active: false }), 60); | |
| } | |
| return data.active ? data : null; | |
| } catch (error) { | |
| logger.error("OAuth introspection error", { error }); | |
| return null; | |
| } | |
| } | |
| /** @public used by auto_charge.ts (disabled, Autumn handles auto-recharge) */ | |
| export async function getACUC( | |
| api_key: string, | |
| cacheOnly = false, | |
| useCache = true, | |
| mode?: RateLimiterMode, | |
| ): Promise<AuthCreditUsageChunk | null> { | |
| let isExtract = | |
| mode === RateLimiterMode.Extract || | |
| mode === RateLimiterMode.ExtractStatus || | |
| mode === RateLimiterMode.ExtractAgentPreview; | |
| if (api_key === config.PREVIEW_TOKEN) { | |
| const acuc = mockPreviewACUC(api_key, isExtract); | |
| acuc.is_extract = isExtract; | |
| return acuc; | |
| } | |
| if (config.USE_DB_AUTHENTICATION !== true) { | |
| const acuc = mockACUC(); | |
| acuc.is_extract = isExtract; | |
| return acuc; | |
| } | |
| const cacheKeyACUC = `acuc_${api_key}_${isExtract ? "extract" : "scrape"}`; | |
| if (useCache) { | |
| const cachedACUC = await getValue(cacheKeyACUC); | |
| if (cachedACUC !== null) { | |
| return JSON.parse(cachedACUC); | |
| } | |
| } | |
| if (!cacheOnly) { | |
| let data; | |
| let error; | |
| let retries = 0; | |
| const maxRetries = 5; | |
| while (retries < maxRetries) { | |
| const client = | |
| Math.random() > 2 / 3 ? supabase_rr_service : supabase_service; | |
| ({ data, error } = await client.rpc( | |
| "auth_credit_usage_chunk_47", | |
| { | |
| input_key: api_key, | |
| i_is_extract: isExtract, | |
| tally_untallied_credits: true, | |
| }, | |
| { get: true }, | |
| )); | |
| if (!error) { | |
| break; | |
| } | |
| logger.warn( | |
| `Failed to retrieve authentication and credit usage data after ${retries}, trying again...`, | |
| { error }, | |
| ); | |
| retries++; | |
| if (retries === maxRetries) { | |
| throw new Error( | |
| "Failed to retrieve authentication and credit usage data after 3 attempts: " + | |
| JSON.stringify(error), | |
| ); | |
| } | |
| // Wait for a short time before retrying | |
| await new Promise(resolve => setTimeout(resolve, 200)); | |
| } | |
| const chunk: AuthCreditUsageChunk | null = | |
| data.length === 0 ? null : data[0].team_id === null ? null : data[0]; | |
| if (chunk) { | |
| chunk.is_extract = isExtract; | |
| } | |
| // NOTE: Should we cache null chunks? - mogery | |
| if (chunk !== null && useCache) { | |
| setCachedACUC(api_key, isExtract, chunk); | |
| } | |
| return chunk; | |
| } else { | |
| return null; | |
| } | |
| } | |
| export async function setCachedACUCTeam( | |
| team_id: string, | |
| is_extract: boolean, | |
| acuc: | |
| | AuthCreditUsageChunkFromTeam | |
| | null | |
| | (( | |
| acuc: AuthCreditUsageChunkFromTeam, | |
| ) => AuthCreditUsageChunkFromTeam | null), | |
| ) { | |
| const cacheKeyACUC = `acuc_team_${team_id}_${is_extract ? "extract" : "scrape"}`; | |
| const redLockKey = `lock_${cacheKeyACUC}`; | |
| try { | |
| await redlock.using([redLockKey], 10000, {}, async signal => { | |
| if (typeof acuc === "function") { | |
| acuc = acuc(JSON.parse((await getValue(cacheKeyACUC)) ?? "null")); | |
| if (acuc === null) { | |
| if (signal.aborted) { | |
| throw signal.error; | |
| } | |
| return; | |
| } | |
| } | |
| if (signal.aborted) { | |
| throw signal.error; | |
| } | |
| // Cache for 10 minutes. - mogery | |
| await setValue(cacheKeyACUC, JSON.stringify(acuc), 600, true); | |
| }); | |
| } catch (error) { | |
| logger.error(`Error updating cached ACUC ${cacheKeyACUC}: ${error}`); | |
| } | |
| } | |
| export async function getACUCTeam( | |
| team_id: string, | |
| cacheOnly = false, | |
| useCache = true, | |
| mode?: RateLimiterMode, | |
| ): Promise<AuthCreditUsageChunkFromTeam | null> { | |
| let isExtract = | |
| mode === RateLimiterMode.Extract || | |
| mode === RateLimiterMode.ExtractStatus || | |
| mode === RateLimiterMode.ExtractAgentPreview; | |
| if (team_id.startsWith("preview")) { | |
| const acuc = mockPreviewACUC(team_id, isExtract); | |
| return acuc; | |
| } | |
| if (config.USE_DB_AUTHENTICATION !== true) { | |
| const acuc = mockACUC(); | |
| acuc.is_extract = isExtract; | |
| return acuc; | |
| } | |
| const cacheKeyACUC = `acuc_team_${team_id}_${isExtract ? "extract" : "scrape"}`; | |
| if (useCache) { | |
| const cachedACUC = await getValue(cacheKeyACUC); | |
| if (cachedACUC !== null) { | |
| return JSON.parse(cachedACUC); | |
| } | |
| } | |
| if (!cacheOnly) { | |
| let data; | |
| let error; | |
| let retries = 0; | |
| const maxRetries = 5; | |
| while (retries < maxRetries) { | |
| const client = | |
| Math.random() > 2 / 3 ? supabase_rr_service : supabase_service; | |
| ({ data, error } = await client.rpc( | |
| "auth_credit_usage_chunk_47_from_team", | |
| { | |
| input_team: team_id, | |
| i_is_extract: isExtract, | |
| tally_untallied_credits: true, | |
| }, | |
| { get: true }, | |
| )); | |
| if (!error) { | |
| break; | |
| } | |
| logger.warn( | |
| `Failed to retrieve authentication and credit usage data after ${retries}, trying again...`, | |
| { error }, | |
| ); | |
| retries++; | |
| if (retries === maxRetries) { | |
| throw new Error( | |
| "Failed to retrieve authentication and credit usage data after 3 attempts: " + | |
| JSON.stringify(error), | |
| ); | |
| } | |
| // Wait for a short time before retrying | |
| await new Promise(resolve => setTimeout(resolve, 200)); | |
| } | |
| const chunk: AuthCreditUsageChunk | null = | |
| data.length === 0 ? null : data[0].team_id === null ? null : data[0]; | |
| // NOTE: Should we cache null chunks? - mogery | |
| if (chunk !== null && useCache) { | |
| setCachedACUCTeam(team_id, isExtract, chunk); | |
| } | |
| return chunk ? { ...chunk, is_extract: isExtract } : null; | |
| } else { | |
| return null; | |
| } | |
| } | |
| export async function clearACUC(api_key: string): Promise<void> { | |
| // Delete cache for all rate limiter modes | |
| const modes = [true, false]; | |
| await Promise.all( | |
| modes.map(async mode => { | |
| const cacheKey = `acuc_${api_key}_${mode ? "extract" : "scrape"}`; | |
| await deleteKey(cacheKey); | |
| }), | |
| ); | |
| // Also clear the base cache key | |
| await deleteKey(`acuc_${api_key}`); | |
| } | |
| export async function clearACUCTeam(team_id: string): Promise<void> { | |
| // Delete cache for all rate limiter modes | |
| const modes = [true, false]; | |
| await Promise.all( | |
| modes.map(async mode => { | |
| const cacheKey = `acuc_team_${team_id}_${mode ? "extract" : "scrape"}`; | |
| await deleteKey(cacheKey); | |
| }), | |
| ); | |
| // Also clear the base cache key | |
| await deleteKey(`acuc_team_${team_id}`); | |
| // Add team to billed_teams set so tally gets updated too | |
| await getRedisConnection().sadd("billed_teams", team_id); | |
| } | |
| export async function authenticateUser( | |
| req, | |
| res, | |
| mode?: RateLimiterMode, | |
| ): Promise<AuthResponse> { | |
| return withAuth(supaAuthenticateUser, { | |
| success: true, | |
| chunk: null, | |
| team_id: "bypass", | |
| org_id: null, | |
| })(req, res, mode); | |
| } | |
| /** | |
| * Backfills org_id for stale cached auth chunks so Autumn check gating can run. | |
| */ | |
| async function ensureChunkOrgId( | |
| apiKey: string, | |
| chunk: AuthCreditUsageChunk | null, | |
| ): Promise<AuthCreditUsageChunk | null> { | |
| if (!chunk || chunk.org_id || config.USE_DB_AUTHENTICATION !== true) { | |
| return chunk; | |
| } | |
| const { data, error } = await supabase_rr_service | |
| .from("teams") | |
| .select("org_id") | |
| .eq("id", chunk.team_id) | |
| .single(); | |
| if (error || !data?.org_id) { | |
| logger.warn("Failed to backfill org_id for auth chunk", { | |
| teamId: chunk.team_id, | |
| error, | |
| }); | |
| return chunk; | |
| } | |
| chunk.org_id = data.org_id; | |
| await setCachedACUC(apiKey, !!chunk.is_extract, chunk); | |
| return chunk; | |
| } | |
| async function supaAuthenticateUser( | |
| req, | |
| res, | |
| mode?: RateLimiterMode, | |
| ): Promise<AuthResponse> { | |
| const authHeader = | |
| req.headers.authorization ?? | |
| (req.headers["sec-websocket-protocol"] | |
| ? `Bearer ${req.headers["sec-websocket-protocol"]}` | |
| : null); | |
| if (!authHeader) { | |
| return { success: false, error: "Unauthorized", status: 401 }; | |
| } | |
| const token = authHeader.split(" ")[1]; // Extract the token from "Bearer <token>" | |
| if (!token) { | |
| return { | |
| success: false, | |
| error: "Unauthorized: Token missing", | |
| status: 401, | |
| }; | |
| } | |
| const incomingIP = (req.headers["x-preview-ip"] || | |
| req.headers["x-forwarded-for"] || | |
| req.socket.remoteAddress) as string; | |
| const iptoken = incomingIP + token; | |
| let rateLimiter: RateLimiterRedis; | |
| let subscriptionData: { team_id: string } | null = null; | |
| let normalizedApi: string; | |
| let teamId: string | null = null; | |
| let priceId: string | null = null; | |
| let chunk: AuthCreditUsageChunk | null = null; | |
| if (token == "this_is_just_a_preview_token") { | |
| throw new Error( | |
| "Unauthenticated Playground calls are temporarily disabled due to abuse. Please sign up.", | |
| ); | |
| } | |
| if (token == config.PREVIEW_TOKEN) { | |
| if (mode == RateLimiterMode.CrawlStatus) { | |
| rateLimiter = getRateLimiter(RateLimiterMode.CrawlStatus, token); | |
| } else if (mode == RateLimiterMode.ExtractStatus) { | |
| rateLimiter = getRateLimiter(RateLimiterMode.ExtractStatus, token); | |
| } else { | |
| rateLimiter = getRateLimiter(RateLimiterMode.Preview, token); | |
| } | |
| teamId = `preview_${iptoken}`; | |
| } else if (token.startsWith("fco_")) { | |
| // OAuth access token — resolve via introspection endpoint | |
| const introspection = await resolveOAuthToken(token); | |
| if (!introspection) { | |
| return { | |
| success: false, | |
| error: "Unauthorized: Invalid or expired OAuth token", | |
| status: 401, | |
| }; | |
| } | |
| // Use the resolved fc- API key to get the normal ACUC chunk | |
| const resolvedApi = parseApi(introspection.api_key); | |
| chunk = await getACUC(resolvedApi, false, true, RateLimiterMode.Scrape); | |
| chunk = await ensureChunkOrgId(resolvedApi, chunk); | |
| if (chunk === null) { | |
| return { | |
| success: false, | |
| error: "Unauthorized: Invalid token", | |
| status: 401, | |
| }; | |
| } | |
| teamId = chunk.team_id; | |
| priceId = chunk.price_id; | |
| subscriptionData = { | |
| team_id: teamId, | |
| }; | |
| rateLimiter = getRateLimiter( | |
| mode ?? RateLimiterMode.Crawl, | |
| chunk.rate_limits, | |
| ); | |
| } else { | |
| normalizedApi = parseApi(token); | |
| if (!normalizedApiIsUuid(normalizedApi)) { | |
| return { | |
| success: false, | |
| error: "Unauthorized: Invalid token", | |
| status: 401, | |
| }; | |
| } | |
| chunk = await getACUC(normalizedApi, false, true, RateLimiterMode.Scrape); | |
| chunk = await ensureChunkOrgId(normalizedApi, chunk); | |
| if (chunk === null) { | |
| return { | |
| success: false, | |
| error: "Unauthorized: Invalid token", | |
| status: 401, | |
| }; | |
| } | |
| teamId = chunk.team_id; | |
| priceId = chunk.price_id; | |
| subscriptionData = { | |
| team_id: teamId, | |
| }; | |
| rateLimiter = getRateLimiter( | |
| mode ?? RateLimiterMode.Crawl, | |
| chunk.rate_limits, | |
| ); | |
| } | |
| const team_endpoint_token = token === config.PREVIEW_TOKEN ? iptoken : teamId; | |
| try { | |
| await rateLimiter.consume(team_endpoint_token); | |
| } catch (rateLimiterRes) { | |
| // logger.error(`Rate limit exceeded: ${rateLimiterRes}`, { | |
| // teamId, | |
| // priceId, | |
| // mode, | |
| // rateLimits: chunk?.rate_limits, | |
| // rateLimiterRes, | |
| // }); | |
| const secs = Math.round(rateLimiterRes.msBeforeNext / 1000) || 1; | |
| const retryDate = new Date(Date.now() + rateLimiterRes.msBeforeNext); | |
| // We can only send a rate limit email every 7 days, send notification already has the date in between checking | |
| // const startDate = new Date(); | |
| // const endDate = new Date(); | |
| // endDate.setDate(endDate.getDate() + 7); | |
| // await sendNotification(team_id, NotificationType.RATE_LIMIT_REACHED, startDate.toISOString(), endDate.toISOString()); | |
| return { | |
| success: false, | |
| error: `Rate limit exceeded. Consumed (req/min): ${rateLimiterRes.consumedPoints}, Remaining (req/min): ${rateLimiterRes.remainingPoints}. Upgrade your plan at https://firecrawl.dev/pricing for increased rate limits or please retry after ${secs}s, resets at ${retryDate}`, | |
| status: 429, | |
| }; | |
| } | |
| if ( | |
| token === config.PREVIEW_TOKEN && | |
| (mode === RateLimiterMode.Scrape || | |
| mode === RateLimiterMode.Preview || | |
| mode === RateLimiterMode.Map || | |
| mode === RateLimiterMode.Crawl || | |
| mode === RateLimiterMode.CrawlStatus || | |
| mode === RateLimiterMode.Extract || | |
| mode === RateLimiterMode.Search) | |
| ) { | |
| return { | |
| success: true, | |
| team_id: `preview_${iptoken}`, | |
| org_id: null, | |
| chunk: null, | |
| }; | |
| // check the origin of the request and make sure its from firecrawl.dev | |
| // const origin = req.headers.origin; | |
| // if (origin && origin.includes("firecrawl.dev")){ | |
| // return { success: true, team_id: "preview" }; | |
| // } | |
| // if(config.ENV !== "production") { | |
| // return { success: true, team_id: "preview" }; | |
| // } | |
| // return { success: false, error: "Unauthorized: Invalid token", status: 401 }; | |
| } | |
| // Check if this is an agent-provisioned key and attach sponsor status | |
| if (chunk && chunk.api_key_id) { | |
| try { | |
| const sponsorStatus = await getAgentSponsorStatus({ | |
| apiKeyId: chunk.api_key_id, | |
| }); | |
| if (sponsorStatus) { | |
| chunk._agentSponsor = { | |
| status: sponsorStatus.status, | |
| verification_deadline: sponsorStatus.verification_deadline, | |
| email: sponsorStatus.email, | |
| }; | |
| } | |
| } catch (err) { | |
| logger.warn("Failed to check agent sponsor status", { | |
| error: err, | |
| api_key_id: chunk.api_key_id, | |
| }); | |
| } | |
| } | |
| return { | |
| success: true, | |
| team_id: teamId ?? undefined, | |
| org_id: chunk?.org_id ?? null, | |
| chunk, | |
| }; | |
| } | |