Spaces:
Sleeping
Sleeping
File size: 2,880 Bytes
097eaf0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 | import { Queue } from "bullmq";
import { config } from "../config";
import { logger } from "../lib/logger";
import IORedis from "ioredis";
import type { DeepResearchServiceOptions } from "../lib/deep-research/deep-research-service";
import { addExtractJob, ExtractJobData } from "./extract-queue";
let loggingQueue: Queue;
let indexQueue: Queue;
let deepResearchQueue: Queue;
let generateLlmsTxtQueue: Queue;
let billingQueue: Queue;
let precrawlQueue: Queue;
let redisConnection: IORedis;
export function getRedisConnection(): IORedis {
if (!redisConnection) {
redisConnection = new IORedis(config.REDIS_URL!, {
maxRetriesPerRequest: null,
});
redisConnection.on("connect", () => logger.info("Redis connected"));
redisConnection.on("reconnecting", () => logger.warn("Redis reconnecting"));
redisConnection.on("error", err => logger.warn("Redis error", { err }));
}
return redisConnection;
}
const generateLlmsTxtQueueName = "{generateLlmsTxtQueue}";
const deepResearchQueueName = "{deepResearchQueue}";
const billingQueueName = "{billingQueue}";
export const precrawlQueueName = "{precrawlQueue}";
export async function addExtractJobToQueue(
extractId: string,
data: ExtractJobData,
): Promise<void> {
await addExtractJob(extractId, data);
}
export function getGenerateLlmsTxtQueue() {
if (!generateLlmsTxtQueue) {
generateLlmsTxtQueue = new Queue(generateLlmsTxtQueueName, {
connection: getRedisConnection(),
defaultJobOptions: {
removeOnComplete: {
age: 90000, // 25 hours
},
removeOnFail: {
age: 90000, // 25 hours
},
},
});
}
return generateLlmsTxtQueue;
}
export function getDeepResearchQueue() {
if (!deepResearchQueue) {
deepResearchQueue = new Queue<DeepResearchServiceOptions>(
deepResearchQueueName,
{
connection: getRedisConnection(),
defaultJobOptions: {
removeOnComplete: {
age: 90000, // 25 hours
},
removeOnFail: {
age: 90000, // 25 hours
},
},
},
);
}
return deepResearchQueue;
}
export function getBillingQueue() {
if (!billingQueue) {
billingQueue = new Queue(billingQueueName, {
connection: getRedisConnection(),
defaultJobOptions: {
removeOnComplete: {
age: 60, // 1 minute
},
removeOnFail: {
age: 3600, // 1 hour
},
},
});
}
return billingQueue;
}
export function getPrecrawlQueue() {
if (!precrawlQueue) {
precrawlQueue = new Queue(precrawlQueueName, {
connection: getRedisConnection(),
defaultJobOptions: {
removeOnComplete: {
age: 24 * 60 * 60, // 1 day
},
removeOnFail: {
age: 24 * 60 * 60, // 1 day
},
},
});
}
return precrawlQueue;
}
|