Spaces:
Sleeping
Sleeping
File size: 2,123 Bytes
097eaf0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 | import { RateLimiterMode } from "../types";
import { getACUCTeam } from "../controllers/auth";
import { redisEvictConnection } from "../services/redis";
import { logger } from "./logger";
const SET_KEY_PREFIX = "limit_team_id:";
export async function addJobPriority(team_id, job_id) {
try {
const setKey = SET_KEY_PREFIX + team_id;
// Add scrape job id to the set
await redisEvictConnection.sadd(setKey, job_id);
// This approach will reset the expiration time to 60 seconds every time a new job is added to the set.
await redisEvictConnection.expire(setKey, 60);
} catch (e) {
logger.error(`Add job priority (sadd) failed: ${team_id}, ${job_id}`);
}
}
export async function deleteJobPriority(team_id, job_id) {
try {
const setKey = SET_KEY_PREFIX + team_id;
// remove job_id from the set
await redisEvictConnection.srem(setKey, job_id);
} catch (e) {
logger.error(`Delete job priority (srem) failed: ${team_id}, ${job_id}`);
}
}
export async function getJobPriority({
team_id,
basePriority = 10,
}: {
team_id: string;
basePriority?: number;
from_extract?: boolean;
}): Promise<number> {
if (team_id === "d97c4ceb-290b-4957-8432-2b2a02727d95") {
return 50;
}
try {
const acuc = await getACUCTeam(
team_id,
false,
true,
RateLimiterMode.Scrape,
);
const setKey = SET_KEY_PREFIX + team_id;
// Get the length of the set
const setLength = await redisEvictConnection.scard(setKey);
// Determine the priority based on the plan and set length
let planModifier = acuc?.plan_priority.planModifier ?? 1;
let bucketLimit = acuc?.plan_priority.bucketLimit ?? 25;
// if length set is smaller than set, just return base priority
if (setLength <= bucketLimit) {
return basePriority;
} else {
// If not, we keep base priority + planModifier
return Math.ceil(
basePriority + Math.ceil((setLength - bucketLimit) * planModifier),
);
}
} catch (e) {
logger.error(`Get job priority failed: ${team_id}, ${basePriority}`);
return basePriority;
}
}
|