Spaces:
Sleeping
Sleeping
File size: 2,543 Bytes
097eaf0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 | import type { Logger } from "winston";
import {
ScrapeRetryLimitError,
ScrapeRetryLimitReason,
ScrapeRetryStats,
} from "./error";
type RetryReason = Exclude<ScrapeRetryLimitReason, "global">;
interface ScrapeRetryTrackerConfig {
maxAttempts: number;
maxFeatureToggles: number;
maxFeatureRemovals: number;
maxPdfPrefetches: number;
maxDocumentPrefetches: number;
}
export class ScrapeRetryTracker {
private stats: ScrapeRetryStats = {
totalAttempts: 0,
addFeatureAttempts: 0,
removeFeatureAttempts: 0,
pdfAntibotAttempts: 0,
documentAntibotAttempts: 0,
};
constructor(
private readonly config: ScrapeRetryTrackerConfig,
private readonly logger: Logger,
) {}
getSnapshot(): ScrapeRetryStats {
return { ...this.stats };
}
record(reason: RetryReason, lastError: unknown) {
this.stats.totalAttempts += 1;
if (this.stats.totalAttempts > this.config.maxAttempts) {
this.throwLimit("global", lastError);
}
switch (reason) {
case "feature_toggle":
this.stats.addFeatureAttempts += 1;
if (this.stats.addFeatureAttempts > this.config.maxFeatureToggles) {
this.throwLimit(reason, lastError);
}
break;
case "feature_removal":
this.stats.removeFeatureAttempts += 1;
if (this.stats.removeFeatureAttempts > this.config.maxFeatureRemovals) {
this.throwLimit(reason, lastError);
}
break;
case "pdf_antibot":
this.stats.pdfAntibotAttempts += 1;
if (this.stats.pdfAntibotAttempts > this.config.maxPdfPrefetches) {
this.throwLimit(reason, lastError);
}
break;
case "document_antibot":
this.stats.documentAntibotAttempts += 1;
if (
this.stats.documentAntibotAttempts > this.config.maxDocumentPrefetches
) {
this.throwLimit(reason, lastError);
}
break;
}
this.logger.warn("scrapeURL retrying after handled error", {
reason,
retryStats: this.getSnapshot(),
lastError:
lastError instanceof Error ? lastError.message : (lastError ?? null),
});
}
private throwLimit(reason: ScrapeRetryLimitReason, lastError: unknown) {
const snapshot = this.getSnapshot();
this.logger.error("scrapeURL retry limit reached", {
reason,
retryStats: snapshot,
lastError:
lastError instanceof Error ? lastError.message : (lastError ?? null),
});
throw new ScrapeRetryLimitError(reason, snapshot);
}
}
|