Spaces:
Sleeping
Sleeping
File size: 3,327 Bytes
097eaf0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 | import { ApiError } from "@google-cloud/storage";
import { logger } from "./logger";
import { config } from "../config";
import crypto from "crypto";
import { storage } from "./gcs-jobs";
type PdfCacheProvider = "runpod" | "firepdf";
// Cache shape — markdown/html are required; pagesProcessed is optional so
// pre-existing entries (written before the field existed) round-trip cleanly
// and the caller can fall back to its own page-count signal on a stale hit.
type CachedPdfResult = {
markdown: string;
html: string;
pagesProcessed?: number;
};
const PROVIDER_PREFIXES: Record<PdfCacheProvider, string> = {
runpod: "pdf-cache-v2/",
firepdf: "pdf-cache-firepdf/",
};
export function createPdfCacheKey(pdfContent: string | Buffer): string {
return crypto.createHash("sha256").update(pdfContent).digest("hex");
}
export async function savePdfResultToCache(
pdfContent: string,
result: CachedPdfResult,
provider: PdfCacheProvider = "runpod",
variant?: string,
): Promise<string | null> {
try {
if (!config.GCS_BUCKET_NAME) {
return null;
}
const prefix = PROVIDER_PREFIXES[provider];
const cacheKey = createPdfCacheKey(pdfContent);
const objectKey = variant ? `${cacheKey}-${variant}` : cacheKey;
const bucket = storage.bucket(config.GCS_BUCKET_NAME);
const blob = bucket.file(`${prefix}${objectKey}.json`);
for (let i = 0; i < 3; i++) {
try {
await blob.save(JSON.stringify(result), {
contentType: "application/json",
metadata: {
source: `${provider}_pdf_conversion`,
cache_type: "pdf_markdown",
created_at: new Date().toISOString(),
},
});
logger.info(`Saved PDF result to GCS cache`, {
cacheKey,
provider,
});
return cacheKey;
} catch (error) {
if (i === 2) {
throw error;
} else {
logger.error(`Error saving PDF result to GCS cache, retrying`, {
error,
cacheKey,
provider,
i,
});
}
}
}
return cacheKey;
} catch (error) {
logger.error(`Error saving PDF result to GCS cache`, {
error,
provider,
});
return null;
}
}
export async function getPdfResultFromCache(
pdfContent: string,
provider: PdfCacheProvider = "runpod",
variant?: string,
): Promise<CachedPdfResult | null> {
try {
if (!config.GCS_BUCKET_NAME) {
return null;
}
const prefix = PROVIDER_PREFIXES[provider];
const cacheKey = createPdfCacheKey(pdfContent);
const objectKey = variant ? `${cacheKey}-${variant}` : cacheKey;
const bucket = storage.bucket(config.GCS_BUCKET_NAME);
const blob = bucket.file(`${prefix}${objectKey}.json`);
const [content] = await blob.download();
const result = JSON.parse(content.toString());
logger.info(`Retrieved PDF result from GCS cache`, {
cacheKey,
provider,
});
return {
...result,
};
} catch (error) {
if (
error instanceof ApiError &&
error.code === 404 &&
error.message.includes("No such object:")
) {
return null;
}
logger.error(`Error retrieving PDF result from GCS cache`, {
error,
provider,
});
return null;
}
}
|