vomebook commited on
Commit
2e07dd2
·
verified ·
1 Parent(s): 79a4fe0

feat(reader): support v3 DjVu primary via image stream and text layer

Browse files

validateReadingManifest accepts primary.kind=djvu (source-scoped objects/<sha>/<ns>/document.djvu in pdf-pages-v2); pdfPrimarySource returns '' for non-PDF primaries so PDF.js never loads DjVu; isBucketPath accepts document.djvu; assetFields rejects .djvu sidecar links.

static/reader/core/reader-contract.js CHANGED
@@ -305,7 +305,7 @@
305
  function isBucketPath(path, versioned = false, bucket = "vomebook/pdf-pages-v2") {
306
  if (bucket === "vomebook/reader-assets-v2") return staticBucketPathPattern.test(path) || v2ChapterPathPattern.test(path) || v2ObjectPathPattern.test(path);
307
  if (bucket !== "vomebook/pdf-pages-v2") return false;
308
- return /^objects\/[0-9a-f]{2}\/[0-9a-f]{64}\/[0-9a-f]{16}\/document\.pdf$/.test(path) ||
309
  (versioned ? versionedBucketPathPattern : bucketPathPattern).test(path) ||
310
  v3BucketPathPattern.test(path) ||
311
  staticBucketPathPattern.test(path) ||
@@ -403,6 +403,7 @@
403
  if (
404
  !asset ||
405
  asset.s !== 2 ||
 
406
  !Object.prototype.hasOwnProperty.call(assetModes, asset.m) ||
407
  !(asset.b ? bucket : bucket || assetPrimaryPattern.test(path)) ||
408
  (asset?.b && bucketName !== "vomebook/pdf-pages-v2" && bucketName !== "vomebook/reader-assets-v2") ||
 
305
  function isBucketPath(path, versioned = false, bucket = "vomebook/pdf-pages-v2") {
306
  if (bucket === "vomebook/reader-assets-v2") return staticBucketPathPattern.test(path) || v2ChapterPathPattern.test(path) || v2ObjectPathPattern.test(path);
307
  if (bucket !== "vomebook/pdf-pages-v2") return false;
308
+ return /^objects\/[0-9a-f]{2}\/[0-9a-f]{64}\/[0-9a-f]{16}\/document\.(?:pdf|djvu)$/.test(path) ||
309
  (versioned ? versionedBucketPathPattern : bucketPathPattern).test(path) ||
310
  v3BucketPathPattern.test(path) ||
311
  staticBucketPathPattern.test(path) ||
 
403
  if (
404
  !asset ||
405
  asset.s !== 2 ||
406
+ path.endsWith(".djvu") ||
407
  !Object.prototype.hasOwnProperty.call(assetModes, asset.m) ||
408
  !(asset.b ? bucket : bucket || assetPrimaryPattern.test(path)) ||
409
  (asset?.b && bucketName !== "vomebook/pdf-pages-v2" && bucketName !== "vomebook/reader-assets-v2") ||
static/reader/pdf/reader-pdf-renderer.mjs CHANGED
@@ -114,7 +114,9 @@ export function createReaderPdfRenderer(readerContext) {
114
 
115
  function pdfPrimarySource() {
116
  if (readerContext.capability.mode !== "pdf-pages") return "";
117
- if (readerContext.pdfV3Manifest) return readerContext.v3ResourceUrl(readerContext.pdfV3Manifest.primary.resource);
 
 
118
  const candidate = readerContext.normalizeSourceUrl(readerContext.params.get("pdf_document") || readerContext.resolvedReaderData?.pdf_document ||
119
  (readerContext.resolvedReaderData?.original_extension === "pdf" || /\.pdf(?:$|[?#])/i.test(readerContext.downloadUrl)
120
  ? readerContext.downloadUrl : ""));
 
114
 
115
  function pdfPrimarySource() {
116
  if (readerContext.capability.mode !== "pdf-pages") return "";
117
+ // A non-PDF (DjVu) primary keeps the image stream plus v3 text layer.
118
+ if (readerContext.pdfV3Manifest)
119
+ return readerContext.pdfV3Manifest.primary.kind === "pdf" ? readerContext.v3ResourceUrl(readerContext.pdfV3Manifest.primary.resource) : "";
120
  const candidate = readerContext.normalizeSourceUrl(readerContext.params.get("pdf_document") || readerContext.resolvedReaderData?.pdf_document ||
121
  (readerContext.resolvedReaderData?.original_extension === "pdf" || /\.pdf(?:$|[?#])/i.test(readerContext.downloadUrl)
122
  ? readerContext.downloadUrl : ""));
static/reader/pdf/reader-v3.mjs CHANGED
@@ -177,10 +177,15 @@ export function validateReadingManifest(manifest, maxPages) {
177
  if (!manifest || manifest.version !== 3 || manifest.kind !== "pdf-reading" ||
178
  !sha(manifest.source_sha256) || !sha(manifest.generation) || !integer(manifest.page_count) ||
179
  manifest.page_count > maxPages || typeof manifest.source_key !== "string" ||
180
- manifest.primary?.kind !== "pdf" || manifest.primary.status !== "ready" ||
181
  typeof manifest.preview?.complete !== "boolean" || manifest.preview.page_count !== manifest.page_count) invalid();
182
  validateResource(manifest.primary.resource);
183
- if (!manifest.primary.resource.path.endsWith(".pdf")) invalid();
 
 
 
 
 
184
  validateResource(manifest.page_map, manifest.source_sha256);
185
  if (!manifest.page_map.path.endsWith("/page-map.json.gz")) invalid();
186
  partitions(manifest.preview.partitions, manifest.page_count, manifest.preview.complete, manifest.source_sha256);
 
177
  if (!manifest || manifest.version !== 3 || manifest.kind !== "pdf-reading" ||
178
  !sha(manifest.source_sha256) || !sha(manifest.generation) || !integer(manifest.page_count) ||
179
  manifest.page_count > maxPages || typeof manifest.source_key !== "string" ||
180
+ !["pdf", "djvu"].includes(manifest.primary?.kind) || manifest.primary.status !== "ready" ||
181
  typeof manifest.preview?.complete !== "boolean" || manifest.preview.page_count !== manifest.page_count) invalid();
182
  validateResource(manifest.primary.resource);
183
+ // Browsers cannot render DjVu: such books read from the preview stream and text layer.
184
+ if (manifest.primary.kind === "djvu"
185
+ ? manifest.primary.resource.bucket !== "vomebook/pdf-pages-v2" ||
186
+ !manifest.primary.resource.path.startsWith(`objects/${manifest.source_sha256.slice(0, 2)}/${manifest.source_sha256}/`) ||
187
+ !manifest.primary.resource.path.endsWith("/document.djvu")
188
+ : !manifest.primary.resource.path.endsWith(".pdf")) invalid();
189
  validateResource(manifest.page_map, manifest.source_sha256);
190
  if (!manifest.page_map.path.endsWith("/page-map.json.gz")) invalid();
191
  partitions(manifest.preview.partitions, manifest.page_count, manifest.preview.complete, manifest.source_sha256);
tests/test_reader_pdf_scroll_scheduling.mjs CHANGED
@@ -1,5 +1,7 @@
1
  import assert from "node:assert/strict";
 
2
  import path from "node:path";
 
3
  import { pathToFileURL, fileURLToPath } from "node:url";
4
 
5
  const root = path.resolve(process.argv[2] || path.join(path.dirname(fileURLToPath(import.meta.url)), ".."));
@@ -140,4 +142,53 @@ for (const outcome of ["promoted", "pdf", "cancelled"]) {
140
  }
141
  } finally {Object.defineProperty(globalThis,'performance',{configurable:true,value:nativePerformance});}
142
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
143
  console.log("PDF scroll scheduling: bounded reads, live demand/fallback promotion, motion grace, cancellation and directional neighbors passed", root);
 
1
  import assert from "node:assert/strict";
2
+ import fs from "node:fs";
3
  import path from "node:path";
4
+ import vm from "node:vm";
5
  import { pathToFileURL, fileURLToPath } from "node:url";
6
 
7
  const root = path.resolve(process.argv[2] || path.join(path.dirname(fileURLToPath(import.meta.url)), ".."));
 
142
  }
143
  } finally {Object.defineProperty(globalThis,'performance',{configurable:true,value:nativePerformance});}
144
  }
145
+ {
146
+ // DjVu primary: the manifest validates, its primary is never handed to PDF.js,
147
+ // and only the exact versioned document.djvu bucket path is accepted.
148
+ const sandbox = { self: {}, TextEncoder, URLSearchParams, URL };
149
+ vm.runInNewContext(fs.readFileSync(path.join(root, "static/reader/core/reader-contract.js"), "utf8"), sandbox);
150
+ const contract = globalThis.VoiceOfMLReader = sandbox.self.VoiceOfMLReader;
151
+ const { validateReadingManifest } = await load("pdf/reader-v3.mjs");
152
+ const source = "a".repeat(64), base = `objects/aa/${source}/`;
153
+ const ref = (p, bucket = "vomebook/pdf-pages-v2") => ({ bucket, path: base + p, sha256: "b".repeat(64), bytes: 10, role: "runtime" });
154
+ const manifest = (kind, primaryPath) => ({
155
+ version: 3, kind: "pdf-reading", source_sha256: source, source_key: "repo\0book.djvu", generation: "c".repeat(64),
156
+ page_count: 2, page_map: ref("d".repeat(16) + "/page-map.json.gz"),
157
+ primary: { kind, status: "ready", resource: ref(primaryPath) },
158
+ preview: { page_count: 2, complete: true, ready_pages: 2, partitions: [
159
+ { start: 1, end: 2, complete: true, resource: ref("d".repeat(16) + "/preview/partition-000001-000128-manifest.json") }] },
160
+ text_layer: ref("e".repeat(16) + "/text-layer-manifest.json"),
161
+ components: { document: "ready", preview: "ready", text: "ready", correction: "pending", searchable_pdf: "skipped" }
162
+ });
163
+ const djvuPath = "f".repeat(16) + "/document.djvu";
164
+ assert.equal(validateReadingManifest(manifest("djvu", djvuPath), 100).primary.kind, "djvu");
165
+ assert.equal(validateReadingManifest(manifest("pdf", "f".repeat(16) + "/document.pdf"), 100).primary.kind, "pdf");
166
+ for (const [kind, primaryPath] of [["pdf", djvuPath], ["djvu", "f".repeat(16) + "/document.pdf"], ["djvu", "document.djvu"],
167
+ ["djvu", "f".repeat(16) + "/book.djvu"], ["epub", djvuPath]])
168
+ assert.throws(() => validateReadingManifest(manifest(kind, primaryPath), 100), /PDF_V3_INVALID/, `${kind} ${primaryPath}`);
169
+ const foreign = manifest("djvu", djvuPath);
170
+ foreign.primary.resource.path = `objects/bb/${"b".repeat(64)}/${djvuPath}`;
171
+ assert.throws(() => validateReadingManifest(foreign, 100), /PDF_V3_INVALID/, "DjVu primary must belong to its source");
172
+ const otherBucket = manifest("djvu", djvuPath);
173
+ otherBucket.primary.resource.bucket = "vomebook/reader-assets-v2";
174
+ assert.throws(() => validateReadingManifest(otherBucket, 100), /PDF_V3_INVALID/);
175
+
176
+ assert.equal(contract.isBucketPath(base + djvuPath, true), true);
177
+ assert.equal(contract.isBucketPath(base + "f".repeat(16) + "/document.pdf", true), true);
178
+ for (const bad of [base + "document.djvu", base + "f".repeat(16) + "/x/document.djvu", base + "f".repeat(16) + "/book.djvu",
179
+ base + "f".repeat(16) + "/document.djvu.pdf", base + "f".repeat(16) + "/../document.djvu", `objects/aa/${source}.djvu`])
180
+ assert.equal(contract.isBucketPath(bad, true), false, bad);
181
+ assert.equal(contract.isBucketPath(base + djvuPath, true, "vomebook/reader-assets-v2"), false);
182
+ assert.equal(contract.isBucketPath(base + djvuPath, true, "vomebook/pdf-optimized"), false);
183
+ assert.equal(contract.assetFields({ s: 2, m: "p", p: base + djvuPath, b: "vomebook/pdf-pages-v2" }, "/api/reader-bucket-resource"), null,
184
+ "a DjVu original is never a sidecar PDF Reader link");
185
+
186
+ const reader = context(false);
187
+ Object.assign(reader, { capability: { mode: "pdf-pages" }, v3ResourceUrl: r => "/api/reader-bucket-resource?path=" + r.path });
188
+ reader.pdfV3Manifest = manifest("djvu", djvuPath);
189
+ assert.equal(reader.pdfPrimarySource(), "", "DjVu primary must not reach PDF.js");
190
+ reader.pdfV3Manifest = manifest("pdf", "f".repeat(16) + "/document.pdf");
191
+ assert.match(reader.pdfPrimarySource(), /document\.pdf$/);
192
+ delete globalThis.VoiceOfMLReader;
193
+ }
194
  console.log("PDF scroll scheduling: bounded reads, live demand/fallback promotion, motion grace, cancellation and directional neighbors passed", root);