Fix Reader source IDs and bound PDF opening reads

app.py

@@ -257,9 +257,23 @@ BROWSER_FILE_ITEM_KEY_MAP = {
 READER_SHORT_ID_RE = re.compile(r"^[0-9a-z]{13}$")
 
 
+def canonical_reader_source_url(value: str) -> str:
+    # Match reader-contract.js byte normalization without decoding path separators
+    # or treating an encoded percent sign as a second escape sequence.
+    def path_replace(match):
+        def token_replace(token_match):
+            token = token_match.group()
+            if re.fullmatch(r"%[0-9a-fA-F]{2}", token):
+                char = chr(int(token[1:], 16))
+                return char if re.fullmatch(r"[A-Za-z0-9._~-]", char) else token.upper()
+            return quote(token, safe="")
+        return match[1] + re.sub(r"%[0-9a-fA-F]{2}|[^A-Za-z0-9._~/-]", token_replace, match[2])
+    return re.sub(r"^(https://huggingface\.co/datasets/VoiceOfML/)([^?#]*)", path_replace, value)
+
+
 def reader_short_id(value: str) -> str:
     digest = 1469598103934665603
-    for byte in value.encode("utf-8"):
+    for byte in canonical_reader_source_url(value).encode("utf-8"):
         digest = ((digest ^ byte) * 1099511628211) & ((1 << 64) - 1)
     return _base36(digest).rjust(13, "0")
 
@@ -1845,7 +1859,7 @@ async def api_reader_resolve(id: str = Query(..., min_length=13, max_length=13))
     for index in reader_source_id_indices.get(id, []):
         record = records[index]
         matches.append(reader_resolved_record(record, assets.get("f", {}).get(build_reader_asset_key(record), {})))
-    if len({match["download"] for match in matches}) > 1:
+    if len({canonical_reader_source_url(match["download"]) for match in matches}) > 1:
         return JSONResponse({"error": "阅读 ID 不唯一"}, status_code=409)
     matches = matches[:1]
     if matches:

static/reader-contract.js

@@ -369,9 +369,23 @@
       params.set("folder_url", record.FolderUrl || record.folderUrl);
     return (basePath || "/static/reader.html") + "?" + params.toString();
   }
+  function canonicalSourceUrl(value) {
+    // Normalize path bytes, not decoded URLs: %2F must remain distinct from /,
+    // and %2528 must never become %28. Query/fragment delimiters stay intact.
+    return value.replace(/^(https:\/\/huggingface\.co\/datasets\/VoiceOfML\/)([^?#]*)/, (_, prefix, path) =>
+      prefix + path.replace(/%[0-9a-fA-F]{2}|[^A-Za-z0-9._~/-]/gu, (token) => {
+        if (/^%[0-9a-f]{2}$/i.test(token)) {
+          const char = String.fromCharCode(parseInt(token.slice(1), 16));
+          return /^[A-Za-z0-9._~-]$/.test(char) ? char : token.toUpperCase();
+        }
+        return encodeURIComponent(token).replace(/[!'()*]/g, (char) =>
+          "%" + char.charCodeAt(0).toString(16).toUpperCase());
+      })
+    );
+  }
   function shortSourceId(value) {
     let hash = 1469598103934665603n;
-    for (const byte of new TextEncoder().encode(value)) {
+    for (const byte of new TextEncoder().encode(canonicalSourceUrl(value))) {
       hash ^= BigInt(byte);
       hash = BigInt.asUintN(64, hash * 1099511628211n);
     }
@@ -389,6 +403,7 @@
     isBucketPath,
     pdfPageSource,
     txtRelativePath,
+    canonicalSourceUrl,
     shortSourceId,
     readerUrl
   });

static/reader.js

@@ -3761,22 +3761,6 @@ function preloadPdfEngine() {
   if (!pdfEnginePreload) pdfEnginePreload = loadPdfEngine();
   return pdfEnginePreload;
 }
-function pdfRangeChunkSize() {
-  try {
-    // The range-optimized artifact has page-oriented object placement. A
-    // smaller chunk avoids pulling unrelated objects before the first page;
-    // original PDFs retain the conservative default because some require
-    // larger sequential reads or the complete file.
-    const source = decodeURIComponent(sourceUrl);
-    if (
-      /\/pdf-range-v1-[0-9a-f]{16}-(?:objects|objects-linearized)\/document\.pdf(?:$|[?#])/i.test(
-        source
-      )
-    )
-      return 262144;
-  } catch (_) {}
-  return 1048576;
-}
 async function loadPdfEngine() {
   for (let attempt = 0; attempt < 3; attempt++) {
     assertReaderActive();
@@ -3824,9 +3808,11 @@ function loadPdfDocument() {
     pdfjs.GlobalWorkerOptions.workerSrc = PDFJS_WORKER_URL;
     const options = (url) => ({
       url,
-      rangeChunkSize: pdfRangeChunkSize(),
+      // Bound demand reads for original and converted PDFs alike. Streaming
+      // otherwise keeps the full download running alongside these ranges.
+      rangeChunkSize: 262144,
       disableAutoFetch: true,
-      disableStream: false,
+      disableStream: true,
       wasmUrl: PDFJS_WASM_URL,
       cMapUrl: PDFJS_CMAP_URL,
       cMapPacked: true,

tests/reader-contract-vectors.json

@@ -5,6 +5,24 @@
     ["书/😀.epub", "1tgzoefbsl8ia"],
     ["https://huggingface.co/datasets/VoiceOfML/Test/resolve/main/a%20b.txt", "1ckyy8f8m7pim"]
   ],
+  "canonicalPaths": [
+    ["Kſ.pdf", "%E2%84%AA%C5%BF.pdf"],
+    ["a (1)!'*.pdf", "a%20%281%29%21%27%2A.pdf"],
+    ["a%20%281%29%21%27%2a.pdf", "a%20%281%29%21%27%2A.pdf"],
+    ["中文/😀.pdf", "%E4%B8%AD%E6%96%87/%F0%9F%98%80.pdf"],
+    ["%e4%b8%ad%e6%96%87/%f0%9f%98%80.pdf", "%E4%B8%AD%E6%96%87/%F0%9F%98%80.pdf"],
+    ["%41%7a%30%2d%2e%5f%7e.pdf", "Az0-._~.pdf"],
+    ["a%2fb.pdf", "a%2Fb.pdf"],
+    ["a/b.pdf", "a/b.pdf"],
+    ["a%2528.pdf", "a%2528.pdf"],
+    ["100% done%.pdf", "100%25%20done%25.pdf"],
+    ["a+b;:@=&.pdf", "a%2Bb%3B%3A%40%3D%26.pdf"],
+    ["a%23%3f.pdf?download=true#page=2", "a%23%3F.pdf?download=true#page=2"]
+  ],
+  "reportedBook": {
+    "path": "马列之声ebook小组第九批成果集成(2018.9.1)/马列之声ebook小组第九批成果集成(2018.9.1)part 1a/2018.8 part 1/20世纪的社会主义 刘俊奇著 2007.09_c__.pdf",
+    "id": "398vk0yyy8m29"
+  },
   "ocr": [
     [{"Folder": ["edition.v2"], "File": "scan", "Extension": ""}, "edition.v2/scan.txt"],
     [{"Folder": ["edition.v2"], "File": "book.v1", "Extension": "pdf"}, "edition.v2/book.v1.txt"],

tests/test_fastapi_endpoints.py

@@ -166,6 +166,26 @@ class FastApiEndpointTests(unittest.TestCase):
         response = self.client.get("/api/reader-resolve", params={"id": "0ym7e7wbcxau0"})
         self.assertEqual(response.status_code, 404)
 
+    def test_reader_source_id_normalizes_frontend_and_backend_paths(self):
+        folders = ["马列之声ebook小组第九批成果集成(2018.9.1)",
+                   "马列之声ebook小组第九批成果集成(2018.9.1)part 1a", "2018.8 part 1"]
+        record = {"Repo": "VoiceOfML/VOMEBOOK", "Folder": folders,
+                  "File": "20世纪的社会主义 刘俊奇著 2007.09_c__", "Extension": "pdf"}
+        canonical = app.build_record_link(record)
+        for link in (canonical, canonical.replace("%28", "(").replace("%29", ")")):
+            app.records = [{**record, "Link": link}]
+            app.build_indexes(build_words=False)
+            with mock.patch.object(app, "get_reader_assets", mock.AsyncMock(return_value={"v": 1, "f": {}})):
+                response = self.client.get("/api/reader-resolve", params={"id": "398vk0yyy8m29"})
+                self.assertEqual(response.status_code, 200)
+                self.assertEqual(response.json()["download"], link)
+                self.assertEqual(self.client.get("/api/reader-resolve", params={"id": "1np0zrhual4h1"}).status_code, 404)
+        # Different spellings of one source are not a hash collision.
+        app.records.append({**record, "Link": canonical})
+        app.build_indexes(build_words=False)
+        with mock.patch.object(app, "get_reader_assets", mock.AsyncMock(return_value={"v": 1, "f": {}})):
+            self.assertEqual(self.client.get("/api/reader-resolve", params={"id": "398vk0yyy8m29"}).status_code, 200)
+
     def test_reader_resolve_uses_cached_assets_without_waiting_for_refresh(self):
         app.records = [{"Repo": "VoiceOfML/Test", "File": "book29", "Extension": "txt", "Folder": []}]
         app.build_indexes(build_words=False)

tests/test_reader_backend.py

@@ -57,6 +57,12 @@ class ReaderArchiveValidationTests(unittest.TestCase):
         vectors = json.loads(Path(__file__).with_name("reader-contract-vectors.json").read_text())
         for value, expected in vectors["ids"]:
             self.assertEqual(app.reader_short_id(value), expected)
+        prefix = "https://huggingface.co/datasets/VoiceOfML/VOMEBOOK/resolve/main/"
+        for value, expected in vectors["canonicalPaths"]:
+            self.assertEqual(app.canonical_reader_source_url(prefix + value), prefix + expected)
+            self.assertEqual(app.canonical_reader_source_url(prefix + expected), prefix + expected)
+            self.assertEqual(app.reader_short_id(prefix + value), app.reader_short_id(prefix + expected))
+        self.assertEqual(app.reader_short_id(prefix + vectors["reportedBook"]["path"]), vectors["reportedBook"]["id"])
         for record, expected in vectors["ocr"]:
             self.assertEqual(app.build_txt_relative_path(record), expected)
 

tests/test_reader_pdf_loading.py

@@ -1,6 +1,9 @@
 """Actual module imports and PDF.js with cold cache and transport faults."""
 import contextlib
 import json
+import re
+import threading
+import time
 import unittest
 import urllib.parse
 from unittest.mock import patch
@@ -78,6 +81,55 @@ class PdfLoadingTests(unittest.TestCase):
           return entries.findIndex((entry) => entry.name.includes('/api/reader-resolve?'));
         }'''))
 
+    def test_large_original_pdf_renders_using_bounded_ranges_and_stops_transfer(self):
+        # A valid large stream object before the xref, not padding after EOF.
+        small = minimal_pdf()
+        xref = small.index(b'xref\n')
+        attachment = b'x' * (4 * 1024 * 1024)
+        extra = b'6 0 obj\n<< /Length ' + str(len(attachment)).encode() + b' >>\nstream\n' + attachment + b'\nendstream\nendobj\n'
+        suffix = small[xref:].replace(b'0 6\n', b'0 7\n').replace(b'trailer\n', f'{xref:010d} 00000 n \ntrailer\n'.encode()).replace(b'/Size 6', b'/Size 7')
+        suffix = re.sub(rb'startxref\n\d+', b'startxref\n' + str(xref + len(extra)).encode(), suffix)
+        payload = small[:xref] + extra + suffix
+        requests, stopped = [], threading.Event()
+        original = StaticHandler.do_GET
+        def serve(handler):
+            if not handler.path.startswith('/range-fixture.pdf'):
+                return original(handler)
+            match = re.fullmatch(r'bytes=(\d+)-(\d*)', handler.headers.get('Range', ''))
+            start, end = (int(match[1]), min(int(match[2] or len(payload)-1), len(payload)-1)) if match else (0, len(payload)-1)
+            request = {'range': (start, end) if match else None, 'sent': 0}
+            requests.append(request)
+            handler.send_response(206 if match else 200)
+            handler.send_header('Content-Type', 'application/pdf')
+            handler.send_header('Content-Length', str(end-start+1))
+            handler.send_header('Accept-Ranges', 'bytes')
+            if match:
+                handler.send_header('Content-Range', f'bytes {start}-{end}/{len(payload)}')
+            handler.end_headers()
+            try:
+                for offset in range(start, end+1, 16384):
+                    chunk = payload[offset:min(offset+16384, end+1)]
+                    handler.wfile.write(chunk)
+                    handler.wfile.flush()
+                    request['sent'] += len(chunk)
+                    time.sleep(.005)
+            except (BrokenPipeError, ConnectionResetError):
+                pass
+            finally:
+                if not match: stopped.set()
+        self.page.unroute('**/api/reader-content**')
+        self.page.add_init_script('''const originalFetch = window.fetch;
+          window.fetch = (url, init) => originalFetch(String(url).includes('/api/reader-content?') ? '/range-fixture.pdf' : url, init);''')
+        with patch.object(StaticHandler, 'do_GET', serve):
+            self.open()
+            self.ready()
+            self.assertTrue(stopped.wait(2), 'full-file transfer must be cancelled')
+            ranges = [item['range'] for item in requests if item['range']]
+            self.assertIn((0, 262143), ranges)
+            self.assertTrue(any(end == len(payload)-1 for _, end in ranges))
+            self.assertLess(sum(item['sent'] for item in requests), 1024 * 1024)
+            self.assertEqual(len(self.modules), 1)
+
     def test_published_range_artifact_selects_small_chunk(self):
         source = 'https://huggingface.co/datasets/vomebook/Reader-Assets/resolve/main/' \
             'objects/aa/' + 'a' * 64 + '/pdf-range-v1-0123456789abcdef-objects/document.pdf'

tests/test_reader_regression.js

@@ -14,6 +14,19 @@ vm.runInNewContext(contract, sandbox)
 const assets = sandbox.self.VoiceOfMLReader
 const vectors = require('./reader-contract-vectors.json')
 for (const [value, expected] of vectors.ids) assert.strictEqual(assets.shortSourceId(value), expected)
+const idPrefix = 'https://huggingface.co/datasets/VoiceOfML/VOMEBOOK/resolve/main/'
+for (const [value, expected] of vectors.canonicalPaths) {
+  assert.strictEqual(assets.canonicalSourceUrl(idPrefix + value), idPrefix + expected)
+  assert.strictEqual(assets.canonicalSourceUrl(idPrefix + expected), idPrefix + expected)
+  assert.strictEqual(assets.shortSourceId(idPrefix + value), assets.shortSourceId(idPrefix + expected))
+}
+for (const bookPath of [vectors.reportedBook.path, vectors.reportedBook.path.split('/').map(encodeURIComponent).join('/')]) {
+  const link = idPrefix + bookPath
+  assert.strictEqual(assets.shortSourceId(link), vectors.reportedBook.id)
+  assert.strictEqual(new URL(assets.readerUrl({ Link: link, Extension: 'pdf' }), 'https://site.test').searchParams.get('id'), vectors.reportedBook.id)
+}
+assert.notStrictEqual(assets.shortSourceId(idPrefix + 'a%2Fb.pdf'), assets.shortSourceId(idPrefix + 'a/b.pdf'))
+assert.notStrictEqual(assets.shortSourceId(idPrefix + 'a%2528.pdf'), assets.shortSourceId(idPrefix + 'a%28.pdf'))
 for (const [record, expected] of vectors.ocr) {
   const relative = [...record.Folder, record.File + (record.Extension ? '.' + record.Extension : '')].join('/')
   assert.strictEqual(assets.txtRelativePath(relative), expected)

tests/test_static_contract.py

@@ -74,10 +74,10 @@ class StaticFrontendContractTests(unittest.TestCase):
         self.assertIn("backToTopPending = true", self.js)
         self.assertIn("backToTopPending = false", self.js)
 
-    def test_pdf_reader_allows_streaming_without_auto_fetch(self):
+    def test_pdf_reader_uses_demand_ranges_without_full_file_streaming(self):
         reader = (ROOT / "static/reader.js").read_text(encoding="utf-8")
         self.assertIn("disableAutoFetch: true", reader)
-        self.assertIn("disableStream: false", reader)
+        self.assertIn("disableStream: true", reader)
 
     def test_desktop_random_actions_match_labels_and_empty_state_behavior(self):
         self.assertIn('id="random-book-btn" class="random-txt-btn" title="随机下本书"', self.html)
@@ -162,7 +162,7 @@ class StaticFrontendContractTests(unittest.TestCase):
         assert_source_contains(self, 'cMapUrl: PDFJS_CMAP_URL', reader)
         assert_source_contains(self, 'standardFontDataUrl: PDFJS_STANDARD_FONT_URL', reader)
         assert_source_contains(self, 'disableAutoFetch: true', reader)
-        assert_source_contains(self, 'disableStream: false', reader)
+        assert_source_contains(self, 'disableStream: true', reader)
         self.assertIn('.reader-content[data-mode="pdf-pages"] .reader-page > img', reader_css)
         self.assertIn("'wasm-unsafe-eval'", reader_html)
         assert_source_contains(self, '"gb18030", "big5", "windows-1251", "windows-1252"', reader)