tests/test_reader_regression.js

const fs = require('fs')
const path = require('path')
const assert = require('assert')
const vm = require('vm')
const { TextEncoder } = require('util')
const { assertCode, readReaderSource } = require('./reader_source_contract')

const root = path.resolve(__dirname, '..')
const reader = readReaderSource(root)
const contract = fs.readFileSync(path.join(root, 'static/reader/core/reader-contract.js'), 'utf8')
const sandbox = { self: {}, TextEncoder, URLSearchParams, URL }
vm.runInNewContext(contract, sandbox)
const pdfText = {}
vm.runInNewContext(fs.readFileSync(path.join(root, 'static/reader/pdf/reader-pdf-text.js'), 'utf8').replace(/^export /gm, ''), pdfText)
for (const [raw, expected] of [
  ['', ''], ['hello world', 'hello world'], ['a\n b', 'a\n b'],
  ['\u624b \u673a', '\u624b\u673a'], ['\u624b\u3002\u673a', '\u624b\u3002\u673a'],
  ['\u6bdb \u6cfd\n\u4e1c hello world', '\u6bdb\u6cfd\u4e1c hello world'],
  ['\ud840\udc00 \ud840\udc01', '\ud840\udc00\ud840\udc01'],
  ['\u624b\t\n \u673a a b \u624b \u673a', '\u624b\u673a a b \u624b\u673a'],
]) {
  for (let repeat = 0; repeat < 3; repeat++) {
    assert.strictEqual(pdfText.normalizePdfSearchText(raw), expected)
    const mapped = pdfText.pdfSearchText(raw)
    assert.strictEqual(mapped.text, expected)
    assert.strictEqual(Array.from({length: mapped.text.length}, (_, index) => raw[pdfText.pdfSourceOffset(mapped, index)]).join(''), expected)
  }
}

const unchanged = pdfText.pdfSearchText('unchanged text '.repeat(10000))
assert.strictEqual(unchanged.offsets, null)
const sparse = pdfText.pdfSearchText('prefix '.repeat(10000) + '\u624b \u673a' + ' suffix'.repeat(10000))
assert.strictEqual(sparse.offsets.byteLength, 8)
for (const raw of ['\u624b \u673a \u624b\n\u673a', '\ud840\udc00 \ud840\udc01\r\n\u624b \u673a']) {
  const deleted = new Set()
  for (const match of raw.matchAll(/(?<=\p{Script=Han})\s+(?=\p{Script=Han})/gu))
    for (let i=match.index;i<match.index+match[0].length;i++) deleted.add(i)
  const oracle = Array.from({length:raw.length}, (_,i)=>i).filter(i=>!deleted.has(i))
  const mapped = pdfText.pdfSearchText(raw)
  assert.deepStrictEqual(Array.from({length:mapped.text.length}, (_,i)=>pdfText.pdfSourceOffset(mapped,i)), oracle)
}

const assets = sandbox.self.VoiceOfMLReader
const vectors = require('./reader-contract-vectors.json')
for (const [value, expected] of vectors.ids) assert.strictEqual(assets.shortSourceId(value), expected)
const idPrefix = 'https://huggingface.co/datasets/VoiceOfML/VOMEBOOK/resolve/main/'
for (const [value, expected] of vectors.canonicalPaths) {
  assert.strictEqual(assets.canonicalSourceUrl(idPrefix + value), idPrefix + expected)
  assert.strictEqual(assets.canonicalSourceUrl(idPrefix + expected), idPrefix + expected)
  assert.strictEqual(assets.shortSourceId(idPrefix + value), assets.shortSourceId(idPrefix + expected))
}
for (const bookPath of [vectors.reportedBook.path, vectors.reportedBook.path.split('/').map(encodeURIComponent).join('/')]) {
  const link = idPrefix + bookPath
  assert.strictEqual(assets.shortSourceId(link), vectors.reportedBook.id)
  assert.strictEqual(new URL(assets.readerUrl({ Link: link, Extension: 'pdf' }), 'https://site.test').searchParams.get('id'), vectors.reportedBook.id)
}
assert.notStrictEqual(assets.shortSourceId(idPrefix + 'a%2Fb.pdf'), assets.shortSourceId(idPrefix + 'a/b.pdf'))
assert.notStrictEqual(assets.shortSourceId(idPrefix + 'a%2528.pdf'), assets.shortSourceId(idPrefix + 'a%28.pdf'))
for (const [record, expected] of vectors.ocr) {
  const relative = [...record.Folder, record.File + (record.Extension ? '.' + record.Extension : '')].join('/')
  assert.strictEqual(assets.txtRelativePath(relative), expected)
}
const assetRoot = 'objects/aa/' + 'a'.repeat(64) + '/'
const legacyPrefix = '/datasets/vomebook/Reader-Assets/resolve/main/' // deleted dataset, kept only as a negative fixture
// Dataset-era object paths without a bucket are no longer readable.
for (const directory of ['', 'converter-v1/', 'b'.repeat(16) + '/', 'b'.repeat(16) + '/converter-v1/']) {
  const path = assetRoot + directory + 'linearized.pdf'
  assert.strictEqual(assets.assetFields({s: 2, m: 'p', p: path}, '/api/reader-bucket-resource'), null)
}
for (const extension of ['epub', 'mobi', 'azw3', 'fb2', 'chm']) {
  const path = 'chapters/ebook/' + extension + '/' + 'c'.repeat(64) + '/' + 'b'.repeat(16) + '/chapter-manifest.json'
  const fields = assets.assetFields({s: 2, m: 'e', p: path, b: 'vomebook/reader-assets-v2',
    c: path, cb: 'vomebook/reader-assets-v2'}, '/api/reader-bucket-resource')
  assert.strictEqual(fields.ReaderExtension, 'epub-chapters')
  assert.strictEqual(fields.ReaderLink, '/api/reader-bucket-resource?path=' + encodeURIComponent(path))
}
const bucket = assetRoot + 'b'.repeat(16) + '/page-manifest.json'
for (const suffix of ['b'.repeat(16) + '/']) {
  const url = 'https://huggingface.co/buckets/vomebook/pdf-pages-v2/resolve/' + assetRoot + suffix + 'page-manifest.json'
  const parsed = assets.pdfPageSource(url, 'https://site.test', 'https://api.test')
  assert.strictEqual(parsed.root, (assetRoot + suffix).slice(0, -1))
  assert.strictEqual(parsed.pageUrl(2), url.replace('page-manifest.json', 'pages/page-000002.webp'))
  assert.throws(() => parsed.pageUrl(0), /PDF_PAGE_INVALID/)
}
assert.ok(assets.pdfPageSource('https://api.test/api/reader-bucket-resource?path=' + encodeURIComponent(bucket), 'https://site.test', 'https://api.test'))
for (const url of [
  'https://evil.test/buckets/vomebook/pdf-pages-v2/resolve/' + bucket,
  'https://api.test/api/reader-bucket-resource?path=' + encodeURIComponent(assetRoot + 'page-manifest.json'),
])
  assert.strictEqual(assets.pdfPageSource(url, 'https://site.test', 'https://api.test'), null)
// The deleted dataset form is no longer a source for any Reader surface.
for (const url of [
  'https://huggingface.co/datasets/vomebook/Reader-Assets/resolve/main/' + bucket,
  'https://hf-mirror.com/datasets/vomebook/Reader-Assets/revision/main/' + bucket,
])
  assert.strictEqual(assets.pdfPageSource(url, 'https://site.test', 'https://api.test'), null)
assert.strictEqual(assets.assetFields({s: 2, m: 'p', p: bucket}, '/api/reader-bucket-resource').ReaderLink,
  '/api/reader-bucket-resource?path=' + encodeURIComponent(bucket))
assert.strictEqual(assets.assetFields({s: 2, m: 'p', p: bucket, b: 'vomebook/pdf-pages-v2'}, '/api/reader-bucket-resource').ReaderLink,
  '/api/reader-bucket-resource?path=' + encodeURIComponent(bucket))
const nativeMediaPath = 'media/audio/native/' + 'a'.repeat(64) + '/audio.flac'
const nativeMedia = assets.assetFields({s: 2, m: 'a', p: nativeMediaPath, b: 'vomebook/reader-assets-v2'}, '/api/reader-bucket-resource')
assert.strictEqual(nativeMedia.ReaderExtension, 'audio')
assert.strictEqual(nativeMedia.ReaderLink,
  '/api/reader-bucket-resource?path=' + encodeURIComponent(nativeMediaPath))
assert.strictEqual(assets.isBucketPath(nativeMediaPath, true, 'vomebook/reader-assets-v2'), true)
const swfPath = 'media/swf/native/' + 'a'.repeat(64) + '/document.swf'
const swf = assets.assetFields({s: 2, m: 'f', p: swfPath, b: 'vomebook/reader-assets-v2'}, '/api/reader-bucket-resource')
assert.strictEqual(swf.ReaderExtension, 'swf')
assert.strictEqual(assets.capability('swf').mode, 'swf')
assert.strictEqual(swf.ReaderLink,
  '/api/reader-bucket-resource?path=' + encodeURIComponent(swfPath))
for (const path of ['derived/migration/' + 'a'.repeat(32) + '/document.pdf', 'documents/pdf/ppt/' + 'a'.repeat(64) + '/document.pdf']) {
  const fields = assets.assetFields({s: 2, m: 'p', p: path, b: path.startsWith('derived/') ? 'vomebook/pdf-pages-v2' : 'vomebook/reader-assets-v2'}, '/api/reader-bucket-resource')
  assert.strictEqual(fields.ReaderLink, '/api/reader-bucket-resource?path=' + encodeURIComponent(path))
  assert.strictEqual(fields.ReaderExtension, 'pdf')
}
assert.strictEqual(assets.assetFields({s: 2, m: 'p', p: bucket, b: 'vomebook/pdf-pages'}), null)
const optimized = assetRoot + 'pdf-range-v1-document/document.pdf'
assert.strictEqual(assets.assetFields({s: 2, m: 'p', p: optimized, b: 'vomebook/pdf-optimized'}), null)
assert.strictEqual(assets.assetFields({s: 2, m: 'p', p: legacyPrefix + 'pdf_manifest.json'}), null)
assert.strictEqual(assets.assetFields({s: 2, m: 'p', p: legacyPrefix + assetRoot + 'epub-chapters/chapters/chapter-0012.xhtml'}), null)
for (const path of [assetRoot + '../linearized.pdf', assetRoot + 'private.json', 'objects/invalid/document.pdf']) {
  assert.strictEqual(assets.assetFields({s: 2, m: 'p', p: path}), null)
}

for (const extension of ['pdf', 'pdf-pages', 'txt', 'md', 'markdown', 'html', 'htm', 'epub', 'mobi', 'azw3', 'fb2', 'fbz', 'jpg', 'png', 'mp3', 'm4a', 'flac', 'mpga', 'audio', 'mp4', 'mov', 'video']) {
  assert.ok(sandbox.self.VoiceOfMLReader.capability(extension).mode, extension)
}
for (const extension of ['epub', 'mobi', 'azw3', 'fb2', 'fbz']) assert.strictEqual(sandbox.self.VoiceOfMLReader.capability(extension).mode, 'foliate')
for (const extension of sandbox.self.VoiceOfMLReader.articleExtensions) assert.deepStrictEqual(Object.keys(sandbox.self.VoiceOfMLReader.capability(extension).features).sort(), ['bookmarks', 'media', 'pagination', 'search', 'toc', 'zoom'])
assert.strictEqual(sandbox.self.VoiceOfMLReader.capability('pdf-pages').mode, 'pdf-pages')
assert.strictEqual(sandbox.self.VoiceOfMLReader.capability('pdf-pages').features.toc, true)
assert.strictEqual(sandbox.self.VoiceOfMLReader.capability('epub-chapters').features.search, true)
for (const extension of ['mp3', 'm4a', 'flac', 'mpga', 'audio']) assert.strictEqual(sandbox.self.VoiceOfMLReader.capability(extension).mode, 'audio')
for (const extension of ['mp4', 'mov', 'video']) assert.strictEqual(sandbox.self.VoiceOfMLReader.capability(extension).mode, 'video')
assert.strictEqual(sandbox.self.VoiceOfMLReader.capability('zip').mode, null)
assert.strictEqual(sandbox.self.VoiceOfMLReader.capability('docx').mode, null)
assert.strictEqual(sandbox.self.VoiceOfMLReader.capability('epub-chapters').mode, 'epub-chapters')
const convertedUrl = sandbox.self.VoiceOfMLReader.readerUrl({
  Link: 'https://download.example.test/books/book.mobi',
  File: '原始书名', Extension: 'mobi', ReaderExtension: 'epub',
  DownloadLink: 'https://download.example.test/books/book.mobi',
  ReaderChapterManifest: 'https://voiceofml-search.hf.space/api/reader-bucket-resource?path=' + encodeURIComponent('chapters/ebook/mobi/' + 'c'.repeat(64) + '/' + 'b'.repeat(16) + '/chapter-manifest.json'),
}, '/reader.html')
const convertedParams = new URL(`https://example.test${convertedUrl}`).searchParams
assert.strictEqual(convertedParams.get('ext'), 'epub')
assert.strictEqual(convertedParams.get('title'), '原始书名.mobi')
assert.strictEqual(convertedParams.get('download'), 'https://download.example.test/books/book.mobi')
assert.strictEqual(convertedParams.get('fallback'), null)
assert.strictEqual(convertedParams.get('chapter_manifest'), 'https://voiceofml-search.hf.space/api/reader-bucket-resource?path=' + encodeURIComponent('chapters/ebook/mobi/' + 'c'.repeat(64) + '/' + 'b'.repeat(16) + '/chapter-manifest.json'))
const pageAssetUrl = sandbox.self.VoiceOfMLReader.readerUrl({
  ReaderLink: 'https://voiceofml-search.hf.space/api/reader-bucket-resource?path=' + encodeURIComponent('objects/aa/' + 'a'.repeat(64) + '/1234567890abcdef/page-manifest.json'),
  ReaderExtension: 'pdf-pages', Extension: 'mobi', File: '原始书名',
})
const pageAssetParams = new URL(`https://example.test${pageAssetUrl}`).searchParams
assert.strictEqual(pageAssetParams.get('url'), 'https://voiceofml-search.hf.space/api/reader-bucket-resource?path=' + encodeURIComponent('objects/aa/' + 'a'.repeat(64) + '/1234567890abcdef/page-manifest.json'))
assert.strictEqual(pageAssetParams.get('ext'), 'pdf-pages')
const sharedAsset = 'https://voiceofml-search.hf.space/api/reader-bucket-resource?path=' + encodeURIComponent('chapters/ebook/chm/' + 'a'.repeat(64) + '/' + 'b'.repeat(16) + '/chapter-manifest.json')
const sourceIds = ['Teachers', 'MLMRL-Hub'].map(repo => {
  const record = { Link: `https://huggingface.co/datasets/VoiceOfML/${repo}/resolve/main/book.mobi`, File: 'book', Extension: 'mobi' }
  const params = value => new URL(`https://example.test${sandbox.self.VoiceOfMLReader.readerUrl(value)}`).searchParams
  const original = params(record).get('id')
  const converted = params({ ...record, ReaderLink: sharedAsset, ReaderExtension: 'epub-chapters' })
  assert.strictEqual(converted.get('id'), original)
  assert.match(original, /^[a-z0-9]{13}$/)
  assert.strictEqual(converted.get('ext'), 'epub-chapters')
  return original
})
assert.notStrictEqual(sourceIds[0], sourceIds[1])
const normalizedUrl = sandbox.self.VoiceOfMLReader.readerUrl({ Link: 'https://download.example.test/book.TXT', File: '书名', Extension: 'TXT', ReaderExtension: 'TXT' }, '/reader.html')
const normalizedParams = new URL(`https://example.test${normalizedUrl}`).searchParams
assert.strictEqual(normalizedParams.get('ext'), 'TXT')
assert.strictEqual(normalizedParams.get('title'), '书名.TXT')
assert.strictEqual(sandbox.self.VoiceOfMLReader.readerUrl(null, '/reader.html'), '')
assert.strictEqual(sandbox.self.VoiceOfMLReader.readerUrl({ Link: 'https://download.example.test/archive.zip', Extension: 'zip' }, '/reader.html'), '')
assert.strictEqual(sandbox.self.VoiceOfMLReader.readerUrl({ Link: 'https://download.example.test/book.txt', Extension: '.txt' }, '/reader.html'), '')
assert.strictEqual(sandbox.self.VoiceOfMLReader.capability('PDF').readerMode, sandbox.self.VoiceOfMLReader.ReaderMode.ORIGINAL)
assert.strictEqual(sandbox.self.VoiceOfMLReader.capability('PDF').article, true)
assert.strictEqual(sandbox.self.VoiceOfMLReader.capability('zip').readerMode, sandbox.self.VoiceOfMLReader.ReaderMode.UNSUPPORTED)
assert.strictEqual(sandbox.self.VoiceOfMLReader.capability('zip').article, false)
for (const extension of ['pdf', 'pdf-pages', 'epub', 'epub-chapters', 'mobi', 'azw3', 'fb2', 'fbz', 'txt', 'md', 'html', 'jpg', 'png', 'webp', 'mp3', 'm4a', 'flac', 'mp4', 'mov']) assert.ok(sandbox.self.VoiceOfMLReader.articleExtensions.includes(extension), extension)
assert.ok(!sandbox.self.VoiceOfMLReader.articleExtensions.includes('docx'))
assert.strictEqual(Object.isFrozen(sandbox.self.VoiceOfMLReader.articleExtensions), true)
assert.strictEqual(Object.isFrozen(sandbox.self.VoiceOfMLReader.capability('pdf')), true)
const immutableRecord = { Link: 'https://download.example.test/book.txt', File: 'book', Extension: 'txt' }
const immutableBefore = JSON.stringify(immutableRecord)
const defaultReaderUrl = sandbox.self.VoiceOfMLReader.readerUrl(immutableRecord)
assert.ok(defaultReaderUrl.startsWith('/static/reader.html?'))
assert.strictEqual(JSON.stringify(immutableRecord), immutableBefore)
assert.strictEqual((reader.match(/function setToc\(/g) || []).length, 1)
assertCode(reader, 'const previousList = document.querySelector("#toc-list"), list = previousList.cloneNode(false)')
assertCode(reader, 'previousList.replaceWith(list)')
assert.strictEqual((reader.match(/function activateFoliateTocEntry\(/g) || []).length, 1)
assert.match(reader, /reader-source:/)
assert.match(reader, /reader-zoom/)
assert.match(reader, /readerRuntime\.track\(readerRequestManager\)/)
assert.match(reader, /readerPath[\s\S]*voice-reader-navigate/)
assert.match(reader, /title: historyTitle|title,\s*extension/)
console.log('reader regression contracts passed')