diff --git a/.prettierrc.json b/.prettierrc.json
index b463fa0a306db0954fd055675d0b8a1622afe5dd..e22dd1dd9bacb0ab12d1d9ceacc6d0525d188333 100644
--- a/.prettierrc.json
+++ b/.prettierrc.json
@@ -3,7 +3,8 @@
{
"files": [
"static/reader*.{js,css,html}",
- "scripts/check_reader_release.mjs"
+ "static/reader/**/*.mjs",
+ "scripts/build/check_reader_release.mjs"
],
"options": { "printWidth": 100, "trailingComma": "none" }
}
diff --git a/Dockerfile b/Dockerfile
index d1070f4414ce117fef50f4fe25be75591d1f4899..a08c46c71a2fb2a87d30fe2e6cfb19a3749ed0e0 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -11,10 +11,10 @@ RUN npm install --global "esbuild@${ESBUILD_VERSION}" \
&& test "$(esbuild --version)" = "${ESBUILD_VERSION}"
COPY static/ /build/static/
-COPY scripts/build_static_assets.mjs /build/build_static_assets.mjs
-COPY scripts/copy_reader_vendor.mjs /build/copy_reader_vendor.mjs
-COPY scripts/check_reader_release.mjs /build/check_reader_release.mjs
-COPY scripts/compose_app.mjs /build/compose_app.mjs
+COPY scripts/build/build_static_assets.mjs /build/build_static_assets.mjs
+COPY scripts/build/copy_reader_vendor.mjs /build/copy_reader_vendor.mjs
+COPY scripts/build/check_reader_release.mjs /build/check_reader_release.mjs
+COPY scripts/build/compose_app.mjs /build/compose_app.mjs
RUN mkdir -p /out \
&& cp -a static/. /out/ \
@@ -22,7 +22,8 @@ RUN mkdir -p /out \
&& node /build/compose_app.mjs > /build/app-composed.js \
&& esbuild /build/app-composed.js --minify --target=es2020 --charset=utf8 --legal-comments=none --outfile=/out/app.js \
&& for file in $(node -e 'console.log(require("./static/reader-resources.js").buildFiles("hf").filter(f => f !== "app.js").join(" "))'); do \
- esbuild "static/$file" --minify --target=es2020 --charset=utf8 --legal-comments=none --outfile="/out/$file" || exit 1; \
+ flags=$(node -e 'console.log(require("./static/reader-resources.js").bundleArguments(process.argv[1]).join(" "))' "$file"); \
+ esbuild "static/$file" $flags --minify --target=es2020 --charset=utf8 --legal-comments=none --outfile="/out/$file" || exit 1; \
done \
&& node /build/build_static_assets.mjs \
&& esbuild /out/sw.js --minify --target=es2019 --charset=utf8 --legal-comments=none --outfile=/out/sw.min.js \
@@ -34,9 +35,9 @@ FROM python:3.11.17-slim AS search-index-build
WORKDIR /build
RUN apt-get update && apt-get install -y --no-install-recommends gcc libc6-dev \
&& rm -rf /var/lib/apt/lists/*
-COPY search_index_native.c .
-COPY scripts/build_search_index_native.py scripts/build_search_index_native.py
-RUN python scripts/build_search_index_native.py --output-dir /native
+COPY backend/native/search_index_native.c backend/native/search_index_native.c
+COPY scripts/build/build_search_index_native.py scripts/build/build_search_index_native.py
+RUN python scripts/build/build_search_index_native.py --output-dir /native
FROM python:3.11.17-slim
@@ -49,7 +50,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt
-COPY app.py data_formats.py proxy_streams.py record_metadata.py upstream_http.py search_execution.py search_indexes.py reader_assets.py reader_v3_catalog.py .
+COPY app.py .
+COPY backend/ backend/
COPY --from=search-index-build /native/ /app/
COPY --from=static-assets /out/ static/
COPY data/ data/
diff --git a/app.py b/app.py
index 0edb966dde4c902dd033379c399b76a450a8ee97..0d2859a09279a1578695258df2fd83f8c3439a99 100644
--- a/app.py
+++ b/app.py
@@ -1,2925 +1,3 @@
-import asyncio
-import gzip
-import hashlib
-import json
-import posixpath
-import random
-import re
-import sys
-import time
-import functools
-import threading
-import weakref
-from collections import OrderedDict
-from array import array
-from uuid import uuid4
-import io
-import zipfile
-import zlib
-from pathlib import Path
-from typing import Annotated, Literal, Optional, Sequence
-from urllib.parse import unquote, quote, urlparse, parse_qs
-import aiohttp
-from huggingface_hub import HfFileSystem
-import jieba
-from fastapi import FastAPI, HTTPException, Query, Request
-from fastapi.responses import FileResponse, HTMLResponse, JSONResponse, PlainTextResponse, Response
-from fastapi.staticfiles import StaticFiles
-from fastapi.middleware.cors import CORSMiddleware
-from fastapi.middleware.gzip import GZipMiddleware
-from pydantic import BaseModel, Field
-from contextlib import asynccontextmanager
-from proxy_streams import OwnedStreamingResponse, SharedRangeFlight, UpstreamLease
-from search_execution import SearchBusy, SearchExecutor
-from search_indexes import (
- NGRAM_MAGIC, WORD_INDEX_MAGIC, WORD_INDEX_TOKENIZER,
- decode_postings, decode_varint, load_ngram_index, load_word_index,
- resolve_postings,
-)
-from reader_assets import (
- READER_ASSET_FILENAMES, READER_BUCKET_PATH_RE, decode_reader_assets,
-)
-from reader_v3_catalog import load_reading_catalog, merge_sidecar as merge_v3_sidecar
-from data_formats import (
- decode_browser_entry, decode_folder_browser_data, decode_folder_tree_data,
- decode_search_payload, decode_tree_node,
-)
-from record_metadata import (
- READER_ASSET_MODES,
- add_reader_assets, build_browser_file_name, build_reader_asset_key,
- build_record_link, build_record_path_url, build_record_relative_path,
- build_txt_relative_path, enrich_browser_entry, enrich_reader_record,
- reader_asset_url, reader_extension_for_asset, reader_resolved_record,
-)
-from upstream_http import (
- ALLOWED_DOWNLOAD_HOSTS, ALLOWED_DOWNLOAD_HOST_SUFFIXES, MAX_REDIRECTS,
- READER_ASSET_PATH_RE, READER_ASSET_SOURCE_RE, VOICEOFML_READER_SOURCE_RE,
- decode_url_path, is_allowed_download_host, is_immutable_reader_asset,
- normalize_download_url, open_download_response, source_url_scope,
- validate_download_url, validate_path_safety, validate_reader_redirect_url,
- validate_reader_source_url, validate_voiceofml_redirect_url, validate_voiceofml_source_url,
-)
-MAX_ARCHIVE_NAME_LENGTH = 240
-DATA_PATH = Path("data/search_data.json.gz")
-NGRAM_PATHS = {2: Path("data/search_ngrams_2.bin.gz"), 3: Path("data/search_ngrams_3.bin.gz")}
-WORD_INDEX_PATH = Path("data/search_words.bin.gz")
-FOLDER_TREE_PATH = Path("data/folder_tree.json.gz")
-FOLDER_BROWSER_PATH = Path("data/folder_browser.json.gz")
-TXT_DIR = Path("txt")
-TXT_SPACE_RAW_BASE = "https://huggingface.co/spaces/VoiceOfML/Search/raw/main/txt"
-TXT_WARM_INTERVAL_SECONDS = 600
-TXT_WARM_IDLE_SECONDS = 600
-READER_ASSETS_URL = "https://huggingface.co/buckets/vomebook/reader-assets-v2/resolve/reader-index/reader_assets.json.gz"
-READER_ASSETS_CACHE_TTL_SECONDS = 300
-READER_ASSETS_MAX_COMPRESSED_BYTES = 5 * 1024 * 1024
-READER_ASSETS_MAX_DECOMPRESSED_BYTES = 50 * 1024 * 1024
-READER_ASSET_ROUTING_CONTRACT_VERSION = "reader-buckets-v2"
-records: list[dict] = []
-folder_tree_data: dict[str, list[dict]] = {}
-folder_browser_data: dict[str, dict[str, dict]] = {}
-word_index = {}
-folder_index = {}
-extension_counts = {}
-repo_counts = {}
-repo_list = []
-extension_list = []
-repo_extension_counts = {}
-vocab_by_len = {}
-repo_records_map = {}
-sorted_by_name = []
-sorted_by_size = []
-repo_sorted_by_name = {}
-repo_sorted_by_size = {}
-txt_record_indices = []
-repo_txt_record_indices = {}
-READER_EXTENSIONS = frozenset({"pdf", "djvu", "caj", "kdh", "epub", "mobi", "azw", "azw3", "fb2", "fbz", "html", "htm", "txt", "md", "markdown", "vcf", "ini", "jpg", "jpeg", "png", "gif", "bmp", "tif", "tiff", "webp", "mp3", "mp4", "wav", "m4a", "flac", "mov", "mpga", "wma", "ape", "amr"})
-reader_record_indices = []
-repo_reader_record_indices = {}
-reader_record_key_indices = {}
-reader_source_id_indices = {}
-_reader_indexes_ready = False
-_reader_indexes_lock = threading.Lock()
-_reader_index_build = None
-READER_INDEX_BATCH_RECORDS = 256
-READER_INDEX_BATCH_SECONDS = .008
-READER_INDEX_WARM_DELAY_SECONDS = 5
-READER_INDEX_IDLE_SECONDS = 5
-initial_payload_global = None
-initial_payload_by_repo = {}
-API_CACHE_TTL_SECONDS = 120
-RESPONSE_CACHE_BYTES_MAX = 16 * 1024 * 1024
-RESPONSE_CACHE_ENTRY_BYTES_MAX = 2 * 1024 * 1024
-_api_response_cache_lock = threading.Lock()
-api_response_cache = {}
-_data_generation = 0
-SEARCH_CACHE_TTL_SECONDS = 120
-_search_cache: dict[tuple, tuple[float, dict, int]] = {}
-_search_instance = uuid4().hex
-# Bump when matching, filtering, ordering or public record semantics change.
-SEARCH_SEMANTICS_VERSION = "hf-search-v2"
-_search_generation = None
-_search_logged_generation = None
-_search_order_cache = OrderedDict()
-SEARCH_ORDER_CACHE_MAX = 32
-SEARCH_ORDER_CACHE_BYTES_MAX = 16 * 1024 * 1024
-SEARCH_ORDER_CACHE_TTL_SECONDS = 600
-_search_anchor_cache = OrderedDict()
-SEARCH_ANCHOR_ORDER_MAX = 8
-SEARCH_ANCHOR_PER_ORDER_MAX = 8
-SEARCH_ANCHOR_KEY_MAX = 4096
-METADATA_CACHE_HEADERS = {
- "Cache-Control": "public, max-age=300, stale-while-revalidate=1800",
-}
-ngram_postings: dict[int, dict[str, bytes]] = {}
-ngram_posting_buffers: dict[int, bytes] = {}
-injected_html_cache: dict[str | None, str] = {}
-last_user_request_at = time.monotonic()
-reader_assets_cache = {"v": 1, "f": {}}
-reader_assets_cache_at = 0.0
-reader_assets_revision = hashlib.sha256(b'{"v":1,"f":{}}').hexdigest()
-reader_join_cache_key = None
-reader_join_cache = {"all": {}, "repos": {}}
+"""Uvicorn entry point for the search service."""
-def _cached(store, key, builder, ttl, limit):
- now = time.monotonic()
- cached = store.get(key)
- if cached and now - cached[0] < ttl:
- store[key] = store.pop(key)
- return cached[1]
- value = builder()
- store.pop(key, None)
- for old_key, entry in list(store.items()):
- if now - entry[0] >= ttl:
- del store[old_key]
- size = (len(value) if isinstance(value, bytes) else len(json.dumps(value, ensure_ascii=False, separators=(",", ":")).encode("utf-8"))) + len(str(key).encode("utf-8"))
- if size > RESPONSE_CACHE_ENTRY_BYTES_MAX:
- return value
- total = sum(entry[2] if len(entry) > 2 else len(json.dumps(entry[1]).encode("utf-8")) for entry in store.values())
- while store and (len(store) >= limit or total + size > RESPONSE_CACHE_BYTES_MAX):
- oldest_key = next(iter(store))
- entry = store.pop(oldest_key)
- total -= entry[2] if len(entry) > 2 else len(json.dumps(entry[1]).encode("utf-8"))
- if size <= RESPONSE_CACHE_BYTES_MAX:
- store[key] = (now, value, size)
- return value
-
-
-def cached_payload(key, builder):
- with _api_response_cache_lock:
- return _cached(api_response_cache, key, builder, API_CACHE_TTL_SECONDS, 500)
-
-
-def cached_json_response(key, builder, headers=None):
- # Encode once, measure the actual body and reuse it within the existing budget.
- body = cached_payload(("json", key), lambda: JSONResponse(builder()).body)
- return Response(body, media_type="application/json", headers=headers)
-
-
-async def get_reader_assets(*, wait: bool = True) -> dict:
- if reader_assets_cache_at != 0 and time.monotonic() - reader_assets_cache_at < READER_ASSETS_CACHE_TTL_SECONDS:
- return reader_assets_cache
- task = getattr(app.state, "reader_assets_task", None)
- if task is None or task.done():
- task = asyncio.create_task(refresh_reader_assets())
- app.state.reader_assets_task = task
- # Search can enrich with the last snapshot while an owned refresh runs.
- return await asyncio.shield(task) if wait else reader_assets_cache
-
-
-async def refresh_reader_assets() -> dict:
- global reader_assets_cache, reader_assets_cache_at, reader_assets_revision
- now = time.monotonic()
- if reader_assets_cache_at != 0 and now - reader_assets_cache_at < READER_ASSETS_CACHE_TTL_SECONDS:
- return reader_assets_cache
- async with app.state.reader_assets_lock:
- now = time.monotonic()
- if reader_assets_cache_at != 0 and now - reader_assets_cache_at < READER_ASSETS_CACHE_TTL_SECONDS:
- return reader_assets_cache
- response = None
- try:
- for url in (READER_ASSETS_URL,):
- response = await app.state.http_session.get(
- f"{url}?v={int(time.time())}",
- timeout=aiohttp.ClientTimeout(total=20, connect=8, sock_read=10),
- )
- if response.status == 200:
- break
- if response.status != 404:
- raise RuntimeError(f"reader assets HTTP {response.status}")
- response.release()
- response = None
- if response is None:
- raise RuntimeError("reader assets bucket and legacy sources unavailable")
- chunks = bytearray()
- async for chunk in response.content.iter_chunked(64 * 1024):
- chunks.extend(chunk)
- if len(chunks) > READER_ASSETS_MAX_COMPRESSED_BYTES:
- raise ValueError("reader assets response is too large")
- raw = bytes(chunks)
- decoded = await asyncio.to_thread(decode_reader_assets, raw)
- v3_catalog = await load_reading_catalog(app.state.http_session)
- if v3_catalog is None and reader_assets_cache.get("v3"):
- raise ValueError("previously published v3 pointer disappeared")
- decoded = merge_v3_sidecar(decoded, v3_catalog)
- reader_assets_cache = decoded
- revision_body = raw + json.dumps(decoded["v3"], sort_keys=True).encode() if v3_catalog else raw
- reader_assets_revision = hashlib.sha256(revision_body).hexdigest()
- reader_assets_cache_at = time.monotonic()
- except Exception as exc:
- print(f"reader assets refresh failed: {exc}")
- # Retain last-known assets, but retry failures after a short backoff.
- reader_assets_cache_at = time.monotonic() - READER_ASSETS_CACHE_TTL_SECONDS + 5
- finally:
- if response is not None:
- response.release()
- return reader_assets_cache
-
-def records_digest() -> bytes:
- digest = hashlib.sha256()
- for record in records:
- for value in (
- record.get("_file_lower", ""),
- record.get("_repo_lower", ""),
- record.get("_folder_path_lower", ""),
- ):
- digest.update(value.encode("utf-8"))
- digest.update(b"\0")
- return digest.digest()
-
-
-def word_postings(token: str) -> set[int]:
- value = word_index.get(token)
- return set(value) if value is not None else set()
-
-
-def word_posting_count(token: str) -> int:
- value = word_index.get(token)
- return len(value) if value is not None else 0
-
-def literal_candidates(query: str, repos=None) -> set[int] | None:
- width = 3 if len(query) >= 3 and ngram_postings.get(3) else 2
- postings = ngram_postings.get(width)
- if not postings or len(query) < width:
- return None
- entries = []
- for gram in dict.fromkeys(query[index:index + width] for index in range(len(query) - width + 1)):
- encoded = postings.get(gram)
- if encoded is None:
- return set()
- entries.append(encoded)
- # VNG3 entries carry validated counts; materialized test postings have lengths.
- entries.sort(key=lambda entry: entry[2] if isinstance(entry, tuple) else len(entry))
- candidates = None
- allowed = None
- if repos:
- allowed = set()
- for repo in repos:
- allowed.update(repo_records_map.get(repo, []))
- for encoded in entries:
- matches = resolve_postings(encoded, ngram_posting_buffers.get(width, b""))
- if candidates is None:
- candidates = matches
- if allowed is not None:
- candidates.intersection_update(allowed)
- else:
- candidates.intersection_update(matches)
- if not candidates:
- return set()
- return candidates or set()
-INITIAL_PAGE_SIZE = 100
-SIDEBAR_DATA_PATH = Path("data/sidebar")
-RECORD_KEY_MAP = {
- "r": "Repo",
- "f": "File",
- "e": "Extension",
- "d": "Folder",
- "s": "Size",
- "t": "HasTxt",
-}
-TREE_KEY_MAP = {
- "n": "name",
- "c": "count",
- "df": "hasDirectFiles",
- "ch": "children",
-}
-BROWSER_ENTRY_KEY_MAP = {
- "d": "folders",
- "f": "files",
-}
-BROWSER_FOLDER_ITEM_KEY_MAP = {
- "n": "name",
- "c": "count",
-}
-BROWSER_FILE_ITEM_KEY_MAP = {
- "n": "name",
- "e": "ext",
- "t": "hasTxt",
- "s": "size",
-}
-
-READER_SHORT_ID_RE = re.compile(r"^[0-9a-z]{13}$")
-# Only lowercase escapes, encoded unreserved ASCII and malformed percent signs
-# require normalization. Canonical reserved/UTF-8 escapes can pass through.
-READER_ESCAPE_NORMALIZATION_RE = re.compile(
- r"%(?:[0-9a-fA-F][a-f]|[a-f][0-9a-fA-F]|2[DE]|3[0-9]|4[1-9A-F]|5[0-9AF]|6[1-9A-F]|7[0-9AE])"
- r"|%(?![0-9a-fA-F]{2})"
-)
-
-
-def canonical_reader_source_url(value: str) -> str:
- # Match reader-contract.js byte normalization without decoding path separators
- # or treating an encoded percent sign as a second escape sequence.
- def path_replace(match):
- def token_replace(token_match):
- token = token_match.group()
- if len(token) == 3:
- char = chr(int(token[1:], 16))
- return char if char in "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._~-" else token.upper()
- return "%25"
- # Quote once, then skip already-canonical reserved and UTF-8 escapes.
- path = quote(match[2], safe="/%")
- return match[1] + READER_ESCAPE_NORMALIZATION_RE.sub(token_replace, path)
- return re.sub(r"^(https://huggingface\.co/datasets/VoiceOfML/)([^?#]*)", path_replace, value)
-
-
-def reader_short_id(value: str) -> str:
- digest = 1469598103934665603
- for byte in canonical_reader_source_url(value).encode("utf-8"):
- digest = ((digest ^ byte) * 1099511628211) & ((1 << 64) - 1)
- return _base36(digest).rjust(13, "0")
-
-
-def _base36(value: int) -> str:
- alphabet = "0123456789abcdefghijklmnopqrstuvwxyz"
- output = ""
- while value:
- value, remainder = divmod(value, 36)
- output = alphabet[remainder] + output
- return output or "0"
-
-
-def sanitize_archive_name(value) -> str:
- name = str(value or "file").replace("\\", "/").rsplit("/", 1)[-1]
- name = "".join(ch for ch in name if ord(ch) >= 32 and ch != "\x7f").strip()
- if not name or name in (".", ".."):
- name = "file"
- return name[:MAX_ARCHIVE_NAME_LENGTH]
-
-def tokenize(text: str) -> list[str]:
- tokens = []
- text_lower = text.lower()
- alpha_tokens = re.findall(r"[a-z0-9]+", text_lower)
- tokens.extend(alpha_tokens)
- chinese_text = re.sub(r"[a-z0-9\s]+", " ", text_lower)
- chinese_text = re.sub(r"[^\u4e00-\u9fff\u3400-\u4dbf\s]+", " ", chinese_text)
- chinese_tokens = jieba.lcut(chinese_text)
- tokens.extend([t.strip() for t in chinese_tokens if t.strip()])
- chinese_token_set = set(chinese_tokens)
- for ch in text_lower:
- if "\u4e00" <= ch <= "\u9fff" or "\u3400" <= ch <= "\u4dbf":
- if ch not in chinese_token_set:
- tokens.append(ch)
- return list(set(tokens))
-
-def compile_exact_matcher(query: str):
- if "*" not in query and "?" not in query:
- lower = query.lower()
- return lambda text: lower in text.lower()
- # Bitset NFA: each bit is a pattern position. Collapse adjacent stars so
- # epsilon closure needs one shift. No input can cause regex backtracking.
- pattern = re.sub(r"\*+", "*", query)
- stars = questions = 0
- literals = {}
- for index, char in enumerate(pattern):
- bit = 1 << index
- if char == "*":
- stars |= bit
- elif char == "?":
- questions |= bit
- else:
- literals[char] = literals.get(char, 0) | bit
- # Single-character regexes retain Python IGNORECASE's Unicode semantics.
- literals = [(re.compile(re.escape(char), re.IGNORECASE), bits)
- for char, bits in literals.items()]
- masks = {}
- end = 1 << len(pattern)
- start = 1 | ((1 & stars) << 1)
-
- def matches(text):
- # Every literal is mandatory; C-level scans reject unrelated fields early.
- if any(regex.search(text) is None for regex, _ in literals):
- return False
- active = start
- if active & end:
- return True
- for char in text:
- mask = masks.get(char)
- if mask is None:
- mask = sum(bits for regex, bits in literals if regex.fullmatch(char))
- if len(masks) >= 512:
- masks.clear()
- masks[char] = mask
- wildcard = char != "\n"
- active = ((active & (mask | (questions if wildcard else 0))) << 1) | (
- active & stars if wildcard else 0) | 1
- active |= (active & stars) << 1
- if active & end:
- return True
- return False
- return matches
-
-
-def matches_exact_query(text: str, query: str) -> bool:
- return compile_exact_matcher(query)(text)
-
-def edit_distance(s1: str, s2: str, max_dist: int = 2) -> int:
- if abs(len(s1) - len(s2)) > max_dist:
- return 999
- prev = list(range(len(s2) + 1))
- for i, c1 in enumerate(s1):
- curr = [i + 1]
- for j, c2 in enumerate(s2):
- cost = 0 if c1 == c2 else 1
- curr.append(min(prev[j + 1] + 1, curr[j] + 1, prev[j] + cost))
- if min(curr) > max_dist:
- return 999
- prev = curr
- return prev[-1]
-
-def could_be_fuzzy(tok: str, word: str, max_dist: int = 2) -> bool:
- if abs(len(tok) - len(word)) > max_dist:
- return False
- if len(tok) < 4:
- return True
- counts: dict[str, int] = {}
- for ch in tok:
- counts[ch] = counts.get(ch, 0) + 1
- diff = 0
- for ch in word:
- if counts.get(ch, 0) > 0:
- counts[ch] -= 1
- else:
- diff += 1
- if diff > 2 * max_dist:
- return False
- for remaining in counts.values():
- diff += remaining
- if diff > 2 * max_dist:
- return False
- return True
-
-
-def is_fuzzy_match(token: str, word: str) -> bool:
- return could_be_fuzzy(token, word, 2) and edit_distance(token, word, max_dist=2) <= 2
-
-
-def matches_file_token(record: dict, token: str) -> bool:
- return token in record.get("_file_lower", "") or token in record.get("_repo_lower", "")
-
-
-def scoped_word_postings(token: str, search_folders: bool):
- postings = word_postings(token)
- if search_folders:
- return postings
- return {index for index in postings if matches_file_token(records[index], token)}
-
-
-def matches_file_query(record: dict, tokens, fuzzy_tokens) -> bool:
- file_tokens = tokenize(record.get("_file_lower", "")) if fuzzy_tokens else []
- return all(matches_file_token(record, token) or (
- token in fuzzy_tokens and any(is_fuzzy_match(token, word) for word in file_tokens)
- ) for token in tokens)
-
-
-def build_word_index():
- global word_index, vocab_by_len
- word_index = {}
- for idx, rec in enumerate(records):
- file_name = rec.get("File", "")
- folders = rec.get("Folder", [])
- for tok in tokenize(" ".join([file_name] + folders)):
- word_index.setdefault(tok, set()).add(idx)
- vocab_by_len = {}
- for word in word_index:
- vocab_by_len.setdefault(len(word), []).append(word)
-
-
-def build_vocab_by_len():
- global vocab_by_len
- vocab_by_len = {}
- for word in word_index:
- vocab_by_len.setdefault(len(word), []).append(word)
-
-
-def _advance_reader_indexes(limit=READER_INDEX_BATCH_RECORDS, budget=READER_INDEX_BATCH_SECONDS):
- global reader_record_indices, repo_reader_record_indices
- global reader_record_key_indices, reader_source_id_indices, _reader_indexes_ready
- global _reader_index_build
- with _reader_indexes_lock:
- if _reader_indexes_ready:
- return True
- if _reader_index_build is None or _reader_index_build["generation"] != _data_generation:
- _reader_index_build = {"generation": _data_generation, "records": records, "next": 0,
- "indices": [], "repos": {}, "keys": {}, "ids": {}}
- build = _reader_index_build
- deadline = time.perf_counter() + budget
- end = min(len(build["records"]), build["next"] + limit)
- while build["next"] < end:
- idx = build["next"]
- rec = build["records"][idx]
- repo = rec.get("Repo", "")
- build["keys"][build_reader_asset_key(rec)] = idx
- source_id = reader_short_id(rec.get("Link") or build_record_link(rec))
- build["ids"].setdefault(source_id, []).append(idx)
- if rec.get("_ext_lower", "") in READER_EXTENSIONS:
- build["indices"].append(idx)
- build["repos"].setdefault(repo, []).append(idx)
- build["next"] += 1
- if time.perf_counter() >= deadline:
- break
- if build["next"] < len(build["records"]):
- return False
- if build["generation"] != _data_generation or build["records"] is not records:
- _reader_index_build = None
- return False
- reader_record_indices = array('I', build["indices"])
- repo_reader_record_indices = {key: array('I', indices) for key, indices in build["repos"].items()}
- reader_record_key_indices = build["keys"]
- reader_source_id_indices = build["ids"]
- _reader_indexes_ready = True
- _reader_index_build = None
- return True
-
-
-def _build_reader_indexes():
- while not _advance_reader_indexes(len(records), float("inf")):
- pass
-
-
-def ensure_reader_indexes():
- global _reader_indexes_ready
- if _reader_indexes_ready:
- return
- _build_reader_indexes()
-
-
-async def ensure_reader_indexes_async():
- """Join the shared background build, or start one for an early Reader request."""
- if _reader_indexes_ready:
- return
- task = getattr(app.state, "reader_index_build_task", None)
- if task is None or task.done():
- async def build():
- while not _reader_indexes_ready:
- executor = getattr(app.state, "search_executor", None)
- if executor is None:
- await asyncio.to_thread(_advance_reader_indexes)
- else:
- try:
- await executor.run(_advance_reader_indexes, background=True)
- except SearchBusy:
- await asyncio.sleep(.02)
- continue
- await asyncio.sleep(0)
- task = asyncio.create_task(build())
- app.state.reader_index_build_task = task
- await asyncio.shield(task)
-
-
-async def warm_reader_indexes():
- """Prepare Reader routing during a quiet period without delaying startup."""
- await asyncio.sleep(READER_INDEX_WARM_DELAY_SECONDS)
- while not _reader_indexes_ready:
- while time.monotonic() - last_user_request_at < READER_INDEX_IDLE_SECONDS:
- await asyncio.sleep(READER_INDEX_IDLE_SECONDS)
- try:
- await ensure_reader_indexes_async()
- except SearchBusy:
- await asyncio.sleep(READER_INDEX_IDLE_SECONDS)
-
-
-def build_indexes(build_words=True, build_reader=True):
- global _reader_index_build
- global reader_source_id_indices, _search_generation
- global _data_generation, word_index, folder_index, extension_counts, repo_counts, repo_list, extension_list, repo_extension_counts, repo_records_map, sorted_by_name, sorted_by_size, repo_sorted_by_name, repo_sorted_by_size, txt_record_indices, repo_txt_record_indices, reader_record_indices, repo_reader_record_indices, reader_record_key_indices, _reader_indexes_ready
- _data_generation += 1
- _search_generation = None
- _search_order_cache.clear()
- _search_anchor_cache.clear()
- _search_cache.clear()
- word_index = {}
- folder_index = {}
- extension_counts = {}
- repo_extension_counts = {}
- repo_counts = {}
- repo_records_map = {}
- repo_sorted_by_name = {}
- repo_sorted_by_size = {}
- txt_record_indices = []
- repo_txt_record_indices = {}
- reader_record_indices = []
- repo_reader_record_indices = {}
- reader_record_key_indices = {}
- reader_source_id_indices = {}
- _reader_indexes_ready = False
- _reader_index_build = None
- for idx, rec in enumerate(records):
- repo = rec.get("Repo", "")
- repo_counts[repo] = repo_counts.get(repo, 0) + 1
- ext = (rec.get("Extension") or "").lower()
- if ext:
- extension_counts[ext] = extension_counts.get(ext, 0) + 1
- repo_ext = repo_extension_counts.setdefault(repo, {})
- repo_ext[ext] = repo_ext.get(ext, 0) + 1
- file_name = rec.get("File", "")
- folders = rec.get("Folder", [])
- if repo not in folder_index:
- folder_index[repo] = {}
- for depth in range(len(folders) + 1):
- folder_path = "/".join(folders[:depth]) if depth > 0 else ""
- folder_index[repo].setdefault(folder_path, []).append(idx)
- repo_records_map.setdefault(repo, []).append(idx)
- if rec.get("HasTxt"):
- txt_record_indices.append(idx)
- repo_txt_record_indices.setdefault(repo, []).append(idx)
- folders_lower = [f.lower() for f in (folders or [])]
- rec["_file_lower"] = file_name.lower()
- rec["_folders_lower"] = folders_lower
- rec["_folder_text"] = " ".join(folders_lower)
- rec["_folder_path_lower"] = "/".join(folders_lower)
- rec["_folder_path"] = "/".join(folders)
- rec["_ext_lower"] = ext
- rec["_repo_lower"] = repo.lower()
- repo_list = sorted(
- [{"name": r, "count": c} for r, c in repo_counts.items()],
- key=lambda x: x["name"],
- )
- if build_words:
- build_word_index()
- else:
- word_index = {}
- global vocab_by_len
- vocab_by_len = {}
- extension_list = sorted(extension_counts.keys())
- all_indices = list(range(len(records)))
- sorted_by_name = sorted(all_indices, key=lambda i: (records[i].get("File") or "").lower())
- sorted_by_size = sorted(all_indices, key=lambda i: (
- -(records[i].get("Size") if isinstance(records[i].get("Size"), (int, float)) else 0),
- (records[i].get("File") or "").lower(),
- ))
- for repo, repo_indices in repo_records_map.items():
- repo_sorted_by_name[repo] = sorted(repo_indices, key=lambda i: (records[i].get("File") or "").lower())
- repo_sorted_by_size[repo] = sorted(repo_indices, key=lambda i: (
- -(records[i].get("Size") if isinstance(records[i].get("Size"), (int, float)) else 0),
- (records[i].get("File") or "").lower(),
- ))
- # These complete, ordered postings are read-only after publication.
- # Packed IDs avoid retaining a Python pointer (and often an int) per slot.
- sorted_by_name = array('I', sorted_by_name)
- sorted_by_size = array('I', sorted_by_size)
- txt_record_indices = array('I', txt_record_indices)
- for mapping in (repo_records_map, repo_sorted_by_name, repo_sorted_by_size,
- repo_txt_record_indices, *folder_index.values()):
- for key, indices in mapping.items():
- mapping[key] = array('I', indices)
- if build_reader:
- _build_reader_indexes()
-
-def search_data_generation(raw: bytes, index_paths) -> str:
- digest = hashlib.sha256()
- digest.update(SEARCH_SEMANTICS_VERSION.encode())
- digest.update(str(sys.version_info[:3]).encode())
- digest.update(jieba.__version__.encode())
- digest.update(hashlib.sha256(raw).digest())
- for path in index_paths:
- # Index content matters too; file timestamps and process IDs do not.
- digest.update(path.name.encode())
- digest.update(hashlib.sha256(path.read_bytes()).digest() if path.exists() else b"missing")
- return "api:" + digest.hexdigest()
-
-
-def load_data():
- global records, folder_tree_data, folder_browser_data, ngram_postings, ngram_posting_buffers, word_index, _search_generation
- start = time.perf_counter()
- stage_start = start
- timings = {}
- if DATA_PATH.exists():
- raw = gzip.decompress(DATA_PATH.read_bytes())
- records = decode_search_payload(json.loads(raw.decode("utf-8")))
- else:
- print("⚠ 数据文件不存在")
- records = []
- return
- if FOLDER_TREE_PATH.exists():
- folder_tree_data = decode_folder_tree_data(json.loads(gzip.decompress(FOLDER_TREE_PATH.read_bytes()).decode("utf-8")))
- else:
- folder_tree_data = {}
- if FOLDER_BROWSER_PATH.exists():
- folder_browser_data = decode_folder_browser_data(json.loads(gzip.decompress(FOLDER_BROWSER_PATH.read_bytes()).decode("utf-8")))
- else:
- folder_browser_data = {}
- timings["data_ms"] = round((time.perf_counter() - stage_start) * 1000, 2)
- print(f"📖 已加载 {len(records)} 条记录 ({time.perf_counter() - start:.2f}s)")
- stage_start = time.perf_counter()
- # Reader ID/pool indexes are built on first Reader use; normal search does
- # not need to scan every record to prepare them during cold startup.
- build_indexes(build_words=False, build_reader=False)
- digest = records_digest()
- timings["metadata_ms"] = round((time.perf_counter() - stage_start) * 1000, 2)
- stage_start = time.perf_counter()
- word_index = load_word_index(WORD_INDEX_PATH, len(records), digest)
- word_index_source = "generated" if word_index else "rebuilt"
- if word_index:
- build_vocab_by_len()
- else:
- build_word_index()
- timings["words_ms"] = round((time.perf_counter() - stage_start) * 1000, 2)
- stage_start = time.perf_counter()
- loaded_ngrams = {
- width: load_ngram_index(path, width, len(records), digest)
- for width, path in NGRAM_PATHS.items()
- }
- ngram_postings = {width: loaded[0] for width, loaded in loaded_ngrams.items()}
- ngram_posting_buffers = {width: loaded[1] for width, loaded in loaded_ngrams.items()}
- timings["ngrams_ms"] = round((time.perf_counter() - stage_start) * 1000, 2)
- stage_start = time.perf_counter()
- _search_generation = search_data_generation(raw, [WORD_INDEX_PATH, *NGRAM_PATHS.values()])
- timings["fingerprint_ms"] = round((time.perf_counter() - stage_start) * 1000, 2)
- print(f"🧭 已加载词索引: {len(word_index)}")
- print(f"🔎 已加载 ngram 索引: 2={len(ngram_postings[2])}, 3={len(ngram_postings[3])}")
- build_initial_payloads()
- precompute_injected_html()
- del raw
- stage_start = time.perf_counter()
- # A generated postings file avoids corpus tokenization, but query tokenization
- # still needs jieba's dictionary. Prepare it before accepting the first query,
- # after releasing the decoded source and finishing index-loading temporaries.
- jieba.initialize()
- timings["tokenizer_ms"] = round((time.perf_counter() - stage_start) * 1000, 2)
- timings["total_ms"] = round((time.perf_counter() - start) * 1000, 2)
- print("search_performance " + json.dumps({"stage": "startup", "word_index": word_index_source, **timings}))
-
-def score_record(rec_idx, query_tokens, search_folders=True):
- rec = records[rec_idx]
- score = 0
- file_name = rec.get("_file_lower", "")
- repo = rec.get("_repo_lower", "")
- folder_text = rec.get("_folder_text", "")
- for tok in query_tokens:
- if tok in file_name:
- score += 3
- if search_folders and tok in folder_text:
- score += 2
- if tok in repo:
- score += 1
- return score
-
-def apply_filters(indices, repos=None, extensions=None, folders=None, min_size=None, max_size=None, folder_match_mode="prefix"):
- repos_set = set(repos) if repos else None
- extensions_lower = {e.lower() for e in extensions} if extensions else None
- folder_set = {f.strip("/") for f in folders} if folders else None
- if folder_match_mode != "exact" and folder_set and "" in folder_set:
- # Unlike the Worker, an empty API prefix includes the complete scope.
- folder_set = None
- folder_prefix = next(iter(folder_set)) + "/" if folder_set and len(folder_set) == 1 and folder_match_mode != "exact" else None
- result = []
- for idx in indices:
- rec = records[idx]
- if repos_set and rec.get("Repo") not in repos_set:
- continue
- if extensions_lower:
- if rec.get("_ext_lower", "") not in extensions_lower:
- continue
- if folder_set:
- rec_path = rec["_folder_path"] if "_folder_path" in rec else "/".join(rec.get("Folder", []) or [])
- matched = rec_path in folder_set
- if not matched and folder_match_mode != "exact":
- if folder_prefix is not None:
- matched = rec_path.startswith(folder_prefix)
- else:
- end = rec_path.rfind("/")
- while end > 0:
- if rec_path[:end] in folder_set:
- matched = True
- break
- end = rec_path.rfind("/", 0, end)
- if not matched:
- continue
- size = rec.get("Size")
- if size and isinstance(size, (int, float)) and size > 0:
- if min_size is not None and size < min_size:
- continue
- if max_size is not None and size > max_size:
- continue
- result.append(idx)
- return result
-
-def public_record(record: dict) -> dict:
- item = {
- key: value
- for key, value in record.items()
- if isinstance(key, str) and not key.startswith("_")
- }
- item.setdefault("Repo", "")
- item.setdefault("File", "")
- item.setdefault("Extension", "")
- item.setdefault("Folder", [])
- item.setdefault("Size", "")
- item.setdefault("HasTxt", False)
- item["Link"] = record.get("Link") or build_record_link(record)
- item["Path"] = record.get("Path") or build_record_path_url(record)
- return item
-
-
-def trim_results(raw):
- keys = ("Repo", "File", "Extension", "Link", "Path", "Folder", "Size", "HasTxt")
- return [
- {key: public.get(key, "" if key != "Folder" else []) for key in keys}
- for public in (public_record(record) for record in raw)
- ]
-
-def sort_key_name(idx: int):
- return (records[idx].get("File") or "").lower()
-
-def sort_key_size(idx: int):
- return (
- -(records[idx].get("Size") if isinstance(records[idx].get("Size"), (int, float)) else 0),
- (records[idx].get("File") or "").lower(),
- )
-
-def paginate_indices(indices: Sequence[int], page: int, page_size: int) -> tuple[int, list[dict]]:
- total = len(indices)
- start = (page - 1) * page_size
- return total, [records[i] for i in indices[start:start + page_size]]
-
-def get_initial_indices(repos=None) -> Sequence[int]:
- if repos:
- indices = []
- for repo in dict.fromkeys(repos):
- indices.extend(repo_records_map.get(repo, []))
- return indices
- return range(len(records))
-
-def search_generation() -> str:
- return _search_generation or f"api:{_search_instance}:{_data_generation}"
-
-
-def build_response(results: list[dict], total: int, page: int, page_size: int) -> dict:
- return {
- "results": trim_results(results),
- "total": total,
- "page": page,
- "page_size": page_size,
- "generation": search_generation(),
- }
-
-
-def trim_initial_results(raw: list[dict]) -> list[dict]:
- keys = ("Repo", "File", "Extension", "Folder", "Size", "HasTxt")
- return [
- {key: record.get(key, [] if key == "Folder" else "") for key in keys}
- for record in raw
- ]
-
-def build_initial_payload(repo: str | None = None) -> dict:
- if repo:
- indices = repo_records_map.get(repo, [])
- result_records = [records[i] for i in indices[:INITIAL_PAGE_SIZE]]
- mode = "repo"
- total = len(indices)
- else:
- result_records = records[:INITIAL_PAGE_SIZE]
- mode = "global"
- total = len(records)
- return {
- "mode": mode,
- "repo": repo,
- "sort": "relevance",
- "page": 1,
- "page_size": INITIAL_PAGE_SIZE,
- "total": total,
- "results": trim_initial_results(result_records),
- "generation": search_generation(),
- }
-
-def build_initial_payloads():
- global initial_payload_global, initial_payload_by_repo
- initial_payload_global = build_initial_payload(None)
- initial_payload_by_repo = {
- repo: build_initial_payload(repo)
- for repo in repo_records_map
- }
-
-def precompute_injected_html():
- global injected_html_cache
- index_path = Path("static/index.html")
- if not index_path.exists():
- injected_html_cache = {}
- return
- html = index_path.read_text(encoding="utf-8")
- injected_html_cache = {None: inject_initial_payload(html, None)}
- for repo in repo_records_map:
- injected_html_cache[repo] = inject_initial_payload(html, repo.split("/")[-1])
-
-def load_sidebar_payload(path: Path):
- if not path.exists() or not path.is_file():
- return None
- try:
- return json.loads(path.read_text(encoding="utf-8"))
- except Exception:
- return None
-
-def build_global_bootstrap():
- sidebar = load_sidebar_payload(SIDEBAR_DATA_PATH / "global.json") or {}
- return {
- "generation": _data_generation,
- "repos": repo_list,
- "extensions": [{"name": name, "count": extension_counts[name]} for name in sorted(extension_counts)],
- "random_txt": {"available": bool(txt_record_indices), "count": len(txt_record_indices)},
- "sidebar": sidebar,
- }
-
-def inject_initial_payload(html: str, repo_short: str | None = None) -> str:
- repo = f"VoiceOfML/{repo_short}" if repo_short else None
- payload = initial_payload_by_repo.get(repo) if repo else initial_payload_global
- if not payload:
- return html
- script = (
- '"
- )
- app_script = re.search(r'', html)
- if app_script:
- return html[:app_script.start()] + script + "\n" + app_script.group(0) + html[app_script.end():]
- return html.replace("", script + "\n")
-
-def apply_mixed_folder_filters(indices, self_folders=None, subtree_folders=None):
- self_set = {str(path).strip("/") for path in (self_folders or [])}
- subtree_set = {str(path).strip("/") for path in (subtree_folders or []) if str(path).strip("/")}
- if not self_set and not subtree_set:
- return indices
- result = []
- for idx in indices:
- rec = records[idx]
- rec_folders = rec.get("Folder", []) or []
- rec_path = "/".join(rec_folders)
- if rec_path in self_set:
- result.append(idx)
- continue
- matched = False
- for depth in range(1, len(rec_folders) + 1):
- prefix = "/".join(rec_folders[:depth])
- if prefix in subtree_set:
- matched = True
- break
- if matched:
- result.append(idx)
- return result
-
-def search(q="", repos=None, extensions=None, folders=None, min_size=None, max_size=None, page=1, page_size=100, search_folders=True, sort="relevance", exact=False, folder_match_mode="prefix", folder_selfs=None, folder_subtrees=None, anchor_id=None):
- global _search_logged_generation
- started = time.perf_counter()
- q = q.strip()
- order_key = _search_order_key(q, repos, extensions, folders, min_size, max_size,
- search_folders, sort, exact, folder_match_mode,
- folder_selfs, folder_subtrees)
- key = order_key + (page, page_size, anchor_id)
- cached_page = _search_cache.get(key)
- cache_hit = bool(cached_page and time.monotonic() - cached_page[0] < SEARCH_CACHE_TTL_SECONDS)
- result = _cached(_search_cache, key,
- lambda: _search_uncached(q, repos, extensions, folders,
- min_size, max_size, page, page_size,
- search_folders, sort, exact,
- folder_match_mode, folder_selfs,
- folder_subtrees, anchor_id),
- SEARCH_CACHE_TTL_SECONDS, 500)
- cached_order = _search_order_cache.get(order_key)
- if cached_order is not None and time.monotonic() - cached_order[0] < SEARCH_ORDER_CACHE_TTL_SECONDS:
- _search_order_cache[order_key] = (time.monotonic(), cached_order[1])
- _search_order_cache.move_to_end(order_key)
- duration_ms = (time.perf_counter() - started) * 1000
- first = _search_logged_generation != _data_generation
- if first or duration_ms >= 250:
- _search_logged_generation = _data_generation
- # Operational timing only; never log queries, paths, anchors or URLs.
- print("search_performance " + json.dumps({"stage": "search", "first": first,
- "duration_ms": round(duration_ms, 2), "page_cache_hit": cache_hit}))
- return result
-
-
-def _search_order_key(q, repos, extensions, folders, min_size, max_size, search_folders,
- sort, exact, folder_match_mode, folder_selfs, folder_subtrees):
- return (_data_generation, q.strip(), tuple(repos or ()), tuple(extensions or ()),
- tuple(folders or ()), min_size, max_size, search_folders, sort, exact,
- folder_match_mode, tuple(folder_selfs or ()), tuple(folder_subtrees or ()))
-
-
-def _get_search_order(q, repos, extensions, folders, min_size, max_size, search_folders,
- sort, exact, folder_match_mode, folder_selfs, folder_subtrees):
- key = _search_order_key(q, repos, extensions, folders, min_size, max_size,
- search_folders, sort, exact, folder_match_mode,
- folder_selfs, folder_subtrees)
- now = time.monotonic()
- for old_key, (created, _) in list(_search_order_cache.items()):
- if now - created >= SEARCH_ORDER_CACHE_TTL_SECONDS:
- del _search_order_cache[old_key]
- cached = _search_order_cache.get(key)
- if cached is not None:
- _search_order_cache[key] = (now, cached[1])
- _search_order_cache.move_to_end(key)
- return cached[1]
- indices = _search_order_uncached(q, repos, extensions, folders, min_size, max_size,
- search_folders, sort, exact, folder_match_mode,
- folder_selfs, folder_subtrees)
- # Oversized queries still return all results; only caching is skipped.
- packed = indices if isinstance(indices, array) else array('I', indices)
- size = len(packed) * packed.itemsize
- if size <= SEARCH_ORDER_CACHE_BYTES_MAX:
- while _search_order_cache and (len(_search_order_cache) >= SEARCH_ORDER_CACHE_MAX or
- sum(len(v[1]) * v[1].itemsize for v in _search_order_cache.values()) + size > SEARCH_ORDER_CACHE_BYTES_MAX):
- _search_order_cache.popitem(last=False)
- _search_order_cache[key] = (now, packed)
- return packed
-
-
-def _find_search_anchor(indices, anchor_id):
- owner = id(indices)
- cached = _search_anchor_cache.get(owner)
- if cached is not None and cached[0]() is not indices:
- del _search_anchor_cache[owner]
- cached = None
- if cached is not None and anchor_id in cached[1]:
- cached[1].move_to_end(anchor_id)
- _search_anchor_cache.move_to_end(owner)
- return cached[1][anchor_id]
- anchor_repo, separator, anchor_path = anchor_id.partition('\0')
- index = -1
- if separator:
- for offset, idx in enumerate(indices):
- rec = records[idx]
- repo = str(rec.get('Repo', ''))
- if repo == anchor_repo:
- matches = build_record_relative_path(rec) == anchor_path
- elif '\0' in repo and anchor_id.startswith(f'{repo}\0'):
- # Preserve raw-key matching for injected repository names.
- matches = f'{repo}\0{build_record_relative_path(rec)}' == anchor_id
- else:
- continue
- if matches:
- index = offset
- break
- if len(anchor_id) <= SEARCH_ANCHOR_KEY_MAX:
- lookups = cached[1] if cached is not None else OrderedDict()
- lookups[anchor_id] = index
- while len(lookups) > SEARCH_ANCHOR_PER_ORDER_MAX:
- lookups.popitem(last=False)
- # Do not keep a complete order alive after its normal cache eviction.
- _search_anchor_cache[owner] = (weakref.ref(indices), lookups)
- _search_anchor_cache.move_to_end(owner)
- while len(_search_anchor_cache) > SEARCH_ANCHOR_ORDER_MAX:
- _search_anchor_cache.popitem(last=False)
- return index
-
-
-def _search_uncached(q="", repos=None, extensions=None, folders=None, min_size=None, max_size=None, page=1, page_size=100, search_folders=True, sort="relevance", exact=False, folder_match_mode="prefix", folder_selfs=None, folder_subtrees=None, anchor_id=None):
- indices = _get_search_order(q, repos, extensions, folders, min_size, max_size,
- search_folders, sort, exact, folder_match_mode,
- folder_selfs, folder_subtrees)
- total, results = paginate_indices(indices, page, page_size)
- response = build_response(results, total, page, page_size)
- if anchor_id is not None:
- response['anchor_index'] = _find_search_anchor(indices, anchor_id)
- return response
-
-
-def _search_order_uncached(q="", repos=None, extensions=None, folders=None, min_size=None, max_size=None, search_folders=True, sort="relevance", exact=False, folder_match_mode="prefix", folder_selfs=None, folder_subtrees=None):
- q = q.strip()
-
- if exact and q:
- matched_indices = []
- _q_lower = q.lower()
- _is_match = compile_exact_matcher(q)
- candidate_indices = get_initial_indices(repos)
- if set(q) <= {"*"}:
- # A star-only pattern matches every string, including an empty field.
- matched_indices = list(candidate_indices)
- elif "*" not in q and "?" not in q:
- candidates = literal_candidates(_q_lower, repos)
- if candidates is not None:
- candidate_indices = sorted(candidates)
- if not matched_indices:
- for idx in candidate_indices:
- rec = records[idx]
- file_name = rec.get("_file_lower", "")
- folder_path = rec.get("_folder_path_lower", "")
- repo_name = rec.get("_repo_lower", "")
- if _is_match(file_name) or _is_match(repo_name) or (search_folders and _is_match(folder_path)):
- matched_indices.append(idx)
- if folder_match_mode == "mixed":
- filtered = apply_mixed_folder_filters(matched_indices, folder_selfs, folder_subtrees)
- filtered = apply_filters(filtered, None, extensions, None, min_size, max_size)
- else:
- filtered = apply_filters(matched_indices, None, extensions, folders, min_size, max_size, folder_match_mode)
- if sort == "name":
- filtered.sort(key=sort_key_name)
- elif sort == "size":
- filtered.sort(key=sort_key_size)
- return filtered
- if not q:
- has_record_filters = extensions or folders or min_size is not None or max_size is not None or folder_match_mode == "mixed"
- can_use_precomputed = not has_record_filters and (not repos or len(repos) == 1)
- if can_use_precomputed:
- if repos and len(repos) == 1:
- repo = repos[0]
- if sort == "name":
- indices = repo_sorted_by_name.get(repo, [])
- elif sort == "size":
- indices = repo_sorted_by_size.get(repo, [])
- else:
- indices = repo_records_map.get(repo, [])
- else:
- if sort == "name":
- indices = sorted_by_name
- elif sort == "size":
- indices = sorted_by_size
- else:
- indices = range(len(records))
- return indices
- else:
- indices = get_initial_indices(repos)
- if folder_match_mode == "mixed":
- filtered = apply_mixed_folder_filters(indices, folder_selfs, folder_subtrees)
- filtered = apply_filters(filtered, repos, extensions, None, min_size, max_size)
- else:
- filtered = apply_filters(indices, repos, extensions, folders, min_size, max_size, folder_match_mode)
- if sort == "name":
- filtered.sort(key=sort_key_name)
- elif sort == "size":
- filtered.sort(key=sort_key_size)
- return filtered
- query_tokens = tokenize(q)
- fuzzy_candidate_limit = 200 if len(query_tokens) == 1 else None
-
- posting_cache = {}
- def query_postings(token):
- if token not in posting_cache:
- posting_cache[token] = scoped_word_postings(token, search_folders=search_folders)
- return posting_cache[token]
-
- exact_constraint = None
- for tok in query_tokens:
- if tok not in word_index:
- continue
- postings = query_postings(tok)
- if not postings:
- continue
- exact_constraint = set(postings) if exact_constraint is None else exact_constraint.intersection(postings)
- token_candidates = []
- fuzzy_tokens = set()
- for tok in query_tokens:
- candidates = query_postings(tok) if tok in word_index else set()
- if not candidates:
- fuzzy_tokens.add(tok)
- fuzzy_candidates = set()
- if exact_constraint is not None:
- for idx in sorted(exact_constraint):
- rec = records[idx]
- record_tokens = tokenize(" ".join([rec.get("File", ""),
- *((rec.get("Folder", []) or []) if search_folders else [])]))
- if any(is_fuzzy_match(tok, word) for word in record_tokens):
- fuzzy_candidates.add(idx)
- if fuzzy_candidate_limit is not None and len(fuzzy_candidates) >= fuzzy_candidate_limit:
- break
- else:
- tok_len = len(tok)
- for delta in (-2, -1, 0, 1, 2):
- for vocab_word in vocab_by_len.get(tok_len + delta, []):
- if is_fuzzy_match(tok, vocab_word):
- fuzzy_candidates.update(query_postings(vocab_word))
- if fuzzy_candidate_limit is not None and len(fuzzy_candidates) >= fuzzy_candidate_limit:
- break
- if fuzzy_candidate_limit is not None and len(fuzzy_candidates) >= fuzzy_candidate_limit:
- break
- candidates = fuzzy_candidates
- if not candidates:
- token_candidates = []
- break
- token_candidates.append(candidates)
- if token_candidates:
- token_candidates.sort(key=len)
- matched = set(token_candidates[0])
- for candidates in token_candidates[1:]:
- matched.intersection_update(candidates)
- if not matched:
- break
- all_matched = list(matched)
- else:
- all_matched = []
- literal_query = q.lower()
- if literal_query:
- literal_matches = []
- initial_indices = get_initial_indices(repos)
- candidates = literal_candidates(literal_query, repos)
- if candidates is not None:
- initial_indices = sorted(candidates)
- skip = set(all_matched)
- for idx in initial_indices:
- if skip and idx in skip:
- continue
- rec = records[idx]
- file_name = rec.get("_file_lower", "")
- folder_path = rec.get("_folder_path_lower", "")
- repo_name = rec.get("_repo_lower", "")
- if literal_query in file_name or literal_query in repo_name or (search_folders and literal_query in folder_path):
- literal_matches.append(idx)
- if literal_matches:
- all_matched.extend(idx for idx in literal_matches if idx not in skip)
- if not search_folders and query_tokens and all_matched:
- all_matched = [idx for idx in all_matched
- if matches_file_query(records[idx], query_tokens, fuzzy_tokens)]
- if folder_match_mode == "mixed":
- filtered = apply_mixed_folder_filters(all_matched, folder_selfs, folder_subtrees)
- filtered = apply_filters(filtered, repos, extensions, None, min_size, max_size)
- else:
- filtered = apply_filters(all_matched, repos, extensions, folders, min_size, max_size, folder_match_mode)
- if sort == "name":
- filtered.sort(key=sort_key_name)
- return filtered
- if sort == "size":
- filtered.sort(key=sort_key_size)
- return filtered
- scored = [(idx, score_record(idx, query_tokens, search_folders)) for idx in filtered]
- if sort == "relevance":
- scored.sort(key=lambda x: (-x[1], x[0]))
- return [idx for idx, _ in scored]
-
-def build_filter_folder_tree(repo: str) -> list[dict]:
- if repo in folder_tree_data:
- return folder_tree_data[repo]
- if repo not in folder_index:
- return []
- repo_name_short = repo.split("/")[-1]
- tree_root = {
- "name": repo_name_short,
- "path": "",
- "children": [],
- "count": 0,
- "hasDirectFiles": False,
- "hasChildren": False,
- "showSelfToggle": False,
- "isRoot": True,
- }
- node_map = {"": tree_root}
- for path, idx_list in folder_index[repo].items():
- if path == "":
- tree_root["count"] = len(idx_list)
- continue
- parts = path.split("/")
- for depth in range(1, len(parts) + 1):
- key = "/".join(parts[:depth])
- if key not in node_map:
- parent_key = "/".join(parts[:depth - 1]) if depth > 1 else ""
- parent = node_map[parent_key]
- node = {
- "name": parts[depth - 1],
- "path": key,
- "children": [],
- "count": 0,
- "hasDirectFiles": False,
- "hasChildren": False,
- "showSelfToggle": False,
- }
- node_map[key] = node
- parent["children"].append(node)
- node_map[path]["count"] = len(idx_list)
- direct_file_dirs = set()
- for idx in repo_records_map.get(repo, []):
- rec = records[idx]
- folders = rec.get("Folder", []) or []
- direct_file_dirs.add("/".join(folders))
- for path_key, node in node_map.items():
- node["hasDirectFiles"] = path_key in direct_file_dirs
- node["children"].sort(key=lambda child: child["name"])
- node["hasChildren"] = len(node["children"]) > 0
- node["showSelfToggle"] = bool(path_key and node["hasDirectFiles"] and node["hasChildren"])
- return [tree_root]
-
-def get_folder_contents(repo: str, path: str) -> dict:
- if repo in folder_browser_data:
- repo_browser = folder_browser_data.get(repo, {}) or {}
- if path in repo_browser:
- return enrich_browser_entry(repo, path, repo_browser[path])
- if path == "" and "" in repo_browser:
- return enrich_browser_entry(repo, path, repo_browser[""])
- path_parts = [p for p in path.strip("/").split("/") if p]
- path_depth = len(path_parts)
- repo_folders = folder_index.get(repo)
- candidate_indices = (repo_folders.get("/".join(path_parts), [])
- if repo_folders is not None else repo_records_map.get(repo, []))
-
- def is_under(rec_folders, prefix):
- return rec_folders[:len(prefix)] == prefix
- matching = [records[i] for i in candidate_indices
- if is_under(records[i].get("Folder", []) or [], path_parts)]
- subfolder_names = set()
- subfolder_counts = {}
- for rec in matching:
- folders = rec.get("Folder", []) or []
- if len(folders) > path_depth:
- name = folders[path_depth]
- subfolder_names.add(name)
- subfolder_counts[name] = subfolder_counts.get(name, 0) + 1
- result_folders = []
- for fname in sorted(subfolder_names):
- full_path = "/".join(path_parts + [fname]) if path_parts else fname
- result_folders.append({"name": fname, "path": full_path, "count": subfolder_counts.get(fname, 0)})
- direct_files = []
- for rec in matching:
- folders = rec.get("Folder", []) or []
- if len(folders) == path_depth:
- direct_files.append({
- "name": rec.get("File", ""),
- "ext": rec.get("Extension", ""),
- "link": rec.get("Link") or build_record_link(rec),
- "path": rec.get("Path") or build_record_path_url(rec),
- "hasTxt": rec.get("HasTxt", False),
- "size": rec.get("Size", ""),
- })
- direct_files.sort(key=lambda f: f["name"])
- return {"folders": result_folders, "files": direct_files, "current_path": path}
-
-DOWNLOAD_CONNECTION_WARM_INTERVAL_SECONDS = 30
-
-
-async def warm_download_connection(app: FastAPI):
- # Use the download pool, but never queue speculative work behind real transfers.
- semaphore = app.state.upstream_semaphore
- if semaphore.locked():
- return
- try:
- async with semaphore:
- async with app.state.http_session.head(
- "https://huggingface.co/robots.txt",
- allow_redirects=False,
- timeout=aiohttp.ClientTimeout(total=4, connect=3, sock_read=3),
- ):
- pass
- except (aiohttp.ClientError, asyncio.TimeoutError):
- # A failed warmup must not affect ping or the later download attempt.
- pass
-
-
-def schedule_download_connection_warm(app: FastAPI):
- state = app.state
- if not getattr(state, "download_warm_enabled", False):
- return
- task = state.download_warm_task
- now = time.monotonic()
- if task is not None and not task.done():
- return
- if now - state.download_warm_at < DOWNLOAD_CONNECTION_WARM_INTERVAL_SECONDS:
- return
- state.download_warm_at = now
- state.download_warm_task = asyncio.create_task(warm_download_connection(app))
-
-
-async def warm_txt_proxy_loop(app: FastAPI):
- while True:
- await asyncio.sleep(TXT_WARM_INTERVAL_SECONDS)
- if time.monotonic() - last_user_request_at < TXT_WARM_IDLE_SECONDS:
- continue
- pool = txt_record_indices
- if not pool:
- continue
- rec = records[random.choice(pool)]
- remote_path = build_txt_relative_path(rec)
- remote_url = TXT_SPACE_RAW_BASE + "/" + quote(remote_path, safe="/")
- try:
- async with app.state.http_session.get(
- remote_url,
- headers={"Range": "bytes=0-0", "Accept-Encoding": "identity"},
- timeout=aiohttp.ClientTimeout(total=15, connect=5, sock_read=5),
- ) as response:
- await response.content.read(1)
- except asyncio.CancelledError:
- raise
- except Exception as e:
- print(f"txt 预热异常: {e}")
-@asynccontextmanager
-
-async def lifespan(app: FastAPI):
- global last_user_request_at
- load_data()
- app.state.search_executor = SearchExecutor()
- app.state.reader_index_build_task = None
- app.state.reader_index_warm_task = None
- app.state.http_session = aiohttp.ClientSession(
- timeout=aiohttp.ClientTimeout(total=300, connect=15),
- connector=aiohttp.TCPConnector(keepalive_timeout=60, ttl_dns_cache=300),
- headers={"User-Agent": "VoiceOfML-Search/1.0"},
- auto_decompress=True,
- )
- app.state.upstream_semaphore = asyncio.Semaphore(8)
- app.state.download_warm_enabled = True
- app.state.download_warm_task = None
- app.state.download_warm_at = float("-inf")
- app.state.reader_assets_lock = asyncio.Lock()
- app.state.reader_assets_task = None
- initialize_reader_bucket_state(app)
- initialize_reader_archive_state(app)
- app.state.reader_range_cache = OrderedDict()
- app.state.reader_range_cache_bytes = 0
- app.state.reader_range_flights = {}
- app.state.reader_range_flight_owners = set()
- last_user_request_at = time.monotonic()
- app.state.reader_index_warm_task = asyncio.create_task(warm_reader_indexes())
- app.state.txt_warm_task = asyncio.create_task(warm_txt_proxy_loop(app))
- try:
- yield
- finally:
- if app.state.reader_index_warm_task is not None:
- app.state.reader_index_warm_task.cancel()
- await asyncio.gather(app.state.reader_index_warm_task, return_exceptions=True)
- app.state.reader_index_warm_task = None
- if app.state.reader_index_build_task is not None:
- app.state.reader_index_build_task.cancel()
- await asyncio.gather(app.state.reader_index_build_task, return_exceptions=True)
- app.state.reader_index_build_task = None
- await app.state.search_executor.close()
- del app.state.search_executor
- if app.state.reader_assets_task is not None:
- app.state.reader_assets_task.cancel()
- await asyncio.gather(app.state.reader_assets_task, return_exceptions=True)
- app.state.reader_assets_task = None
- app.state.download_warm_enabled = False
- if app.state.download_warm_task is not None:
- app.state.download_warm_task.cancel()
- await asyncio.gather(app.state.download_warm_task, return_exceptions=True)
- app.state.download_warm_task = None
- app.state.txt_warm_task.cancel()
- try:
- await app.state.txt_warm_task
- except asyncio.CancelledError:
- pass
- await shutdown_reader_bucket_state(app)
- await shutdown_reader_archive_state(app)
- await shutdown_reader_range_flights()
- app.state.reader_range_cache.clear()
- app.state.reader_range_cache_bytes = 0
- await app.state.http_session.close()
-app = FastAPI(title="VoiceOfML Search", version="2.0", lifespan=lifespan)
-app.add_middleware(GZipMiddleware, minimum_size=500)
-app.add_middleware(
- CORSMiddleware,
- allow_origins=["*"],
- allow_methods=["*"],
- allow_headers=["*"],
- expose_headers=["Accept-Ranges", "Content-Range", "Content-Length", "ETag", "Last-Modified", "Server-Timing"],
-)
-
-@app.middleware("http")
-async def track_user_activity(request: Request, call_next):
- global last_user_request_at
- if request.url.path not in ("/health", "/favicon.ico", "/api/ping"):
- last_user_request_at = time.monotonic()
- return await call_next(request)
-
-SearchText = Annotated[str, Field(max_length=256)]
-SearchListItem = Annotated[str, Field(max_length=512)]
-SearchList = Annotated[list[SearchListItem], Field(max_length=100)]
-FolderSearchList = Annotated[list[SearchListItem], Field(max_length=10000)]
-
-
-class SearchRequest(BaseModel):
- q: SearchText = ""
- repos: Optional[SearchList] = None
- extensions: Optional[SearchList] = None
- folders: Optional[FolderSearchList] = None
- folder_match_mode: Literal["prefix", "exact", "mixed"] = "prefix"
- folder_selfs: Optional[FolderSearchList] = None
- folder_subtrees: Optional[FolderSearchList] = None
- min_size: Optional[int] = Field(default=None, ge=0)
- max_size: Optional[int] = Field(default=None, ge=0)
- page: int = Field(default=1, ge=1)
- page_size: int = Field(default=100, ge=1, le=500)
- search_folders: bool = True
- sort: Literal["relevance", "name", "size"] = "relevance"
- exact: bool = False
- anchor_id: Optional[str] = Field(default=None, max_length=16384)
-
-
-def normalize_search_list(values: list[str] | None, lowercase: bool = False) -> list[str] | None:
- if not values:
- return None
- return list(dict.fromkeys(value.strip().lower() if lowercase else value.strip() for value in values))
-
-
-async def build_search_api_response(body: SearchRequest, repos: list[str] | None) -> Response:
- params = body.model_dump(exclude={"repos"})
- for name in ("extensions", "folders", "folder_selfs", "folder_subtrees"):
- params[name] = normalize_search_list(params[name], lowercase=name == "extensions")
- compute = functools.partial(search, repos=repos, **params)
- try:
- executor = getattr(app.state, "search_executor", None)
- if executor is None:
- # Direct endpoint tests without ASGI lifespan own a temporary pool.
- async with SearchExecutor() as executor:
- payload, queued_ms, search_ms = await executor.run(compute)
- else:
- payload, queued_ms, search_ms = await executor.run(compute)
- except SearchBusy:
- return JSONResponse({"error": "搜索繁忙，请稍后重试"}, status_code=503,
- headers={"Retry-After": "1"})
- started = time.perf_counter()
- assets = await get_reader_assets(wait=False)
- assets_ms = (time.perf_counter() - started) * 1000
- started = time.perf_counter()
- key = ("search", _data_generation, reader_assets_revision, READER_ASSET_ROUTING_CONTRACT_VERSION, tuple(repos or ()),
- json.dumps(params, sort_keys=True, ensure_ascii=False))
- response = cached_json_response(key, lambda: add_reader_assets(payload, assets))
- response.headers["Server-Timing"] = (
- f"queue;dur={queued_ms:.2f}, search;dur={search_ms:.2f}, "
- f"reader_assets;dur={assets_ms:.2f}, response;dur={(time.perf_counter() - started) * 1000:.2f}")
- return response
-
-
-@app.post("/api/search")
-async def api_search(body: SearchRequest):
- return await build_search_api_response(body, normalize_search_list(body.repos))
-
-
-@app.post("/api/search/{repo_name}")
-async def api_search_repo(repo_name: str, body: SearchRequest):
- full_repo = f"VoiceOfML/{repo_name}"
- if full_repo not in repo_counts:
- return JSONResponse({"error": f"仓库 {full_repo} 不存在", "results": [], "total": 0}, status_code=404)
- return await build_search_api_response(body, [full_repo])
-
-
-@app.get("/api/repos")
-
-def api_repos():
- return cached_json_response(("repos", _data_generation), lambda: repo_list, METADATA_CACHE_HEADERS)
-@app.get("/api/reader-assets")
-
-async def api_reader_assets():
- assets = await get_reader_assets()
- cache_control = "public, max-age=300" if assets.get("f") else "no-store"
- return JSONResponse(assets, headers={"Cache-Control": cache_control})
-@app.get("/api/reader-v3-status")
-async def api_reader_v3_status():
- assets = await get_reader_assets()
- metadata = assets.get("v3")
- return JSONResponse({"version": 1, "surface": "hf", "active": metadata is not None,
- **(metadata or {})}, headers={"Cache-Control": "no-store"})
-@app.get("/api/extensions")
-
-def api_extensions(repo: Optional[str] = Query(default=None)):
- def build():
- if repo:
- full_repo = f"VoiceOfML/{repo}"
- ext_count = repo_extension_counts.get(full_repo, {})
- else:
- ext_count = extension_counts
- sorted_exts = sorted(ext_count.keys())
- return [{"name": e, "count": ext_count[e]} for e in sorted_exts]
- return cached_json_response(("extensions", _data_generation, repo or ""), build, METADATA_CACHE_HEADERS)
-@app.get("/api/folders/{repo_name}")
-
-def api_folders(repo_name: str):
- full_repo = f"VoiceOfML/{repo_name}"
- if full_repo not in repo_counts:
- return JSONResponse({"error": f"仓库 {full_repo} 不存在"}, status_code=404)
- return cached_json_response(("folders", _data_generation, full_repo), lambda: build_filter_folder_tree(full_repo))
-@app.get("/api/folders/{repo_name}/contents")
-
-def api_folder_contents(repo_name: str, path: str = Query(default="")):
- full_repo = f"VoiceOfML/{repo_name}"
- if full_repo not in repo_counts:
- return JSONResponse({"error": f"仓库 {full_repo} 不存在"}, status_code=404)
- return cached_json_response(("folder_contents", _data_generation, full_repo, path), lambda: get_folder_contents(full_repo, path))
-@app.get("/api/random")
-
-def api_random(repo: Optional[str] = Query(default=None)):
- if repo:
- full_repo = f"VoiceOfML/{repo}"
- repo_indices = repo_records_map.get(full_repo, [])
- if not repo_indices:
- return JSONResponse({"error": "无可用记录"}, status_code=404)
- rec = public_record(records[random.choice(repo_indices)])
- else:
- if not records:
- return JSONResponse({"error": "无可用记录"}, status_code=404)
- rec = public_record(random.choice(records))
- return JSONResponse(rec)
-
-@app.get("/api/random-txt")
-def api_random_txt(repo: Optional[str] = Query(default=None)):
- if repo:
- full_repo = f"VoiceOfML/{repo}"
- pool = repo_txt_record_indices.get(full_repo, [])
- else:
- pool = txt_record_indices
- if not pool:
- return JSONResponse({"error": "无可用 txt"}, status_code=404)
- return JSONResponse(public_record(records[random.choice(pool)]))
-
-@app.get("/api/random-txt/status")
-def api_random_txt_status(repo: Optional[str] = Query(default=None)):
- if repo:
- full_repo = f"VoiceOfML/{repo}"
- count = len(repo_txt_record_indices.get(full_repo, []))
- else:
- count = len(txt_record_indices)
- return JSONResponse({"available": count > 0, "count": count}, headers=METADATA_CACHE_HEADERS)
-
-def converted_reader_indices(assets: dict, full_repo: str | None = None) -> dict[int, dict]:
- global reader_join_cache_key, reader_join_cache
- ensure_reader_indexes()
- cache_key = (_data_generation, reader_assets_revision, READER_ASSET_ROUTING_CONTRACT_VERSION)
- if reader_join_cache_key != cache_key:
- converted = {}
- for key, asset in assets.get("f", {}).items():
- if not isinstance(asset, dict) or asset.get("s") != 2:
- continue
- index = reader_record_key_indices.get(key)
- if index is not None:
- converted[index] = asset
- by_repo = {}
- for index, asset in converted.items():
- repo = records[index].get("Repo", "")
- by_repo.setdefault(repo, {})[index] = asset
- reader_join_cache = {"all": converted, "repos": by_repo}
- reader_join_cache_key = cache_key
- if full_repo:
- return reader_join_cache["repos"].get(full_repo, {})
- return reader_join_cache["all"]
-
-
-async def get_reader_pool(repo: str | None = None) -> tuple[list[int], dict[int, dict]]:
- await ensure_reader_indexes_async()
- full_repo = f"VoiceOfML/{repo}" if repo else None
- original = repo_reader_record_indices.get(full_repo, []) if full_repo else reader_record_indices
- # Random selection uses the complete last published snapshot while refresh
- # proceeds; a cold process must still await its first asset generation.
- assets = await get_reader_assets(wait=not bool(reader_assets_cache.get("f")))
- converted = converted_reader_indices(assets, full_repo)
- pool = dict.fromkeys(original)
- pool.update(dict.fromkeys(converted))
- return list(pool), converted
-
-
-@app.get("/api/random-reader")
-async def api_random_reader(repo: Optional[str] = Query(default=None)):
- started = time.perf_counter()
- pool, converted = await get_reader_pool(repo)
- if not pool:
- return JSONResponse({"error": "无可读文档"}, status_code=404)
- index = random.choice(pool)
- rec = public_record(records[index])
- asset = converted.get(index)
- if asset:
- rec = enrich_reader_record(rec, asset)
- return JSONResponse(rec, headers={"Server-Timing": f"reader_select;dur={(time.perf_counter() - started) * 1000:.2f}"})
-
-@app.get("/api/random-reader/status")
-async def api_random_reader_status(repo: Optional[str] = Query(default=None)):
- pool, _ = await get_reader_pool(repo)
- count = len(pool)
- return JSONResponse({"available": count > 0, "count": count}, headers=METADATA_CACHE_HEADERS)
-
-@app.get("/api/bootstrap")
-def api_bootstrap():
- return JSONResponse(
- cached_payload(("bootstrap", _data_generation), build_global_bootstrap),
- headers=METADATA_CACHE_HEADERS,
- )
-
-@app.get("/api/ping")
-async def api_ping():
- schedule_download_connection_warm(app)
- return PlainTextResponse(status_code=204, headers={"Cache-Control": "no-store"})
-
-
-READER_UPSTREAM_RESPONSE_HEADERS = (
- "Content-Type", "Content-Length", "Content-Range", "Accept-Ranges",
- "ETag", "Last-Modified", "Cache-Control",
-)
-READER_IMMUTABLE_CACHE_CONTROL = "public, max-age=31536000, immutable"
-READER_ISOLATION_HEADERS = {
- "Content-Security-Policy": "sandbox; default-src 'none'; style-src 'unsafe-inline'; img-src data:; base-uri 'none'; form-action 'none'",
- "X-Content-Type-Options": "nosniff",
-}
-READER_DOCUMENT_MAX_BYTES = 512 * 1024 * 1024
-READER_RESOURCE_MAX_ARCHIVE_BYTES = 512 * 1024 * 1024
-READER_RESOURCE_MAX_BYTES = 64 * 1024 * 1024
-READER_BUCKET_CACHE_TTL_SECONDS = 3600
-READER_BUCKET_CACHE_MAX_BYTES = 32 * 1024 * 1024
-READER_BUCKET_CACHE_MAX_ITEM_BYTES = 8 * 1024 * 1024
-READER_BUCKET_MAX_CONCURRENT_READS = 8
-READER_BUCKET_MAX_INFLIGHT = 32
-READER_OCR_CACHE_MAX_BYTES = 32 * 1024 * 1024
-READER_OCR_CACHE_MAX_ITEMS = 4
-READER_ARCHIVE_MAX_ENTRIES = 20_000
-READER_ARCHIVE_MAX_EXPANDED_BYTES = 1536 * 1024 * 1024
-READER_ARCHIVE_MAX_COMPRESSION_RATIO = 200
-READER_ARCHIVE_MAX_CENTRAL_DIRECTORY_BYTES = 8 * 1024 * 1024
-READER_ARCHIVE_MAX_METADATA_BYTES = 16 * 1024 * 1024
-READER_ARCHIVE_CACHE_TTL_SECONDS = 900
-READER_ARCHIVE_CACHE_MAX_BYTES = 32 * 1024 * 1024
-READER_ARCHIVE_CACHE_MAX_ENTRIES = 4
-READER_ARCHIVE_MAX_CONCURRENT_LOADS = 2
-READER_ARCHIVE_MAX_INFLIGHT = 16
-READER_ARCHIVE_MAX_WORKERS = 2
-READER_ARCHIVE_MEMBER_RE = re.compile(r"^[^/\x00]+(?:/[^/\x00]+)*(?:/)?$")
-READER_RANGE_CACHE_TTL_SECONDS = 900
-READER_RANGE_CACHE_MAX_BYTES = 32 * 1024 * 1024
-READER_RANGE_CACHE_MAX_ITEM_BYTES = 2 * 1024 * 1024
-READER_RANGE_CACHE_MAX_ENTRIES = 256
-READER_RANGE_MAX_FLIGHTS = 32
-
-
-class ReaderArchiveLimitError(ValueError):
- pass
-
-
-class ReaderArchiveInvalidError(ValueError):
- pass
-
-
-class ReaderDocumentLimitError(ValueError):
- pass
-
-
-def validate_reader_archive(archive: bytes) -> None:
- if len(archive) > READER_RESOURCE_MAX_ARCHIVE_BYTES:
- raise ReaderArchiveLimitError("阅读文件过大")
- try:
- # Use the stdlib's EOCD/ZIP64 parser before ZipFile allocates its directory.
- end_record = zipfile._EndRecData(io.BytesIO(archive))
- if not end_record:
- raise ReaderArchiveInvalidError("阅读文件不是有效 ZIP")
- central_directory_size = end_record[zipfile._ECD_SIZE]
- directory_offset = end_record[zipfile._ECD_OFFSET]
- entry_count = end_record[zipfile._ECD_ENTRIES_TOTAL]
- if (end_record[zipfile._ECD_DISK_NUMBER] != 0
- or end_record[zipfile._ECD_DISK_START] != 0
- or end_record[zipfile._ECD_ENTRIES_THIS_DISK] != entry_count
- or len(end_record[zipfile._ECD_COMMENT]) != end_record[zipfile._ECD_COMMENT_SIZE]
- or (end_record[zipfile._ECD_SIGNATURE] != zipfile.stringEndArchive64
- and (central_directory_size == 0xffffffff or directory_offset == 0xffffffff))):
- raise ReaderArchiveInvalidError("阅读文件元数据无效")
- if central_directory_size > READER_ARCHIVE_MAX_CENTRAL_DIRECTORY_BYTES:
- raise ReaderArchiveLimitError("阅读文件元数据过大")
- if entry_count > READER_ARCHIVE_MAX_ENTRIES:
- raise ReaderArchiveLimitError("阅读文件条目过多")
- if directory_offset + central_directory_size > len(archive):
- raise ReaderArchiveInvalidError("阅读文件元数据无效")
- with zipfile.ZipFile(io.BytesIO(archive)) as book_zip:
- infos = book_zip.infolist()
- if len(infos) > READER_ARCHIVE_MAX_ENTRIES:
- raise ReaderArchiveLimitError("阅读文件条目过多")
-
- if len(infos) != entry_count:
- raise ReaderArchiveInvalidError("阅读文件条目数无效")
- central_directory_offset = book_zip.start_dir
- archive_comment = book_zip.comment
- if (
- not isinstance(central_directory_size, int)
- or isinstance(central_directory_size, bool)
- or central_directory_size < 0
- or central_directory_size > READER_ARCHIVE_MAX_CENTRAL_DIRECTORY_BYTES
- or not isinstance(central_directory_offset, int)
- or isinstance(central_directory_offset, bool)
- or central_directory_offset < 0
- or central_directory_offset > len(archive)
- or central_directory_size > len(archive) - central_directory_offset
- or not isinstance(archive_comment, bytes)
- ):
- raise ReaderArchiveInvalidError("阅读文件元数据无效")
-
- expanded = 0
- compressed = 0
- metadata_bytes = len(archive_comment)
- for info in infos:
- filename = info.orig_filename
- extra = info.extra
- comment = info.comment
- if not isinstance(filename, str) or not isinstance(extra, bytes) or not isinstance(comment, bytes):
- raise ReaderArchiveInvalidError("阅读文件元数据无效")
- name = filename.replace("\\", "/")
- name_parts = name.split("/")
- if (not name or name.startswith("/") or "\x00" in name
- or any(part in {".", ".."} for part in name_parts)
- or any(not part for part in name_parts[:-1])
- or not READER_ARCHIVE_MEMBER_RE.fullmatch(name)):
- raise ReaderArchiveInvalidError("阅读文件包含不安全路径")
- try:
- filename_bytes = len(filename.encode("utf-8", "surrogatepass"))
- except (UnicodeError, AttributeError) as exc:
- raise ReaderArchiveInvalidError("阅读文件元数据无效") from exc
- metadata_bytes += 46 + filename_bytes + len(extra) + len(comment)
- if metadata_bytes > READER_ARCHIVE_MAX_METADATA_BYTES:
- raise ReaderArchiveLimitError("阅读文件元数据过大")
-
- file_size = info.file_size
- compressed_size = info.compress_size
- header_offset = info.header_offset
- if (
- not isinstance(file_size, int)
- or isinstance(file_size, bool)
- or file_size < 0
- or not isinstance(compressed_size, int)
- or isinstance(compressed_size, bool)
- or compressed_size < 0
- or compressed_size > len(archive)
- or not isinstance(header_offset, int)
- or isinstance(header_offset, bool)
- or header_offset < 0
- or header_offset >= len(archive)
- or header_offset >= central_directory_offset
- ):
- raise ReaderArchiveInvalidError("阅读文件条目无效")
- expanded += file_size
- compressed += compressed_size
- if expanded > READER_ARCHIVE_MAX_EXPANDED_BYTES:
- raise ReaderArchiveLimitError("阅读文件展开后过大")
- if file_size and (compressed_size == 0 or file_size > compressed_size * READER_ARCHIVE_MAX_COMPRESSION_RATIO):
- raise ReaderArchiveLimitError("阅读文件压缩比过高")
-
- if compressed == 0:
- if expanded:
- raise ReaderArchiveLimitError("阅读文件压缩比过高")
- elif expanded > compressed * READER_ARCHIVE_MAX_COMPRESSION_RATIO:
- raise ReaderArchiveLimitError("阅读文件压缩比过高")
- if compressed > len(archive):
- raise ReaderArchiveInvalidError("阅读文件条目无效")
- except (ReaderArchiveLimitError, ReaderArchiveInvalidError):
- raise
- except Exception as exc:
- raise ReaderArchiveInvalidError("阅读文件不是有效 ZIP") from exc
-
-
-READER_RESOURCE_TYPES = {
- "json": "application/json", "xhtml": "application/xhtml+xml", "html": "text/html", "txt": "text/plain",
- "md": "text/markdown", "docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
- "pdf": "application/pdf", "epub": "application/epub+zip",
- "css": "text/css", "jpg": "image/jpeg", "jpeg": "image/jpeg", "png": "image/png",
- "gif": "image/gif", "bmp": "image/bmp", "webp": "image/webp", "jxl": "image/jxl", "svg": "image/svg+xml", "ttf": "font/ttf",
- "otf": "font/otf", "woff": "font/woff", "woff2": "font/woff2", "mp3": "audio/mpeg",
- "wav": "audio/wav", "m4a": "audio/mp4", "flac": "audio/flac", "mpga": "audio/mpeg",
- "mp4": "video/mp4", "mov": "video/quicktime", "swf": "application/x-shockwave-flash", "gz": "application/gzip",
-}
-def initialize_reader_bucket_state(target_app: FastAPI) -> None:
- target_app.state.reader_bucket_cache = OrderedDict()
- target_app.state.reader_bucket_cache_bytes = 0
- target_app.state.reader_bucket_inflight = {}
- target_app.state.reader_bucket_workers = set()
- target_app.state.reader_bucket_semaphore = asyncio.Semaphore(READER_BUCKET_MAX_CONCURRENT_READS)
- target_app.state.reader_ocr_cache = OrderedDict()
- target_app.state.reader_ocr_cache_bytes = 0
-
-
-async def shutdown_reader_bucket_state(target_app: FastAPI) -> None:
- tasks = list(target_app.state.reader_bucket_inflight.values())
- target_app.state.reader_bucket_inflight.clear()
- for task in tasks:
- task.cancel()
- if tasks:
- await asyncio.gather(*tasks, return_exceptions=True)
- workers = list(target_app.state.reader_bucket_workers)
- if workers:
- await asyncio.gather(*workers, return_exceptions=True)
- target_app.state.reader_bucket_cache.clear()
- target_app.state.reader_bucket_cache_bytes = 0
- target_app.state.reader_ocr_cache.clear()
- target_app.state.reader_ocr_cache_bytes = 0
-
-
-def is_reader_bucket_cacheable(path: str) -> bool:
- return (
- path.endswith("/page-manifest.json") or path.endswith("/ocr-manifest.json")
- or path.endswith("/ocr/book-text.json.gz") or path.endswith("/pages/page-000001.webp")
- or (path.startswith("objects/") and path.rsplit("/", 1)[-1].startswith("document."))
- or (path.startswith("chapters/ebook/") and any(
- path.endswith(suffix) for suffix in (
- "/chapter-manifest.json", "/epub-search-index.json.gz",
- )))
- )
-
-
-def reader_bucket_cache_control(path: str) -> str:
- # Page paths contain the source digest and an optional asset version, so a
- # generated image can be cached immutably even though the manifest moves.
- if re.search(r"/pages/page-[0-9]{6}\.(?:webp|jxl)$", path):
- return "public, max-age=31536000, immutable"
- if re.fullmatch(r"objects/[0-9a-f]{2}/[0-9a-f]{64}/[0-9a-f]{16}/(?:preview|text)/.+", path):
- return "public, max-age=31536000, immutable"
- if path.startswith("objects/") and path.rsplit("/", 1)[-1].startswith("document."):
- return "public, max-age=31536000, immutable"
- if "/ebook-chapters/" in path:
- return "public, max-age=31536000, immutable"
- return "public, max-age=3600"
-
-
-def reader_bucket_for_path(path: str) -> str:
- if (path.startswith(("documents/", "chapters/", "media/", "pages/image/", "native/"))):
- return "vomebook/reader-assets-v2"
- return "vomebook/pdf-pages-v2"
-
-
-def reader_bucket_source_url(path: str) -> str:
- bucket = reader_bucket_for_path(path)
- return f"https://huggingface.co/buckets/{bucket}/resolve/{quote(path, safe='/')}"
-
-
-def validate_reader_bucket_redirect_url(url: str, source_url: str) -> str:
- validate_download_url(url)
- if urlparse(url).hostname == "huggingface.co" and url != source_url:
- raise ValueError("上游重定向超出允许的阅读资源范围")
- return url
-
-
-def read_reader_bucket_resource(path: str, max_bytes: int = READER_RESOURCE_MAX_BYTES) -> bytes:
- filesystem = HfFileSystem()
- bucket = reader_bucket_for_path(path)
- with filesystem.open(f"hf://buckets/{bucket}/{path}", "rb") as stream:
- payload = stream.read(max_bytes + 1)
- if len(payload) > max_bytes:
- raise ValueError("阅读资源过大")
- return payload
-
-
-async def get_reader_bucket_resource(path: str, max_bytes: int = READER_RESOURCE_MAX_BYTES) -> bytes:
- state = app.state
- cache_key = path
- if is_reader_bucket_cacheable(path):
- cached = state.reader_bucket_cache.get(cache_key)
- if cached:
- cached_at, payload = cached
- if time.monotonic() - cached_at < READER_BUCKET_CACHE_TTL_SECONDS:
- state.reader_bucket_cache.move_to_end(cache_key)
- return payload
- state.reader_bucket_cache.pop(cache_key)
- state.reader_bucket_cache_bytes -= len(payload)
-
- task = state.reader_bucket_inflight.get(cache_key)
- if task is None:
- if len(state.reader_bucket_inflight) >= READER_BUCKET_MAX_INFLIGHT:
- raise RuntimeError("too many reader bucket requests")
- async def load() -> bytes:
- try:
- payload = await run_reader_bucket_read(path, max_bytes)
- if is_reader_bucket_cacheable(path) and len(payload) <= READER_BUCKET_CACHE_MAX_ITEM_BYTES:
- previous = state.reader_bucket_cache.pop(cache_key, None)
- if previous:
- state.reader_bucket_cache_bytes -= len(previous[1])
- state.reader_bucket_cache[cache_key] = (time.monotonic(), payload)
- state.reader_bucket_cache_bytes += len(payload)
- while state.reader_bucket_cache_bytes > READER_BUCKET_CACHE_MAX_BYTES:
- _, (_, evicted) = state.reader_bucket_cache.popitem(last=False)
- state.reader_bucket_cache_bytes -= len(evicted)
- return payload
- finally:
- if state.reader_bucket_inflight.get(cache_key) is asyncio.current_task():
- state.reader_bucket_inflight.pop(cache_key, None)
-
- task = asyncio.create_task(load())
- task.add_done_callback(lambda done: None if done.cancelled() else done.exception())
- state.reader_bucket_inflight[cache_key] = task
- return await asyncio.shield(task)
-
-
-async def run_reader_bucket_read(path: str, max_bytes: int) -> bytes:
- state = app.state
- semaphore = state.reader_bucket_semaphore
- await semaphore.acquire()
- worker = asyncio.create_task(asyncio.to_thread(read_reader_bucket_resource, path, max_bytes))
- state.reader_bucket_workers.add(worker)
-
- def finished(done):
- # Cancelling an async caller cannot stop its filesystem thread.
- state.reader_bucket_workers.discard(done)
- semaphore.release()
- if not done.cancelled():
- done.exception()
-
- worker.add_done_callback(finished)
- return await asyncio.shield(worker)
-
-
-def parse_reader_ocr_book(raw: bytes) -> tuple[list[dict], int]:
- """Decode the single v2 book text index and its layout mappings."""
- with gzip.GzipFile(fileobj=io.BytesIO(raw)) as stream:
- decoded = stream.read(READER_RESOURCE_MAX_BYTES + 1)
- if len(decoded) > READER_RESOURCE_MAX_BYTES:
- raise ValueError("PDF OCR text index is too large")
- payload = json.loads(decoded)
- pages = payload.get("pages") if isinstance(payload, dict) else None
- if (not isinstance(pages, list) or len(pages) > 200000
- or payload.get("version") != 2 or payload.get("kind") != "pdf-book-text"
- or payload.get("complete") is not True
- or payload.get("offset_unit") != "unicode-codepoint"
- or payload.get("page_count") != len(pages)
- or not re.fullmatch(r"[0-9a-f]{64}", str(payload.get("source_sha256", "")))):
- raise ValueError("invalid PDF OCR text index")
- output = []
- total_bytes = 0
- previous = 0
- for item in pages:
- if (not isinstance(item, dict) or not isinstance(item.get("page"), int)
- or item["page"] < 1 or item["page"] <= previous
- or not isinstance(item.get("text"), str)):
- raise ValueError("invalid PDF OCR text page")
- text = item["text"]
- total_bytes += len(text.encode("utf-8"))
- if total_bytes > READER_RESOURCE_MAX_BYTES:
- raise ValueError("PDF OCR text index is too large")
- layout = item.get("layout")
- spans = item.get("text_spans")
- if (not isinstance(layout, dict) or layout.get("offset_unit") != "unicode-codepoint"
- or not isinstance(spans, list)):
- raise ValueError("invalid PDF text layout mapping")
- previous_span_end = 0
- for span in spans:
- box = span.get("box") if isinstance(span, dict) else None
- if (not isinstance(span, dict) or not isinstance(span.get("start"), int)
- or not isinstance(span.get("end"), int) or span["start"] < previous_span_end
- or span["end"] <= span["start"] or span["end"] > len(text)
- or not isinstance(box, list) or len(box) != 4
- or any(not isinstance(value, (int, float)) or value < 0 or value > 1 for value in box)):
- raise ValueError("invalid PDF text span")
- previous_span_end = span["end"]
- output.append({"page": item["page"], "text": text, "folded": text.casefold(),
- "layout": layout, "spans": spans})
- previous = item["page"]
- return output, total_bytes
-
-
-async def get_reader_ocr_book(path: str) -> list[dict]:
- state = app.state
- if not hasattr(state, "reader_ocr_cache"):
- state.reader_ocr_cache = OrderedDict()
- state.reader_ocr_cache_bytes = 0
- cached = state.reader_ocr_cache.get(path)
- if cached:
- cached_at, pages, size = cached
- if time.monotonic() - cached_at < READER_BUCKET_CACHE_TTL_SECONDS:
- state.reader_ocr_cache.move_to_end(path)
- return pages
- state.reader_ocr_cache.pop(path)
- state.reader_ocr_cache_bytes -= size
- raw = await get_reader_bucket_resource(path)
- pages, size = await asyncio.to_thread(parse_reader_ocr_book, raw)
- if size <= READER_OCR_CACHE_MAX_BYTES:
- previous = state.reader_ocr_cache.pop(path, None)
- if previous:
- state.reader_ocr_cache_bytes -= previous[2]
- state.reader_ocr_cache[path] = (time.monotonic(), pages, size)
- state.reader_ocr_cache_bytes += size
- while (len(state.reader_ocr_cache) > READER_OCR_CACHE_MAX_ITEMS
- or state.reader_ocr_cache_bytes > READER_OCR_CACHE_MAX_BYTES):
- _, (_, _, evicted_size) = state.reader_ocr_cache.popitem(last=False)
- state.reader_ocr_cache_bytes -= evicted_size
- return pages
-
-
-@app.api_route("/api/reader-bucket-resource", methods=["GET", "HEAD"])
-async def api_reader_bucket_resource(path: str = Query(...), request: Request = None):
- resource_path = unquote(unquote(path)).replace("\\", "/").lstrip("/")
- if not READER_BUCKET_PATH_RE.fullmatch(resource_path):
- return JSONResponse({"error": "不允许的阅读资源路径"}, status_code=400,
- headers={"Cache-Control": "no-store"})
- suffix = resource_path.rsplit(".", 1)[-1].lower()
- media_type = READER_RESOURCE_TYPES.get(suffix, "application/octet-stream")
- # Uncached body resources must release transport on close, including office,
- # text and chapter bodies. Shared metadata/index cache reads remain bounded.
- if request is not None and (suffix == "pdf" or media_type.startswith(("audio/", "video/", "image/"))
- or not is_reader_bucket_cacheable(resource_path)):
- source_url = reader_bucket_source_url(resource_path)
- return await stream_reader_document(
- request, source_url,
- url_validator=lambda url: validate_reader_bucket_redirect_url(url, source_url),
- validation_scope=(validate_reader_bucket_redirect_url, source_url),
- cache_control=reader_bucket_cache_control(resource_path),
- max_bytes=(None if media_type.startswith(("audio/", "video/")) else
- READER_DOCUMENT_MAX_BYTES if suffix == "pdf" else READER_RESOURCE_MAX_BYTES),
- media_type=media_type,
- )
- try:
- payload = await get_reader_bucket_resource(resource_path)
- except FileNotFoundError:
- return JSONResponse({"error": "阅读资源不存在"}, status_code=404,
- headers={"Cache-Control": "no-store"})
- except Exception:
- return JSONResponse({"error": "阅读资源加载失败"}, status_code=502,
- headers={"Cache-Control": "no-store"})
- return Response(content=b"" if request is not None and request.method == "HEAD" else payload,
- media_type=media_type,
- headers={**READER_ISOLATION_HEADERS,
- "Cache-Control": reader_bucket_cache_control(resource_path),
- "Content-Length": str(len(payload)), "Content-Encoding": "identity"})
-
-
-def initialize_reader_archive_state(target_app: FastAPI) -> None:
- target_app.state.reader_archive_cache = OrderedDict()
- target_app.state.reader_archive_inflight = {}
- target_app.state.reader_archive_workers = set()
- target_app.state.reader_archive_load_semaphore = asyncio.Semaphore(READER_ARCHIVE_MAX_CONCURRENT_LOADS)
- target_app.state.reader_archive_worker_semaphore = asyncio.Semaphore(READER_ARCHIVE_MAX_WORKERS)
-
-
-async def shutdown_reader_archive_state(target_app: FastAPI) -> None:
- tasks = list(target_app.state.reader_archive_inflight.values())
- for task in tasks:
- task.cancel()
- if tasks:
- await asyncio.gather(*tasks, return_exceptions=True)
- workers = list(target_app.state.reader_archive_workers)
- if workers:
- await asyncio.gather(*workers, return_exceptions=True)
- target_app.state.reader_archive_inflight.clear()
- target_app.state.reader_archive_cache.clear()
-
-
-async def run_reader_archive_worker(function, *args):
- state = app.state
- semaphore = state.reader_archive_worker_semaphore
- await semaphore.acquire()
- task = asyncio.create_task(asyncio.to_thread(function, *args))
- state.reader_archive_workers.add(task)
-
- def finished(done):
- # Cancelling a caller cannot stop its thread or free that thread's slot.
- state.reader_archive_workers.discard(done)
- semaphore.release()
- if not done.cancelled():
- done.exception()
-
- task.add_done_callback(finished)
- return await asyncio.shield(task)
-
-
-def prepare_reader_archive(raw: bytes | bytearray) -> bytes:
- archive = bytes(raw)
- if archive.startswith(b"\x1f\x8b"):
- try:
- with gzip.GzipFile(fileobj=io.BytesIO(archive)) as stream:
- archive = stream.read(READER_RESOURCE_MAX_ARCHIVE_BYTES + 1)
- except (OSError, EOFError, zlib.error) as exc:
- raise ReaderArchiveInvalidError("阅读文件不是有效 ZIP") from exc
- validate_reader_archive(archive)
- return archive
-
-
-async def get_reader_archive(target_url: str) -> bytes:
- state = app.state
- cache = state.reader_archive_cache
- cached = cache.get(target_url)
- if cached:
- if time.monotonic() - cached[0] < READER_ARCHIVE_CACHE_TTL_SECONDS:
- cache.move_to_end(target_url)
- return cached[1]
- cache.pop(target_url)
- task = state.reader_archive_inflight.get(target_url)
- if task is None:
- if len(state.reader_archive_inflight) >= READER_ARCHIVE_MAX_INFLIGHT:
- raise HTTPException(503, "阅读资源请求过多")
-
- async def load():
- try:
- async with state.reader_archive_load_semaphore:
- async with state.upstream_semaphore:
- upstream = await open_download_response(
- state.http_session, target_url,
- aiohttp.ClientTimeout(total=180, connect=15, sock_read=60),
- url_validator=validate_reader_redirect_url,
- )
- try:
- if upstream.status != 200:
- raise HTTPException(upstream.status, f"上游阅读文件加载失败: HTTP {upstream.status}")
- chunks = bytearray()
- async for chunk in upstream.content.iter_chunked(64 * 1024):
- chunks.extend(chunk)
- if len(chunks) > READER_RESOURCE_MAX_ARCHIVE_BYTES:
- raise ReaderArchiveLimitError("阅读文件过大")
- finally:
- upstream.release()
- archive = await run_reader_archive_worker(prepare_reader_archive, chunks)
- if len(archive) <= READER_ARCHIVE_CACHE_MAX_BYTES:
- cache[target_url] = (time.monotonic(), archive)
- cache.move_to_end(target_url)
- while (len(cache) > READER_ARCHIVE_CACHE_MAX_ENTRIES
- or sum(len(item[1]) for item in cache.values()) > READER_ARCHIVE_CACHE_MAX_BYTES):
- cache.popitem(last=False)
- return archive
- finally:
- state.reader_archive_inflight.pop(target_url, None)
-
- task = asyncio.create_task(load())
- task.add_done_callback(lambda done: None if done.cancelled() else done.exception())
- state.reader_archive_inflight[target_url] = task
- return await asyncio.shield(task)
-
-
-def read_reader_archive_member(archive: bytes, resource_path: str) -> bytes:
- with zipfile.ZipFile(io.BytesIO(archive)) as book_zip:
- names = {name.replace("\\", "/"): name for name in book_zip.namelist()}
- info = book_zip.getinfo(names[resource_path])
- if info.file_size > READER_RESOURCE_MAX_BYTES:
- raise ReaderArchiveLimitError("阅读资源过大")
- return book_zip.read(info)
-
-
-@app.get("/api/reader-resource")
-async def api_reader_resource(book: str = Query(...), path: str = Query(...)):
- try:
- target_url = validate_reader_source_url(book)
- except ValueError as exc:
- return JSONResponse({"error": str(exc)}, status_code=403)
- resource_path = path.replace("\\", "/")
- if (not resource_path or len(resource_path) > 512
- or "\x00" in resource_path
- or any(part in {"", ".", ".."} for part in resource_path.split("/"))):
- return JSONResponse({"error": "不允许的阅读资源路径"}, status_code=400)
- try:
- archive = await get_reader_archive(target_url)
- except HTTPException as exc:
- return JSONResponse({"error": exc.detail}, status_code=exc.status_code, headers={"Cache-Control": "no-store"})
- except ReaderArchiveLimitError as exc:
- return JSONResponse({"error": str(exc)}, status_code=413)
- except ReaderArchiveInvalidError as exc:
- return JSONResponse({"error": str(exc)}, status_code=400)
- except Exception as exc:
- return JSONResponse({"error": str(exc) or "上游阅读文件加载失败"}, status_code=502)
- try:
- payload = await run_reader_archive_worker(read_reader_archive_member, archive, resource_path)
- except KeyError:
- return JSONResponse({"error": "阅读资源不存在"}, status_code=404)
- except ReaderArchiveLimitError as exc:
- return JSONResponse({"error": str(exc)}, status_code=413)
- except (zipfile.BadZipFile, EOFError, IndexError, OverflowError,
- RuntimeError, NotImplementedError, OSError, zlib.error):
- return JSONResponse({"error": "阅读资源损坏"}, status_code=400)
- suffix = resource_path.rsplit(".", 1)[-1].lower() if "." in resource_path else ""
- return Response(content=payload, media_type=READER_RESOURCE_TYPES.get(suffix, "application/octet-stream"), headers={**READER_ISOLATION_HEADERS, "Cache-Control": "public, max-age=3600"})
-
-
-@app.get("/api/reader-resolve")
-async def api_reader_resolve(id: str = Query(..., min_length=13, max_length=13)):
- await ensure_reader_indexes_async()
- if not READER_SHORT_ID_RE.fullmatch(id):
- return JSONResponse({"error": "无效阅读 ID"}, status_code=400)
- # A cold resolution must await conversion metadata; otherwise unsupported
- # originals (such as PS) become cached as the Reader's document source.
- assets = await get_reader_assets(wait=not bool(reader_assets_cache.get("f")))
- matches = []
- for index in reader_source_id_indices.get(id, []):
- record = records[index]
- matches.append(reader_resolved_record(record, assets.get("f", {}).get(build_reader_asset_key(record), {})))
- if len({canonical_reader_source_url(match["download"]) for match in matches}) > 1:
- return JSONResponse({"error": "阅读 ID 不唯一"}, status_code=409)
- matches = matches[:1]
- if matches:
- return JSONResponse(matches[0], headers={"Cache-Control":
- "public, max-age=300" if assets.get("f") else "no-store"})
- return JSONResponse({"error": "阅读文件不存在"}, status_code=404)
-
-
-@app.get("/api/reader-ocr-search")
-async def api_reader_ocr_search(
- id: str = Query(..., min_length=13, max_length=13),
- q: str = Query(..., min_length=1, max_length=256),
- page: int = Query(1, ge=1, le=1000000),
- page_size: int = Query(50, ge=1, le=100),
-):
- """Search one PDF's immutable OCR text without loading it into app memory."""
- resolved = await api_reader_resolve(id)
- if resolved.status_code != 200:
- return resolved
- try:
- data = json.loads(resolved.body)
- manifest_url = str(data.get("ocr_manifest") or "")
- parsed = urlparse(manifest_url)
- parameters = parse_qs(parsed.query)
- manifest_path = parameters.get("path", [""])[0]
- if (parsed.path != "/api/reader-bucket-resource" or set(parameters) != {"path"}
- or not READER_BUCKET_PATH_RE.fullmatch(manifest_path)
- or not manifest_path.endswith("/ocr-manifest.json")):
- return JSONResponse({"error": "本书暂无 OCR 索引"}, status_code=404)
- manifest = json.loads(await get_reader_bucket_resource(manifest_path))
- book = manifest.get("book_text") if isinstance(manifest, dict) else None
- book_path = book.get("path") if isinstance(book, dict) else ""
- if (not isinstance(book_path, str) or not READER_BUCKET_PATH_RE.fullmatch(book_path)
- or not book_path.endswith("/book-text.json.gz")):
- return JSONResponse({"error": "OCR 索引损坏"}, status_code=502)
- pages = await get_reader_ocr_book(book_path)
- needle = q.casefold()
- matches = []
- for item in pages:
- page_number, text, folded = item["page"], item["text"], item["folded"]
- start = 0
- while True:
- found = folded.find(needle, start)
- if found < 0:
- break
- end = found + len(q)
- snippet_start = max(0, found - 72)
- snippet_end = min(len(text), end + 88)
- begin = found
- end = found + len(q)
- boxes = [span["box"] for span in item["spans"]
- if span["end"] > begin and span["start"] < end]
- matches.append({
- "page": page_number, "start": found, "length": len(q),
- "snippet": text[snippet_start:snippet_end],
- "snippet_start": found - snippet_start, "boxes": boxes,
- })
- start = end
- total = len(matches)
- offset = (page - 1) * page_size
- return JSONResponse({
- "version": 2, "kind": "pdf-book-text-search", "page": page, "page_size": page_size, "total": total,
- "results": matches[offset:offset + page_size],
- }, headers={"Cache-Control": "no-store"})
- except (ValueError, TypeError, KeyError, EOFError, gzip.BadGzipFile, zlib.error):
- return JSONResponse({"error": "OCR 索引损坏"}, status_code=502)
-
-
-@app.api_route("/api/reader-content", methods=["GET", "HEAD"])
-async def api_reader_content(request: Request, url: Optional[str] = Query(default=None), id: Optional[str] = Query(default=None)):
- if id and not url:
- resolved = await api_reader_resolve(id)
- if resolved.status_code != 200:
- return resolved
- try:
- url = json.loads(resolved.body).get("url")
- except (AttributeError, TypeError, ValueError):
- return JSONResponse({"error": "阅读文件解析失败"}, status_code=502)
- if not url:
- return JSONResponse({"error": "缺少阅读文件地址"}, status_code=400)
- try:
- parsed = urlparse(url)
- except ValueError as exc:
- return JSONResponse({"error": str(exc)}, status_code=403)
- is_bucket_proxy = (
- (not parsed.scheme and not parsed.netloc)
- or (parsed.scheme == "https" and parsed.netloc == "voiceofml-search.hf.space")
- )
- if is_bucket_proxy and parsed.path == "/api/reader-bucket-resource" and not parsed.fragment:
- parameters = parse_qs(parsed.query)
- if (set(parameters) != {"path"} or len(parameters["path"]) != 1):
- return JSONResponse({"error": "不允许的阅读资源路径"}, status_code=400)
- return await api_reader_bucket_resource(parameters["path"][0], request)
- bucket_prefix = next((prefix for prefix in (
- "/buckets/vomebook/pdf-pages-v2/resolve/",
- "/buckets/vomebook/reader-assets-v2/resolve/",
- ) if parsed.path.startswith(prefix)), None)
- if (parsed.scheme == "https" and parsed.hostname == "huggingface.co"
- and parsed.netloc.lower() in {"huggingface.co", "huggingface.co:443"}
- and not parsed.username and not parsed.password and not parsed.query and not parsed.fragment
- and bucket_prefix is not None):
- resource_path = unquote(parsed.path[len(bucket_prefix):])
- if not READER_BUCKET_PATH_RE.fullmatch(resource_path):
- return JSONResponse({"error": "不允许的阅读资源路径"}, status_code=400)
- return await api_reader_bucket_resource(resource_path, request)
- try:
- target_url = validate_reader_source_url(url)
- except ValueError as exc:
- return JSONResponse({"error": str(exc)}, status_code=403)
- source_type = READER_RESOURCE_TYPES.get(unquote(urlparse(target_url).path).rsplit(".", 1)[-1].lower(), "")
- if source_type.startswith(("audio/", "video/")):
- return await stream_reader_document(request, target_url, media_type=source_type)
- return await stream_reader_document(request, target_url)
-
-
-def reader_document_browser_headers(target_url: str, headers: dict, status: int) -> dict:
- is_pdf = (
- headers.get("Content-Type", "").split(";", 1)[0].strip().lower() == "application/pdf"
- or urlparse(target_url).path.lower().endswith(".pdf")
- )
- if status == 206 and is_pdf:
- # Keep browser partial bodies separate from the bounded server cache.
- # The request-side PDF policy bypasses Chromium's same-URL cache lock.
- return {**headers, "Cache-Control": "no-store"}
- return headers
-
-
-async def stream_reader_document(
- request: Request, target_url: str, *,
- url_validator=validate_reader_redirect_url, cache_control: str | None = None,
- max_bytes: int | None = None, media_type: str | None = None, validation_scope=None,
-):
- options = dict(url_validator=url_validator, cache_control=cache_control,
- max_bytes=max_bytes, media_type=media_type, validation_scope=validation_scope)
- if not callable(getattr(request, "is_disconnected", None)):
- return await _stream_reader_document(request, target_url, **options)
-
- stop_watching = asyncio.Event()
-
- async def disconnected():
- while not stop_watching.is_set():
- if await request.is_disconnected():
- return
- if not stop_watching.is_set():
- await asyncio.sleep(0.1)
-
- # StreamingResponse handles disconnects after headers; own the wait for
- # the upstream slot and headers too, so abandoned books cannot occupy it.
- transport = asyncio.create_task(_stream_reader_document(request, target_url, **options))
- watcher = asyncio.create_task(disconnected())
- try:
- await asyncio.wait((transport, watcher), return_when=asyncio.FIRST_COMPLETED)
- if transport.done():
- return transport.result()
- raise asyncio.CancelledError()
- finally:
- stop_watching.set()
- watcher.cancel()
- if not transport.done():
- transport.cancel()
- await asyncio.gather(transport, watcher, return_exceptions=True)
-
-
-async def _stream_reader_document(
- request: Request, target_url: str, *,
- url_validator=validate_reader_redirect_url, cache_control: str | None = None,
- max_bytes: int | None = None, media_type: str | None = None, validation_scope=None, _share_range: bool = True,
-):
- # Native media is streamed, not parsed or buffered as a complete document.
- # Bound transport memory and idle time instead of rejecting large recordings.
- is_media_stream = bool(media_type and media_type.startswith(("audio/", "video/")))
- if max_bytes is None and not is_media_stream:
- max_bytes = READER_DOCUMENT_MAX_BYTES
- forwarded_headers = {
- name: request.headers[name]
- for name in ("Range", "If-Range", "If-None-Match", "If-Modified-Since")
- if name in request.headers
- }
- range_header = forwarded_headers.get("Range")
- # Conditional requests must reach the origin: a cached partial body cannot
- # represent a 200/304 decision made against a different validator.
- has_conditionals = any(
- name in forwarded_headers for name in ("If-Range", "If-None-Match", "If-Modified-Since")
- )
- range_key = (
- (target_url, range_header, max_bytes, media_type)
- if request.method == "GET" and range_header and not has_conditionals
- else None
- )
- if range_key:
- if not hasattr(app.state, "reader_range_cache"):
- app.state.reader_range_cache = OrderedDict()
- app.state.reader_range_cache_bytes = 0
- cached = app.state.reader_range_cache.get(range_key)
- if cached:
- cached_at, payload, cached_headers = cached
- if time.monotonic() - cached_at < READER_RANGE_CACHE_TTL_SECONDS:
- app.state.reader_range_cache.move_to_end(range_key)
- return Response(content=payload, status_code=206,
- headers=reader_proxy_timing(
- reader_document_browser_headers(target_url, cached_headers, 206), cache="hit"))
- app.state.reader_range_cache.pop(range_key, None)
- app.state.reader_range_cache_bytes -= len(payload)
- covered = reader_cached_subrange(range_key)
- if covered is not None:
- payload, cached_headers = covered
- return Response(content=payload, status_code=206, headers=reader_proxy_timing(
- reader_document_browser_headers(target_url, cached_headers, 206), cache="slice"))
- requested = re.fullmatch(r"bytes=([0-9]{1,20})-([0-9]{1,20})", range_header)
- if (_share_range and requested and int(requested[1]) <= int(requested[2])
- and int(requested[2]) - int(requested[1]) + 1 <= READER_RANGE_CACHE_MAX_ITEM_BYTES
- and (urlparse(target_url).path.lower().endswith(".pdf") or media_type == "application/pdf")):
- return await shared_reader_range_response(request, target_url, range_key, dict(
- url_validator=url_validator, cache_control=cache_control,
- max_bytes=max_bytes, media_type=media_type, validation_scope=validation_scope))
- semaphore = app.state.upstream_semaphore
- queued_at = time.monotonic()
- await semaphore.acquire()
- acquired_at = time.monotonic()
- lease = UpstreamLease(semaphore)
- try:
- upstream = lease.response = await open_download_response(
- app.state.http_session,
- target_url,
- aiohttp.ClientTimeout(total=None if is_media_stream else 180, connect=15, sock_read=60),
- method=request.method,
- request_headers=forwarded_headers,
- url_validator=url_validator,
- auto_decompress=False,
- )
- headers_at = time.monotonic()
- # Range offsets and document limits describe identity bytes, not compressed wire bytes.
- if upstream.headers.get("Content-Encoding", "identity").strip().lower() != "identity":
- return JSONResponse({"error": "上游阅读文件编码无效"}, status_code=502)
- headers = {name: upstream.headers[name] for name in READER_UPSTREAM_RESPONSE_HEADERS if name in upstream.headers}
- headers.update(READER_ISOLATION_HEADERS)
- headers["Cache-Control"] = cache_control or (
- READER_IMMUTABLE_CACHE_CONTROL
- if is_immutable_reader_asset(target_url)
- else headers.get("Cache-Control", "public, max-age=300")
- )
- # Gecko treats any Content-Encoding header as compressed media and
- # disables length/range seeking. GZipMiddleware already excludes media.
- if is_media_stream:
- headers.pop("Content-Encoding", None)
- else:
- headers["Content-Encoding"] = "identity"
- if media_type:
- headers["Content-Type"] = media_type
- status = upstream.status
- headers = reader_document_browser_headers(target_url, headers, status)
- headers = reader_proxy_timing(headers, cache="miss", queued=acquired_at - queued_at,
- upstream=headers_at - acquired_at)
- declared_length = upstream.headers.get("Content-Length")
- length = None
- if declared_length is not None:
- try:
- length = int(declared_length)
- except (TypeError, ValueError):
- return JSONResponse({"error": "上游阅读文件长度无效"}, status_code=502)
- if length < 0 or (max_bytes is not None and length > max_bytes):
- return JSONResponse({"error": "阅读文件过大"}, status_code=413)
- content_range = upstream.headers.get("Content-Range")
- if status == 206:
- match = re.fullmatch(r"bytes\s+([0-9]{1,20})-([0-9]{1,20})/([0-9]{1,20})", str(content_range or ""))
- if not match:
- return JSONResponse({"error": "上游阅读文件范围无效"}, status_code=502)
- range_start, range_end, total_length = (int(value) for value in match.groups())
- if (range_start > range_end or range_end >= total_length
- or (length is not None and length != range_end - range_start + 1)):
- return JSONResponse({"error": "上游阅读文件范围无效"}, status_code=502)
- if max_bytes is not None and total_length > max_bytes:
- return JSONResponse({"error": "阅读文件过大"}, status_code=413)
- elif status == 416:
- match = re.fullmatch(r"bytes\s+\*/([0-9]{1,20})", str(content_range or ""))
- if match and max_bytes is not None and int(match.group(1)) > max_bytes:
- return JSONResponse({"error": "阅读文件过大"}, status_code=413)
- if request.method == "HEAD" or status == 304:
- return Response(status_code=status, headers=headers)
- if status not in (200, 206, 416):
- return JSONResponse({"error": f"上游阅读文件加载失败: HTTP {status}"}, status_code=status)
-
- async def stream_reader_content():
- total = 0
- chunks = []
- cacheable = bool(
- range_key and status == 206 and length is not None
- and length <= READER_RANGE_CACHE_MAX_ITEM_BYTES
- )
- try:
- async for chunk in upstream.content.iter_chunked(65536):
- total += len(chunk)
- if max_bytes is not None and total > max_bytes:
- raise ReaderDocumentLimitError("阅读文件过大")
- if cacheable:
- chunks.append(chunk)
- yield chunk
- if cacheable and total == length:
- payload = b"".join(chunks)
- cache = app.state.reader_range_cache
- previous = cache.pop(range_key, None)
- if previous:
- app.state.reader_range_cache_bytes -= len(previous[1])
- cache[range_key] = (time.monotonic(), payload, headers)
- app.state.reader_range_cache_bytes += len(payload)
- while (app.state.reader_range_cache_bytes > READER_RANGE_CACHE_MAX_BYTES
- or len(cache) > READER_RANGE_CACHE_MAX_ENTRIES):
- _, (_, evicted, _) = cache.popitem(last=False)
- app.state.reader_range_cache_bytes -= len(evicted)
- finally:
- lease.close()
-
- return lease.streaming_response(stream_reader_content(), status_code=status, headers=headers)
- except Exception as exc:
- print(f"阅读代理异常: {exc}")
- return JSONResponse({"error": str(exc) or "上游阅读文件加载失败"}, status_code=502)
- finally:
- if not lease.transferred:
- lease.close()
-
-
-@app.api_route("/api/reader-cdn", methods=["GET", "HEAD"])
-async def api_reader_cdn(request: Request, url: str = Query(...)):
- """Resolve a public VoiceOfML asset to its short-lived CDN URL.
-
- The browser can then range-read the CDN directly. Reader clients retain
- /api/reader-content as a fallback for networks that cannot reach the CDN.
- """
- try:
- target_url = validate_reader_source_url(url)
- semaphore = app.state.upstream_semaphore
- await semaphore.acquire()
- lease = UpstreamLease(semaphore)
- try:
- upstream = lease.response = await open_download_response(
- app.state.http_session,
- target_url,
- aiohttp.ClientTimeout(total=30, connect=10, sock_read=15),
- method="HEAD",
- url_validator=validate_reader_redirect_url,
- auto_decompress=False,
- )
- final_url = str(upstream.url)
- validate_reader_redirect_url(final_url)
- if upstream.status >= 400:
- return JSONResponse({"error": f"上游阅读文件加载失败: HTTP {upstream.status}"}, status_code=upstream.status)
- return Response(status_code=307, headers={
- "Location": final_url,
- "Cache-Control": "no-store",
- "Access-Control-Allow-Origin": "*",
- })
- finally:
- lease.close()
- except Exception as exc:
- return JSONResponse({"error": str(exc) or "阅读 CDN 解析失败"}, status_code=502)
-
-
-@app.get("/api/download")
-async def api_download(file: str = Query(...), link: str = Query(...)):
- try:
- target_url = validate_voiceofml_source_url(link)
- except ValueError:
- return JSONResponse({"error": "只允许 HuggingFace 域名"}, status_code=403)
- session = app.state.http_session
- semaphore = app.state.upstream_semaphore
- await semaphore.acquire()
- with UpstreamLease(semaphore) as lease:
- try:
- upstream = lease.response = await open_download_response(
- session,
- target_url,
- aiohttp.ClientTimeout(total=120, connect=15, sock_read=30),
- url_validator=validate_voiceofml_redirect_url,
- )
- except Exception as e:
- print(f"下载代理异常: {e}")
- return JSONResponse({"error": str(e) or "上游下载失败"}, status_code=502)
- if upstream.status != 200:
- return JSONResponse({"error": f"上游下载失败: HTTP {upstream.status}"}, status_code=502)
- encoded_filename = quote(sanitize_archive_name(file), safe="")
- return lease.streaming_response(
- lease.chunks("下载代理异常"),
- media_type="application/octet-stream",
- headers={
- "Content-Disposition": f"attachment; filename*=UTF-8''{encoded_filename}",
- "Cache-Control": "no-cache",
- },
- )
-
-
-@app.get("/api/download/check")
-async def api_download_check(link: str = Query(...)):
- target_url = link
- try:
- target_url = validate_voiceofml_source_url(target_url)
- except ValueError as e:
- return JSONResponse({"error": str(e)}, status_code=403)
- session = app.state.http_session
- semaphore = app.state.upstream_semaphore
- await semaphore.acquire()
- upstream = None
- try:
- upstream = await open_download_response(
- session,
- target_url,
- aiohttp.ClientTimeout(total=30, connect=10, sock_read=10),
- method="HEAD",
- url_validator=validate_voiceofml_redirect_url,
- )
- if upstream.status not in (200, 206):
- return JSONResponse({"error": f"上游下载失败: HTTP {upstream.status}"}, status_code=502)
- return JSONResponse({"ok": True})
- except Exception as e:
- print(f"下载检查异常: {e}")
- return JSONResponse({"error": str(e) or "上游下载失败"}, status_code=502)
- finally:
- if upstream is not None:
- upstream.release()
- semaphore.release()
-TXT_DIR.mkdir(parents=True, exist_ok=True)
-
-def resolve_safe_txt_path(raw_path: str) -> Path:
- decoded_path = unquote(raw_path).replace("\\", "/").strip("/")
- normalized = posixpath.normpath(decoded_path)
- if normalized in ("", ".") or normalized.startswith("../") or normalized == "..":
- raise ValueError("invalid txt path")
- target = (TXT_DIR / normalized).resolve()
- base = TXT_DIR.resolve()
- if target != base and base not in target.parents:
- raise ValueError("invalid txt path")
- return target
-
-@app.get("/txt/{txt_path:path}")
-async def serve_txt_proxy(txt_path: str):
- try:
- local_path = resolve_safe_txt_path(txt_path)
- except ValueError:
- return JSONResponse({"error": "invalid txt path"}, status_code=400)
- if local_path.exists() and local_path.is_file():
- return FileResponse(local_path, media_type="text/plain; charset=utf-8")
- remote_path = unquote(txt_path).replace("\\", "/").strip("/")
- remote_url = TXT_SPACE_RAW_BASE + "/" + quote(remote_path, safe="/")
- session = app.state.http_session
- with UpstreamLease() as lease:
- try:
- upstream = lease.response = await open_download_response(
- session,
- remote_url,
- aiohttp.ClientTimeout(total=60, connect=10, sock_read=20),
- )
- except Exception as e:
- print(f"txt 代理异常: {e}")
- return JSONResponse({"error": str(e) or "txt 读取失败"}, status_code=502)
- if upstream.status != 200:
- status = upstream.status
- return JSONResponse({"error": f"txt 不存在: HTTP {status}"}, status_code=404 if status == 404 else 502)
- return lease.streaming_response(lease.chunks("txt 代理异常"), media_type="text/plain; charset=utf-8")
-
-HASHED_ASSET_RE = re.compile(r"\.[0-9a-f]{12}\.(?:js|css|mjs)$")
-
-
-class HashedStaticFiles(StaticFiles):
- async def get_response(self, path: str, scope):
- response = await super().get_response(path, scope)
- if response.status_code == 200 and HASHED_ASSET_RE.search(path):
- response.headers["Cache-Control"] = "public, max-age=31536000, immutable"
- return response
-
-
-app.mount("/static", HashedStaticFiles(directory="static", html=True), name="static")
-app.mount("/icons", StaticFiles(directory="static/icons"), name="icons")
-@app.get("/manifest.json")
-
-def serve_manifest():
- p = Path("static/manifest.json")
- if p.exists():
- return JSONResponse(json.loads(p.read_text(encoding="utf-8")))
- return JSONResponse({"error": "manifest not found"}, status_code=404)
-@app.get("/sw.js")
-
-def serve_sw():
- p = Path("static/sw.js")
- if p.exists():
- return PlainTextResponse(p.read_text(encoding="utf-8"), media_type="application/javascript")
- return PlainTextResponse("", status_code=404)
-
-@app.get("/data/initial/global.json")
-def serve_initial_global():
- return JSONResponse(cached_payload(("initial_global", _data_generation), lambda: initial_payload_global or build_initial_payload(None)))
-
-@app.get("/data/initial/repos/{repo_name}.json")
-def serve_initial_repo(repo_name: str):
- full_repo = f"VoiceOfML/{repo_name}"
- payload = initial_payload_by_repo.get(full_repo)
- if not payload:
- return JSONResponse({"error": f"仓库 {full_repo} 不存在"}, status_code=404)
- return JSONResponse(cached_payload(("initial_repo", _data_generation, full_repo), lambda: payload))
-
-@app.get("/data/initial/manifest.json")
-def serve_initial_manifest():
- def build():
- urls = ["/data/initial/global.json"]
- for repo in sorted(initial_payload_by_repo):
- urls.append(f"/data/initial/repos/{quote(repo.split('/')[-1], safe='')}.json")
- return {"urls": urls}
- return JSONResponse(cached_payload(("initial_manifest", _data_generation), build))
-
-def read_sidebar_json(path: Path):
- data = load_sidebar_payload(path)
- if data is None:
- return JSONResponse({"error": "sidebar data not found"}, status_code=404)
- return JSONResponse(
- cached_payload(("sidebar_file", _data_generation, str(path)), lambda: data),
- headers=METADATA_CACHE_HEADERS,
- )
-
-@app.get("/data/sidebar/global.json")
-def serve_sidebar_global():
- return read_sidebar_json(SIDEBAR_DATA_PATH / "global.json")
-
-@app.get("/data/sidebar/repos/{repo_name}.json")
-def serve_sidebar_repo(repo_name: str):
- full_repo = f"VoiceOfML/{repo_name}"
- if full_repo not in repo_counts:
- return JSONResponse({"error": f"仓库 {full_repo} 不存在"}, status_code=404)
- return read_sidebar_json(SIDEBAR_DATA_PATH / "repos" / f"{quote(repo_name, safe='')}.json")
-
-@app.get("/data/sidebar/manifest.json")
-def serve_sidebar_manifest():
- return read_sidebar_json(SIDEBAR_DATA_PATH / "manifest.json")
-
-@app.get("/")
-async def serve_root():
- html = injected_html_cache.get(None)
- if html is not None:
- return HTMLResponse(html)
- index_path = Path("static/index.html")
- if index_path.exists():
- return HTMLResponse(inject_initial_payload(index_path.read_text(encoding="utf-8"), None))
- return PlainTextResponse("Frontend shell unavailable", status_code=503)
-
-@app.get("/{rest_of_path:path}")
-
-async def serve_spa(rest_of_path: str):
- index_path = Path("static/index.html")
- if index_path.exists():
- repo_short = rest_of_path.strip("/").split("/")[0] if rest_of_path.strip("/") else None
- if repo_short and f"VoiceOfML/{repo_short}" not in repo_counts:
- return PlainTextResponse("Repository not found", status_code=404)
- cache_key = f"VoiceOfML/{repo_short}" if repo_short else None
- html = injected_html_cache.get(cache_key)
- if html is not None:
- return HTMLResponse(html)
- return HTMLResponse(inject_initial_payload(index_path.read_text(encoding="utf-8"), repo_short))
- return PlainTextResponse("Frontend shell unavailable", status_code=503)
-
-def reader_proxy_timing(headers: dict, *, cache: str, queued: float = 0, upstream: float = 0) -> dict:
- return {**headers, "Server-Timing": (
- f'reader_cache;desc="{cache}", reader_queue;dur={queued * 1000:.2f}, '
- f'reader_headers;dur={upstream * 1000:.2f}'
- )}
-
-
-def reader_cached_subrange(range_key):
- requested = re.fullmatch(r"bytes=([0-9]{1,20})-([0-9]{1,20})", range_key[1])
- if requested is None:
- return None
- start, end = (int(value) for value in requested.groups())
- if start > end or end - start + 1 > READER_RANGE_CACHE_MAX_ITEM_BYTES:
- return None
- cache = app.state.reader_range_cache
- now = time.monotonic()
- for key in reversed(list(cache)):
- cached_at, payload, headers = cache[key]
- if now - cached_at >= READER_RANGE_CACHE_TTL_SECONDS:
- cache.pop(key)
- app.state.reader_range_cache_bytes -= len(payload)
- continue
- if key[0] != range_key[0] or key[2:] != range_key[2:]:
- continue
- match = re.fullmatch(r"bytes ([0-9]+)-([0-9]+)/([0-9]+)", headers.get("Content-Range", ""))
- if match is None:
- continue
- cached_start, cached_end, total = (int(value) for value in match.groups())
- final_end = min(end, total - 1)
- if cached_start <= start <= final_end <= cached_end and len(payload) == cached_end - cached_start + 1:
- cache.move_to_end(key)
- body = payload[start - cached_start:final_end - cached_start + 1]
- return body, {**headers, "Content-Range": f"bytes {start}-{final_end}/{total}",
- "Content-Length": str(len(body))}
- return None
-
-
-async def shutdown_reader_range_flights():
- owners = getattr(app.state, "reader_range_flight_owners", set())
- await asyncio.gather(*(flight.close() for flight in list(owners)), return_exceptions=True)
- owners.clear()
- getattr(app.state, "reader_range_flights", {}).clear()
-
-
-async def shared_reader_range_response(request, target_url, range_key, options):
- if not hasattr(app.state, "reader_range_flights"):
- app.state.reader_range_flights = {}
- app.state.reader_range_flight_owners = set()
- flights = app.state.reader_range_flights
- owners = app.state.reader_range_flight_owners
- policy = options["validation_scope"] if options["validation_scope"] is not None else options["url_validator"]
- key = (range_key, policy, options["cache_control"])
- flight = flights.get(key)
- joined = flight is not None
- if flight is None:
- if len(flights) >= READER_RANGE_MAX_FLIGHTS:
- return await _stream_reader_document(request, target_url, _share_range=False, **options)
-
- def finished(owner):
- if flights.get(key) is owner:
- flights.pop(key, None)
- if owner.done:
- owners.discard(owner)
-
- flight = SharedRangeFlight(
- lambda: _stream_reader_document(request, target_url, _share_range=False, **options),
- READER_RANGE_CACHE_MAX_ITEM_BYTES, finished)
- flights[key] = flight
- owners.add(flight)
- subscription = flight.subscribe()
- shared_at = time.monotonic()
- transferred = False
- try:
- if not await asyncio.shield(flight.ready):
- response = flight.take_response()
- if response is not None:
- return response
- return await _stream_reader_document(request, target_url, _share_range=False, **options)
- headers = dict(flight.headers)
- if joined:
- headers["server-timing"] += f', reader_shared;dur={(time.monotonic() - shared_at) * 1000:.2f};desc="joined"'
- response = OwnedStreamingResponse(subscription.stream(), cleanup=subscription.release,
- status_code=206, headers=headers)
- transferred = True
- return response
- finally:
- if not transferred:
- await subscription.release()
+from backend.app import app
diff --git a/backend/__init__.py b/backend/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..2262b4a0489aa72a65adc2ef5f71ff96954031b2
--- /dev/null
+++ b/backend/__init__.py
@@ -0,0 +1 @@
+"""Search service helpers; the FastAPI entry point remains app:app."""
diff --git a/backend/app.py b/backend/app.py
new file mode 100644
index 0000000000000000000000000000000000000000..c19abef3ade74fc62af51c6f03ae9a3a9d731ded
--- /dev/null
+++ b/backend/app.py
@@ -0,0 +1,2927 @@
+"""FastAPI application, routes and process-owned search state."""
+
+import asyncio
+import gzip
+import hashlib
+import json
+import posixpath
+import random
+import re
+import sys
+import time
+import functools
+import threading
+import weakref
+from collections import OrderedDict
+from array import array
+from uuid import uuid4
+import io
+import zipfile
+import zlib
+from pathlib import Path
+from typing import Annotated, Literal, Optional, Sequence
+from urllib.parse import unquote, quote, urlparse, parse_qs
+import aiohttp
+from huggingface_hub import HfFileSystem
+import jieba
+from fastapi import FastAPI, HTTPException, Query, Request
+from fastapi.responses import FileResponse, HTMLResponse, JSONResponse, PlainTextResponse, Response
+from fastapi.staticfiles import StaticFiles
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.middleware.gzip import GZipMiddleware
+from pydantic import BaseModel, Field
+from contextlib import asynccontextmanager
+from backend.proxy_streams import OwnedStreamingResponse, SharedRangeFlight, UpstreamLease
+from backend.search_execution import SearchBusy, SearchExecutor
+from backend.search_indexes import (
+ NGRAM_MAGIC, WORD_INDEX_MAGIC, WORD_INDEX_TOKENIZER,
+ decode_postings, decode_varint, load_ngram_index, load_word_index,
+ resolve_postings,
+)
+from backend.reader_assets import (
+ READER_ASSET_FILENAMES, READER_BUCKET_PATH_RE, decode_reader_assets,
+)
+from backend.reader_v3_catalog import load_reading_catalog, merge_sidecar as merge_v3_sidecar
+from backend.data_formats import (
+ decode_browser_entry, decode_folder_browser_data, decode_folder_tree_data,
+ decode_search_payload, decode_tree_node,
+)
+from backend.record_metadata import (
+ READER_ASSET_MODES,
+ add_reader_assets, build_browser_file_name, build_reader_asset_key,
+ build_record_link, build_record_path_url, build_record_relative_path,
+ build_txt_relative_path, enrich_browser_entry, enrich_reader_record,
+ reader_asset_url, reader_extension_for_asset, reader_resolved_record,
+)
+from backend.upstream_http import (
+ ALLOWED_DOWNLOAD_HOSTS, ALLOWED_DOWNLOAD_HOST_SUFFIXES, MAX_REDIRECTS,
+ READER_ASSET_PATH_RE, READER_ASSET_SOURCE_RE, VOICEOFML_READER_SOURCE_RE,
+ decode_url_path, is_allowed_download_host, is_immutable_reader_asset,
+ normalize_download_url, open_download_response, source_url_scope,
+ validate_download_url, validate_path_safety, validate_reader_redirect_url,
+ validate_reader_source_url, validate_voiceofml_redirect_url, validate_voiceofml_source_url,
+)
+MAX_ARCHIVE_NAME_LENGTH = 240
+DATA_PATH = Path("data/search_data.json.gz")
+NGRAM_PATHS = {2: Path("data/search_ngrams_2.bin.gz"), 3: Path("data/search_ngrams_3.bin.gz")}
+WORD_INDEX_PATH = Path("data/search_words.bin.gz")
+FOLDER_TREE_PATH = Path("data/folder_tree.json.gz")
+FOLDER_BROWSER_PATH = Path("data/folder_browser.json.gz")
+TXT_DIR = Path("txt")
+TXT_SPACE_RAW_BASE = "https://huggingface.co/spaces/VoiceOfML/Search/raw/main/txt"
+TXT_WARM_INTERVAL_SECONDS = 600
+TXT_WARM_IDLE_SECONDS = 600
+READER_ASSETS_URL = "https://huggingface.co/buckets/vomebook/reader-assets-v2/resolve/reader-index/reader_assets.json.gz"
+READER_ASSETS_CACHE_TTL_SECONDS = 300
+READER_ASSETS_MAX_COMPRESSED_BYTES = 5 * 1024 * 1024
+READER_ASSETS_MAX_DECOMPRESSED_BYTES = 50 * 1024 * 1024
+READER_ASSET_ROUTING_CONTRACT_VERSION = "reader-buckets-v2"
+records: list[dict] = []
+folder_tree_data: dict[str, list[dict]] = {}
+folder_browser_data: dict[str, dict[str, dict]] = {}
+word_index = {}
+folder_index = {}
+extension_counts = {}
+repo_counts = {}
+repo_list = []
+extension_list = []
+repo_extension_counts = {}
+vocab_by_len = {}
+repo_records_map = {}
+sorted_by_name = []
+sorted_by_size = []
+repo_sorted_by_name = {}
+repo_sorted_by_size = {}
+txt_record_indices = []
+repo_txt_record_indices = {}
+READER_EXTENSIONS = frozenset({"pdf", "djvu", "caj", "kdh", "epub", "mobi", "azw", "azw3", "fb2", "fbz", "html", "htm", "txt", "md", "markdown", "vcf", "ini", "jpg", "jpeg", "png", "gif", "bmp", "tif", "tiff", "webp", "mp3", "mp4", "wav", "m4a", "flac", "mov", "mpga", "wma", "ape", "amr"})
+reader_record_indices = []
+repo_reader_record_indices = {}
+reader_record_key_indices = {}
+reader_source_id_indices = {}
+_reader_indexes_ready = False
+_reader_indexes_lock = threading.Lock()
+_reader_index_build = None
+READER_INDEX_BATCH_RECORDS = 256
+READER_INDEX_BATCH_SECONDS = .008
+READER_INDEX_WARM_DELAY_SECONDS = 5
+READER_INDEX_IDLE_SECONDS = 5
+initial_payload_global = None
+initial_payload_by_repo = {}
+API_CACHE_TTL_SECONDS = 120
+RESPONSE_CACHE_BYTES_MAX = 16 * 1024 * 1024
+RESPONSE_CACHE_ENTRY_BYTES_MAX = 2 * 1024 * 1024
+_api_response_cache_lock = threading.Lock()
+api_response_cache = {}
+_data_generation = 0
+SEARCH_CACHE_TTL_SECONDS = 120
+_search_cache: dict[tuple, tuple[float, dict, int]] = {}
+_search_instance = uuid4().hex
+# Bump when matching, filtering, ordering or public record semantics change.
+SEARCH_SEMANTICS_VERSION = "hf-search-v2"
+_search_generation = None
+_search_logged_generation = None
+_search_order_cache = OrderedDict()
+SEARCH_ORDER_CACHE_MAX = 32
+SEARCH_ORDER_CACHE_BYTES_MAX = 16 * 1024 * 1024
+SEARCH_ORDER_CACHE_TTL_SECONDS = 600
+_search_anchor_cache = OrderedDict()
+SEARCH_ANCHOR_ORDER_MAX = 8
+SEARCH_ANCHOR_PER_ORDER_MAX = 8
+SEARCH_ANCHOR_KEY_MAX = 4096
+METADATA_CACHE_HEADERS = {
+ "Cache-Control": "public, max-age=300, stale-while-revalidate=1800",
+}
+ngram_postings: dict[int, dict[str, bytes]] = {}
+ngram_posting_buffers: dict[int, bytes] = {}
+injected_html_cache: dict[str | None, str] = {}
+last_user_request_at = time.monotonic()
+reader_assets_cache = {"v": 1, "f": {}}
+reader_assets_cache_at = 0.0
+reader_assets_revision = hashlib.sha256(b'{"v":1,"f":{}}').hexdigest()
+reader_join_cache_key = None
+reader_join_cache = {"all": {}, "repos": {}}
+
+def _cached(store, key, builder, ttl, limit):
+ now = time.monotonic()
+ cached = store.get(key)
+ if cached and now - cached[0] < ttl:
+ store[key] = store.pop(key)
+ return cached[1]
+ value = builder()
+ store.pop(key, None)
+ for old_key, entry in list(store.items()):
+ if now - entry[0] >= ttl:
+ del store[old_key]
+ size = (len(value) if isinstance(value, bytes) else len(json.dumps(value, ensure_ascii=False, separators=(",", ":")).encode("utf-8"))) + len(str(key).encode("utf-8"))
+ if size > RESPONSE_CACHE_ENTRY_BYTES_MAX:
+ return value
+ total = sum(entry[2] if len(entry) > 2 else len(json.dumps(entry[1]).encode("utf-8")) for entry in store.values())
+ while store and (len(store) >= limit or total + size > RESPONSE_CACHE_BYTES_MAX):
+ oldest_key = next(iter(store))
+ entry = store.pop(oldest_key)
+ total -= entry[2] if len(entry) > 2 else len(json.dumps(entry[1]).encode("utf-8"))
+ if size <= RESPONSE_CACHE_BYTES_MAX:
+ store[key] = (now, value, size)
+ return value
+
+
+def cached_payload(key, builder):
+ with _api_response_cache_lock:
+ return _cached(api_response_cache, key, builder, API_CACHE_TTL_SECONDS, 500)
+
+
+def cached_json_response(key, builder, headers=None):
+ # Encode once, measure the actual body and reuse it within the existing budget.
+ body = cached_payload(("json", key), lambda: JSONResponse(builder()).body)
+ return Response(body, media_type="application/json", headers=headers)
+
+
+async def get_reader_assets(*, wait: bool = True) -> dict:
+ if reader_assets_cache_at != 0 and time.monotonic() - reader_assets_cache_at < READER_ASSETS_CACHE_TTL_SECONDS:
+ return reader_assets_cache
+ task = getattr(app.state, "reader_assets_task", None)
+ if task is None or task.done():
+ task = asyncio.create_task(refresh_reader_assets())
+ app.state.reader_assets_task = task
+ # Search can enrich with the last snapshot while an owned refresh runs.
+ return await asyncio.shield(task) if wait else reader_assets_cache
+
+
+async def refresh_reader_assets() -> dict:
+ global reader_assets_cache, reader_assets_cache_at, reader_assets_revision
+ now = time.monotonic()
+ if reader_assets_cache_at != 0 and now - reader_assets_cache_at < READER_ASSETS_CACHE_TTL_SECONDS:
+ return reader_assets_cache
+ async with app.state.reader_assets_lock:
+ now = time.monotonic()
+ if reader_assets_cache_at != 0 and now - reader_assets_cache_at < READER_ASSETS_CACHE_TTL_SECONDS:
+ return reader_assets_cache
+ response = None
+ try:
+ for url in (READER_ASSETS_URL,):
+ response = await app.state.http_session.get(
+ f"{url}?v={int(time.time())}",
+ timeout=aiohttp.ClientTimeout(total=20, connect=8, sock_read=10),
+ )
+ if response.status == 200:
+ break
+ if response.status != 404:
+ raise RuntimeError(f"reader assets HTTP {response.status}")
+ response.release()
+ response = None
+ if response is None:
+ raise RuntimeError("reader assets bucket and legacy sources unavailable")
+ chunks = bytearray()
+ async for chunk in response.content.iter_chunked(64 * 1024):
+ chunks.extend(chunk)
+ if len(chunks) > READER_ASSETS_MAX_COMPRESSED_BYTES:
+ raise ValueError("reader assets response is too large")
+ raw = bytes(chunks)
+ decoded = await asyncio.to_thread(decode_reader_assets, raw)
+ v3_catalog = await load_reading_catalog(app.state.http_session)
+ if v3_catalog is None and reader_assets_cache.get("v3"):
+ raise ValueError("previously published v3 pointer disappeared")
+ decoded = merge_v3_sidecar(decoded, v3_catalog)
+ reader_assets_cache = decoded
+ revision_body = raw + json.dumps(decoded["v3"], sort_keys=True).encode() if v3_catalog else raw
+ reader_assets_revision = hashlib.sha256(revision_body).hexdigest()
+ reader_assets_cache_at = time.monotonic()
+ except Exception as exc:
+ print(f"reader assets refresh failed: {exc}")
+ # Retain last-known assets, but retry failures after a short backoff.
+ reader_assets_cache_at = time.monotonic() - READER_ASSETS_CACHE_TTL_SECONDS + 5
+ finally:
+ if response is not None:
+ response.release()
+ return reader_assets_cache
+
+def records_digest() -> bytes:
+ digest = hashlib.sha256()
+ for record in records:
+ for value in (
+ record.get("_file_lower", ""),
+ record.get("_repo_lower", ""),
+ record.get("_folder_path_lower", ""),
+ ):
+ digest.update(value.encode("utf-8"))
+ digest.update(b"\0")
+ return digest.digest()
+
+
+def word_postings(token: str) -> set[int]:
+ value = word_index.get(token)
+ return set(value) if value is not None else set()
+
+
+def word_posting_count(token: str) -> int:
+ value = word_index.get(token)
+ return len(value) if value is not None else 0
+
+def literal_candidates(query: str, repos=None) -> set[int] | None:
+ width = 3 if len(query) >= 3 and ngram_postings.get(3) else 2
+ postings = ngram_postings.get(width)
+ if not postings or len(query) < width:
+ return None
+ entries = []
+ for gram in dict.fromkeys(query[index:index + width] for index in range(len(query) - width + 1)):
+ encoded = postings.get(gram)
+ if encoded is None:
+ return set()
+ entries.append(encoded)
+ # VNG3 entries carry validated counts; materialized test postings have lengths.
+ entries.sort(key=lambda entry: entry[2] if isinstance(entry, tuple) else len(entry))
+ candidates = None
+ allowed = None
+ if repos:
+ allowed = set()
+ for repo in repos:
+ allowed.update(repo_records_map.get(repo, []))
+ for encoded in entries:
+ matches = resolve_postings(encoded, ngram_posting_buffers.get(width, b""))
+ if candidates is None:
+ candidates = matches
+ if allowed is not None:
+ candidates.intersection_update(allowed)
+ else:
+ candidates.intersection_update(matches)
+ if not candidates:
+ return set()
+ return candidates or set()
+INITIAL_PAGE_SIZE = 100
+SIDEBAR_DATA_PATH = Path("data/sidebar")
+RECORD_KEY_MAP = {
+ "r": "Repo",
+ "f": "File",
+ "e": "Extension",
+ "d": "Folder",
+ "s": "Size",
+ "t": "HasTxt",
+}
+TREE_KEY_MAP = {
+ "n": "name",
+ "c": "count",
+ "df": "hasDirectFiles",
+ "ch": "children",
+}
+BROWSER_ENTRY_KEY_MAP = {
+ "d": "folders",
+ "f": "files",
+}
+BROWSER_FOLDER_ITEM_KEY_MAP = {
+ "n": "name",
+ "c": "count",
+}
+BROWSER_FILE_ITEM_KEY_MAP = {
+ "n": "name",
+ "e": "ext",
+ "t": "hasTxt",
+ "s": "size",
+}
+
+READER_SHORT_ID_RE = re.compile(r"^[0-9a-z]{13}$")
+# Only lowercase escapes, encoded unreserved ASCII and malformed percent signs
+# require normalization. Canonical reserved/UTF-8 escapes can pass through.
+READER_ESCAPE_NORMALIZATION_RE = re.compile(
+ r"%(?:[0-9a-fA-F][a-f]|[a-f][0-9a-fA-F]|2[DE]|3[0-9]|4[1-9A-F]|5[0-9AF]|6[1-9A-F]|7[0-9AE])"
+ r"|%(?![0-9a-fA-F]{2})"
+)
+
+
+def canonical_reader_source_url(value: str) -> str:
+ # Match reader/core/reader-contract.js byte normalization without decoding path separators
+ # or treating an encoded percent sign as a second escape sequence.
+ def path_replace(match):
+ def token_replace(token_match):
+ token = token_match.group()
+ if len(token) == 3:
+ char = chr(int(token[1:], 16))
+ return char if char in "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._~-" else token.upper()
+ return "%25"
+ # Quote once, then skip already-canonical reserved and UTF-8 escapes.
+ path = quote(match[2], safe="/%")
+ return match[1] + READER_ESCAPE_NORMALIZATION_RE.sub(token_replace, path)
+ return re.sub(r"^(https://huggingface\.co/datasets/VoiceOfML/)([^?#]*)", path_replace, value)
+
+
+def reader_short_id(value: str) -> str:
+ digest = 1469598103934665603
+ for byte in canonical_reader_source_url(value).encode("utf-8"):
+ digest = ((digest ^ byte) * 1099511628211) & ((1 << 64) - 1)
+ return _base36(digest).rjust(13, "0")
+
+
+def _base36(value: int) -> str:
+ alphabet = "0123456789abcdefghijklmnopqrstuvwxyz"
+ output = ""
+ while value:
+ value, remainder = divmod(value, 36)
+ output = alphabet[remainder] + output
+ return output or "0"
+
+
+def sanitize_archive_name(value) -> str:
+ name = str(value or "file").replace("\\", "/").rsplit("/", 1)[-1]
+ name = "".join(ch for ch in name if ord(ch) >= 32 and ch != "\x7f").strip()
+ if not name or name in (".", ".."):
+ name = "file"
+ return name[:MAX_ARCHIVE_NAME_LENGTH]
+
+def tokenize(text: str) -> list[str]:
+ tokens = []
+ text_lower = text.lower()
+ alpha_tokens = re.findall(r"[a-z0-9]+", text_lower)
+ tokens.extend(alpha_tokens)
+ chinese_text = re.sub(r"[a-z0-9\s]+", " ", text_lower)
+ chinese_text = re.sub(r"[^\u4e00-\u9fff\u3400-\u4dbf\s]+", " ", chinese_text)
+ chinese_tokens = jieba.lcut(chinese_text)
+ tokens.extend([t.strip() for t in chinese_tokens if t.strip()])
+ chinese_token_set = set(chinese_tokens)
+ for ch in text_lower:
+ if "\u4e00" <= ch <= "\u9fff" or "\u3400" <= ch <= "\u4dbf":
+ if ch not in chinese_token_set:
+ tokens.append(ch)
+ return list(set(tokens))
+
+def compile_exact_matcher(query: str):
+ if "*" not in query and "?" not in query:
+ lower = query.lower()
+ return lambda text: lower in text.lower()
+ # Bitset NFA: each bit is a pattern position. Collapse adjacent stars so
+ # epsilon closure needs one shift. No input can cause regex backtracking.
+ pattern = re.sub(r"\*+", "*", query)
+ stars = questions = 0
+ literals = {}
+ for index, char in enumerate(pattern):
+ bit = 1 << index
+ if char == "*":
+ stars |= bit
+ elif char == "?":
+ questions |= bit
+ else:
+ literals[char] = literals.get(char, 0) | bit
+ # Single-character regexes retain Python IGNORECASE's Unicode semantics.
+ literals = [(re.compile(re.escape(char), re.IGNORECASE), bits)
+ for char, bits in literals.items()]
+ masks = {}
+ end = 1 << len(pattern)
+ start = 1 | ((1 & stars) << 1)
+
+ def matches(text):
+ # Every literal is mandatory; C-level scans reject unrelated fields early.
+ if any(regex.search(text) is None for regex, _ in literals):
+ return False
+ active = start
+ if active & end:
+ return True
+ for char in text:
+ mask = masks.get(char)
+ if mask is None:
+ mask = sum(bits for regex, bits in literals if regex.fullmatch(char))
+ if len(masks) >= 512:
+ masks.clear()
+ masks[char] = mask
+ wildcard = char != "\n"
+ active = ((active & (mask | (questions if wildcard else 0))) << 1) | (
+ active & stars if wildcard else 0) | 1
+ active |= (active & stars) << 1
+ if active & end:
+ return True
+ return False
+ return matches
+
+
+def matches_exact_query(text: str, query: str) -> bool:
+ return compile_exact_matcher(query)(text)
+
+def edit_distance(s1: str, s2: str, max_dist: int = 2) -> int:
+ if abs(len(s1) - len(s2)) > max_dist:
+ return 999
+ prev = list(range(len(s2) + 1))
+ for i, c1 in enumerate(s1):
+ curr = [i + 1]
+ for j, c2 in enumerate(s2):
+ cost = 0 if c1 == c2 else 1
+ curr.append(min(prev[j + 1] + 1, curr[j] + 1, prev[j] + cost))
+ if min(curr) > max_dist:
+ return 999
+ prev = curr
+ return prev[-1]
+
+def could_be_fuzzy(tok: str, word: str, max_dist: int = 2) -> bool:
+ if abs(len(tok) - len(word)) > max_dist:
+ return False
+ if len(tok) < 4:
+ return True
+ counts: dict[str, int] = {}
+ for ch in tok:
+ counts[ch] = counts.get(ch, 0) + 1
+ diff = 0
+ for ch in word:
+ if counts.get(ch, 0) > 0:
+ counts[ch] -= 1
+ else:
+ diff += 1
+ if diff > 2 * max_dist:
+ return False
+ for remaining in counts.values():
+ diff += remaining
+ if diff > 2 * max_dist:
+ return False
+ return True
+
+
+def is_fuzzy_match(token: str, word: str) -> bool:
+ return could_be_fuzzy(token, word, 2) and edit_distance(token, word, max_dist=2) <= 2
+
+
+def matches_file_token(record: dict, token: str) -> bool:
+ return token in record.get("_file_lower", "") or token in record.get("_repo_lower", "")
+
+
+def scoped_word_postings(token: str, search_folders: bool):
+ postings = word_postings(token)
+ if search_folders:
+ return postings
+ return {index for index in postings if matches_file_token(records[index], token)}
+
+
+def matches_file_query(record: dict, tokens, fuzzy_tokens) -> bool:
+ file_tokens = tokenize(record.get("_file_lower", "")) if fuzzy_tokens else []
+ return all(matches_file_token(record, token) or (
+ token in fuzzy_tokens and any(is_fuzzy_match(token, word) for word in file_tokens)
+ ) for token in tokens)
+
+
+def build_word_index():
+ global word_index, vocab_by_len
+ word_index = {}
+ for idx, rec in enumerate(records):
+ file_name = rec.get("File", "")
+ folders = rec.get("Folder", [])
+ for tok in tokenize(" ".join([file_name] + folders)):
+ word_index.setdefault(tok, set()).add(idx)
+ vocab_by_len = {}
+ for word in word_index:
+ vocab_by_len.setdefault(len(word), []).append(word)
+
+
+def build_vocab_by_len():
+ global vocab_by_len
+ vocab_by_len = {}
+ for word in word_index:
+ vocab_by_len.setdefault(len(word), []).append(word)
+
+
+def _advance_reader_indexes(limit=READER_INDEX_BATCH_RECORDS, budget=READER_INDEX_BATCH_SECONDS):
+ global reader_record_indices, repo_reader_record_indices
+ global reader_record_key_indices, reader_source_id_indices, _reader_indexes_ready
+ global _reader_index_build
+ with _reader_indexes_lock:
+ if _reader_indexes_ready:
+ return True
+ if _reader_index_build is None or _reader_index_build["generation"] != _data_generation:
+ _reader_index_build = {"generation": _data_generation, "records": records, "next": 0,
+ "indices": [], "repos": {}, "keys": {}, "ids": {}}
+ build = _reader_index_build
+ deadline = time.perf_counter() + budget
+ end = min(len(build["records"]), build["next"] + limit)
+ while build["next"] < end:
+ idx = build["next"]
+ rec = build["records"][idx]
+ repo = rec.get("Repo", "")
+ build["keys"][build_reader_asset_key(rec)] = idx
+ source_id = reader_short_id(rec.get("Link") or build_record_link(rec))
+ build["ids"].setdefault(source_id, []).append(idx)
+ if rec.get("_ext_lower", "") in READER_EXTENSIONS:
+ build["indices"].append(idx)
+ build["repos"].setdefault(repo, []).append(idx)
+ build["next"] += 1
+ if time.perf_counter() >= deadline:
+ break
+ if build["next"] < len(build["records"]):
+ return False
+ if build["generation"] != _data_generation or build["records"] is not records:
+ _reader_index_build = None
+ return False
+ reader_record_indices = array('I', build["indices"])
+ repo_reader_record_indices = {key: array('I', indices) for key, indices in build["repos"].items()}
+ reader_record_key_indices = build["keys"]
+ reader_source_id_indices = build["ids"]
+ _reader_indexes_ready = True
+ _reader_index_build = None
+ return True
+
+
+def _build_reader_indexes():
+ while not _advance_reader_indexes(len(records), float("inf")):
+ pass
+
+
+def ensure_reader_indexes():
+ global _reader_indexes_ready
+ if _reader_indexes_ready:
+ return
+ _build_reader_indexes()
+
+
+async def ensure_reader_indexes_async():
+ """Join the shared background build, or start one for an early Reader request."""
+ if _reader_indexes_ready:
+ return
+ task = getattr(app.state, "reader_index_build_task", None)
+ if task is None or task.done():
+ async def build():
+ while not _reader_indexes_ready:
+ executor = getattr(app.state, "search_executor", None)
+ if executor is None:
+ await asyncio.to_thread(_advance_reader_indexes)
+ else:
+ try:
+ await executor.run(_advance_reader_indexes, background=True)
+ except SearchBusy:
+ await asyncio.sleep(.02)
+ continue
+ await asyncio.sleep(0)
+ task = asyncio.create_task(build())
+ app.state.reader_index_build_task = task
+ await asyncio.shield(task)
+
+
+async def warm_reader_indexes():
+ """Prepare Reader routing during a quiet period without delaying startup."""
+ await asyncio.sleep(READER_INDEX_WARM_DELAY_SECONDS)
+ while not _reader_indexes_ready:
+ while time.monotonic() - last_user_request_at < READER_INDEX_IDLE_SECONDS:
+ await asyncio.sleep(READER_INDEX_IDLE_SECONDS)
+ try:
+ await ensure_reader_indexes_async()
+ except SearchBusy:
+ await asyncio.sleep(READER_INDEX_IDLE_SECONDS)
+
+
+def build_indexes(build_words=True, build_reader=True):
+ global _reader_index_build
+ global reader_source_id_indices, _search_generation
+ global _data_generation, word_index, folder_index, extension_counts, repo_counts, repo_list, extension_list, repo_extension_counts, repo_records_map, sorted_by_name, sorted_by_size, repo_sorted_by_name, repo_sorted_by_size, txt_record_indices, repo_txt_record_indices, reader_record_indices, repo_reader_record_indices, reader_record_key_indices, _reader_indexes_ready
+ _data_generation += 1
+ _search_generation = None
+ _search_order_cache.clear()
+ _search_anchor_cache.clear()
+ _search_cache.clear()
+ word_index = {}
+ folder_index = {}
+ extension_counts = {}
+ repo_extension_counts = {}
+ repo_counts = {}
+ repo_records_map = {}
+ repo_sorted_by_name = {}
+ repo_sorted_by_size = {}
+ txt_record_indices = []
+ repo_txt_record_indices = {}
+ reader_record_indices = []
+ repo_reader_record_indices = {}
+ reader_record_key_indices = {}
+ reader_source_id_indices = {}
+ _reader_indexes_ready = False
+ _reader_index_build = None
+ for idx, rec in enumerate(records):
+ repo = rec.get("Repo", "")
+ repo_counts[repo] = repo_counts.get(repo, 0) + 1
+ ext = (rec.get("Extension") or "").lower()
+ if ext:
+ extension_counts[ext] = extension_counts.get(ext, 0) + 1
+ repo_ext = repo_extension_counts.setdefault(repo, {})
+ repo_ext[ext] = repo_ext.get(ext, 0) + 1
+ file_name = rec.get("File", "")
+ folders = rec.get("Folder", [])
+ if repo not in folder_index:
+ folder_index[repo] = {}
+ for depth in range(len(folders) + 1):
+ folder_path = "/".join(folders[:depth]) if depth > 0 else ""
+ folder_index[repo].setdefault(folder_path, []).append(idx)
+ repo_records_map.setdefault(repo, []).append(idx)
+ if rec.get("HasTxt"):
+ txt_record_indices.append(idx)
+ repo_txt_record_indices.setdefault(repo, []).append(idx)
+ folders_lower = [f.lower() for f in (folders or [])]
+ rec["_file_lower"] = file_name.lower()
+ rec["_folders_lower"] = folders_lower
+ rec["_folder_text"] = " ".join(folders_lower)
+ rec["_folder_path_lower"] = "/".join(folders_lower)
+ rec["_folder_path"] = "/".join(folders)
+ rec["_ext_lower"] = ext
+ rec["_repo_lower"] = repo.lower()
+ repo_list = sorted(
+ [{"name": r, "count": c} for r, c in repo_counts.items()],
+ key=lambda x: x["name"],
+ )
+ if build_words:
+ build_word_index()
+ else:
+ word_index = {}
+ global vocab_by_len
+ vocab_by_len = {}
+ extension_list = sorted(extension_counts.keys())
+ all_indices = list(range(len(records)))
+ sorted_by_name = sorted(all_indices, key=lambda i: (records[i].get("File") or "").lower())
+ sorted_by_size = sorted(all_indices, key=lambda i: (
+ -(records[i].get("Size") if isinstance(records[i].get("Size"), (int, float)) else 0),
+ (records[i].get("File") or "").lower(),
+ ))
+ for repo, repo_indices in repo_records_map.items():
+ repo_sorted_by_name[repo] = sorted(repo_indices, key=lambda i: (records[i].get("File") or "").lower())
+ repo_sorted_by_size[repo] = sorted(repo_indices, key=lambda i: (
+ -(records[i].get("Size") if isinstance(records[i].get("Size"), (int, float)) else 0),
+ (records[i].get("File") or "").lower(),
+ ))
+ # These complete, ordered postings are read-only after publication.
+ # Packed IDs avoid retaining a Python pointer (and often an int) per slot.
+ sorted_by_name = array('I', sorted_by_name)
+ sorted_by_size = array('I', sorted_by_size)
+ txt_record_indices = array('I', txt_record_indices)
+ for mapping in (repo_records_map, repo_sorted_by_name, repo_sorted_by_size,
+ repo_txt_record_indices, *folder_index.values()):
+ for key, indices in mapping.items():
+ mapping[key] = array('I', indices)
+ if build_reader:
+ _build_reader_indexes()
+
+def search_data_generation(raw: bytes, index_paths) -> str:
+ digest = hashlib.sha256()
+ digest.update(SEARCH_SEMANTICS_VERSION.encode())
+ digest.update(str(sys.version_info[:3]).encode())
+ digest.update(jieba.__version__.encode())
+ digest.update(hashlib.sha256(raw).digest())
+ for path in index_paths:
+ # Index content matters too; file timestamps and process IDs do not.
+ digest.update(path.name.encode())
+ digest.update(hashlib.sha256(path.read_bytes()).digest() if path.exists() else b"missing")
+ return "api:" + digest.hexdigest()
+
+
+def load_data():
+ global records, folder_tree_data, folder_browser_data, ngram_postings, ngram_posting_buffers, word_index, _search_generation
+ start = time.perf_counter()
+ stage_start = start
+ timings = {}
+ if DATA_PATH.exists():
+ raw = gzip.decompress(DATA_PATH.read_bytes())
+ records = decode_search_payload(json.loads(raw.decode("utf-8")))
+ else:
+ print("⚠ 数据文件不存在")
+ records = []
+ return
+ if FOLDER_TREE_PATH.exists():
+ folder_tree_data = decode_folder_tree_data(json.loads(gzip.decompress(FOLDER_TREE_PATH.read_bytes()).decode("utf-8")))
+ else:
+ folder_tree_data = {}
+ if FOLDER_BROWSER_PATH.exists():
+ folder_browser_data = decode_folder_browser_data(json.loads(gzip.decompress(FOLDER_BROWSER_PATH.read_bytes()).decode("utf-8")))
+ else:
+ folder_browser_data = {}
+ timings["data_ms"] = round((time.perf_counter() - stage_start) * 1000, 2)
+ print(f"📖 已加载 {len(records)} 条记录 ({time.perf_counter() - start:.2f}s)")
+ stage_start = time.perf_counter()
+ # Reader ID/pool indexes are built on first Reader use; normal search does
+ # not need to scan every record to prepare them during cold startup.
+ build_indexes(build_words=False, build_reader=False)
+ digest = records_digest()
+ timings["metadata_ms"] = round((time.perf_counter() - stage_start) * 1000, 2)
+ stage_start = time.perf_counter()
+ word_index = load_word_index(WORD_INDEX_PATH, len(records), digest)
+ word_index_source = "generated" if word_index else "rebuilt"
+ if word_index:
+ build_vocab_by_len()
+ else:
+ build_word_index()
+ timings["words_ms"] = round((time.perf_counter() - stage_start) * 1000, 2)
+ stage_start = time.perf_counter()
+ loaded_ngrams = {
+ width: load_ngram_index(path, width, len(records), digest)
+ for width, path in NGRAM_PATHS.items()
+ }
+ ngram_postings = {width: loaded[0] for width, loaded in loaded_ngrams.items()}
+ ngram_posting_buffers = {width: loaded[1] for width, loaded in loaded_ngrams.items()}
+ timings["ngrams_ms"] = round((time.perf_counter() - stage_start) * 1000, 2)
+ stage_start = time.perf_counter()
+ _search_generation = search_data_generation(raw, [WORD_INDEX_PATH, *NGRAM_PATHS.values()])
+ timings["fingerprint_ms"] = round((time.perf_counter() - stage_start) * 1000, 2)
+ print(f"🧭 已加载词索引: {len(word_index)}")
+ print(f"🔎 已加载 ngram 索引: 2={len(ngram_postings[2])}, 3={len(ngram_postings[3])}")
+ build_initial_payloads()
+ precompute_injected_html()
+ del raw
+ stage_start = time.perf_counter()
+ # A generated postings file avoids corpus tokenization, but query tokenization
+ # still needs jieba's dictionary. Prepare it before accepting the first query,
+ # after releasing the decoded source and finishing index-loading temporaries.
+ jieba.initialize()
+ timings["tokenizer_ms"] = round((time.perf_counter() - stage_start) * 1000, 2)
+ timings["total_ms"] = round((time.perf_counter() - start) * 1000, 2)
+ print("search_performance " + json.dumps({"stage": "startup", "word_index": word_index_source, **timings}))
+
+def score_record(rec_idx, query_tokens, search_folders=True):
+ rec = records[rec_idx]
+ score = 0
+ file_name = rec.get("_file_lower", "")
+ repo = rec.get("_repo_lower", "")
+ folder_text = rec.get("_folder_text", "")
+ for tok in query_tokens:
+ if tok in file_name:
+ score += 3
+ if search_folders and tok in folder_text:
+ score += 2
+ if tok in repo:
+ score += 1
+ return score
+
+def apply_filters(indices, repos=None, extensions=None, folders=None, min_size=None, max_size=None, folder_match_mode="prefix"):
+ repos_set = set(repos) if repos else None
+ extensions_lower = {e.lower() for e in extensions} if extensions else None
+ folder_set = {f.strip("/") for f in folders} if folders else None
+ if folder_match_mode != "exact" and folder_set and "" in folder_set:
+ # Unlike the Worker, an empty API prefix includes the complete scope.
+ folder_set = None
+ folder_prefix = next(iter(folder_set)) + "/" if folder_set and len(folder_set) == 1 and folder_match_mode != "exact" else None
+ result = []
+ for idx in indices:
+ rec = records[idx]
+ if repos_set and rec.get("Repo") not in repos_set:
+ continue
+ if extensions_lower:
+ if rec.get("_ext_lower", "") not in extensions_lower:
+ continue
+ if folder_set:
+ rec_path = rec["_folder_path"] if "_folder_path" in rec else "/".join(rec.get("Folder", []) or [])
+ matched = rec_path in folder_set
+ if not matched and folder_match_mode != "exact":
+ if folder_prefix is not None:
+ matched = rec_path.startswith(folder_prefix)
+ else:
+ end = rec_path.rfind("/")
+ while end > 0:
+ if rec_path[:end] in folder_set:
+ matched = True
+ break
+ end = rec_path.rfind("/", 0, end)
+ if not matched:
+ continue
+ size = rec.get("Size")
+ if size and isinstance(size, (int, float)) and size > 0:
+ if min_size is not None and size < min_size:
+ continue
+ if max_size is not None and size > max_size:
+ continue
+ result.append(idx)
+ return result
+
+def public_record(record: dict) -> dict:
+ item = {
+ key: value
+ for key, value in record.items()
+ if isinstance(key, str) and not key.startswith("_")
+ }
+ item.setdefault("Repo", "")
+ item.setdefault("File", "")
+ item.setdefault("Extension", "")
+ item.setdefault("Folder", [])
+ item.setdefault("Size", "")
+ item.setdefault("HasTxt", False)
+ item["Link"] = record.get("Link") or build_record_link(record)
+ item["Path"] = record.get("Path") or build_record_path_url(record)
+ return item
+
+
+def trim_results(raw):
+ keys = ("Repo", "File", "Extension", "Link", "Path", "Folder", "Size", "HasTxt")
+ return [
+ {key: public.get(key, "" if key != "Folder" else []) for key in keys}
+ for public in (public_record(record) for record in raw)
+ ]
+
+def sort_key_name(idx: int):
+ return (records[idx].get("File") or "").lower()
+
+def sort_key_size(idx: int):
+ return (
+ -(records[idx].get("Size") if isinstance(records[idx].get("Size"), (int, float)) else 0),
+ (records[idx].get("File") or "").lower(),
+ )
+
+def paginate_indices(indices: Sequence[int], page: int, page_size: int) -> tuple[int, list[dict]]:
+ total = len(indices)
+ start = (page - 1) * page_size
+ return total, [records[i] for i in indices[start:start + page_size]]
+
+def get_initial_indices(repos=None) -> Sequence[int]:
+ if repos:
+ indices = []
+ for repo in dict.fromkeys(repos):
+ indices.extend(repo_records_map.get(repo, []))
+ return indices
+ return range(len(records))
+
+def search_generation() -> str:
+ return _search_generation or f"api:{_search_instance}:{_data_generation}"
+
+
+def build_response(results: list[dict], total: int, page: int, page_size: int) -> dict:
+ return {
+ "results": trim_results(results),
+ "total": total,
+ "page": page,
+ "page_size": page_size,
+ "generation": search_generation(),
+ }
+
+
+def trim_initial_results(raw: list[dict]) -> list[dict]:
+ keys = ("Repo", "File", "Extension", "Folder", "Size", "HasTxt")
+ return [
+ {key: record.get(key, [] if key == "Folder" else "") for key in keys}
+ for record in raw
+ ]
+
+def build_initial_payload(repo: str | None = None) -> dict:
+ if repo:
+ indices = repo_records_map.get(repo, [])
+ result_records = [records[i] for i in indices[:INITIAL_PAGE_SIZE]]
+ mode = "repo"
+ total = len(indices)
+ else:
+ result_records = records[:INITIAL_PAGE_SIZE]
+ mode = "global"
+ total = len(records)
+ return {
+ "mode": mode,
+ "repo": repo,
+ "sort": "relevance",
+ "page": 1,
+ "page_size": INITIAL_PAGE_SIZE,
+ "total": total,
+ "results": trim_initial_results(result_records),
+ "generation": search_generation(),
+ }
+
+def build_initial_payloads():
+ global initial_payload_global, initial_payload_by_repo
+ initial_payload_global = build_initial_payload(None)
+ initial_payload_by_repo = {
+ repo: build_initial_payload(repo)
+ for repo in repo_records_map
+ }
+
+def precompute_injected_html():
+ global injected_html_cache
+ index_path = Path("static/index.html")
+ if not index_path.exists():
+ injected_html_cache = {}
+ return
+ html = index_path.read_text(encoding="utf-8")
+ injected_html_cache = {None: inject_initial_payload(html, None)}
+ for repo in repo_records_map:
+ injected_html_cache[repo] = inject_initial_payload(html, repo.split("/")[-1])
+
+def load_sidebar_payload(path: Path):
+ if not path.exists() or not path.is_file():
+ return None
+ try:
+ return json.loads(path.read_text(encoding="utf-8"))
+ except Exception:
+ return None
+
+def build_global_bootstrap():
+ sidebar = load_sidebar_payload(SIDEBAR_DATA_PATH / "global.json") or {}
+ return {
+ "generation": _data_generation,
+ "repos": repo_list,
+ "extensions": [{"name": name, "count": extension_counts[name]} for name in sorted(extension_counts)],
+ "random_txt": {"available": bool(txt_record_indices), "count": len(txt_record_indices)},
+ "sidebar": sidebar,
+ }
+
+def inject_initial_payload(html: str, repo_short: str | None = None) -> str:
+ repo = f"VoiceOfML/{repo_short}" if repo_short else None
+ payload = initial_payload_by_repo.get(repo) if repo else initial_payload_global
+ if not payload:
+ return html
+ script = (
+ '