Spaces:
Running on Zero
Running on Zero
| """Real-tokenizer chunk-length measurement. | |
| Chunk-sizing decisions must be made in the same units the embedding model | |
| actually enforces. The v2 chunker used a len(text)//4 approximation, which | |
| measured close to real BERT-tokenizer counts at the median but diverged badly | |
| at the tail (p95 ratio 1.44x) — exactly where it mattered, causing ~51% of | |
| chunks to silently exceed bge-small's 512-token hard limit at embed time. | |
| This loads ONLY the tokenizer (not the full embedding model) — cheap, CPU-only, | |
| no GPU/VRAM contention with the embedder, safe to call from every chunking | |
| function without a performance penalty. | |
| """ | |
| from __future__ import annotations | |
| from .config import EMBED_MODEL | |
| _tokenizer = None | |
| def get_tokenizer(): | |
| global _tokenizer | |
| if _tokenizer is None: | |
| from transformers import AutoTokenizer | |
| _tokenizer = AutoTokenizer.from_pretrained(EMBED_MODEL) | |
| return _tokenizer | |
| def tok_len(text: str) -> int: | |
| """Real, untruncated token count for chunk-sizing decisions (no special | |
| tokens — those add a small constant overhead the embedder pads for | |
| automatically). truncation=False is explicit: verified the library | |
| default already doesn't truncate here, but this must never silently | |
| depend on that — an undercount would defeat the whole point of using | |
| the real tokenizer instead of the old chars/4 approximation.""" | |
| if not text: | |
| return 0 | |
| return len(get_tokenizer()(text, add_special_tokens=False, | |
| truncation=False)["input_ids"]) | |