Buckets:
AnonymousSpottedToad/candidate_bucket / codebase /python /HKUDS_LightRAG /tests /test_query_validation.py
| import pytest | |
| from lightrag.query_validation import ( | |
| _WIDE_CODEPOINT_RANGES, | |
| EmptyQueryError, | |
| RAGQueryTooShortError, | |
| meets_min_rag_query_weight, | |
| rag_query_weight, | |
| validate_query_not_empty, | |
| validate_rag_query, | |
| ) | |
| pytestmark = pytest.mark.offline | |
| def test_rag_query_weight_counts_east_asian_characters_as_two(query, expected): | |
| assert rag_query_weight(query) == expected | |
| def test_validate_rag_query_rejects_weight_below_three(query): | |
| with pytest.raises(ValueError): | |
| validate_rag_query(query) | |
| def test_validate_rag_query_accepts_and_strips_valid_queries(query, expected): | |
| assert validate_rag_query(query) == expected | |
| def test_halfwidth_and_fullwidth_kana_weigh_the_same(halfwidth, fullwidth): | |
| """The same word must not hinge on which encoding the keyboard emits.""" | |
| assert rag_query_weight(halfwidth) == rag_query_weight(fullwidth) | |
| assert validate_rag_query(halfwidth) == halfwidth | |
| def test_validate_query_not_empty_rejects_blank_text(query): | |
| with pytest.raises(EmptyQueryError): | |
| validate_query_not_empty(query) | |
| def test_validate_query_not_empty_allows_any_non_blank_text(query, expected): | |
| """The non-empty rule is mode-independent; the RAG minimum is not.""" | |
| assert validate_query_not_empty(query) == expected | |
| def test_an_empty_rag_query_is_reported_as_empty_not_as_too_short(): | |
| """The user-facing distinction: nothing typed vs. not enough typed.""" | |
| with pytest.raises(EmptyQueryError): | |
| validate_rag_query(" ") | |
| with pytest.raises(RAGQueryTooShortError): | |
| validate_rag_query("ab") | |
| def test_meets_min_rag_query_weight_agrees_with_the_full_walk(query, expected): | |
| assert meets_min_rag_query_weight(query) is expected | |
| assert meets_min_rag_query_weight(query) is (rag_query_weight(query) >= 3) | |
| def test_the_threshold_check_does_not_walk_the_whole_query(): | |
| """A 64 KiB query must not cost 65,536 iterations on the event loop. | |
| `rag_query_weight` walking the full string added ~69 ms of synchronous CPU | |
| to every max-size RAG request; only the comparison against the minimum is | |
| ever needed, and that is decided by the third character at the latest. | |
| """ | |
| consumed = 0 | |
| class _CountingStr(str): | |
| """Counts how far the validator actually reads.""" | |
| def __iter__(self): | |
| nonlocal consumed | |
| for character in str(self): | |
| consumed += 1 | |
| yield character | |
| def strip(self): | |
| return self | |
| assert meets_min_rag_query_weight(_CountingStr("a" * 65536)) is True | |
| assert consumed == 3 | |
| def test_every_cjk_ideograph_plane_is_covered_without_an_edit(codepoint, block): | |
| """Why the table is coarse rather than per assigned character. | |
| Planes 2 and 3 are allocated to CJK ideographs, so one range covers every | |
| extension that exists and every one that will. Enumerating assigned | |
| characters instead meant Extension J shipped in Unicode 17 weighing 1, and | |
| needed an edit in two languages to fix — as would Extension K, and so on. | |
| """ | |
| assert rag_query_weight(chr(codepoint) * 2) == 4, block | |
| def test_the_coarse_table_over_counts_and_that_is_the_deal(query, what): | |
| """Pins the imprecision as deliberate, so it is not "fixed" back. | |
| The ranges are block- and plane-granular, so punctuation, fillers and | |
| unassigned code points inside them weigh 2, and two of them clear the | |
| minimum. Chasing that per character is what made the table need an edit, in | |
| two languages, on every Unicode release. The cost of getting it wrong is one | |
| retrieval that finds nothing, for input nobody types meaning to ask | |
| something — this check is a coarse heuristic for "did the user ask | |
| anything", not a correctness boundary. | |
| """ | |
| assert rag_query_weight(query) == 4, what | |
| assert validate_rag_query(query) == query | |
| # The Unicode blocks the ranges are built from, at their true boundaries. | |
| _SOURCE_BLOCKS = { | |
| "Hangul Jamo": (0x1100, 0x11FF), | |
| "CJK Radicals Supplement": (0x2E80, 0x2EFF), | |
| "Kangxi Radicals": (0x2F00, 0x2FDF), | |
| "CJK Symbols and Punctuation": (0x3000, 0x303F), | |
| "Hiragana": (0x3040, 0x309F), | |
| "Katakana": (0x30A0, 0x30FF), | |
| "Bopomofo": (0x3100, 0x312F), | |
| "Hangul Compatibility Jamo": (0x3130, 0x318F), | |
| "Bopomofo Extended": (0x31A0, 0x31BF), | |
| "Katakana Phonetic Extensions": (0x31F0, 0x31FF), | |
| "CJK Compatibility": (0x3300, 0x33FF), | |
| "CJK Extension A": (0x3400, 0x4DBF), | |
| "CJK Unified Ideographs": (0x4E00, 0x9FFF), | |
| "Hangul Jamo Extended-A": (0xA960, 0xA97F), | |
| "Hangul Syllables": (0xAC00, 0xD7AF), | |
| "Hangul Jamo Extended-B": (0xD7B0, 0xD7FF), | |
| "CJK Compatibility Ideographs": (0xF900, 0xFAFF), | |
| "CJK Compatibility Forms": (0xFE30, 0xFE4F), | |
| "Halfwidth and Fullwidth Forms": (0xFF00, 0xFFEE), | |
| "Ideographic Symbols and Punctuation": (0x16FE0, 0x16FFF), | |
| "Kana Extended-B": (0x1AFF0, 0x1AFFF), | |
| "Kana Supplement": (0x1B000, 0x1B0FF), | |
| "Nushu": (0x1B170, 0x1B2FF), | |
| } | |
| def test_no_range_stops_part_way_through_a_block(name, bounds): | |
| """Coarse means WHOLE blocks — a boundary drawn inside one is a bug. | |
| Ending a range early is how Korean medial and final jamo came to weigh 1: | |
| U+115F was carried over from an older per-character table, where it marked | |
| the choseong filler, and it cut Hangul Jamo in half. Both ends of every | |
| source block must be covered, so a stale interior boundary cannot survive. | |
| """ | |
| start, end = bounds | |
| for edge in (start, end): | |
| assert any(low <= edge <= high for low, high in _WIDE_CODEPOINT_RANGES), ( | |
| f"{name}: U+{edge:04X} not covered" | |
| ) | |
| def test_scripts_that_are_wide_but_not_cjk_are_out(codepoint, script): | |
| """The contract is Chinese, Japanese and Korean — not "East Asian wide". | |
| Tangut, Khitan and Yi are wide and Han-adjacent, and Tangut and Yi were | |
| briefly weighted here, from reading a width table instead of the contract | |
| the docs and all eleven locale strings state. The only thing that came of it | |
| was a report that Tangut was covered inconsistently — half its blocks in, | |
| the Supplement out. Answering that by adding every wide block grows the | |
| table and invites the next report; matching the contract ends the class. | |
| """ | |
| assert rag_query_weight(chr(codepoint) * 2) == 2, script | |
| def test_the_weighted_ranges_are_sorted_and_disjoint(): | |
| for earlier, later in zip(_WIDE_CODEPOINT_RANGES, _WIDE_CODEPOINT_RANGES[1:]): | |
| assert earlier[0] <= earlier[1] | |
| assert earlier[1] < later[0] | |
| def test_the_frontend_range_table_matches_this_one(): | |
| """The two sides are duplicated by necessity and drift silently. | |
| The WebUI pre-validates so the user is told why before a round trip; the | |
| server validates because it is the authority. They only agree if the tables | |
| agree, and a comment asking for that is not a mechanism — the halfwidth kana | |
| gap was exactly this drift, present in both files at once. | |
| """ | |
| import re | |
| from pathlib import Path | |
| source = ( | |
| Path(__file__).resolve().parents[1] | |
| / "lightrag_webui" | |
| / "src" | |
| / "utils" | |
| / "queryValidation.ts" | |
| ).read_text(encoding="utf-8") | |
| def parse(name: str) -> tuple[tuple[int, int], ...]: | |
| table = re.search(rf"{name}[^=]*=\s*\[(.*?)\n\]", source, re.DOTALL) | |
| assert table, f"could not locate {name} in queryValidation.ts" | |
| return tuple( | |
| (int(start, 16), int(end, 16)) | |
| for start, end in re.findall( | |
| r"\[\s*0x([0-9a-fA-F]+)\s*,\s*0x([0-9a-fA-F]+)\s*\]", table.group(1) | |
| ) | |
| ) | |
| assert parse("WIDE_CODEPOINT_RANGES") == _WIDE_CODEPOINT_RANGES | |
Xet Storage Details
- Size:
- 10.5 kB
- Xet hash:
- 9ee81821dfb56459b8a415fe0634d36be41c3e2dd09a41739973ec482063289c
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.