Spaces:
Running
Running
File size: 4,378 Bytes
d30d906 b97b073 d30d906 b97b073 d30d906 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 | """Validation and sanitization helpers for sample-query retrieval data."""
from __future__ import annotations
import copy
import re
import unicodedata
from collections import defaultdict
from typing import Iterable
_SPACE_RE = re.compile(r"\s+")
_TRAILING_PUNCT_RE = re.compile(r"[\s?.!,~。？！]+$")
def normalize_query_text(text: str) -> str:
value = unicodedata.normalize("NFKC", str(text or "")).strip().lower()
value = _TRAILING_PUNCT_RE.sub("", value)
return _SPACE_RE.sub(" ", value)
def conflicting_query_texts(rows: Iterable[dict]) -> set[str]:
"""Return normalized texts assigned to more than one menu."""
menu_ids_by_query: dict[str, set[str]] = defaultdict(set)
for row in rows:
query = normalize_query_text(row.get("query", ""))
menu_id = str(row.get("menu_id", "")).strip()
if query and menu_id:
menu_ids_by_query[query].add(menu_id)
return {
query
for query, menu_ids in menu_ids_by_query.items()
if len(menu_ids) > 1
}
def sanitize_query_rows(
rows: Iterable[dict],
*,
excluded_queries: set[str] | None = None,
) -> tuple[list[dict], dict]:
"""Drop conflicting, held-out, empty, and duplicate query rows."""
source_rows = list(rows)
rows_by_query: dict[str, list[dict]] = defaultdict(list)
for row in source_rows:
normalized = normalize_query_text(row.get("query", ""))
if normalized:
rows_by_query[normalized].append(row)
shared_conflicts = {
query
for query, query_rows in rows_by_query.items()
if len({str(row.get("menu_id", "")).strip() for row in query_rows}) > 1
and all(row.get("allow_shared_query", False) for row in query_rows)
}
conflicts = conflicting_query_texts(source_rows) - shared_conflicts
excluded = {
normalize_query_text(query)
for query in (excluded_queries or set())
if normalize_query_text(query)
}
output = []
seen_pairs = set()
stats = {
"input": len(source_rows),
"empty": 0,
"conflicting": 0,
"held_out": 0,
"duplicate": 0,
"output": 0,
"conflicting_texts": len(conflicts),
"shared_conflicting_texts": len(shared_conflicts),
}
for row in source_rows:
query = normalize_query_text(row.get("query", ""))
menu_id = str(row.get("menu_id", "")).strip()
if not query or not menu_id:
stats["empty"] += 1
continue
if query in conflicts:
stats["conflicting"] += 1
continue
if query in excluded:
stats["held_out"] += 1
continue
pair = (query, menu_id)
if pair in seen_pairs:
stats["duplicate"] += 1
continue
seen_pairs.add(pair)
clean = dict(row)
clean["query"] = str(row.get("query", "")).strip()
clean["menu_id"] = menu_id
output.append(clean)
stats["output"] = len(output)
return output, stats
def sanitize_description_queries(
descriptions: Iterable[dict],
) -> tuple[list[dict], dict]:
"""Remove exact sample-query conflicts without modifying source objects."""
source = list(descriptions)
rows = []
for desc in source:
for query in desc.get("sample_queries", []) or []:
rows.append({"query": query, "menu_id": desc.get("menu_id", "")})
conflicts = conflicting_query_texts(rows)
output = []
removed = 0
duplicates = 0
for desc in source:
clean = copy.deepcopy(desc)
kept = []
seen = set()
for query in desc.get("sample_queries", []) or []:
normalized = normalize_query_text(query)
if not normalized or normalized in conflicts:
removed += 1
continue
if normalized in seen:
duplicates += 1
continue
seen.add(normalized)
kept.append(str(query).strip())
clean["sample_queries"] = kept
output.append(clean)
return output, {
"menus": len(source),
"input_queries": len(rows),
"conflicting_texts": len(conflicts),
"removed_conflicting": removed,
"removed_duplicate": duplicates,
"output_queries": len(rows) - removed - duplicates,
}
|