""" TOKEN OPTIMIZER - Smart Chunk Selection & Context Condensation """ import re from typing import List, Dict class TokenOptimizer: def __init__(self, max_tokens: int = 4000): self.max_tokens = max_tokens def select_best_chunks(self, results: List[Dict], question: str) -> List[Dict]: if not results: return [] scored = [] question_words = set(question.lower().split()) for r in results: content = r['content'].lower() content_words = set(content.split()) overlap = len(question_words & content_words) / max(len(question_words), 1) has_number = bool(re.search(r'\d+\.?\d*\s*(?:mm|m|%|°C|mph)', content)) number_score = 0.3 if has_number else 0 has_table = bool(re.search(r'TABLE|----|\|\s*\d+', content)) table_score = 0.2 if has_table else 0 source = r['metadata'].get('source', '') is_primary = any(x in source.lower() for x in ['2102', '2500', '2049', 'sop']) source_score = 0.2 if is_primary else 0 total_score = overlap + number_score + table_score + source_score scored.append((r, total_score)) scored.sort(key=lambda x: x[1], reverse=True) selected = [] current_tokens = 0 for r, score in scored: chunk_tokens = len(r['content']) // 4 if current_tokens + chunk_tokens <= self.max_tokens: selected.append(r) current_tokens += chunk_tokens else: if chunk_tokens > 200: r['content'] = r['content'][:800] selected.append(r) break return selected def condense_context(self, chunks: List[Dict], max_chars: int = 2000) -> str: if not chunks: return "No relevant context found." context_parts = [] for i, chunk in enumerate(chunks, 1): content = chunk['content'] source = chunk['metadata'].get('source', 'Unknown') sentences = re.split(r'[.!?]+', content) key_sentences = [] for s in sentences: if re.search(r'\d+\.?\d*\s*(?:mm|m|%|°C|mph|km/h)|must|shall|required|minimum|maximum', s): key_sentences.append(s.strip()) elif len(key_sentences) < 2: key_sentences.append(s.strip()) condensed = '. '.join(key_sentences[:3]) if condensed: context_parts.append(f"[{i}] {source}: {condensed}") result = "\n".join(context_parts) return result[:max_chars] _optimizer = TokenOptimizer()