flowmarg / mcp /token_optimizer.py
akash
Deploy to Hugging Face (source code only, no binary files)
2379054
Raw History Blame Contribute Delete
2.89 kB
"""
TOKEN OPTIMIZER - Smart Chunk Selection & Context Condensation
"""
import re
from typing import List, Dict
class TokenOptimizer:
def __init__(self, max_tokens: int = 4000):
self.max_tokens = max_tokens
def select_best_chunks(self, results: List[Dict], question: str) -> List[Dict]:
if not results:
return []
scored = []
question_words = set(question.lower().split())
for r in results:
content = r['content'].lower()
content_words = set(content.split())
overlap = len(question_words & content_words) / max(len(question_words), 1)
has_number = bool(re.search(r'\d+\.?\d*\s*(?:mm|m|%|°C|mph)', content))
number_score = 0.3 if has_number else 0
has_table = bool(re.search(r'TABLE|----|\|\s*\d+', content))
table_score = 0.2 if has_table else 0
source = r['metadata'].get('source', '')
is_primary = any(x in source.lower() for x in ['2102', '2500', '2049', 'sop'])
source_score = 0.2 if is_primary else 0
total_score = overlap + number_score + table_score + source_score
scored.append((r, total_score))
scored.sort(key=lambda x: x[1], reverse=True)
selected = []
current_tokens = 0
for r, score in scored:
chunk_tokens = len(r['content']) // 4
if current_tokens + chunk_tokens <= self.max_tokens:
selected.append(r)
current_tokens += chunk_tokens
else:
if chunk_tokens > 200:
r['content'] = r['content'][:800]
selected.append(r)
break
return selected
def condense_context(self, chunks: List[Dict], max_chars: int = 2000) -> str:
if not chunks:
return "No relevant context found."
context_parts = []
for i, chunk in enumerate(chunks, 1):
content = chunk['content']
source = chunk['metadata'].get('source', 'Unknown')
sentences = re.split(r'[.!?]+', content)
key_sentences = []
for s in sentences:
if re.search(r'\d+\.?\d*\s*(?:mm|m|%|°C|mph|km/h)|must|shall|required|minimum|maximum', s):
key_sentences.append(s.strip())
elif len(key_sentences) < 2:
key_sentences.append(s.strip())
condensed = '. '.join(key_sentences[:3])
if condensed:
context_parts.append(f"[{i}] {source}: {condensed}")
result = "\n".join(context_parts)
return result[:max_chars]
_optimizer = TokenOptimizer()