Download mcp/token_optimizer.py from Surajnitp/flowmarg: direct link, hf CLI and curl.
- Browser
- Download file 2.89 kB
-
https://huggingface.co/spaces/Surajnitp/flowmarg/resolve/main/mcp/token_optimizer.py
- Command line
-
hf download hf://spaces/Surajnitp/flowmarg/mcp/token_optimizer.py
-
curl -L -o token_optimizer.py https://huggingface.co/spaces/Surajnitp/flowmarg/resolve/main/mcp/token_optimizer.py
2.89 kB
| """ | |
| TOKEN OPTIMIZER - Smart Chunk Selection & Context Condensation | |
| """ | |
| import re | |
| from typing import List, Dict | |
| class TokenOptimizer: | |
| def __init__(self, max_tokens: int = 4000): | |
| self.max_tokens = max_tokens | |
| def select_best_chunks(self, results: List[Dict], question: str) -> List[Dict]: | |
| if not results: | |
| return [] | |
| scored = [] | |
| question_words = set(question.lower().split()) | |
| for r in results: | |
| content = r['content'].lower() | |
| content_words = set(content.split()) | |
| overlap = len(question_words & content_words) / max(len(question_words), 1) | |
| has_number = bool(re.search(r'\d+\.?\d*\s*(?:mm|m|%|°C|mph)', content)) | |
| number_score = 0.3 if has_number else 0 | |
| has_table = bool(re.search(r'TABLE|----|\|\s*\d+', content)) | |
| table_score = 0.2 if has_table else 0 | |
| source = r['metadata'].get('source', '') | |
| is_primary = any(x in source.lower() for x in ['2102', '2500', '2049', 'sop']) | |
| source_score = 0.2 if is_primary else 0 | |
| total_score = overlap + number_score + table_score + source_score | |
| scored.append((r, total_score)) | |
| scored.sort(key=lambda x: x[1], reverse=True) | |
| selected = [] | |
| current_tokens = 0 | |
| for r, score in scored: | |
| chunk_tokens = len(r['content']) // 4 | |
| if current_tokens + chunk_tokens <= self.max_tokens: | |
| selected.append(r) | |
| current_tokens += chunk_tokens | |
| else: | |
| if chunk_tokens > 200: | |
| r['content'] = r['content'][:800] | |
| selected.append(r) | |
| break | |
| return selected | |
| def condense_context(self, chunks: List[Dict], max_chars: int = 2000) -> str: | |
| if not chunks: | |
| return "No relevant context found." | |
| context_parts = [] | |
| for i, chunk in enumerate(chunks, 1): | |
| content = chunk['content'] | |
| source = chunk['metadata'].get('source', 'Unknown') | |
| sentences = re.split(r'[.!?]+', content) | |
| key_sentences = [] | |
| for s in sentences: | |
| if re.search(r'\d+\.?\d*\s*(?:mm|m|%|°C|mph|km/h)|must|shall|required|minimum|maximum', s): | |
| key_sentences.append(s.strip()) | |
| elif len(key_sentences) < 2: | |
| key_sentences.append(s.strip()) | |
| condensed = '. '.join(key_sentences[:3]) | |
| if condensed: | |
| context_parts.append(f"[{i}] {source}: {condensed}") | |
| result = "\n".join(context_parts) | |
| return result[:max_chars] | |
| _optimizer = TokenOptimizer() | |