File size: 2,891 Bytes
2379054
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
"""
TOKEN OPTIMIZER - Smart Chunk Selection & Context Condensation
"""

import re
from typing import List, Dict

class TokenOptimizer:
    def __init__(self, max_tokens: int = 4000):
        self.max_tokens = max_tokens
    
    def select_best_chunks(self, results: List[Dict], question: str) -> List[Dict]:
        if not results:
            return []
        
        scored = []
        question_words = set(question.lower().split())
        
        for r in results:
            content = r['content'].lower()
            
            content_words = set(content.split())
            overlap = len(question_words & content_words) / max(len(question_words), 1)
            
            has_number = bool(re.search(r'\d+\.?\d*\s*(?:mm|m|%|°C|mph)', content))
            number_score = 0.3 if has_number else 0
            
            has_table = bool(re.search(r'TABLE|----|\|\s*\d+', content))
            table_score = 0.2 if has_table else 0
            
            source = r['metadata'].get('source', '')
            is_primary = any(x in source.lower() for x in ['2102', '2500', '2049', 'sop'])
            source_score = 0.2 if is_primary else 0
            
            total_score = overlap + number_score + table_score + source_score
            scored.append((r, total_score))
        
        scored.sort(key=lambda x: x[1], reverse=True)
        
        selected = []
        current_tokens = 0
        for r, score in scored:
            chunk_tokens = len(r['content']) // 4
            if current_tokens + chunk_tokens <= self.max_tokens:
                selected.append(r)
                current_tokens += chunk_tokens
            else:
                if chunk_tokens > 200:
                    r['content'] = r['content'][:800]
                    selected.append(r)
                    break
        
        return selected
    
    def condense_context(self, chunks: List[Dict], max_chars: int = 2000) -> str:
        if not chunks:
            return "No relevant context found."
        
        context_parts = []
        for i, chunk in enumerate(chunks, 1):
            content = chunk['content']
            source = chunk['metadata'].get('source', 'Unknown')
            
            sentences = re.split(r'[.!?]+', content)
            key_sentences = []
            
            for s in sentences:
                if re.search(r'\d+\.?\d*\s*(?:mm|m|%|°C|mph|km/h)|must|shall|required|minimum|maximum', s):
                    key_sentences.append(s.strip())
                elif len(key_sentences) < 2:
                    key_sentences.append(s.strip())
            
            condensed = '. '.join(key_sentences[:3])
            
            if condensed:
                context_parts.append(f"[{i}] {source}: {condensed}")
        
        result = "\n".join(context_parts)
        return result[:max_chars]


_optimizer = TokenOptimizer()