import re import nltk from nltk.tokenize import sent_tokenize, word_tokenize from nltk.corpus import stopwords from nltk.stem import WordNetLemmatizer from collections import defaultdict, Counter import torch import json import pickle from typing import List, Dict, Tuple, Set import numpy as np from sklearn.feature_extraction.text import TfidfVectorizer from sklearn.cluster import KMeans import spacy from transformers import pipeline # Download required NLTK data try: nltk.data.find('tokenizers/punkt') except LookupError: nltk.download('punkt') try: nltk.data.find('corpora/stopwords') except LookupError: nltk.download('stopwords') try: nltk.data.find('corpora/wordnet') except LookupError: nltk.download('wordnet') class HierarchicalTextProcessor: """ Automatically breaks down large text files into structured knowledge that can be used by the ZRIA architecture. """ def __init__(self, model_path: str = None): self.lemmatizer = WordNetLemmatizer() self.stop_words = set(stopwords.words('english')) # Load spaCy for advanced NLP try: self.nlp = spacy.load("en_core_web_sm") except OSError: print("Installing spaCy model...") import subprocess subprocess.run(["python", "-m", "spacy", "download", "en_core_web_sm"]) self.nlp = spacy.load("en_core_web_sm") # Sentiment analysis pipeline self.sentiment_analyzer = pipeline("sentiment-analysis", model="distilbert-base-uncased-finetuned-sst-2-english") # Knowledge categories self.knowledge_categories = { 'FACT': 'factual statements, definitions, data', 'CONCEPT': 'abstract concepts, theories, ideas', 'PROCEDURE': 'how-to instructions, methods, steps', 'RELATIONSHIP': 'connections, comparisons, associations', 'ATTRIBUTE': 'properties, characteristics, qualities', 'EVENT': 'historical events, occurrences, timelines', 'LOCATION': 'places, geographical information', 'PERSON': 'people, characters, entities' } # Load existing knowledge base if provided self.knowledge_base = {} if model_path: self.load_knowledge_base(model_path) def process_large_text_file(self, file_path: str, chunk_size: int = 10000) -> Dict: """ Process a large text file by breaking it into manageable chunks and extracting structured knowledge. """ print(f"Processing large text file: {file_path}") with open(file_path, 'r', encoding='utf-8') as f: text = f.read() # Clean and preprocess text = self._clean_text(text) # Break into chunks chunks = self._create_chunks(text, chunk_size) # Process each chunk all_knowledge = [] for i, chunk in enumerate(chunks): print(f"Processing chunk {i+1}/{len(chunks)}") chunk_knowledge = self._extract_knowledge_from_chunk(chunk) all_knowledge.extend(chunk_knowledge) # Consolidate and organize knowledge organized_knowledge = self._organize_knowledge(all_knowledge) return organized_knowledge def _clean_text(self, text: str) -> str: """Clean and normalize text.""" # Remove extra whitespace text = re.sub(r'\s+', ' ', text) # Remove special characters but keep punctuation text = re.sub(r'[^\w\s\.\,\!\?\;\:\-\(\)]', '', text) # Normalize quotes and dashes text = text.replace('"', '"').replace('"', '"') text = text.replace('–', '-').replace('—', '-') return text.strip() def _create_chunks(self, text: str, chunk_size: int) -> List[str]: """Create overlapping chunks of text.""" chunks = [] start = 0 while start < len(text): end = start + chunk_size # Try to break at sentence boundaries if end < len(text): # Look for sentence endings near the chunk boundary for i in range(end, max(start, end - 200), -1): if text[i] in '.!?': end = i + 1 break chunk = text[start:end].strip() if chunk: chunks.append(chunk) # Overlap chunks by 20% start = end - int(chunk_size * 0.2) return chunks def _extract_knowledge_from_chunk(self, chunk: str) -> List[Dict]: """Extract structured knowledge from a text chunk.""" knowledge_items = [] # Split into sentences sentences = sent_tokenize(chunk) for sentence in sentences: if len(sentence.strip()) < 10: # Skip very short sentences continue # Analyze sentence with spaCy doc = self.nlp(sentence) # Determine knowledge category category = self._categorize_sentence(doc) # Extract entities and relationships entities = self._extract_entities(doc) # Analyze sentiment sentiment = self._analyze_sentiment(sentence) # Extract key concepts concepts = self._extract_concepts(doc) # Create knowledge item knowledge_item = { 'text': sentence.strip(), 'category': category, 'entities': entities, 'concepts': concepts, 'sentiment': sentiment, 'confidence': self._calculate_confidence(doc), 'metadata': { 'length': len(sentence), 'word_count': len(doc), 'has_verbs': any(token.pos_ == 'VERB' for token in doc), 'has_nouns': any(token.pos_ == 'NOUN' for token in doc) } } knowledge_items.append(knowledge_item) return knowledge_items def _categorize_sentence(self, doc) -> str: """Categorize sentence into knowledge type.""" text = doc.text.lower() # Simple rule-based categorization if any(word in text for word in ['is', 'are', 'was', 'were', 'means', 'defined']): return 'FACT' elif any(word in text for word in ['how', 'step', 'process', 'method']): return 'PROCEDURE' elif any(word in text for word in ['because', 'therefore', 'however', 'although']): return 'RELATIONSHIP' elif any(word in text for word in ['has', 'contains', 'includes', 'features']): return 'ATTRIBUTE' elif any(word in text for word in ['happened', 'occurred', 'event', 'history']): return 'EVENT' elif any(word in text for word in ['in', 'at', 'location', 'place', 'country']): return 'LOCATION' elif any(word in text for word in ['person', 'people', 'he', 'she', 'they']): return 'PERSON' else: return 'CONCEPT' def _extract_entities(self, doc) -> List[Dict]: """Extract named entities and their relationships.""" entities = [] for ent in doc.ents: entities.append({ 'text': ent.text, 'label': ent.label_, 'start': ent.start_char, 'end': ent.end_char }) return entities def _extract_concepts(self, doc) -> List[str]: """Extract key concepts from the sentence.""" concepts = [] # Extract noun phrases for chunk in doc.noun_chunks: if len(chunk.text.split()) <= 3: # Limit to reasonable length concepts.append(chunk.text.lower()) # Extract important adjectives for token in doc: if token.pos_ == 'ADJ' and token.text.lower() not in self.stop_words: concepts.append(token.text.lower()) return list(set(concepts)) # Remove duplicates def _analyze_sentiment(self, sentence: str) -> Dict: """Analyze sentiment of the sentence.""" try: result = self.sentiment_analyzer(sentence[:512]) # Limit length return { 'label': result[0]['label'], 'score': result[0]['score'] } except: return {'label': 'NEUTRAL', 'score': 0.5} def _calculate_confidence(self, doc) -> float: """Calculate confidence score for the knowledge extraction.""" # Simple heuristic based on sentence structure has_subject = any(token.dep_ == 'nsubj' for token in doc) has_verb = any(token.pos_ == 'VERB' for token in doc) has_object = any(token.dep_ in ['dobj', 'pobj'] for token in doc) confidence = 0.5 # Base confidence if has_subject: confidence += 0.2 if has_verb: confidence += 0.2 if has_object: confidence += 0.1 return min(confidence, 1.0) def _organize_knowledge(self, knowledge_items: List[Dict]) -> Dict: """Organize extracted knowledge into structured format.""" organized = { 'categories': defaultdict(list), 'entities': defaultdict(list), 'concepts': defaultdict(list), 'sentiments': defaultdict(list), 'relationships': [], 'metadata': { 'total_items': len(knowledge_items), 'categories_count': defaultdict(int), 'avg_confidence': np.mean([item['confidence'] for item in knowledge_items]) } } for item in knowledge_items: # Organize by category organized['categories'][item['category']].append(item) organized['metadata']['categories_count'][item['category']] += 1 # Organize by entities for entity in item['entities']: organized['entities'][entity['label']].append({ 'entity': entity, 'context': item['text'], 'category': item['category'] }) # Organize by concepts for concept in item['concepts']: organized['concepts'][concept].append({ 'concept': concept, 'context': item['text'], 'category': item['category'] }) # Organize by sentiment sentiment_label = item['sentiment']['label'] organized['sentiments'][sentiment_label].append(item) return dict(organized) def create_training_data(self, organized_knowledge: Dict) -> List[Tuple]: """ Convert organized knowledge into training data format compatible with the ZRIA architecture. """ training_data = [] # Create KNOW statements for each knowledge item for category, items in organized_knowledge['categories'].items(): for item in items: # Create different types of KNOW statements statements = self._create_knowledge_statements(item) for statement, label in statements: training_data.append(('KNOW', statement, label)) return training_data def _create_knowledge_statements(self, item: Dict) -> List[Tuple[str, int]]: """Create various KNOW statements from a knowledge item.""" statements = [] text = item['text'] sentiment = item['sentiment']['label'] # Convert sentiment to numeric label if sentiment == 'POSITIVE': label = 2 elif sentiment == 'NEGATIVE': label = 0 else: label = 1 # Create different statement variations statements.append((text, label)) # Create simplified versions if len(text.split()) > 10: # Create a shorter version words = text.split() short_version = ' '.join(words[:min(10, len(words))]) statements.append((short_version, label)) return statements def save_knowledge_base(self, organized_knowledge: Dict, file_path: str): """Save the organized knowledge base to disk.""" with open(file_path, 'wb') as f: pickle.dump(organized_knowledge, f) print(f"Knowledge base saved to {file_path}") def load_knowledge_base(self, file_path: str): """Load the organized knowledge base from disk.""" with open(file_path, 'rb') as f: self.knowledge_base = pickle.load(f) print(f"Knowledge base loaded from {file_path}") def get_knowledge_summary(self) -> Dict: """Get a summary of the knowledge base.""" if not self.knowledge_base: return {"error": "No knowledge base loaded"} summary = { 'total_items': self.knowledge_base['metadata']['total_items'], 'categories': dict(self.knowledge_base['metadata']['categories_count']), 'entities': {k: len(v) for k, v in self.knowledge_base['entities'].items()}, 'concepts': len(self.knowledge_base['concepts']), 'sentiments': {k: len(v) for k, v in self.knowledge_base['sentiments'].items()}, 'avg_confidence': self.knowledge_base['metadata']['avg_confidence'] } return summary # Example usage if __name__ == "__main__": processor = HierarchicalTextProcessor() # Process a large text file # knowledge = processor.process_large_text_file("large_text_file.txt") # processor.save_knowledge_base(knowledge, "knowledge_base.pkl") # Create training data # training_data = processor.create_training_data(knowledge) # print(f"Created {len(training_data)} training examples")