File size: 14,792 Bytes
c9e8126 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 | import re
import nltk
from nltk.tokenize import sent_tokenize, word_tokenize
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
from collections import defaultdict, Counter
import torch
import json
import pickle
from typing import List, Dict, Tuple, Set
import numpy as np
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.cluster import KMeans
import spacy
from transformers import pipeline
# Download required NLTK data
try:
nltk.data.find('tokenizers/punkt')
except LookupError:
nltk.download('punkt')
try:
nltk.data.find('corpora/stopwords')
except LookupError:
nltk.download('stopwords')
try:
nltk.data.find('corpora/wordnet')
except LookupError:
nltk.download('wordnet')
class HierarchicalTextProcessor:
"""
Automatically breaks down large text files into structured knowledge
that can be used by the ZRIA architecture.
"""
def __init__(self, model_path: str = None):
self.lemmatizer = WordNetLemmatizer()
self.stop_words = set(stopwords.words('english'))
# Load spaCy for advanced NLP
try:
self.nlp = spacy.load("en_core_web_sm")
except OSError:
print("Installing spaCy model...")
import subprocess
subprocess.run(["python", "-m", "spacy", "download", "en_core_web_sm"])
self.nlp = spacy.load("en_core_web_sm")
# Sentiment analysis pipeline
self.sentiment_analyzer = pipeline("sentiment-analysis", model="distilbert-base-uncased-finetuned-sst-2-english")
# Knowledge categories
self.knowledge_categories = {
'FACT': 'factual statements, definitions, data',
'CONCEPT': 'abstract concepts, theories, ideas',
'PROCEDURE': 'how-to instructions, methods, steps',
'RELATIONSHIP': 'connections, comparisons, associations',
'ATTRIBUTE': 'properties, characteristics, qualities',
'EVENT': 'historical events, occurrences, timelines',
'LOCATION': 'places, geographical information',
'PERSON': 'people, characters, entities'
}
# Load existing knowledge base if provided
self.knowledge_base = {}
if model_path:
self.load_knowledge_base(model_path)
def process_large_text_file(self, file_path: str, chunk_size: int = 10000) -> Dict:
"""
Process a large text file by breaking it into manageable chunks
and extracting structured knowledge.
"""
print(f"Processing large text file: {file_path}")
with open(file_path, 'r', encoding='utf-8') as f:
text = f.read()
# Clean and preprocess
text = self._clean_text(text)
# Break into chunks
chunks = self._create_chunks(text, chunk_size)
# Process each chunk
all_knowledge = []
for i, chunk in enumerate(chunks):
print(f"Processing chunk {i+1}/{len(chunks)}")
chunk_knowledge = self._extract_knowledge_from_chunk(chunk)
all_knowledge.extend(chunk_knowledge)
# Consolidate and organize knowledge
organized_knowledge = self._organize_knowledge(all_knowledge)
return organized_knowledge
def _clean_text(self, text: str) -> str:
"""Clean and normalize text."""
# Remove extra whitespace
text = re.sub(r'\s+', ' ', text)
# Remove special characters but keep punctuation
text = re.sub(r'[^\w\s\.\,\!\?\;\:\-\(\)]', '', text)
# Normalize quotes and dashes
text = text.replace('"', '"').replace('"', '"')
text = text.replace('–', '-').replace('—', '-')
return text.strip()
def _create_chunks(self, text: str, chunk_size: int) -> List[str]:
"""Create overlapping chunks of text."""
chunks = []
start = 0
while start < len(text):
end = start + chunk_size
# Try to break at sentence boundaries
if end < len(text):
# Look for sentence endings near the chunk boundary
for i in range(end, max(start, end - 200), -1):
if text[i] in '.!?':
end = i + 1
break
chunk = text[start:end].strip()
if chunk:
chunks.append(chunk)
# Overlap chunks by 20%
start = end - int(chunk_size * 0.2)
return chunks
def _extract_knowledge_from_chunk(self, chunk: str) -> List[Dict]:
"""Extract structured knowledge from a text chunk."""
knowledge_items = []
# Split into sentences
sentences = sent_tokenize(chunk)
for sentence in sentences:
if len(sentence.strip()) < 10: # Skip very short sentences
continue
# Analyze sentence with spaCy
doc = self.nlp(sentence)
# Determine knowledge category
category = self._categorize_sentence(doc)
# Extract entities and relationships
entities = self._extract_entities(doc)
# Analyze sentiment
sentiment = self._analyze_sentiment(sentence)
# Extract key concepts
concepts = self._extract_concepts(doc)
# Create knowledge item
knowledge_item = {
'text': sentence.strip(),
'category': category,
'entities': entities,
'concepts': concepts,
'sentiment': sentiment,
'confidence': self._calculate_confidence(doc),
'metadata': {
'length': len(sentence),
'word_count': len(doc),
'has_verbs': any(token.pos_ == 'VERB' for token in doc),
'has_nouns': any(token.pos_ == 'NOUN' for token in doc)
}
}
knowledge_items.append(knowledge_item)
return knowledge_items
def _categorize_sentence(self, doc) -> str:
"""Categorize sentence into knowledge type."""
text = doc.text.lower()
# Simple rule-based categorization
if any(word in text for word in ['is', 'are', 'was', 'were', 'means', 'defined']):
return 'FACT'
elif any(word in text for word in ['how', 'step', 'process', 'method']):
return 'PROCEDURE'
elif any(word in text for word in ['because', 'therefore', 'however', 'although']):
return 'RELATIONSHIP'
elif any(word in text for word in ['has', 'contains', 'includes', 'features']):
return 'ATTRIBUTE'
elif any(word in text for word in ['happened', 'occurred', 'event', 'history']):
return 'EVENT'
elif any(word in text for word in ['in', 'at', 'location', 'place', 'country']):
return 'LOCATION'
elif any(word in text for word in ['person', 'people', 'he', 'she', 'they']):
return 'PERSON'
else:
return 'CONCEPT'
def _extract_entities(self, doc) -> List[Dict]:
"""Extract named entities and their relationships."""
entities = []
for ent in doc.ents:
entities.append({
'text': ent.text,
'label': ent.label_,
'start': ent.start_char,
'end': ent.end_char
})
return entities
def _extract_concepts(self, doc) -> List[str]:
"""Extract key concepts from the sentence."""
concepts = []
# Extract noun phrases
for chunk in doc.noun_chunks:
if len(chunk.text.split()) <= 3: # Limit to reasonable length
concepts.append(chunk.text.lower())
# Extract important adjectives
for token in doc:
if token.pos_ == 'ADJ' and token.text.lower() not in self.stop_words:
concepts.append(token.text.lower())
return list(set(concepts)) # Remove duplicates
def _analyze_sentiment(self, sentence: str) -> Dict:
"""Analyze sentiment of the sentence."""
try:
result = self.sentiment_analyzer(sentence[:512]) # Limit length
return {
'label': result[0]['label'],
'score': result[0]['score']
}
except:
return {'label': 'NEUTRAL', 'score': 0.5}
def _calculate_confidence(self, doc) -> float:
"""Calculate confidence score for the knowledge extraction."""
# Simple heuristic based on sentence structure
has_subject = any(token.dep_ == 'nsubj' for token in doc)
has_verb = any(token.pos_ == 'VERB' for token in doc)
has_object = any(token.dep_ in ['dobj', 'pobj'] for token in doc)
confidence = 0.5 # Base confidence
if has_subject:
confidence += 0.2
if has_verb:
confidence += 0.2
if has_object:
confidence += 0.1
return min(confidence, 1.0)
def _organize_knowledge(self, knowledge_items: List[Dict]) -> Dict:
"""Organize extracted knowledge into structured format."""
organized = {
'categories': defaultdict(list),
'entities': defaultdict(list),
'concepts': defaultdict(list),
'sentiments': defaultdict(list),
'relationships': [],
'metadata': {
'total_items': len(knowledge_items),
'categories_count': defaultdict(int),
'avg_confidence': np.mean([item['confidence'] for item in knowledge_items])
}
}
for item in knowledge_items:
# Organize by category
organized['categories'][item['category']].append(item)
organized['metadata']['categories_count'][item['category']] += 1
# Organize by entities
for entity in item['entities']:
organized['entities'][entity['label']].append({
'entity': entity,
'context': item['text'],
'category': item['category']
})
# Organize by concepts
for concept in item['concepts']:
organized['concepts'][concept].append({
'concept': concept,
'context': item['text'],
'category': item['category']
})
# Organize by sentiment
sentiment_label = item['sentiment']['label']
organized['sentiments'][sentiment_label].append(item)
return dict(organized)
def create_training_data(self, organized_knowledge: Dict) -> List[Tuple]:
"""
Convert organized knowledge into training data format
compatible with the ZRIA architecture.
"""
training_data = []
# Create KNOW statements for each knowledge item
for category, items in organized_knowledge['categories'].items():
for item in items:
# Create different types of KNOW statements
statements = self._create_knowledge_statements(item)
for statement, label in statements:
training_data.append(('KNOW', statement, label))
return training_data
def _create_knowledge_statements(self, item: Dict) -> List[Tuple[str, int]]:
"""Create various KNOW statements from a knowledge item."""
statements = []
text = item['text']
sentiment = item['sentiment']['label']
# Convert sentiment to numeric label
if sentiment == 'POSITIVE':
label = 2
elif sentiment == 'NEGATIVE':
label = 0
else:
label = 1
# Create different statement variations
statements.append((text, label))
# Create simplified versions
if len(text.split()) > 10:
# Create a shorter version
words = text.split()
short_version = ' '.join(words[:min(10, len(words))])
statements.append((short_version, label))
return statements
def save_knowledge_base(self, organized_knowledge: Dict, file_path: str):
"""Save the organized knowledge base to disk."""
with open(file_path, 'wb') as f:
pickle.dump(organized_knowledge, f)
print(f"Knowledge base saved to {file_path}")
def load_knowledge_base(self, file_path: str):
"""Load the organized knowledge base from disk."""
with open(file_path, 'rb') as f:
self.knowledge_base = pickle.load(f)
print(f"Knowledge base loaded from {file_path}")
def get_knowledge_summary(self) -> Dict:
"""Get a summary of the knowledge base."""
if not self.knowledge_base:
return {"error": "No knowledge base loaded"}
summary = {
'total_items': self.knowledge_base['metadata']['total_items'],
'categories': dict(self.knowledge_base['metadata']['categories_count']),
'entities': {k: len(v) for k, v in self.knowledge_base['entities'].items()},
'concepts': len(self.knowledge_base['concepts']),
'sentiments': {k: len(v) for k, v in self.knowledge_base['sentiments'].items()},
'avg_confidence': self.knowledge_base['metadata']['avg_confidence']
}
return summary
# Example usage
if __name__ == "__main__":
processor = HierarchicalTextProcessor()
# Process a large text file
# knowledge = processor.process_large_text_file("large_text_file.txt")
# processor.save_knowledge_base(knowledge, "knowledge_base.pkl")
# Create training data
# training_data = processor.create_training_data(knowledge)
# print(f"Created {len(training_data)} training examples") |