ae / text_processor.py
ChadlieOak's picture
Upload 22 files
c9e8126 verified
Raw History Blame Contribute Delete
14.8 kB
import re
import nltk
from nltk.tokenize import sent_tokenize, word_tokenize
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
from collections import defaultdict, Counter
import torch
import json
import pickle
from typing import List, Dict, Tuple, Set
import numpy as np
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.cluster import KMeans
import spacy
from transformers import pipeline
# Download required NLTK data
try:
nltk.data.find('tokenizers/punkt')
except LookupError:
nltk.download('punkt')
try:
nltk.data.find('corpora/stopwords')
except LookupError:
nltk.download('stopwords')
try:
nltk.data.find('corpora/wordnet')
except LookupError:
nltk.download('wordnet')
class HierarchicalTextProcessor:
"""
Automatically breaks down large text files into structured knowledge
that can be used by the ZRIA architecture.
"""
def __init__(self, model_path: str = None):
self.lemmatizer = WordNetLemmatizer()
self.stop_words = set(stopwords.words('english'))
# Load spaCy for advanced NLP
try:
self.nlp = spacy.load("en_core_web_sm")
except OSError:
print("Installing spaCy model...")
import subprocess
subprocess.run(["python", "-m", "spacy", "download", "en_core_web_sm"])
self.nlp = spacy.load("en_core_web_sm")
# Sentiment analysis pipeline
self.sentiment_analyzer = pipeline("sentiment-analysis", model="distilbert-base-uncased-finetuned-sst-2-english")
# Knowledge categories
self.knowledge_categories = {
'FACT': 'factual statements, definitions, data',
'CONCEPT': 'abstract concepts, theories, ideas',
'PROCEDURE': 'how-to instructions, methods, steps',
'RELATIONSHIP': 'connections, comparisons, associations',
'ATTRIBUTE': 'properties, characteristics, qualities',
'EVENT': 'historical events, occurrences, timelines',
'LOCATION': 'places, geographical information',
'PERSON': 'people, characters, entities'
}
# Load existing knowledge base if provided
self.knowledge_base = {}
if model_path:
self.load_knowledge_base(model_path)
def process_large_text_file(self, file_path: str, chunk_size: int = 10000) -> Dict:
"""
Process a large text file by breaking it into manageable chunks
and extracting structured knowledge.
"""
print(f"Processing large text file: {file_path}")
with open(file_path, 'r', encoding='utf-8') as f:
text = f.read()
# Clean and preprocess
text = self._clean_text(text)
# Break into chunks
chunks = self._create_chunks(text, chunk_size)
# Process each chunk
all_knowledge = []
for i, chunk in enumerate(chunks):
print(f"Processing chunk {i+1}/{len(chunks)}")
chunk_knowledge = self._extract_knowledge_from_chunk(chunk)
all_knowledge.extend(chunk_knowledge)
# Consolidate and organize knowledge
organized_knowledge = self._organize_knowledge(all_knowledge)
return organized_knowledge
def _clean_text(self, text: str) -> str:
"""Clean and normalize text."""
# Remove extra whitespace
text = re.sub(r'\s+', ' ', text)
# Remove special characters but keep punctuation
text = re.sub(r'[^\w\s\.\,\!\?\;\:\-\(\)]', '', text)
# Normalize quotes and dashes
text = text.replace('"', '"').replace('"', '"')
text = text.replace('–', '-').replace('—', '-')
return text.strip()
def _create_chunks(self, text: str, chunk_size: int) -> List[str]:
"""Create overlapping chunks of text."""
chunks = []
start = 0
while start < len(text):
end = start + chunk_size
# Try to break at sentence boundaries
if end < len(text):
# Look for sentence endings near the chunk boundary
for i in range(end, max(start, end - 200), -1):
if text[i] in '.!?':
end = i + 1
break
chunk = text[start:end].strip()
if chunk:
chunks.append(chunk)
# Overlap chunks by 20%
start = end - int(chunk_size * 0.2)
return chunks
def _extract_knowledge_from_chunk(self, chunk: str) -> List[Dict]:
"""Extract structured knowledge from a text chunk."""
knowledge_items = []
# Split into sentences
sentences = sent_tokenize(chunk)
for sentence in sentences:
if len(sentence.strip()) < 10: # Skip very short sentences
continue
# Analyze sentence with spaCy
doc = self.nlp(sentence)
# Determine knowledge category
category = self._categorize_sentence(doc)
# Extract entities and relationships
entities = self._extract_entities(doc)
# Analyze sentiment
sentiment = self._analyze_sentiment(sentence)
# Extract key concepts
concepts = self._extract_concepts(doc)
# Create knowledge item
knowledge_item = {
'text': sentence.strip(),
'category': category,
'entities': entities,
'concepts': concepts,
'sentiment': sentiment,
'confidence': self._calculate_confidence(doc),
'metadata': {
'length': len(sentence),
'word_count': len(doc),
'has_verbs': any(token.pos_ == 'VERB' for token in doc),
'has_nouns': any(token.pos_ == 'NOUN' for token in doc)
}
}
knowledge_items.append(knowledge_item)
return knowledge_items
def _categorize_sentence(self, doc) -> str:
"""Categorize sentence into knowledge type."""
text = doc.text.lower()
# Simple rule-based categorization
if any(word in text for word in ['is', 'are', 'was', 'were', 'means', 'defined']):
return 'FACT'
elif any(word in text for word in ['how', 'step', 'process', 'method']):
return 'PROCEDURE'
elif any(word in text for word in ['because', 'therefore', 'however', 'although']):
return 'RELATIONSHIP'
elif any(word in text for word in ['has', 'contains', 'includes', 'features']):
return 'ATTRIBUTE'
elif any(word in text for word in ['happened', 'occurred', 'event', 'history']):
return 'EVENT'
elif any(word in text for word in ['in', 'at', 'location', 'place', 'country']):
return 'LOCATION'
elif any(word in text for word in ['person', 'people', 'he', 'she', 'they']):
return 'PERSON'
else:
return 'CONCEPT'
def _extract_entities(self, doc) -> List[Dict]:
"""Extract named entities and their relationships."""
entities = []
for ent in doc.ents:
entities.append({
'text': ent.text,
'label': ent.label_,
'start': ent.start_char,
'end': ent.end_char
})
return entities
def _extract_concepts(self, doc) -> List[str]:
"""Extract key concepts from the sentence."""
concepts = []
# Extract noun phrases
for chunk in doc.noun_chunks:
if len(chunk.text.split()) <= 3: # Limit to reasonable length
concepts.append(chunk.text.lower())
# Extract important adjectives
for token in doc:
if token.pos_ == 'ADJ' and token.text.lower() not in self.stop_words:
concepts.append(token.text.lower())
return list(set(concepts)) # Remove duplicates
def _analyze_sentiment(self, sentence: str) -> Dict:
"""Analyze sentiment of the sentence."""
try:
result = self.sentiment_analyzer(sentence[:512]) # Limit length
return {
'label': result[0]['label'],
'score': result[0]['score']
}
except:
return {'label': 'NEUTRAL', 'score': 0.5}
def _calculate_confidence(self, doc) -> float:
"""Calculate confidence score for the knowledge extraction."""
# Simple heuristic based on sentence structure
has_subject = any(token.dep_ == 'nsubj' for token in doc)
has_verb = any(token.pos_ == 'VERB' for token in doc)
has_object = any(token.dep_ in ['dobj', 'pobj'] for token in doc)
confidence = 0.5 # Base confidence
if has_subject:
confidence += 0.2
if has_verb:
confidence += 0.2
if has_object:
confidence += 0.1
return min(confidence, 1.0)
def _organize_knowledge(self, knowledge_items: List[Dict]) -> Dict:
"""Organize extracted knowledge into structured format."""
organized = {
'categories': defaultdict(list),
'entities': defaultdict(list),
'concepts': defaultdict(list),
'sentiments': defaultdict(list),
'relationships': [],
'metadata': {
'total_items': len(knowledge_items),
'categories_count': defaultdict(int),
'avg_confidence': np.mean([item['confidence'] for item in knowledge_items])
}
}
for item in knowledge_items:
# Organize by category
organized['categories'][item['category']].append(item)
organized['metadata']['categories_count'][item['category']] += 1
# Organize by entities
for entity in item['entities']:
organized['entities'][entity['label']].append({
'entity': entity,
'context': item['text'],
'category': item['category']
})
# Organize by concepts
for concept in item['concepts']:
organized['concepts'][concept].append({
'concept': concept,
'context': item['text'],
'category': item['category']
})
# Organize by sentiment
sentiment_label = item['sentiment']['label']
organized['sentiments'][sentiment_label].append(item)
return dict(organized)
def create_training_data(self, organized_knowledge: Dict) -> List[Tuple]:
"""
Convert organized knowledge into training data format
compatible with the ZRIA architecture.
"""
training_data = []
# Create KNOW statements for each knowledge item
for category, items in organized_knowledge['categories'].items():
for item in items:
# Create different types of KNOW statements
statements = self._create_knowledge_statements(item)
for statement, label in statements:
training_data.append(('KNOW', statement, label))
return training_data
def _create_knowledge_statements(self, item: Dict) -> List[Tuple[str, int]]:
"""Create various KNOW statements from a knowledge item."""
statements = []
text = item['text']
sentiment = item['sentiment']['label']
# Convert sentiment to numeric label
if sentiment == 'POSITIVE':
label = 2
elif sentiment == 'NEGATIVE':
label = 0
else:
label = 1
# Create different statement variations
statements.append((text, label))
# Create simplified versions
if len(text.split()) > 10:
# Create a shorter version
words = text.split()
short_version = ' '.join(words[:min(10, len(words))])
statements.append((short_version, label))
return statements
def save_knowledge_base(self, organized_knowledge: Dict, file_path: str):
"""Save the organized knowledge base to disk."""
with open(file_path, 'wb') as f:
pickle.dump(organized_knowledge, f)
print(f"Knowledge base saved to {file_path}")
def load_knowledge_base(self, file_path: str):
"""Load the organized knowledge base from disk."""
with open(file_path, 'rb') as f:
self.knowledge_base = pickle.load(f)
print(f"Knowledge base loaded from {file_path}")
def get_knowledge_summary(self) -> Dict:
"""Get a summary of the knowledge base."""
if not self.knowledge_base:
return {"error": "No knowledge base loaded"}
summary = {
'total_items': self.knowledge_base['metadata']['total_items'],
'categories': dict(self.knowledge_base['metadata']['categories_count']),
'entities': {k: len(v) for k, v in self.knowledge_base['entities'].items()},
'concepts': len(self.knowledge_base['concepts']),
'sentiments': {k: len(v) for k, v in self.knowledge_base['sentiments'].items()},
'avg_confidence': self.knowledge_base['metadata']['avg_confidence']
}
return summary
# Example usage
if __name__ == "__main__":
processor = HierarchicalTextProcessor()
# Process a large text file
# knowledge = processor.process_large_text_file("large_text_file.txt")
# processor.save_knowledge_base(knowledge, "knowledge_base.pkl")
# Create training data
# training_data = processor.create_training_data(knowledge)
# print(f"Created {len(training_data)} training examples")