Download text_processor.py from ChadlieOak/ae: direct link, hf CLI and curl.
- Browser
- Download file 14.8 kB
-
https://huggingface.co/ChadlieOak/ae/resolve/main/text_processor.py
- Command line
-
hf download hf://ChadlieOak/ae/text_processor.py
-
curl -L -o text_processor.py https://huggingface.co/ChadlieOak/ae/resolve/main/text_processor.py
14.8 kB
| import re | |
| import nltk | |
| from nltk.tokenize import sent_tokenize, word_tokenize | |
| from nltk.corpus import stopwords | |
| from nltk.stem import WordNetLemmatizer | |
| from collections import defaultdict, Counter | |
| import torch | |
| import json | |
| import pickle | |
| from typing import List, Dict, Tuple, Set | |
| import numpy as np | |
| from sklearn.feature_extraction.text import TfidfVectorizer | |
| from sklearn.cluster import KMeans | |
| import spacy | |
| from transformers import pipeline | |
| # Download required NLTK data | |
| try: | |
| nltk.data.find('tokenizers/punkt') | |
| except LookupError: | |
| nltk.download('punkt') | |
| try: | |
| nltk.data.find('corpora/stopwords') | |
| except LookupError: | |
| nltk.download('stopwords') | |
| try: | |
| nltk.data.find('corpora/wordnet') | |
| except LookupError: | |
| nltk.download('wordnet') | |
| class HierarchicalTextProcessor: | |
| """ | |
| Automatically breaks down large text files into structured knowledge | |
| that can be used by the ZRIA architecture. | |
| """ | |
| def __init__(self, model_path: str = None): | |
| self.lemmatizer = WordNetLemmatizer() | |
| self.stop_words = set(stopwords.words('english')) | |
| # Load spaCy for advanced NLP | |
| try: | |
| self.nlp = spacy.load("en_core_web_sm") | |
| except OSError: | |
| print("Installing spaCy model...") | |
| import subprocess | |
| subprocess.run(["python", "-m", "spacy", "download", "en_core_web_sm"]) | |
| self.nlp = spacy.load("en_core_web_sm") | |
| # Sentiment analysis pipeline | |
| self.sentiment_analyzer = pipeline("sentiment-analysis", model="distilbert-base-uncased-finetuned-sst-2-english") | |
| # Knowledge categories | |
| self.knowledge_categories = { | |
| 'FACT': 'factual statements, definitions, data', | |
| 'CONCEPT': 'abstract concepts, theories, ideas', | |
| 'PROCEDURE': 'how-to instructions, methods, steps', | |
| 'RELATIONSHIP': 'connections, comparisons, associations', | |
| 'ATTRIBUTE': 'properties, characteristics, qualities', | |
| 'EVENT': 'historical events, occurrences, timelines', | |
| 'LOCATION': 'places, geographical information', | |
| 'PERSON': 'people, characters, entities' | |
| } | |
| # Load existing knowledge base if provided | |
| self.knowledge_base = {} | |
| if model_path: | |
| self.load_knowledge_base(model_path) | |
| def process_large_text_file(self, file_path: str, chunk_size: int = 10000) -> Dict: | |
| """ | |
| Process a large text file by breaking it into manageable chunks | |
| and extracting structured knowledge. | |
| """ | |
| print(f"Processing large text file: {file_path}") | |
| with open(file_path, 'r', encoding='utf-8') as f: | |
| text = f.read() | |
| # Clean and preprocess | |
| text = self._clean_text(text) | |
| # Break into chunks | |
| chunks = self._create_chunks(text, chunk_size) | |
| # Process each chunk | |
| all_knowledge = [] | |
| for i, chunk in enumerate(chunks): | |
| print(f"Processing chunk {i+1}/{len(chunks)}") | |
| chunk_knowledge = self._extract_knowledge_from_chunk(chunk) | |
| all_knowledge.extend(chunk_knowledge) | |
| # Consolidate and organize knowledge | |
| organized_knowledge = self._organize_knowledge(all_knowledge) | |
| return organized_knowledge | |
| def _clean_text(self, text: str) -> str: | |
| """Clean and normalize text.""" | |
| # Remove extra whitespace | |
| text = re.sub(r'\s+', ' ', text) | |
| # Remove special characters but keep punctuation | |
| text = re.sub(r'[^\w\s\.\,\!\?\;\:\-\(\)]', '', text) | |
| # Normalize quotes and dashes | |
| text = text.replace('"', '"').replace('"', '"') | |
| text = text.replace('–', '-').replace('—', '-') | |
| return text.strip() | |
| def _create_chunks(self, text: str, chunk_size: int) -> List[str]: | |
| """Create overlapping chunks of text.""" | |
| chunks = [] | |
| start = 0 | |
| while start < len(text): | |
| end = start + chunk_size | |
| # Try to break at sentence boundaries | |
| if end < len(text): | |
| # Look for sentence endings near the chunk boundary | |
| for i in range(end, max(start, end - 200), -1): | |
| if text[i] in '.!?': | |
| end = i + 1 | |
| break | |
| chunk = text[start:end].strip() | |
| if chunk: | |
| chunks.append(chunk) | |
| # Overlap chunks by 20% | |
| start = end - int(chunk_size * 0.2) | |
| return chunks | |
| def _extract_knowledge_from_chunk(self, chunk: str) -> List[Dict]: | |
| """Extract structured knowledge from a text chunk.""" | |
| knowledge_items = [] | |
| # Split into sentences | |
| sentences = sent_tokenize(chunk) | |
| for sentence in sentences: | |
| if len(sentence.strip()) < 10: # Skip very short sentences | |
| continue | |
| # Analyze sentence with spaCy | |
| doc = self.nlp(sentence) | |
| # Determine knowledge category | |
| category = self._categorize_sentence(doc) | |
| # Extract entities and relationships | |
| entities = self._extract_entities(doc) | |
| # Analyze sentiment | |
| sentiment = self._analyze_sentiment(sentence) | |
| # Extract key concepts | |
| concepts = self._extract_concepts(doc) | |
| # Create knowledge item | |
| knowledge_item = { | |
| 'text': sentence.strip(), | |
| 'category': category, | |
| 'entities': entities, | |
| 'concepts': concepts, | |
| 'sentiment': sentiment, | |
| 'confidence': self._calculate_confidence(doc), | |
| 'metadata': { | |
| 'length': len(sentence), | |
| 'word_count': len(doc), | |
| 'has_verbs': any(token.pos_ == 'VERB' for token in doc), | |
| 'has_nouns': any(token.pos_ == 'NOUN' for token in doc) | |
| } | |
| } | |
| knowledge_items.append(knowledge_item) | |
| return knowledge_items | |
| def _categorize_sentence(self, doc) -> str: | |
| """Categorize sentence into knowledge type.""" | |
| text = doc.text.lower() | |
| # Simple rule-based categorization | |
| if any(word in text for word in ['is', 'are', 'was', 'were', 'means', 'defined']): | |
| return 'FACT' | |
| elif any(word in text for word in ['how', 'step', 'process', 'method']): | |
| return 'PROCEDURE' | |
| elif any(word in text for word in ['because', 'therefore', 'however', 'although']): | |
| return 'RELATIONSHIP' | |
| elif any(word in text for word in ['has', 'contains', 'includes', 'features']): | |
| return 'ATTRIBUTE' | |
| elif any(word in text for word in ['happened', 'occurred', 'event', 'history']): | |
| return 'EVENT' | |
| elif any(word in text for word in ['in', 'at', 'location', 'place', 'country']): | |
| return 'LOCATION' | |
| elif any(word in text for word in ['person', 'people', 'he', 'she', 'they']): | |
| return 'PERSON' | |
| else: | |
| return 'CONCEPT' | |
| def _extract_entities(self, doc) -> List[Dict]: | |
| """Extract named entities and their relationships.""" | |
| entities = [] | |
| for ent in doc.ents: | |
| entities.append({ | |
| 'text': ent.text, | |
| 'label': ent.label_, | |
| 'start': ent.start_char, | |
| 'end': ent.end_char | |
| }) | |
| return entities | |
| def _extract_concepts(self, doc) -> List[str]: | |
| """Extract key concepts from the sentence.""" | |
| concepts = [] | |
| # Extract noun phrases | |
| for chunk in doc.noun_chunks: | |
| if len(chunk.text.split()) <= 3: # Limit to reasonable length | |
| concepts.append(chunk.text.lower()) | |
| # Extract important adjectives | |
| for token in doc: | |
| if token.pos_ == 'ADJ' and token.text.lower() not in self.stop_words: | |
| concepts.append(token.text.lower()) | |
| return list(set(concepts)) # Remove duplicates | |
| def _analyze_sentiment(self, sentence: str) -> Dict: | |
| """Analyze sentiment of the sentence.""" | |
| try: | |
| result = self.sentiment_analyzer(sentence[:512]) # Limit length | |
| return { | |
| 'label': result[0]['label'], | |
| 'score': result[0]['score'] | |
| } | |
| except: | |
| return {'label': 'NEUTRAL', 'score': 0.5} | |
| def _calculate_confidence(self, doc) -> float: | |
| """Calculate confidence score for the knowledge extraction.""" | |
| # Simple heuristic based on sentence structure | |
| has_subject = any(token.dep_ == 'nsubj' for token in doc) | |
| has_verb = any(token.pos_ == 'VERB' for token in doc) | |
| has_object = any(token.dep_ in ['dobj', 'pobj'] for token in doc) | |
| confidence = 0.5 # Base confidence | |
| if has_subject: | |
| confidence += 0.2 | |
| if has_verb: | |
| confidence += 0.2 | |
| if has_object: | |
| confidence += 0.1 | |
| return min(confidence, 1.0) | |
| def _organize_knowledge(self, knowledge_items: List[Dict]) -> Dict: | |
| """Organize extracted knowledge into structured format.""" | |
| organized = { | |
| 'categories': defaultdict(list), | |
| 'entities': defaultdict(list), | |
| 'concepts': defaultdict(list), | |
| 'sentiments': defaultdict(list), | |
| 'relationships': [], | |
| 'metadata': { | |
| 'total_items': len(knowledge_items), | |
| 'categories_count': defaultdict(int), | |
| 'avg_confidence': np.mean([item['confidence'] for item in knowledge_items]) | |
| } | |
| } | |
| for item in knowledge_items: | |
| # Organize by category | |
| organized['categories'][item['category']].append(item) | |
| organized['metadata']['categories_count'][item['category']] += 1 | |
| # Organize by entities | |
| for entity in item['entities']: | |
| organized['entities'][entity['label']].append({ | |
| 'entity': entity, | |
| 'context': item['text'], | |
| 'category': item['category'] | |
| }) | |
| # Organize by concepts | |
| for concept in item['concepts']: | |
| organized['concepts'][concept].append({ | |
| 'concept': concept, | |
| 'context': item['text'], | |
| 'category': item['category'] | |
| }) | |
| # Organize by sentiment | |
| sentiment_label = item['sentiment']['label'] | |
| organized['sentiments'][sentiment_label].append(item) | |
| return dict(organized) | |
| def create_training_data(self, organized_knowledge: Dict) -> List[Tuple]: | |
| """ | |
| Convert organized knowledge into training data format | |
| compatible with the ZRIA architecture. | |
| """ | |
| training_data = [] | |
| # Create KNOW statements for each knowledge item | |
| for category, items in organized_knowledge['categories'].items(): | |
| for item in items: | |
| # Create different types of KNOW statements | |
| statements = self._create_knowledge_statements(item) | |
| for statement, label in statements: | |
| training_data.append(('KNOW', statement, label)) | |
| return training_data | |
| def _create_knowledge_statements(self, item: Dict) -> List[Tuple[str, int]]: | |
| """Create various KNOW statements from a knowledge item.""" | |
| statements = [] | |
| text = item['text'] | |
| sentiment = item['sentiment']['label'] | |
| # Convert sentiment to numeric label | |
| if sentiment == 'POSITIVE': | |
| label = 2 | |
| elif sentiment == 'NEGATIVE': | |
| label = 0 | |
| else: | |
| label = 1 | |
| # Create different statement variations | |
| statements.append((text, label)) | |
| # Create simplified versions | |
| if len(text.split()) > 10: | |
| # Create a shorter version | |
| words = text.split() | |
| short_version = ' '.join(words[:min(10, len(words))]) | |
| statements.append((short_version, label)) | |
| return statements | |
| def save_knowledge_base(self, organized_knowledge: Dict, file_path: str): | |
| """Save the organized knowledge base to disk.""" | |
| with open(file_path, 'wb') as f: | |
| pickle.dump(organized_knowledge, f) | |
| print(f"Knowledge base saved to {file_path}") | |
| def load_knowledge_base(self, file_path: str): | |
| """Load the organized knowledge base from disk.""" | |
| with open(file_path, 'rb') as f: | |
| self.knowledge_base = pickle.load(f) | |
| print(f"Knowledge base loaded from {file_path}") | |
| def get_knowledge_summary(self) -> Dict: | |
| """Get a summary of the knowledge base.""" | |
| if not self.knowledge_base: | |
| return {"error": "No knowledge base loaded"} | |
| summary = { | |
| 'total_items': self.knowledge_base['metadata']['total_items'], | |
| 'categories': dict(self.knowledge_base['metadata']['categories_count']), | |
| 'entities': {k: len(v) for k, v in self.knowledge_base['entities'].items()}, | |
| 'concepts': len(self.knowledge_base['concepts']), | |
| 'sentiments': {k: len(v) for k, v in self.knowledge_base['sentiments'].items()}, | |
| 'avg_confidence': self.knowledge_base['metadata']['avg_confidence'] | |
| } | |
| return summary | |
| # Example usage | |
| if __name__ == "__main__": | |
| processor = HierarchicalTextProcessor() | |
| # Process a large text file | |
| # knowledge = processor.process_large_text_file("large_text_file.txt") | |
| # processor.save_knowledge_base(knowledge, "knowledge_base.pkl") | |
| # Create training data | |
| # training_data = processor.create_training_data(knowledge) | |
| # print(f"Created {len(training_data)} training examples") |