#!/usr/bin/env python3 """ Complete Example: Processing Large Text Files and Creating Persistent ZRIA Models This example demonstrates: 1. Processing large text files into structured knowledge 2. Training the ZRIA model on the processed data 3. Saving the model for persistent use 4. Interactive prompting without retraining """ import os import sys from text_processor import HierarchicalTextProcessor from persistent_model import PersistentZRIAModel, create_and_train_model_from_text_file, load_and_use_model def create_sample_text_file(): """Create a sample large text file for demonstration.""" sample_text = """ Artificial Intelligence and Machine Learning Artificial intelligence (AI) is a branch of computer science that aims to create intelligent machines. These machines can perform tasks that typically require human intelligence, such as visual perception, speech recognition, decision-making, and language translation. Machine learning is a subset of AI that focuses on the development of algorithms and statistical models that enable computers to improve their performance on a specific task through experience. The process of learning begins with observations or data, such as examples, direct experience, or instruction. Deep learning, a subset of machine learning, uses neural networks with multiple layers to model and understand complex patterns. These neural networks are inspired by the human brain's structure and function, consisting of interconnected nodes that process information. Natural Language Processing (NLP) is a field of AI that focuses on the interaction between computers and human language. It involves the development of algorithms and models that can understand, interpret, and generate human language in a meaningful way. Computer Vision is another important area of AI that deals with how computers can gain high-level understanding from digital images or videos. It involves methods for acquiring, processing, analyzing, and understanding digital images to extract meaningful information. The applications of AI are vast and growing rapidly. In healthcare, AI is being used for disease diagnosis, drug discovery, and personalized medicine. In finance, it's used for fraud detection, algorithmic trading, and risk assessment. In transportation, AI powers autonomous vehicles and traffic management systems. However, AI also presents significant challenges and concerns. Issues of privacy, security, bias, and job displacement are important considerations. The development of ethical AI systems that are fair, transparent, and accountable is crucial for the responsible deployment of these technologies. The future of AI holds tremendous promise. As technology continues to advance, we can expect AI systems to become more sophisticated, capable, and integrated into our daily lives. The key is to ensure that these developments benefit humanity while addressing the associated risks and challenges. Neural networks are computational models inspired by biological neural networks. They consist of interconnected nodes, or neurons, that process information and can learn to recognize patterns. Each connection between neurons has an associated weight that is adjusted during the learning process. Supervised learning is a type of machine learning where the algorithm learns from labeled training data. The goal is to learn a mapping from inputs to outputs that can be used to make predictions on new, unseen data. Common supervised learning tasks include classification and regression. Unsupervised learning, on the other hand, deals with unlabeled data. The algorithm must find hidden patterns or structures in the data without any guidance. Clustering and dimensionality reduction are common unsupervised learning tasks. Reinforcement learning is a type of machine learning where an agent learns to make decisions by taking actions in an environment to achieve maximum cumulative reward. It's particularly useful for problems where the optimal strategy is not known in advance. The field of AI is constantly evolving, with new techniques and applications emerging regularly. Researchers are working on developing more efficient algorithms, better models, and more robust systems that can handle real-world challenges. Transfer learning is an important concept in machine learning where knowledge gained from training on one task is applied to a related task. This approach can significantly reduce the amount of training data and computational resources needed for new tasks. Explainable AI (XAI) is an emerging field that focuses on making AI systems more transparent and interpretable. This is crucial for building trust in AI systems and ensuring they can be used responsibly in critical applications. Edge computing and AI are becoming increasingly important as more devices need to process data locally. This approach reduces latency, improves privacy, and reduces bandwidth requirements. The integration of AI with other technologies, such as the Internet of Things (IoT), blockchain, and quantum computing, is creating new possibilities and applications. These combinations are driving innovation across multiple industries. As AI continues to advance, it's important to consider the societal implications and ensure that these technologies are developed and deployed in ways that benefit all of humanity. This includes addressing issues of accessibility, fairness, and sustainability. """ with open("sample_large_text.txt", "w", encoding="utf-8") as f: f.write(sample_text) print("Created sample_large_text.txt") return "sample_large_text.txt" def demonstrate_text_processing(): """Demonstrate the text processing pipeline.""" print("\n=== Text Processing Demonstration ===") # Create sample text file text_file = create_sample_text_file() # Initialize processor processor = HierarchicalTextProcessor() # Process the text file print(f"Processing {text_file}...") knowledge = processor.process_large_text_file(text_file, chunk_size=1000) # Show knowledge summary summary = processor.get_knowledge_summary() print("\nKnowledge Base Summary:") print(f"Total items: {summary['total_items']}") print(f"Categories: {summary['categories']}") print(f"Entities: {summary['entities']}") print(f"Concepts: {len(summary['concepts'])}") print(f"Sentiments: {summary['sentiments']}") print(f"Average confidence: {summary['avg_confidence']:.3f}") # Create training data training_data = processor.create_training_data(knowledge) print(f"\nCreated {len(training_data)} training examples") # Show some examples print("\nSample training examples:") for i, (prefix, text, label) in enumerate(training_data[:5]): print(f"{i+1}. {prefix}: {text[:50]}... -> {label}") return knowledge, training_data def demonstrate_model_training(knowledge, training_data): """Demonstrate model training and saving.""" print("\n=== Model Training Demonstration ===") # Create and train model model = create_and_train_model_from_text_file("sample_large_text.txt", "demo_model") return model def demonstrate_interactive_usage(): """Demonstrate interactive usage of the trained model.""" print("\n=== Interactive Usage Demonstration ===") # Load the trained model model = load_and_use_model("demo_model") if model is None: print("No trained model found. Please run the training demonstration first.") return # Test some prompts test_prompts = [ "artificial intelligence is amazing", "machine learning algorithms are complex", "count the vowels in this sentence", "how many vowels in artificial intelligence", "deep learning is fascinating", "this technology is terrible", "neural networks are powerful tools" ] print("\nTesting various prompts:") for prompt in test_prompts: result = model.prompt(prompt, 'auto') print(f"\nInput: {prompt}") print(f"Task: {result['task_type']}") if result['task_type'] == 'KNOW': print(f"Sentiment: {result['prediction']}") else: print(f"Action Value: {result['prediction']:.2f}") print(f"Route Probability: {result['route_probability']:.3f}") # Start interactive mode print("\nStarting interactive mode...") print("(You can type 'quit' to exit)") model.interactive_mode() def demonstrate_batch_processing(): """Demonstrate batch processing capabilities.""" print("\n=== Batch Processing Demonstration ===") model = load_and_use_model("demo_model") if model is None: print("No trained model found.") return # Batch of texts to process batch_texts = [ "artificial intelligence is revolutionary", "count vowels in this example", "machine learning is difficult", "how many vowels in neural networks", "this is a neutral statement", "deep learning is amazing", "count the vowels in artificial intelligence" ] print("Processing batch of texts...") results = model.batch_prompt(batch_texts, 'auto') print("\nBatch Results:") for i, (text, result) in enumerate(zip(batch_texts, results)): print(f"{i+1}. {text}") print(f" Task: {result['task_type']}") if result['task_type'] == 'KNOW': print(f" Sentiment: {result['prediction']}") else: print(f" Action Value: {result['prediction']:.2f}") print(f" Route Probability: {result['route_probability']:.3f}") print() def main(): """Run the complete demonstration.""" print("=== ZRIA Model System Complete Demonstration ===") print("This demonstration shows the complete pipeline from text processing to interactive usage.") try: # Step 1: Text Processing knowledge, training_data = demonstrate_text_processing() # Step 2: Model Training model = demonstrate_model_training(knowledge, training_data) # Step 3: Interactive Usage demonstrate_interactive_usage() # Step 4: Batch Processing demonstrate_batch_processing() print("\n=== Demonstration Complete ===") print("The model is now saved and can be reused without retraining!") print("You can load it anytime using: load_and_use_model('demo_model')") except Exception as e: print(f"Error during demonstration: {e}") import traceback traceback.print_exc() if __name__ == "__main__": main()