""" Parallel Flashcard Processor This module processes semantic chunks using multiple flashcard agents concurrently to generate flashcards efficiently from large documents. """ import os import json import asyncio from datetime import datetime from typing import List, Dict, Any, Optional from pathlib import Path import logging from concurrent.futures import ThreadPoolExecutor from agents.flashcard_agent import FlashcardAgent, FlashcardGeneration # Set up logging logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') logger = logging.getLogger(__name__) class ParallelFlashcardProcessor: """ Processes semantic chunks using multiple concurrent flashcard agents. """ def __init__(self, max_concurrent_agents: int = 10): """ Initialize the parallel processor. Args: max_concurrent_agents: Maximum number of concurrent flashcard agents to run """ self.max_concurrent_agents = max_concurrent_agents self.semaphore = asyncio.Semaphore(max_concurrent_agents) async def process_page_with_semaphore(self, agent: FlashcardAgent, chunks: List[Dict[str, Any]], source_file: str, page_index: int) -> Optional[FlashcardGeneration]: """ Process a single page with semaphore control for concurrency. Args: agent: FlashcardAgent instance chunks: List of chunks for this page source_file: Source file name page_index: Page index number Returns: FlashcardGeneration or None if failed """ async with self.semaphore: try: logger.info(f"Processing {source_file} page {page_index} with {len(chunks)} chunks") result = await agent.generate_flashcards_for_page(chunks, source_file, page_index) logger.info(f"Completed {source_file} page {page_index} - Generated {result.total_flashcards} flashcards") return result except Exception as e: logger.error(f"Failed to process {source_file} page {page_index}: {e}") return None async def process_document(self, file_name: str, file_data: Dict[str, Any]) -> List[FlashcardGeneration]: """ Process all pages of a document concurrently. Args: file_name: Name of the source file file_data: File data containing chunks Returns: List of FlashcardGeneration objects """ chunks = file_data.get('chunks', []) if not chunks: logger.warning(f"No chunks found for {file_name}") return [] # Group chunks by page pages = {} for chunk in chunks: page_index = chunk['metadata']['page_index'] if page_index not in pages: pages[page_index] = [] pages[page_index].append(chunk) logger.info(f"Processing {file_name} with {len(pages)} pages") # Create tasks for concurrent processing tasks = [] for page_index, page_chunks in pages.items(): agent = FlashcardAgent() # Create a new agent instance for each task task = self.process_page_with_semaphore(agent, page_chunks, file_name, page_index) tasks.append(task) # Process all pages concurrently results = await asyncio.gather(*tasks, return_exceptions=True) # Filter out None results and exceptions valid_results = [] for result in results: if isinstance(result, FlashcardGeneration): valid_results.append(result) elif isinstance(result, Exception): logger.error(f"Task failed with exception: {result}") logger.info(f"Completed {file_name} - Generated flashcards for {len(valid_results)} pages") return valid_results async def process_all_documents(self, chunks_file_path: str, output_file_path: str) -> Dict[str, Any]: """ Process all documents from the semantic chunks file. Args: chunks_file_path: Path to semantic_chunks.json output_file_path: Path to save the generated flashcards Returns: Dictionary containing processing results and statistics """ try: # Load semantic chunks logger.info(f"Loading semantic chunks from {chunks_file_path}") with open(chunks_file_path, 'r', encoding='utf-8') as f: data = json.load(f) files_data = data.get('files', {}) if not files_data: logger.error("No files found in semantic chunks data") return {"error": "No files found"} logger.info(f"Found {len(files_data)} files to process") # Process all documents all_results = [] total_flashcards = 0 for file_name, file_data in files_data.items(): document_results = await self.process_document(file_name, file_data) all_results.extend(document_results) total_flashcards += sum(result.total_flashcards for result in document_results) # Prepare output data output_data = { "processing_info": { "processed_at": datetime.now().isoformat(), "total_documents": len(files_data), "total_pages_processed": len(all_results), "total_flashcards_generated": total_flashcards, "max_concurrent_agents": self.max_concurrent_agents }, "flashcard_generations": [] } # Convert results to dictionaries for JSON serialization for result in all_results: flashcard_dicts = [] for flashcard in result.flashcards: flashcard_dicts.append({ "id": flashcard.id, "question": flashcard.question, "answer": flashcard.answer, "keyword_explanation": flashcard.keyword_explanation, "metadata": flashcard.metadata }) output_data["flashcard_generations"].append({ "source_file": result.source_file, "page_index": result.page_index, "total_flashcards": result.total_flashcards, "flashcards": flashcard_dicts }) # Save results to file logger.info(f"Saving results to {output_file_path}") os.makedirs(os.path.dirname(output_file_path), exist_ok=True) with open(output_file_path, 'w', encoding='utf-8') as f: json.dump(output_data, f, indent=2, ensure_ascii=False) logger.info(f"Processing complete! Generated {total_flashcards} flashcards from {len(all_results)} pages") return { "success": True, "total_documents": len(files_data), "total_pages": len(all_results), "total_flashcards": total_flashcards, "output_file": output_file_path } except Exception as e: logger.error(f"Error in processing all documents: {e}") return {"error": str(e)} def process_chunks_sync(self, chunks_file_path: str, output_file_path: str) -> Dict[str, Any]: """ Synchronous wrapper for processing chunks. Args: chunks_file_path: Path to semantic_chunks.json output_file_path: Path to save the generated flashcards Returns: Dictionary containing processing results """ return asyncio.run(self.process_all_documents(chunks_file_path, output_file_path)) async def main(): """ Main function to run the flashcard processor. """ # Define paths current_dir = Path(__file__).parent chunks_file = current_dir / "../ingestion_pipeline/chunked_output/semantic_chunks.json" output_file = current_dir / "output/generated_flashcards.json" # Create processor and run processor = ParallelFlashcardProcessor(max_concurrent_agents=10) logger.info("Starting parallel flashcard processing...") result = await processor.process_all_documents(str(chunks_file), str(output_file)) if "error" in result: logger.error(f"Processing failed: {result['error']}") else: logger.info(f"Processing completed successfully: {result}") if __name__ == "__main__": asyncio.run(main())