Comprhensive_papers / backend /agentic_system /flashcard_processor.py
h19overflow's picture
Upload folder using huggingface_hub
776bb1e verified
Raw History Blame Contribute Delete
9.26 kB
"""
Parallel Flashcard Processor
This module processes semantic chunks using multiple flashcard agents concurrently
to generate flashcards efficiently from large documents.
"""
import os
import json
import asyncio
from datetime import datetime
from typing import List, Dict, Any, Optional
from pathlib import Path
import logging
from concurrent.futures import ThreadPoolExecutor
from agents.flashcard_agent import FlashcardAgent, FlashcardGeneration
# Set up logging
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
logger = logging.getLogger(__name__)
class ParallelFlashcardProcessor:
"""
Processes semantic chunks using multiple concurrent flashcard agents.
"""
def __init__(self, max_concurrent_agents: int = 10):
"""
Initialize the parallel processor.
Args:
max_concurrent_agents: Maximum number of concurrent flashcard agents to run
"""
self.max_concurrent_agents = max_concurrent_agents
self.semaphore = asyncio.Semaphore(max_concurrent_agents)
async def process_page_with_semaphore(self, agent: FlashcardAgent, chunks: List[Dict[str, Any]],
source_file: str, page_index: int) -> Optional[FlashcardGeneration]:
"""
Process a single page with semaphore control for concurrency.
Args:
agent: FlashcardAgent instance
chunks: List of chunks for this page
source_file: Source file name
page_index: Page index number
Returns:
FlashcardGeneration or None if failed
"""
async with self.semaphore:
try:
logger.info(f"Processing {source_file} page {page_index} with {len(chunks)} chunks")
result = await agent.generate_flashcards_for_page(chunks, source_file, page_index)
logger.info(f"Completed {source_file} page {page_index} - Generated {result.total_flashcards} flashcards")
return result
except Exception as e:
logger.error(f"Failed to process {source_file} page {page_index}: {e}")
return None
async def process_document(self, file_name: str, file_data: Dict[str, Any]) -> List[FlashcardGeneration]:
"""
Process all pages of a document concurrently.
Args:
file_name: Name of the source file
file_data: File data containing chunks
Returns:
List of FlashcardGeneration objects
"""
chunks = file_data.get('chunks', [])
if not chunks:
logger.warning(f"No chunks found for {file_name}")
return []
# Group chunks by page
pages = {}
for chunk in chunks:
page_index = chunk['metadata']['page_index']
if page_index not in pages:
pages[page_index] = []
pages[page_index].append(chunk)
logger.info(f"Processing {file_name} with {len(pages)} pages")
# Create tasks for concurrent processing
tasks = []
for page_index, page_chunks in pages.items():
agent = FlashcardAgent() # Create a new agent instance for each task
task = self.process_page_with_semaphore(agent, page_chunks, file_name, page_index)
tasks.append(task)
# Process all pages concurrently
results = await asyncio.gather(*tasks, return_exceptions=True)
# Filter out None results and exceptions
valid_results = []
for result in results:
if isinstance(result, FlashcardGeneration):
valid_results.append(result)
elif isinstance(result, Exception):
logger.error(f"Task failed with exception: {result}")
logger.info(f"Completed {file_name} - Generated flashcards for {len(valid_results)} pages")
return valid_results
async def process_all_documents(self, chunks_file_path: str, output_file_path: str) -> Dict[str, Any]:
"""
Process all documents from the semantic chunks file.
Args:
chunks_file_path: Path to semantic_chunks.json
output_file_path: Path to save the generated flashcards
Returns:
Dictionary containing processing results and statistics
"""
try:
# Load semantic chunks
logger.info(f"Loading semantic chunks from {chunks_file_path}")
with open(chunks_file_path, 'r', encoding='utf-8') as f:
data = json.load(f)
files_data = data.get('files', {})
if not files_data:
logger.error("No files found in semantic chunks data")
return {"error": "No files found"}
logger.info(f"Found {len(files_data)} files to process")
# Process all documents
all_results = []
total_flashcards = 0
for file_name, file_data in files_data.items():
document_results = await self.process_document(file_name, file_data)
all_results.extend(document_results)
total_flashcards += sum(result.total_flashcards for result in document_results)
# Prepare output data
output_data = {
"processing_info": {
"processed_at": datetime.now().isoformat(),
"total_documents": len(files_data),
"total_pages_processed": len(all_results),
"total_flashcards_generated": total_flashcards,
"max_concurrent_agents": self.max_concurrent_agents
},
"flashcard_generations": []
}
# Convert results to dictionaries for JSON serialization
for result in all_results:
flashcard_dicts = []
for flashcard in result.flashcards:
flashcard_dicts.append({
"id": flashcard.id,
"question": flashcard.question,
"answer": flashcard.answer,
"keyword_explanation": flashcard.keyword_explanation,
"metadata": flashcard.metadata
})
output_data["flashcard_generations"].append({
"source_file": result.source_file,
"page_index": result.page_index,
"total_flashcards": result.total_flashcards,
"flashcards": flashcard_dicts
})
# Save results to file
logger.info(f"Saving results to {output_file_path}")
os.makedirs(os.path.dirname(output_file_path), exist_ok=True)
with open(output_file_path, 'w', encoding='utf-8') as f:
json.dump(output_data, f, indent=2, ensure_ascii=False)
logger.info(f"Processing complete! Generated {total_flashcards} flashcards from {len(all_results)} pages")
return {
"success": True,
"total_documents": len(files_data),
"total_pages": len(all_results),
"total_flashcards": total_flashcards,
"output_file": output_file_path
}
except Exception as e:
logger.error(f"Error in processing all documents: {e}")
return {"error": str(e)}
def process_chunks_sync(self, chunks_file_path: str, output_file_path: str) -> Dict[str, Any]:
"""
Synchronous wrapper for processing chunks.
Args:
chunks_file_path: Path to semantic_chunks.json
output_file_path: Path to save the generated flashcards
Returns:
Dictionary containing processing results
"""
return asyncio.run(self.process_all_documents(chunks_file_path, output_file_path))
async def main():
"""
Main function to run the flashcard processor.
"""
# Define paths
current_dir = Path(__file__).parent
chunks_file = current_dir / "../ingestion_pipeline/chunked_output/semantic_chunks.json"
output_file = current_dir / "output/generated_flashcards.json"
# Create processor and run
processor = ParallelFlashcardProcessor(max_concurrent_agents=10)
logger.info("Starting parallel flashcard processing...")
result = await processor.process_all_documents(str(chunks_file), str(output_file))
if "error" in result:
logger.error(f"Processing failed: {result['error']}")
else:
logger.info(f"Processing completed successfully: {result}")
if __name__ == "__main__":
asyncio.run(main())