Spaces:
Runtime error
Runtime error
Download backend/agentic_system/flashcard_processor.py from h19overflow/Comprhensive_papers: direct link, hf CLI and curl.
- Browser
- Download file 9.26 kB
-
https://huggingface.co/spaces/h19overflow/Comprhensive_papers/resolve/main/backend/agentic_system/flashcard_processor.py
- Command line
-
hf download hf://spaces/h19overflow/Comprhensive_papers/backend/agentic_system/flashcard_processor.py
-
curl -L -o flashcard_processor.py https://huggingface.co/spaces/h19overflow/Comprhensive_papers/resolve/main/backend/agentic_system/flashcard_processor.py
9.26 kB
| """ | |
| Parallel Flashcard Processor | |
| This module processes semantic chunks using multiple flashcard agents concurrently | |
| to generate flashcards efficiently from large documents. | |
| """ | |
| import os | |
| import json | |
| import asyncio | |
| from datetime import datetime | |
| from typing import List, Dict, Any, Optional | |
| from pathlib import Path | |
| import logging | |
| from concurrent.futures import ThreadPoolExecutor | |
| from agents.flashcard_agent import FlashcardAgent, FlashcardGeneration | |
| # Set up logging | |
| logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') | |
| logger = logging.getLogger(__name__) | |
| class ParallelFlashcardProcessor: | |
| """ | |
| Processes semantic chunks using multiple concurrent flashcard agents. | |
| """ | |
| def __init__(self, max_concurrent_agents: int = 10): | |
| """ | |
| Initialize the parallel processor. | |
| Args: | |
| max_concurrent_agents: Maximum number of concurrent flashcard agents to run | |
| """ | |
| self.max_concurrent_agents = max_concurrent_agents | |
| self.semaphore = asyncio.Semaphore(max_concurrent_agents) | |
| async def process_page_with_semaphore(self, agent: FlashcardAgent, chunks: List[Dict[str, Any]], | |
| source_file: str, page_index: int) -> Optional[FlashcardGeneration]: | |
| """ | |
| Process a single page with semaphore control for concurrency. | |
| Args: | |
| agent: FlashcardAgent instance | |
| chunks: List of chunks for this page | |
| source_file: Source file name | |
| page_index: Page index number | |
| Returns: | |
| FlashcardGeneration or None if failed | |
| """ | |
| async with self.semaphore: | |
| try: | |
| logger.info(f"Processing {source_file} page {page_index} with {len(chunks)} chunks") | |
| result = await agent.generate_flashcards_for_page(chunks, source_file, page_index) | |
| logger.info(f"Completed {source_file} page {page_index} - Generated {result.total_flashcards} flashcards") | |
| return result | |
| except Exception as e: | |
| logger.error(f"Failed to process {source_file} page {page_index}: {e}") | |
| return None | |
| async def process_document(self, file_name: str, file_data: Dict[str, Any]) -> List[FlashcardGeneration]: | |
| """ | |
| Process all pages of a document concurrently. | |
| Args: | |
| file_name: Name of the source file | |
| file_data: File data containing chunks | |
| Returns: | |
| List of FlashcardGeneration objects | |
| """ | |
| chunks = file_data.get('chunks', []) | |
| if not chunks: | |
| logger.warning(f"No chunks found for {file_name}") | |
| return [] | |
| # Group chunks by page | |
| pages = {} | |
| for chunk in chunks: | |
| page_index = chunk['metadata']['page_index'] | |
| if page_index not in pages: | |
| pages[page_index] = [] | |
| pages[page_index].append(chunk) | |
| logger.info(f"Processing {file_name} with {len(pages)} pages") | |
| # Create tasks for concurrent processing | |
| tasks = [] | |
| for page_index, page_chunks in pages.items(): | |
| agent = FlashcardAgent() # Create a new agent instance for each task | |
| task = self.process_page_with_semaphore(agent, page_chunks, file_name, page_index) | |
| tasks.append(task) | |
| # Process all pages concurrently | |
| results = await asyncio.gather(*tasks, return_exceptions=True) | |
| # Filter out None results and exceptions | |
| valid_results = [] | |
| for result in results: | |
| if isinstance(result, FlashcardGeneration): | |
| valid_results.append(result) | |
| elif isinstance(result, Exception): | |
| logger.error(f"Task failed with exception: {result}") | |
| logger.info(f"Completed {file_name} - Generated flashcards for {len(valid_results)} pages") | |
| return valid_results | |
| async def process_all_documents(self, chunks_file_path: str, output_file_path: str) -> Dict[str, Any]: | |
| """ | |
| Process all documents from the semantic chunks file. | |
| Args: | |
| chunks_file_path: Path to semantic_chunks.json | |
| output_file_path: Path to save the generated flashcards | |
| Returns: | |
| Dictionary containing processing results and statistics | |
| """ | |
| try: | |
| # Load semantic chunks | |
| logger.info(f"Loading semantic chunks from {chunks_file_path}") | |
| with open(chunks_file_path, 'r', encoding='utf-8') as f: | |
| data = json.load(f) | |
| files_data = data.get('files', {}) | |
| if not files_data: | |
| logger.error("No files found in semantic chunks data") | |
| return {"error": "No files found"} | |
| logger.info(f"Found {len(files_data)} files to process") | |
| # Process all documents | |
| all_results = [] | |
| total_flashcards = 0 | |
| for file_name, file_data in files_data.items(): | |
| document_results = await self.process_document(file_name, file_data) | |
| all_results.extend(document_results) | |
| total_flashcards += sum(result.total_flashcards for result in document_results) | |
| # Prepare output data | |
| output_data = { | |
| "processing_info": { | |
| "processed_at": datetime.now().isoformat(), | |
| "total_documents": len(files_data), | |
| "total_pages_processed": len(all_results), | |
| "total_flashcards_generated": total_flashcards, | |
| "max_concurrent_agents": self.max_concurrent_agents | |
| }, | |
| "flashcard_generations": [] | |
| } | |
| # Convert results to dictionaries for JSON serialization | |
| for result in all_results: | |
| flashcard_dicts = [] | |
| for flashcard in result.flashcards: | |
| flashcard_dicts.append({ | |
| "id": flashcard.id, | |
| "question": flashcard.question, | |
| "answer": flashcard.answer, | |
| "keyword_explanation": flashcard.keyword_explanation, | |
| "metadata": flashcard.metadata | |
| }) | |
| output_data["flashcard_generations"].append({ | |
| "source_file": result.source_file, | |
| "page_index": result.page_index, | |
| "total_flashcards": result.total_flashcards, | |
| "flashcards": flashcard_dicts | |
| }) | |
| # Save results to file | |
| logger.info(f"Saving results to {output_file_path}") | |
| os.makedirs(os.path.dirname(output_file_path), exist_ok=True) | |
| with open(output_file_path, 'w', encoding='utf-8') as f: | |
| json.dump(output_data, f, indent=2, ensure_ascii=False) | |
| logger.info(f"Processing complete! Generated {total_flashcards} flashcards from {len(all_results)} pages") | |
| return { | |
| "success": True, | |
| "total_documents": len(files_data), | |
| "total_pages": len(all_results), | |
| "total_flashcards": total_flashcards, | |
| "output_file": output_file_path | |
| } | |
| except Exception as e: | |
| logger.error(f"Error in processing all documents: {e}") | |
| return {"error": str(e)} | |
| def process_chunks_sync(self, chunks_file_path: str, output_file_path: str) -> Dict[str, Any]: | |
| """ | |
| Synchronous wrapper for processing chunks. | |
| Args: | |
| chunks_file_path: Path to semantic_chunks.json | |
| output_file_path: Path to save the generated flashcards | |
| Returns: | |
| Dictionary containing processing results | |
| """ | |
| return asyncio.run(self.process_all_documents(chunks_file_path, output_file_path)) | |
| async def main(): | |
| """ | |
| Main function to run the flashcard processor. | |
| """ | |
| # Define paths | |
| current_dir = Path(__file__).parent | |
| chunks_file = current_dir / "../ingestion_pipeline/chunked_output/semantic_chunks.json" | |
| output_file = current_dir / "output/generated_flashcards.json" | |
| # Create processor and run | |
| processor = ParallelFlashcardProcessor(max_concurrent_agents=10) | |
| logger.info("Starting parallel flashcard processing...") | |
| result = await processor.process_all_documents(str(chunks_file), str(output_file)) | |
| if "error" in result: | |
| logger.error(f"Processing failed: {result['error']}") | |
| else: | |
| logger.info(f"Processing completed successfully: {result}") | |
| if __name__ == "__main__": | |
| asyncio.run(main()) |