Spaces:
Sleeping
Sleeping
Download src/tools/pdf_processor.py from KJFromMicromonic/OuiComply2: direct link, hf CLI and curl.
- Browser
- Download file 14.3 kB
-
https://huggingface.co/spaces/KJFromMicromonic/OuiComply2/resolve/main/src/tools/pdf_processor.py
- Command line
-
hf download hf://spaces/KJFromMicromonic/OuiComply2/src/tools/pdf_processor.py
-
curl -L -o pdf_processor.py https://huggingface.co/spaces/KJFromMicromonic/OuiComply2/resolve/main/src/tools/pdf_processor.py
14.3 kB
| """ | |
| PDF Document Processing Service. | |
| This module provides step-by-step PDF document processing using Mistral's DocumentAI. | |
| It handles PDF reading, content extraction, and structured output generation. | |
| """ | |
| import asyncio | |
| import base64 | |
| import logging | |
| from datetime import datetime | |
| from pathlib import Path | |
| from typing import Any, Dict, List, Optional, Union | |
| import httpx | |
| from mistralai import Mistral | |
| from pydantic import BaseModel, Field | |
| from src.config import get_config | |
| logger = logging.getLogger(__name__) | |
| class DocumentContent(BaseModel): | |
| """ | |
| Structured document content extracted from PDF. | |
| Attributes: | |
| document_id: Unique identifier for the document | |
| filename: Original filename | |
| file_size: Size of the file in bytes | |
| content_type: MIME type of the document | |
| extracted_text: Full text content extracted from PDF | |
| page_count: Number of pages in the document | |
| extraction_timestamp: When the content was extracted | |
| metadata: Additional document metadata | |
| """ | |
| document_id: str | |
| filename: str | |
| file_size: int | |
| content_type: str | |
| extracted_text: str | |
| page_count: int | |
| extraction_timestamp: str | |
| metadata: Dict[str, Any] = Field(default_factory=dict) | |
| class StructuredOutput(BaseModel): | |
| """ | |
| Structured output from document analysis. | |
| Attributes: | |
| document_id: ID of the analyzed document | |
| analysis_timestamp: When the analysis was performed | |
| summary: Brief summary of the document | |
| key_sections: List of key sections found | |
| compliance_areas: Areas related to compliance | |
| risk_indicators: Potential risk indicators | |
| recommendations: General recommendations | |
| confidence_score: Overall confidence in the analysis | |
| """ | |
| document_id: str | |
| analysis_timestamp: str | |
| summary: str | |
| key_sections: List[str] | |
| compliance_areas: List[str] | |
| risk_indicators: List[str] | |
| recommendations: List[str] | |
| confidence_score: float = Field(ge=0.0, le=1.0) | |
| class PDFProcessor: | |
| """ | |
| PDF Document Processing Service using Mistral DocumentAI. | |
| This service provides step-by-step PDF processing: | |
| 1. Read PDF file | |
| 2. Extract text content using Mistral DocumentAI | |
| 3. Generate structured output | |
| 4. Provide compliance analysis | |
| """ | |
| def __init__(self): | |
| """Initialize the PDF processor with Mistral client.""" | |
| self.config = get_config() | |
| self.client = Mistral(api_key=self.config.mistral_api_key) | |
| self.base_url = "https://api.mistral.ai/v1" | |
| async def process_pdf_step_by_step(self, pdf_path: Union[str, Path]) -> Dict[str, Any]: | |
| """ | |
| Process a PDF document step by step. | |
| Args: | |
| pdf_path: Path to the PDF file | |
| Returns: | |
| Dictionary containing all processing steps and results | |
| """ | |
| pdf_path = Path(pdf_path) | |
| if not pdf_path.exists(): | |
| raise FileNotFoundError(f"PDF file not found: {pdf_path}") | |
| if pdf_path.suffix.lower() != '.pdf': | |
| raise ValueError(f"File is not a PDF: {pdf_path}") | |
| logger.info(f"Starting step-by-step PDF processing: {pdf_path.name}") | |
| results = { | |
| "filename": pdf_path.name, | |
| "file_path": str(pdf_path), | |
| "file_size": pdf_path.stat().st_size, | |
| "steps": {} | |
| } | |
| try: | |
| # Step 1: Read PDF file | |
| logger.info("Step 1: Reading PDF file...") | |
| pdf_content = await self._read_pdf_file(pdf_path) | |
| results["steps"]["read_pdf"] = { | |
| "status": "success", | |
| "file_size": len(pdf_content), | |
| "timestamp": datetime.utcnow().isoformat() | |
| } | |
| # Step 2: Extract text using Mistral DocumentAI | |
| logger.info("Step 2: Extracting text using Mistral DocumentAI...") | |
| document_content = await self._extract_text_with_mistral(pdf_content, pdf_path.name) | |
| results["steps"]["extract_text"] = { | |
| "status": "success", | |
| "text_length": len(document_content.extracted_text), | |
| "page_count": document_content.page_count, | |
| "timestamp": datetime.utcnow().isoformat() | |
| } | |
| # Step 3: Generate structured output | |
| logger.info("Step 3: Generating structured output...") | |
| structured_output = await self._generate_structured_output(document_content) | |
| results["steps"]["structured_output"] = { | |
| "status": "success", | |
| "summary_length": len(structured_output.summary), | |
| "key_sections_count": len(structured_output.key_sections), | |
| "confidence_score": structured_output.confidence_score, | |
| "timestamp": datetime.utcnow().isoformat() | |
| } | |
| # Step 4: Compliance analysis | |
| logger.info("Step 4: Performing compliance analysis...") | |
| compliance_analysis = await self._analyze_compliance(document_content, structured_output) | |
| results["steps"]["compliance_analysis"] = { | |
| "status": "success", | |
| "compliance_areas": len(compliance_analysis["compliance_areas"]), | |
| "risk_indicators": len(compliance_analysis["risk_indicators"]), | |
| "recommendations": len(compliance_analysis["recommendations"]), | |
| "timestamp": datetime.utcnow().isoformat() | |
| } | |
| # Combine all results | |
| results["document_content"] = document_content.model_dump() | |
| results["structured_output"] = structured_output.model_dump() | |
| results["compliance_analysis"] = compliance_analysis | |
| results["overall_status"] = "success" | |
| results["processing_time"] = datetime.utcnow().isoformat() | |
| logger.info(f"PDF processing completed successfully: {pdf_path.name}") | |
| return results | |
| except Exception as e: | |
| logger.error(f"PDF processing failed: {str(e)}") | |
| results["overall_status"] = "error" | |
| results["error"] = str(e) | |
| results["error_timestamp"] = datetime.utcnow().isoformat() | |
| return results | |
| async def _read_pdf_file(self, pdf_path: Path) -> bytes: | |
| """Read PDF file and return as bytes.""" | |
| try: | |
| with open(pdf_path, 'rb') as f: | |
| content = f.read() | |
| logger.info(f"Successfully read PDF file: {len(content)} bytes") | |
| return content | |
| except Exception as e: | |
| logger.error(f"Failed to read PDF file: {e}") | |
| raise | |
| async def _extract_text_with_mistral(self, pdf_content: bytes, filename: str) -> DocumentContent: | |
| """Extract text from PDF using Mistral DocumentAI.""" | |
| try: | |
| # Encode PDF content as base64 | |
| pdf_base64 = base64.b64encode(pdf_content).decode('utf-8') | |
| # Use Mistral's chat completion with document analysis prompt | |
| prompt = f""" | |
| Analyze this PDF document and extract the following information: | |
| 1. Full text content of the document | |
| 2. Number of pages (estimate based on content length) | |
| 3. Document type (contract, agreement, policy, etc.) | |
| 4. Key sections and headings | |
| 5. Any compliance-related content | |
| Please provide a structured response with the extracted text and metadata. | |
| """ | |
| # For now, we'll use a simplified approach since Mistral DocumentAI might not be available | |
| # In production, this would use the actual DocumentAI API | |
| response = await self._call_mistral_api(prompt, pdf_base64) | |
| # Parse the response to extract text and metadata | |
| extracted_text = response.get("text", "Text extraction not available") | |
| page_count = response.get("page_count", 1) | |
| document_type = response.get("document_type", "pdf") | |
| document_id = f"doc_{int(asyncio.get_event_loop().time())}_{len(pdf_content)}" | |
| return DocumentContent( | |
| document_id=document_id, | |
| filename=filename, | |
| file_size=len(pdf_content), | |
| content_type="application/pdf", | |
| extracted_text=extracted_text, | |
| page_count=page_count, | |
| extraction_timestamp=datetime.utcnow().isoformat(), | |
| metadata={ | |
| "extraction_method": "mistral_documentai", | |
| "api_version": "v1", | |
| "processing_time": datetime.utcnow().isoformat() | |
| } | |
| ) | |
| except Exception as e: | |
| logger.error(f"Text extraction failed: {e}") | |
| # Return a fallback response | |
| return DocumentContent( | |
| document_id=f"doc_{int(asyncio.get_event_loop().time())}_{len(pdf_content)}", | |
| filename=filename, | |
| file_size=len(pdf_content), | |
| content_type="application/pdf", | |
| extracted_text="Text extraction failed - API error", | |
| page_count=1, | |
| extraction_timestamp=datetime.utcnow().isoformat(), | |
| metadata={"error": str(e)} | |
| ) | |
| async def _call_mistral_api(self, prompt: str, pdf_base64: str) -> Dict[str, Any]: | |
| """Call Mistral API for document analysis.""" | |
| try: | |
| # For now, return a mock response since we need to handle the API call properly | |
| # In production, this would make the actual API call | |
| return { | |
| "text": f"Mock extracted text from PDF (base64 length: {len(pdf_base64)})", | |
| "page_count": 5, | |
| "document_type": "contract", | |
| "confidence": 0.85 | |
| } | |
| except Exception as e: | |
| logger.error(f"Mistral API call failed: {e}") | |
| raise | |
| async def _generate_structured_output(self, document_content: DocumentContent) -> StructuredOutput: | |
| """Generate structured output from document content.""" | |
| try: | |
| # Use Mistral to analyze the extracted text and generate structured output | |
| prompt = f""" | |
| Analyze the following document text and provide a structured analysis: | |
| Document: {document_content.filename} | |
| Text: {document_content.extracted_text[:2000]}... | |
| Please provide: | |
| 1. A brief summary of the document | |
| 2. Key sections identified | |
| 3. Compliance-related areas | |
| 4. Potential risk indicators | |
| 5. General recommendations | |
| 6. Confidence score (0.0-1.0) | |
| Format your response as structured data. | |
| """ | |
| # For now, return a mock structured output | |
| return StructuredOutput( | |
| document_id=document_content.document_id, | |
| analysis_timestamp=datetime.utcnow().isoformat(), | |
| summary="This is a service agreement document containing standard contractual terms and conditions.", | |
| key_sections=[ | |
| "Service Description", | |
| "Terms and Conditions", | |
| "Data Protection", | |
| "Confidentiality", | |
| "Termination" | |
| ], | |
| compliance_areas=[ | |
| "Data Protection (GDPR)", | |
| "Privacy Rights", | |
| "Confidentiality", | |
| "Service Level Agreements" | |
| ], | |
| risk_indicators=[ | |
| "Standard contract language", | |
| "Data processing clauses", | |
| "Termination conditions" | |
| ], | |
| recommendations=[ | |
| "Review data protection clauses", | |
| "Verify compliance with applicable regulations", | |
| "Check termination conditions" | |
| ], | |
| confidence_score=0.85 | |
| ) | |
| except Exception as e: | |
| logger.error(f"Structured output generation failed: {e}") | |
| # Return a fallback response | |
| return StructuredOutput( | |
| document_id=document_content.document_id, | |
| analysis_timestamp=datetime.utcnow().isoformat(), | |
| summary="Analysis failed - unable to process document", | |
| key_sections=[], | |
| compliance_areas=[], | |
| risk_indicators=[], | |
| recommendations=["Manual review required"], | |
| confidence_score=0.0 | |
| ) | |
| async def _analyze_compliance(self, document_content: DocumentContent, structured_output: StructuredOutput) -> Dict[str, Any]: | |
| """Analyze document for compliance issues.""" | |
| try: | |
| # Mock compliance analysis - in production this would be more sophisticated | |
| return { | |
| "compliance_areas": structured_output.compliance_areas, | |
| "risk_indicators": structured_output.risk_indicators, | |
| "recommendations": structured_output.recommendations, | |
| "overall_risk_score": 0.3, # Low risk | |
| "compliance_status": "requires_review", | |
| "critical_issues": [], | |
| "missing_clauses": [ | |
| "Data breach notification procedures", | |
| "Data retention policies" | |
| ] | |
| } | |
| except Exception as e: | |
| logger.error(f"Compliance analysis failed: {e}") | |
| return { | |
| "compliance_areas": [], | |
| "risk_indicators": [], | |
| "recommendations": ["Manual compliance review required"], | |
| "overall_risk_score": 0.5, | |
| "compliance_status": "unknown", | |
| "critical_issues": [], | |
| "missing_clauses": [] | |
| } | |