Spaces:

Hwandji
/

SAAP

Sleeping

SAAP / backend /api /document_upload.py

Add full SAAP platform with Docker SDK

63bfc55 7 months ago

8.56 kB

"""
SAAP Document Upload API
Handles file uploads, parsing, and privacy detection for multi-agent chat
"""
import logging
from typing import Optional
from fastapi import APIRouter, UploadFile, File, HTTPException, Form
from fastapi.responses import JSONResponse
from pydantic import BaseModel

from services.document_parser import document_parser
from services.privacy_detector import privacy_detector

logger = logging.getLogger(__name__)

router = APIRouter(prefix="/api/v1/documents", tags=["documents"])


class DocumentUploadResponse(BaseModel):
    """Response model for document upload"""
    success: bool
    document_id: str
    filename: str
    file_type: str
    file_size: int
    char_count: int
    has_sensitive_data: bool
    sensitive_data_types: list[str]
    content_preview: str
    full_content: str  # 📄 Full document content for chat context
    error: Optional[str] = None


@router.post("/upload", response_model=DocumentUploadResponse)
async def upload_document(
    file: UploadFile = File(...),
    user_message: Optional[str] = Form(None)
):
    """
    Upload and parse a document (PDF, DOCX, TXT)
    
    - Extracts text content from document
    - Analyzes for sensitive/private data
    - Returns parsed content and privacy analysis
    
    Args:
        file: Uploaded file (PDF, DOCX, TXT)
        user_message: Optional user message with context about the document
    
    Returns:
        DocumentUploadResponse with parsed content and privacy analysis
    """
    try:
        logger.info(f"📤 Document upload started: {file.filename}")
        
        # Read file content
        file_content = await file.read()
        
        # Parse document
        parse_result = document_parser.parse_document(
            file_data=file_content,
            filename=file.filename,
            mime_type=file.content_type
        )
        
        if not parse_result["success"]:
            logger.error(f"❌ Document parsing failed: {parse_result['error']}")
            raise HTTPException(
                status_code=400,
                detail=f"Failed to parse document: {parse_result['error']}"
            )
        
        extracted_text = parse_result["content"]
        metadata = parse_result["metadata"]
        
        # Analyze for sensitive data
        from services.privacy_detector import analyze_document_privacy
        privacy_level, privacy_details = analyze_document_privacy(
            document_text=extracted_text,
            filename=file.filename
        )
        
        # Convert privacy level to boolean + types for response
        has_sensitive_data = privacy_level.value in ["private", "confidential"]
        
        # Extract detected types from details
        sensitive_types = []
        if privacy_details.get("keyword_matches"):
            sensitive_types.extend([cat for cat, _ in privacy_details["keyword_matches"]])
        if privacy_details.get("pattern_matches"):
            sensitive_types.extend([cat for _, cat in privacy_details["pattern_matches"]])
        if privacy_details.get("document_indicators"):
            sensitive_types.extend(privacy_details["document_indicators"])
        
        # Remove duplicates
        sensitive_types = list(set(sensitive_types))
        
        # Create content preview (first 500 characters)
        content_preview = extracted_text[:500]
        if len(extracted_text) > 500:
            content_preview += "..."
        
        # Generate document ID (simple hash for now)
        import hashlib
        doc_id = hashlib.sha256(file_content).hexdigest()[:16]
        
        logger.info(
            f"✅ Document processed successfully: {file.filename} "
            f"({metadata['char_count']} chars, "
            f"sensitive: {has_sensitive_data})"
        )
        
        return DocumentUploadResponse(
            success=True,
            document_id=doc_id,
            filename=metadata["filename"],
            file_type=metadata["file_type"],
            file_size=metadata["file_size"],
            char_count=metadata["char_count"],
            has_sensitive_data=has_sensitive_data,
            sensitive_data_types=sensitive_types,
            content_preview=content_preview,
            full_content=extracted_text,  # 📄 Full document content for chat context
            error=None
        )
    
    except HTTPException:
        raise
    
    except Exception as e:
        logger.error(f"❌ Document upload error: {e}", exc_info=True)
        raise HTTPException(
            status_code=500,
            detail=f"Internal server error: {str(e)}"
        )


@router.post("/analyze")
async def analyze_document_content(
    file: UploadFile = File(...),
):
    """
    Analyze document for sensitive data without full parsing
    
    Quick privacy check before full processing
    
    Args:
        file: Uploaded file
    
    Returns:
        Privacy analysis results
    """
    try:
        logger.info(f"🔍 Document privacy analysis: {file.filename}")
        
        # Read file content
        file_content = await file.read()
        
        # Parse document
        parse_result = document_parser.parse_document(
            file_data=file_content,
            filename=file.filename,
            mime_type=file.content_type
        )
        
        if not parse_result["success"]:
            raise HTTPException(
                status_code=400,
                detail=f"Failed to parse document: {parse_result['error']}"
            )
        
        # Analyze for sensitive data
        from services.privacy_detector import analyze_document_privacy
        privacy_level, privacy_details = analyze_document_privacy(
            document_text=parse_result["content"],
            filename=file.filename
        )
        
        # Convert privacy level to boolean
        has_sensitive_data = privacy_level.value in ["private", "confidential"]
        
        # Extract detected types
        sensitive_types = []
        if privacy_details.get("keyword_matches"):
            sensitive_types.extend([cat for cat, _ in privacy_details["keyword_matches"]])
        if privacy_details.get("pattern_matches"):
            sensitive_types.extend([cat for _, cat in privacy_details["pattern_matches"]])
        if privacy_details.get("document_indicators"):
            sensitive_types.extend(privacy_details["document_indicators"])
        sensitive_types = list(set(sensitive_types))
        
        logger.info(
            f"✅ Privacy analysis complete: {file.filename} "
            f"(sensitive: {has_sensitive_data})"
        )
        
        return JSONResponse(content={
            "success": True,
            "filename": file.filename,
            "privacy_level": privacy_level.value,
            "has_sensitive_data": has_sensitive_data,
            "sensitive_data_types": sensitive_types,
            "reason": privacy_details.get("reason", "unknown"),
            "details": privacy_details
        })
    
    except HTTPException:
        raise
    
    except Exception as e:
        logger.error(f"❌ Document analysis error: {e}", exc_info=True)
        raise HTTPException(
            status_code=500,
            detail=f"Internal server error: {str(e)}"
        )


@router.get("/supported-formats")
async def get_supported_formats():
    """
    Get list of supported document formats
    
    Returns:
        List of supported file formats and their MIME types
    """
    return JSONResponse(content={
        "success": True,
        "supported_formats": [
            {
                "extension": "pdf",
                "mime_type": "application/pdf",
                "description": "Adobe PDF Document",
                "max_size_mb": 10
            },
            {
                "extension": "docx",
                "mime_type": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
                "description": "Microsoft Word Document (2007+)",
                "max_size_mb": 10
            },
            {
                "extension": "doc",
                "mime_type": "application/msword",
                "description": "Microsoft Word Document (Legacy)",
                "max_size_mb": 10
            },
            {
                "extension": "txt",
                "mime_type": "text/plain",
                "description": "Plain Text Document",
                "max_size_mb": 10
            }
        ],
        "max_file_size_bytes": document_parser.MAX_FILE_SIZE
    })

	"""
	SAAP Document Upload API
	Handles file uploads, parsing, and privacy detection for multi-agent chat
	"""
	import logging
	from typing import Optional
	from fastapi import APIRouter, UploadFile, File, HTTPException, Form
	from fastapi.responses import JSONResponse
	from pydantic import BaseModel

	from services.document_parser import document_parser
	from services.privacy_detector import privacy_detector

	logger = logging.getLogger(__name__)

	router = APIRouter(prefix="/api/v1/documents", tags=["documents"])


	class DocumentUploadResponse(BaseModel):
	"""Response model for document upload"""
	success: bool
	document_id: str
	filename: str
	file_type: str
	file_size: int
	char_count: int
	has_sensitive_data: bool
	sensitive_data_types: list[str]
	content_preview: str
	full_content: str # 📄 Full document content for chat context
	error: Optional[str] = None


	@router.post("/upload", response_model=DocumentUploadResponse)
	async def upload_document(
	file: UploadFile = File(...),
	user_message: Optional[str] = Form(None)
	):
	"""
	Upload and parse a document (PDF, DOCX, TXT)

	- Extracts text content from document
	- Analyzes for sensitive/private data
	- Returns parsed content and privacy analysis

	Args:
	file: Uploaded file (PDF, DOCX, TXT)
	user_message: Optional user message with context about the document

	Returns:
	DocumentUploadResponse with parsed content and privacy analysis
	"""
	try:
	logger.info(f"📤 Document upload started: {file.filename}")

	# Read file content
	file_content = await file.read()

	# Parse document
	parse_result = document_parser.parse_document(
	file_data=file_content,
	filename=file.filename,
	mime_type=file.content_type
	)

	if not parse_result["success"]:
	logger.error(f"❌ Document parsing failed: {parse_result['error']}")
	raise HTTPException(
	status_code=400,
	detail=f"Failed to parse document: {parse_result['error']}"
	)

	extracted_text = parse_result["content"]
	metadata = parse_result["metadata"]

	# Analyze for sensitive data
	from services.privacy_detector import analyze_document_privacy
	privacy_level, privacy_details = analyze_document_privacy(
	document_text=extracted_text,
	filename=file.filename
	)

	# Convert privacy level to boolean + types for response
	has_sensitive_data = privacy_level.value in ["private", "confidential"]

	# Extract detected types from details
	sensitive_types = []
	if privacy_details.get("keyword_matches"):
	sensitive_types.extend([cat for cat, _ in privacy_details["keyword_matches"]])
	if privacy_details.get("pattern_matches"):
	sensitive_types.extend([cat for _, cat in privacy_details["pattern_matches"]])
	if privacy_details.get("document_indicators"):
	sensitive_types.extend(privacy_details["document_indicators"])

	# Remove duplicates
	sensitive_types = list(set(sensitive_types))

	# Create content preview (first 500 characters)
	content_preview = extracted_text[:500]
	if len(extracted_text) > 500:
	content_preview += "..."

	# Generate document ID (simple hash for now)
	import hashlib
	doc_id = hashlib.sha256(file_content).hexdigest()[:16]

	logger.info(
	f"✅ Document processed successfully: {file.filename} "
	f"({metadata['char_count']} chars, "
	f"sensitive: {has_sensitive_data})"
	)

	return DocumentUploadResponse(
	success=True,
	document_id=doc_id,
	filename=metadata["filename"],
	file_type=metadata["file_type"],
	file_size=metadata["file_size"],
	char_count=metadata["char_count"],
	has_sensitive_data=has_sensitive_data,
	sensitive_data_types=sensitive_types,
	content_preview=content_preview,
	full_content=extracted_text, # 📄 Full document content for chat context
	error=None
	)

	except HTTPException:
	raise

	except Exception as e:
	logger.error(f"❌ Document upload error: {e}", exc_info=True)
	raise HTTPException(
	status_code=500,
	detail=f"Internal server error: {str(e)}"
	)


	@router.post("/analyze")
	async def analyze_document_content(
	file: UploadFile = File(...),
	):
	"""
	Analyze document for sensitive data without full parsing

	Quick privacy check before full processing

	Args:
	file: Uploaded file

	Returns:
	Privacy analysis results
	"""
	try:
	logger.info(f"🔍 Document privacy analysis: {file.filename}")

	# Read file content
	file_content = await file.read()

	# Parse document
	parse_result = document_parser.parse_document(
	file_data=file_content,
	filename=file.filename,
	mime_type=file.content_type
	)

	if not parse_result["success"]:
	raise HTTPException(
	status_code=400,
	detail=f"Failed to parse document: {parse_result['error']}"
	)

	# Analyze for sensitive data
	from services.privacy_detector import analyze_document_privacy
	privacy_level, privacy_details = analyze_document_privacy(
	document_text=parse_result["content"],
	filename=file.filename
	)

	# Convert privacy level to boolean
	has_sensitive_data = privacy_level.value in ["private", "confidential"]

	# Extract detected types
	sensitive_types = []
	if privacy_details.get("keyword_matches"):
	sensitive_types.extend([cat for cat, _ in privacy_details["keyword_matches"]])
	if privacy_details.get("pattern_matches"):
	sensitive_types.extend([cat for _, cat in privacy_details["pattern_matches"]])
	if privacy_details.get("document_indicators"):
	sensitive_types.extend(privacy_details["document_indicators"])
	sensitive_types = list(set(sensitive_types))

	logger.info(
	f"✅ Privacy analysis complete: {file.filename} "
	f"(sensitive: {has_sensitive_data})"
	)

	return JSONResponse(content={
	"success": True,
	"filename": file.filename,
	"privacy_level": privacy_level.value,
	"has_sensitive_data": has_sensitive_data,
	"sensitive_data_types": sensitive_types,
	"reason": privacy_details.get("reason", "unknown"),
	"details": privacy_details
	})

	except HTTPException:
	raise

	except Exception as e:
	logger.error(f"❌ Document analysis error: {e}", exc_info=True)
	raise HTTPException(
	status_code=500,
	detail=f"Internal server error: {str(e)}"
	)


	@router.get("/supported-formats")
	async def get_supported_formats():
	"""
	Get list of supported document formats

	Returns:
	List of supported file formats and their MIME types
	"""
	return JSONResponse(content={
	"success": True,
	"supported_formats": [
	{
	"extension": "pdf",
	"mime_type": "application/pdf",
	"description": "Adobe PDF Document",
	"max_size_mb": 10
	},
	{
	"extension": "docx",
	"mime_type": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
	"description": "Microsoft Word Document (2007+)",
	"max_size_mb": 10
	},
	{
	"extension": "doc",
	"mime_type": "application/msword",
	"description": "Microsoft Word Document (Legacy)",
	"max_size_mb": 10
	},
	{
	"extension": "txt",
	"mime_type": "text/plain",
	"description": "Plain Text Document",
	"max_size_mb": 10
	}
	],
	"max_file_size_bytes": document_parser.MAX_FILE_SIZE
	})