kyrlos adel
cva1
7cfdc4c
Raw History Blame Contribute Delete
27.1 kB
from typing import List, Optional
from fastapi import APIRouter, HTTPException, Depends, UploadFile, File, BackgroundTasks
from pydantic import BaseModel, Field, validator
from core.utils.auth_utils import verify_and_get_user_id_from_jwt, require_agent_access, AuthorizedAgentAccess
from core.services.supabase import DBConnection
from .file_processor import FileProcessor
from core.utils.logger import logger
from core.utils.config import config
from .validation import FileNameValidator, ValidationError, validate_folder_name_unique, validate_file_name_unique_in_folder
# Constants
MAX_TOTAL_FILE_SIZE = 50 * 1024 * 1024 # 50MB total limit per user
router = APIRouter(prefix="/knowledge-base", tags=["knowledge-base"])
# Helper function to check total file size limit
async def check_total_file_size_limit(account_id: str, new_file_size: int):
"""Check if adding a new file would exceed the total file size limit."""
try:
client = await DBConnection().client
# Get total size of all current entries for this account
result = await client.table('knowledge_base_entries').select(
'file_size'
).eq('account_id', account_id).eq('is_active', True).execute()
current_total_size = sum(entry['file_size'] for entry in result.data)
new_total_size = current_total_size + new_file_size
if new_total_size > MAX_TOTAL_FILE_SIZE:
current_mb = current_total_size / (1024 * 1024)
new_mb = new_file_size / (1024 * 1024)
limit_mb = MAX_TOTAL_FILE_SIZE / (1024 * 1024)
raise HTTPException(
status_code=413,
detail=f"File size limit exceeded. Current total: {current_mb:.1f}MB, New file: {new_mb:.1f}MB, Limit: {limit_mb}MB"
)
return True
except HTTPException:
raise
except Exception as e:
logger.error(f"Error checking file size limit: {str(e)}")
raise HTTPException(status_code=500, detail="Failed to check file size limit")
# Models
class FolderRequest(BaseModel):
name: str = Field(..., min_length=1, max_length=255)
description: Optional[str] = None
@validator('name')
def validate_folder_name(cls, v):
is_valid, error_message = FileNameValidator.validate_name(v, "folder")
if not is_valid:
raise ValueError(error_message)
return FileNameValidator.sanitize_name(v)
class UpdateFolderRequest(BaseModel):
name: Optional[str] = Field(None, min_length=1, max_length=255)
description: Optional[str] = None
@validator('name')
def validate_folder_name(cls, v):
if v is not None:
is_valid, error_message = FileNameValidator.validate_name(v, "folder")
if not is_valid:
raise ValueError(error_message)
return FileNameValidator.sanitize_name(v)
return v
class FolderResponse(BaseModel):
folder_id: str
name: str
description: Optional[str]
entry_count: int
created_at: str
class EntryResponse(BaseModel):
entry_id: str
filename: str
summary: str
file_size: int
created_at: str
class UpdateEntryRequest(BaseModel):
summary: str = Field(..., min_length=1, max_length=1000)
class AgentAssignmentRequest(BaseModel):
folder_ids: List[str]
db = DBConnection()
file_processor = FileProcessor()
# Folder management
@router.get("/folders", response_model=List[FolderResponse])
async def get_folders(user_id: str = Depends(verify_and_get_user_id_from_jwt)):
"""Get all knowledge base folders for user."""
if not config.ENABLE_KNOWLEDGE_BASE:
return []
try:
client = await db.client
account_id = user_id
result = await client.table('knowledge_base_folders').select(
'folder_id, name, description, created_at'
).eq('account_id', account_id).order('created_at', desc=True).execute()
folders = []
for folder_data in result.data:
# Count entries in folder
count_result = await client.table('knowledge_base_entries').select(
'entry_id', count='exact'
).eq('folder_id', folder_data['folder_id']).execute()
folders.append(FolderResponse(
folder_id=folder_data['folder_id'],
name=folder_data['name'],
description=folder_data['description'],
entry_count=count_result.count or 0,
created_at=folder_data['created_at']
))
return folders
except Exception as e:
logger.error(f"Error getting folders: {str(e)}")
raise HTTPException(status_code=500, detail="Failed to retrieve folders")
@router.post("/folders", response_model=FolderResponse)
async def create_folder(
folder_data: FolderRequest,
user_id: str = Depends(verify_and_get_user_id_from_jwt)
):
"""Create a new knowledge base folder."""
if not config.ENABLE_KNOWLEDGE_BASE:
raise HTTPException(status_code=503, detail="Knowledge base feature is currently disabled")
try:
client = await db.client
account_id = user_id
# Get existing folder names to check for conflicts
existing_result = await client.table('knowledge_base_folders').select('name').eq('account_id', account_id).execute()
existing_names = [folder['name'] for folder in existing_result.data]
# Generate unique name if there's a conflict
final_name = FileNameValidator.generate_unique_name(folder_data.name, existing_names, "folder")
insert_data = {
'account_id': account_id,
'name': final_name,
'description': folder_data.description
}
result = await client.table('knowledge_base_folders').insert(insert_data).execute()
if not result.data:
raise HTTPException(status_code=500, detail="Failed to create folder")
created_folder = result.data[0]
return FolderResponse(
folder_id=created_folder['folder_id'],
name=created_folder['name'],
description=created_folder['description'],
entry_count=0,
created_at=created_folder['created_at']
)
except ValidationError:
raise
except HTTPException:
raise
except Exception as e:
logger.error(f"Error creating folder: {str(e)}")
raise HTTPException(status_code=500, detail="Failed to create folder")
@router.put("/folders/{folder_id}", response_model=FolderResponse)
async def update_folder(
folder_id: str,
folder_data: UpdateFolderRequest,
user_id: str = Depends(verify_and_get_user_id_from_jwt)
):
"""Update a knowledge base folder."""
if not config.ENABLE_KNOWLEDGE_BASE:
raise HTTPException(status_code=503, detail="Knowledge base feature is currently disabled")
try:
client = await db.client
account_id = user_id
# Verify ownership and get current folder
folder_result = await client.table('knowledge_base_folders').select(
'folder_id, name, description, created_at'
).eq('folder_id', folder_id).eq('account_id', account_id).execute()
if not folder_result.data:
raise HTTPException(status_code=404, detail="Folder not found")
current_folder = folder_result.data[0]
# Build update data with only provided fields
update_data = {}
if folder_data.name is not None:
# Validate name uniqueness (excluding current folder)
await validate_folder_name_unique(folder_data.name, account_id, folder_id)
update_data['name'] = folder_data.name
if folder_data.description is not None:
update_data['description'] = folder_data.description
# If no fields to update, return current folder
if not update_data:
# Count entries in folder
count_result = await client.table('knowledge_base_entries').select(
'entry_id', count='exact'
).eq('folder_id', folder_id).execute()
return FolderResponse(
folder_id=current_folder['folder_id'],
name=current_folder['name'],
description=current_folder['description'],
entry_count=count_result.count or 0,
created_at=current_folder['created_at']
)
# Update folder
result = await client.table('knowledge_base_folders').update(
update_data
).eq('folder_id', folder_id).execute()
if not result.data:
raise HTTPException(status_code=500, detail="Failed to update folder")
updated_folder = result.data[0]
# Count entries in folder
count_result = await client.table('knowledge_base_entries').select(
'entry_id', count='exact'
).eq('folder_id', folder_id).execute()
return FolderResponse(
folder_id=updated_folder['folder_id'],
name=updated_folder['name'],
description=updated_folder['description'],
entry_count=count_result.count or 0,
created_at=updated_folder['created_at']
)
except ValidationError:
raise
except HTTPException:
raise
except Exception as e:
logger.error(f"Error updating folder: {str(e)}")
raise HTTPException(status_code=500, detail="Failed to update folder")
@router.delete("/folders/{folder_id}")
async def delete_folder(
folder_id: str,
user_id: str = Depends(verify_and_get_user_id_from_jwt)
):
"""Delete a knowledge base folder and all its entries."""
if not config.ENABLE_KNOWLEDGE_BASE:
raise HTTPException(status_code=503, detail="Knowledge base feature is currently disabled")
try:
client = await db.client
account_id = user_id
# Verify ownership
folder_result = await client.table('knowledge_base_folders').select(
'folder_id'
).eq('folder_id', folder_id).eq('account_id', account_id).execute()
if not folder_result.data:
raise HTTPException(status_code=404, detail="Folder not found")
# Get all entries in the folder to delete their files from S3
entries_result = await client.table('knowledge_base_entries').select(
'entry_id, file_path'
).eq('folder_id', folder_id).execute()
# Delete all files from S3 storage
if entries_result.data:
file_paths = [entry['file_path'] for entry in entries_result.data]
try:
await client.storage.from_('file-uploads').remove(file_paths)
logger.info(f"Deleted {len(file_paths)} files from S3 for folder {folder_id}")
except Exception as e:
logger.warning(f"Failed to delete some files from S3: {str(e)}")
# Delete folder (cascade will handle entries and assignments in DB)
await client.table('knowledge_base_folders').delete().eq('folder_id', folder_id).execute()
return {"success": True}
except HTTPException:
raise
except Exception as e:
logger.error(f"Error deleting folder: {str(e)}")
raise HTTPException(status_code=500, detail="Failed to delete folder")
# File upload
@router.post("/folders/{folder_id}/upload")
async def upload_file(
folder_id: str,
background_tasks: BackgroundTasks,
file: UploadFile = File(...),
user_id: str = Depends(verify_and_get_user_id_from_jwt)
):
"""Upload a file to a knowledge base folder."""
if not config.ENABLE_KNOWLEDGE_BASE:
raise HTTPException(status_code=503, detail="Knowledge base feature is currently disabled")
import time
start_time = time.time()
logger.info(f"[UPLOAD] Starting upload for file: {file.filename}")
try:
client = await db.client
account_id = user_id
# Verify folder ownership
t1 = time.time()
folder_result = await client.table('knowledge_base_folders').select(
'folder_id'
).eq('folder_id', folder_id).eq('account_id', account_id).execute()
logger.info(f"[UPLOAD] Folder verification took: {time.time() - t1:.2f}s")
if not folder_result.data:
raise HTTPException(status_code=404, detail="Folder not found")
# Validate and sanitize filename
if not file.filename:
raise ValidationError("Filename is required")
t2 = time.time()
is_valid, error_message = FileNameValidator.validate_name(file.filename, "file")
if not is_valid:
raise ValidationError(error_message)
logger.info(f"[UPLOAD] Filename validation took: {time.time() - t2:.2f}s")
# Read file content
t3 = time.time()
file_content = await file.read()
logger.info(f"[UPLOAD] File read ({len(file_content)} bytes) took: {time.time() - t3:.2f}s")
# Check total file size limit before processing
t4 = time.time()
await check_total_file_size_limit(account_id, len(file_content))
logger.info(f"[UPLOAD] Size limit check took: {time.time() - t4:.2f}s")
# Generate unique filename if there's a conflict
t5 = time.time()
final_filename = await validate_file_name_unique_in_folder(file.filename, folder_id)
logger.info(f"[UPLOAD] Filename uniqueness check took: {time.time() - t5:.2f}s")
# Process file in background
t6 = time.time()
result = await file_processor.process_file_fast(
account_id=account_id,
folder_id=folder_id,
file_content=file_content,
filename=final_filename,
mime_type=file.content_type or 'application/octet-stream',
background_tasks=background_tasks
)
logger.info(f"[UPLOAD] File processing took: {time.time() - t6:.2f}s")
if not result['success']:
raise HTTPException(status_code=400, detail=result['error'])
# Add info about filename changes
if final_filename != file.filename:
result['filename_changed'] = True
result['original_filename'] = file.filename
result['final_filename'] = final_filename
logger.info(f"[UPLOAD] Total upload time: {time.time() - start_time:.2f}s")
return result
except ValidationError:
raise
except HTTPException:
raise
except Exception as e:
logger.error(f"Error uploading file: {str(e)}")
raise HTTPException(status_code=500, detail="Failed to upload file")
# Entries
@router.get("/folders/{folder_id}/entries", response_model=List[EntryResponse])
async def get_folder_entries(
folder_id: str,
user_id: str = Depends(verify_and_get_user_id_from_jwt)
):
"""Get all entries in a folder."""
if not config.ENABLE_KNOWLEDGE_BASE:
return []
try:
client = await db.client
account_id = user_id
# Verify folder ownership
folder_result = await client.table('knowledge_base_folders').select(
'folder_id'
).eq('folder_id', folder_id).eq('account_id', account_id).execute()
if not folder_result.data:
raise HTTPException(status_code=404, detail="Folder not found")
result = await client.table('knowledge_base_entries').select(
'entry_id, filename, summary, file_size, created_at'
).eq('folder_id', folder_id).eq('is_active', True).order('created_at', desc=True).execute()
return [
EntryResponse(
entry_id=entry['entry_id'],
filename=entry['filename'],
summary=entry['summary'],
file_size=entry['file_size'],
created_at=entry['created_at']
)
for entry in result.data
]
except HTTPException:
raise
except Exception as e:
logger.error(f"Error getting folder entries: {str(e)}")
raise HTTPException(status_code=500, detail="Failed to retrieve entries")
@router.delete("/entries/{entry_id}")
async def delete_entry(
entry_id: str,
user_id: str = Depends(verify_and_get_user_id_from_jwt)
):
"""Delete a knowledge base entry."""
if not config.ENABLE_KNOWLEDGE_BASE:
raise HTTPException(status_code=503, detail="Knowledge base feature is currently disabled")
try:
client = await db.client
account_id = user_id
# Verify ownership
entry_result = await client.table('knowledge_base_entries').select(
'entry_id, file_path'
).eq('entry_id', entry_id).eq('account_id', account_id).execute()
if not entry_result.data:
raise HTTPException(status_code=404, detail="Entry not found")
entry = entry_result.data[0]
# Delete from S3
try:
await client.storage.from_('file-uploads').remove([entry['file_path']])
except Exception as e:
logger.warning(f"Failed to delete file from S3: {str(e)}")
# Delete from database
await client.table('knowledge_base_entries').delete().eq('entry_id', entry_id).execute()
# Invalidate KB context cache for all agents that had this entry assigned
try:
from core.cache.runtime_cache import invalidate_kb_context_cache
assignments_result = await client.table('agent_knowledge_entry_assignments') \
.select('agent_id').eq('entry_id', entry_id).execute()
agent_ids = set(row['agent_id'] for row in assignments_result.data) if assignments_result.data else set()
for agent_id in agent_ids:
await invalidate_kb_context_cache(agent_id)
logger.debug(f"๐Ÿ—‘๏ธ Invalidated KB cache for {len(agent_ids)} agents after entry deletion")
except Exception as e:
logger.warning(f"Failed to invalidate KB cache after entry deletion: {e}")
return {"success": True}
except HTTPException:
raise
except Exception as e:
logger.error(f"Error deleting entry: {str(e)}")
raise HTTPException(status_code=500, detail="Failed to delete entry")
@router.patch("/entries/{entry_id}", response_model=EntryResponse)
async def update_entry(
entry_id: str,
request: UpdateEntryRequest,
user_id: str = Depends(verify_and_get_user_id_from_jwt)
):
"""Update a knowledge base entry summary."""
if not config.ENABLE_KNOWLEDGE_BASE:
raise HTTPException(status_code=503, detail="Knowledge base feature is currently disabled")
try:
client = await db.client
account_id = user_id
# Verify ownership and get current entry
entry_result = await client.table('knowledge_base_entries').select(
'entry_id, filename, summary, file_size, created_at, account_id'
).eq('entry_id', entry_id).eq('account_id', account_id).execute()
if not entry_result.data:
raise HTTPException(status_code=404, detail="Entry not found")
# Update the summary
update_result = await client.table('knowledge_base_entries').update({
'summary': request.summary
}).eq('entry_id', entry_id).execute()
if not update_result.data:
raise HTTPException(status_code=500, detail="Failed to update entry")
# Invalidate KB context cache for all agents that have this entry assigned
try:
from core.cache.runtime_cache import invalidate_kb_context_cache
assignments_result = await client.table('agent_knowledge_entry_assignments') \
.select('agent_id').eq('entry_id', entry_id).execute()
agent_ids = set(row['agent_id'] for row in assignments_result.data) if assignments_result.data else set()
for agent_id in agent_ids:
await invalidate_kb_context_cache(agent_id)
logger.debug(f"๐Ÿ—‘๏ธ Invalidated KB cache for {len(agent_ids)} agents after entry update")
except Exception as e:
logger.warning(f"Failed to invalidate KB cache after entry update: {e}")
# Return the updated entry
updated_entry = update_result.data[0]
return EntryResponse(
entry_id=updated_entry['entry_id'],
filename=updated_entry['filename'],
summary=updated_entry['summary'],
file_size=updated_entry['file_size'],
created_at=updated_entry['created_at']
)
except HTTPException:
raise
except Exception as e:
logger.error(f"Error updating entry: {str(e)}")
raise HTTPException(status_code=500, detail="Failed to update entry")
# Agent assignments
@router.get("/agents/{agent_id}/assignments")
async def get_agent_assignments(
agent_id: str,
auth: AuthorizedAgentAccess = Depends(require_agent_access)
):
"""Get entry assignments for an agent."""
if not config.ENABLE_KNOWLEDGE_BASE:
return {}
try:
client = await DBConnection().client
# Get file-level assignments only
file_result = await client.table('agent_knowledge_entry_assignments').select(
'entry_id, enabled'
).eq('agent_id', agent_id).execute()
return {row['entry_id']: row['enabled'] for row in file_result.data}
except Exception as e:
logger.error(f"Error getting agent assignments: {str(e)}")
raise HTTPException(status_code=500, detail="Failed to retrieve agent assignments")
@router.post("/agents/{agent_id}/assignments")
async def update_agent_assignments(
agent_id: str,
assignment_data: dict,
auth: AuthorizedAgentAccess = Depends(require_agent_access)
):
"""Update agent entry assignments."""
if not config.ENABLE_KNOWLEDGE_BASE:
raise HTTPException(status_code=503, detail="Knowledge base feature is currently disabled")
try:
client = await db.client
account_id = auth.user_id
entry_ids = assignment_data.get('entry_ids', [])
# Clear existing assignments
await client.table('agent_knowledge_entry_assignments').delete().eq('agent_id', agent_id).execute()
# Insert new entry assignments
for entry_id in entry_ids:
await client.table('agent_knowledge_entry_assignments').insert({
'agent_id': agent_id,
'entry_id': entry_id,
'account_id': account_id,
'enabled': True
}).execute()
# Invalidate agent config cache and KB context cache (knowledge base assignments changed)
try:
from core.cache.runtime_cache import invalidate_agent_config_cache, invalidate_kb_context_cache
await invalidate_agent_config_cache(agent_id)
await invalidate_kb_context_cache(agent_id)
logger.debug(f"๐Ÿ—‘๏ธ Invalidated cache for agent {agent_id} after knowledge base update")
except Exception as e:
logger.warning(f"Failed to invalidate cache for agent {agent_id}: {e}")
return {"success": True, "message": "Assignments updated successfully"}
except Exception as e:
logger.error(f"Error updating agent assignments: {str(e)}")
raise HTTPException(status_code=500, detail="Failed to update assignments")
class FolderMoveRequest(BaseModel):
folder_id: str
# File operations
@router.put("/entries/{entry_id}/move")
async def move_file(
entry_id: str,
request: FolderMoveRequest,
user_id: str = Depends(verify_and_get_user_id_from_jwt)
):
"""Move a file to a different folder."""
if not config.ENABLE_KNOWLEDGE_BASE:
raise HTTPException(status_code=503, detail="Knowledge base feature is currently disabled")
try:
client = await db.client
account_id = user_id
# Get current entry details including file path and filename
entry_result = await client.table('knowledge_base_entries').select(
'entry_id, folder_id, file_path, filename'
).eq('entry_id', entry_id).execute()
if not entry_result.data:
raise HTTPException(status_code=404, detail="File not found")
entry = entry_result.data[0]
current_folder_id = entry['folder_id']
current_file_path = entry['file_path']
filename = entry['filename']
# If already in the target folder, no need to move
if current_folder_id == request.folder_id:
return {"success": True, "message": "File is already in the target folder"}
# Verify target folder belongs to user
folder_result = await client.table('knowledge_base_folders').select(
'folder_id'
).eq('folder_id', request.folder_id).eq('account_id', account_id).execute()
if not folder_result.data:
raise HTTPException(status_code=404, detail="Target folder not found")
# Sanitize filename for storage (same logic as file processor)
sanitized_filename = file_processor.sanitize_filename(filename)
# Create new file path
new_file_path = f"knowledge-base/{request.folder_id}/{entry_id}/{sanitized_filename}"
# Move file in storage
try:
# Copy file to new location
copy_result = await client.storage.from_('file-uploads').copy(
current_file_path, new_file_path
)
# Remove old file
await client.storage.from_('file-uploads').remove([current_file_path])
except Exception as storage_error:
logger.error(f"Error moving file in storage: {str(storage_error)}")
raise HTTPException(status_code=500, detail="Failed to move file in storage")
# Update the database with new folder and file path
await client.table('knowledge_base_entries').update({
'folder_id': request.folder_id,
'file_path': new_file_path
}).eq('entry_id', entry_id).execute()
return {"success": True, "message": "File moved successfully"}
except HTTPException:
raise
except Exception as e:
logger.error(f"Error moving file: {str(e)}")
raise HTTPException(status_code=500, detail="Failed to move file")