Download core/files/upload_handler.py from Kyrlos11/nniback: direct link, hf CLI and curl.
- Browser
- Download file 8.08 kB
-
https://huggingface.co/spaces/Kyrlos11/nniback/resolve/main/core/files/upload_handler.py
- Command line
-
hf download hf://spaces/Kyrlos11/nniback/core/files/upload_handler.py
-
curl -L -o upload_handler.py https://huggingface.co/spaces/Kyrlos11/nniback/resolve/main/core/files/upload_handler.py
8.08 kB
| import asyncio | |
| import json | |
| import re | |
| from typing import Optional, List, Tuple, Dict, Any | |
| from fastapi import UploadFile | |
| from core.services import redis | |
| from core.services.supabase import DBConnection | |
| from core.utils.logger import logger | |
| from core.utils.sandbox_utils import generate_unique_filename, get_uploads_directory | |
| from core.utils.files_utils import normalize_filename | |
| from core.sandbox.resolver import resolve_sandbox | |
| db = DBConnection() | |
| async def fast_parse_files(files: List[UploadFile], prompt: str = "") -> Tuple[str, List[Tuple[str, bytes, str, Optional[str]]]]: | |
| from core.utils.fast_parse import parse, FileType, format_file_size | |
| if not files: | |
| return prompt, [] | |
| message_content = prompt | |
| files_for_upload: List[Tuple[str, bytes, str, Optional[str]]] = [] | |
| file_refs = [] | |
| existing_refs = set() | |
| if prompt: | |
| existing_matches = re.findall(r'\[(?:Uploaded File|Attached|Image):\s*([^\]]+)\]', prompt) | |
| for match in existing_matches: | |
| normalized = match.replace('/workspace/', '') if match.startswith('/workspace/') else match | |
| existing_refs.add(normalized.lower()) | |
| for file in files: | |
| if not file.filename: | |
| continue | |
| try: | |
| # Normalize filename for Unicode compatibility (macOS screenshots, etc.) | |
| original_filename = normalize_filename(file.filename.replace('/', '_').replace('\\', '_')) | |
| content_bytes = await file.read() | |
| mime_type = file.content_type or "application/octet-stream" | |
| result = parse(content_bytes, original_filename, mime_type) | |
| parsed_content = None | |
| if result.success and result.file_type != FileType.IMAGE: | |
| parsed_content = result.content | |
| if len(parsed_content) > 100000: | |
| parsed_content = parsed_content[:100000] | |
| files_for_upload.append((original_filename, content_bytes, mime_type, parsed_content)) | |
| file_path = f"uploads/{original_filename}" | |
| if file_path.lower() not in existing_refs: | |
| file_refs.append(f"[Attached: {original_filename} ({format_file_size(result.file_size)}) -> {file_path}]") | |
| logger.debug(f"Fast-parsed {original_filename}: {result.char_count} chars, type={result.file_type.name}") | |
| except Exception as e: | |
| logger.error(f"Error fast-parsing file {file.filename}: {str(e)}", exc_info=True) | |
| file_path = f"uploads/{file.filename}" | |
| if file_path.lower() not in existing_refs: | |
| file_refs.append(f"[Attached: {file.filename} -> {file_path}]") | |
| finally: | |
| await file.seek(0) | |
| if file_refs: | |
| message_content = prompt + "\n\n" + "\n".join(file_refs) if prompt else "\n".join(file_refs) | |
| return message_content, files_for_upload | |
| async def upload_files_to_sandbox( | |
| project_id: str, | |
| thread_id: str, | |
| files_data: List[Tuple[str, bytes, str, Optional[str]]], | |
| account_id: Optional[str] = None, | |
| ): | |
| if not files_data: | |
| return | |
| logger.info(f"[UPLOAD] Uploading files for project {project_id} ({len(files_data)} files)") | |
| try: | |
| client = await db.client | |
| if not account_id: | |
| result = await client.table('projects').select('account_id').eq('project_id', project_id).execute() | |
| if result.data: | |
| account_id = str(result.data[0].get('account_id')) | |
| sandbox_info = await resolve_sandbox( | |
| project_id=project_id, | |
| account_id=account_id, | |
| db_client=client, | |
| require_started=True | |
| ) | |
| if not sandbox_info: | |
| logger.info(f"[UPLOAD] No sandbox for project {project_id} - files cached in Redis only") | |
| return | |
| logger.info(f"[UPLOAD] Sandbox {sandbox_info.sandbox_id} ready, uploading {len(files_data)} files...") | |
| uploads_dir = get_uploads_directory() | |
| uploaded_count = 0 | |
| for filename, content_bytes, mime_type, _ in files_data: | |
| try: | |
| unique_filename = await generate_unique_filename(sandbox_info.sandbox, uploads_dir, filename) | |
| target_path = f"{uploads_dir}/{unique_filename}" | |
| if hasattr(sandbox_info.sandbox, 'fs') and hasattr(sandbox_info.sandbox.fs, 'upload_file'): | |
| await sandbox_info.sandbox.fs.upload_file(content_bytes, target_path) | |
| uploaded_count += 1 | |
| logger.debug(f"[UPLOAD] Complete: {filename} -> {target_path}") | |
| except Exception as e: | |
| logger.warning(f"[UPLOAD] Failed for {filename}: {str(e)}") | |
| logger.info(f"[UPLOAD] Complete: {uploaded_count}/{len(files_data)} files to sandbox {sandbox_info.sandbox_id}") | |
| except Exception as e: | |
| logger.warning(f"[UPLOAD] Error for project {project_id}: {str(e)}") | |
| async def get_cached_file_context(thread_id: str) -> Optional[List[Dict[str, Any]]]: | |
| try: | |
| cache_key = f"file_context:{thread_id}" | |
| cached = await redis.get(cache_key) | |
| if cached: | |
| return json.loads(cached) | |
| except Exception as e: | |
| logger.warning(f"Failed to retrieve cached file context for {thread_id}: {e}") | |
| return None | |
| def format_file_context_for_agent(files: List[Dict[str, Any]]) -> str: | |
| if not files: | |
| return "" | |
| parts = ["\n\n--- ATTACHED FILE CONTENTS ---"] | |
| for f in files: | |
| filename = f.get("filename", "unknown") | |
| content = f.get("content", "") | |
| size = f.get("size", 0) | |
| if content: | |
| parts.append(f"\n### {filename} ({size:,} bytes)\n```\n{content}\n```") | |
| parts.append("\n--- END OF ATTACHED FILES ---\n") | |
| return "\n".join(parts) | |
| async def handle_file_uploads_fast( | |
| files: List[UploadFile], | |
| project_id: str, | |
| prompt: str = "", | |
| thread_id: Optional[str] = None, | |
| account_id: Optional[str] = None, | |
| ) -> str: | |
| message_content, files_data = await fast_parse_files(files, prompt) | |
| if files_data: | |
| tid = thread_id or project_id | |
| parsed_contents = [] | |
| for filename, content_bytes, mime_type, parsed_content in files_data: | |
| if parsed_content: | |
| parsed_contents.append({ | |
| "filename": filename, | |
| "content": parsed_content, | |
| "mime_type": mime_type, | |
| "size": len(content_bytes) | |
| }) | |
| if parsed_contents: | |
| try: | |
| cache_key = f"file_context:{tid}" | |
| await redis.set(cache_key, json.dumps(parsed_contents), ex=3600) | |
| logger.info(f"[UPLOAD] Cached {len(parsed_contents)} parsed files for thread {tid}") | |
| except Exception as cache_error: | |
| logger.warning(f"Failed to cache parsed files: {cache_error}") | |
| if project_id: | |
| await upload_files_to_sandbox(project_id, tid, files_data, account_id) | |
| return message_content | |
| async def ensure_sandbox_for_thread(client, project_id: str, files: Optional[List[Any]] = None): | |
| result = await client.table('projects').select('account_id').eq('project_id', project_id).execute() | |
| if not result.data: | |
| logger.warning(f"Project {project_id} not found") | |
| return None, None | |
| account_id = str(result.data[0].get('account_id')) if result.data[0].get('account_id') else None | |
| if not files or len(files) == 0: | |
| logger.debug(f"No files to upload for project {project_id}") | |
| return None, None | |
| sandbox_info = await resolve_sandbox( | |
| project_id=project_id, | |
| account_id=account_id, | |
| db_client=client, | |
| require_started=True | |
| ) | |
| if sandbox_info: | |
| return sandbox_info.sandbox, sandbox_info.sandbox_id | |
| return None, None | |