"""Unified file manager for Research and Docs sources in the dashboard. Provides a unified API over both ``research/`` and ``docs/`` directories with proper MIME type detection, binary file serving, and directory trees. """ from __future__ import annotations import json import re import subprocess from http import HTTPStatus from io import BytesIO from pathlib import Path from typing import Any, Iterator, cast from urllib.parse import unquote from .docs_source import DocumentationSource, create_docs_source from .research_source import ResearchSource # --------------------------------------------------------------------------- # MIME type map # --------------------------------------------------------------------------- _MEDIA_TYPES: dict[str, str] = { ".html": "text/html; charset=utf-8", ".css": "text/css; charset=utf-8", ".js": "text/javascript; charset=utf-8", ".svg": "image/svg+xml", ".ico": "image/x-icon", ".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg", ".gif": "image/gif", ".webp": "image/webp", ".bmp": "image/bmp", ".mp4": "video/mp4", ".webm": "video/webm", ".ogg": "video/ogg", ".mov": "video/quicktime", ".avi": "video/x-msvideo", ".mp3": "audio/mpeg", ".wav": "audio/wav", ".flac": "audio/flac", ".aac": "audio/aac", ".m4a": "audio/mp4", ".opus": "audio/opus", ".pdf": "application/pdf", ".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", ".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", } _BINARY_EXTENSIONS = frozenset( { ".png", ".jpg", ".jpeg", ".gif", ".webp", ".svg", ".mp4", ".webm", ".ogg", ".mov", ".avi", ".mp3", ".wav", ".flac", ".aac", ".m4a", ".opus", ".pdf", ".docx", ".xlsx", ".xls", ".ico", ".bmp", } ) def _mime_type(ext: str) -> str: return _MEDIA_TYPES.get(ext.lower(), "application/octet-stream") # --------------------------------------------------------------------------- # Exceptions # --------------------------------------------------------------------------- class FileManagerError(Exception): """Base error for file manager operations.""" class SourceNotAvailableError(FileManagerError): """Raised when a requested source is not configured.""" class InvalidSourceError(FileManagerError): """Raised when an unknown source name is given.""" class PathTraversalError(FileManagerError): """Raised on path traversal attempts.""" # --------------------------------------------------------------------------- # Markdown export helpers # --------------------------------------------------------------------------- def _escape_html(value: str) -> str: return ( value.replace("&", "&") .replace("<", "<") .replace(">", ">") .replace('"', """) ) def _markdown_to_html(content: str, title: str = "Document") -> str: """Convert a simple Markdown document to a standalone HTML page.""" lines = content.splitlines() html_parts: list[str] = [] in_code = False code_lines: list[str] = [] for raw_line in lines: line = raw_line.rstrip() if line.startswith("```"): if in_code: html_parts.append( "
"
+ _escape_html("\n".join(code_lines))
+ ""
)
code_lines = []
in_code = not in_code
continue
if in_code:
code_lines.append(raw_line)
continue
# Heading
match = re.match(r"^(#{1,6})\s+(.*)$", line)
if match:
level = len(match.group(1))
html_parts.append(f"{para}
") # Wrap loose list items in\1", html)
html = re.sub(r"\*\*(.+?)\*\*", r"\1", html)
html = re.sub(r"__(.+?)__", r"\1", html)
html = re.sub(r"\*(.+?)\*", r"\1", html)
html = re.sub(r"_(.+?)_", r"\1", html)
html = re.sub(r"\[([^\]]+)\]\(([^)]+)\)", r'\1', html)
escaped = escaped.replace(key, html)
return escaped
def _iter_markdown_blocks(content: str) -> Iterator[dict[str, Any]]:
"""Yield Markdown blocks for DOCX export."""
lines = content.splitlines()
in_code = False
code_lines: list[str] = []
for raw_line in lines:
line = raw_line.rstrip()
if line.startswith("```"):
if in_code:
yield {"type": "code", "text": "\n".join(code_lines)}
code_lines = []
in_code = not in_code
continue
if in_code:
code_lines.append(raw_line)
continue
match = re.match(r"^(#{1,6})\s+(.*)$", line)
if match:
yield {
"type": "heading",
"level": len(match.group(1)),
"text": match.group(2),
}
continue
if re.match(r"^\s*[-*+\d]\.\s+", line):
text = re.sub(r"^\s*[-*+\d]\.\s+", "", line)
yield {"type": "list", "text": text}
continue
if not line.strip():
continue
yield {"type": "paragraph", "text": line}
# ---------------------------------------------------------------------------
# File Manager
# ---------------------------------------------------------------------------
class FileManager:
"""Unified file manager for Research and Docs sources.
Provides directory tree listing, file search, statistics, and
content retrieval (text as JSON, binary as raw bytes).
"""
def __init__(
self,
research_source: ResearchSource | None,
docs_source: DocumentationSource | None,
default_docs_root: Path,
) -> None:
self._research = research_source
self._docs = docs_source
self._default_docs_root = default_docs_root
# ------------------------------------------------------------------
# Source resolution
# ------------------------------------------------------------------
def _resolve_source(self, source: str) -> tuple[Any, Path, str]:
if source == "research":
s = self._research
if s is None or not s.is_available():
raise SourceNotAvailableError("Research source is not configured.")
return s, s.root(), "research"
if source == "docs":
docs = self._docs or create_docs_source(self._default_docs_root)
return docs, docs.docs_root, "docs"
raise InvalidSourceError(f"Unknown source: {source}")
def _root(self, source: str) -> Path:
"""Return the filesystem root for a source."""
return self._resolve_source(source)[1]
# ------------------------------------------------------------------
# Directory tree
# ------------------------------------------------------------------
def get_tree(self, source: str = "research") -> dict[str, Any]:
"""Return the full directory tree for a source."""
try:
_obj, root, source_name = self._resolve_source(source)
except FileManagerError as exc:
return {"available": False, "error": str(exc), "children": []}
def _build(path: Path, rel_prefix: str = "") -> dict[str, Any]:
children: list[dict[str, Any]] = []
try:
entries = sorted(path.iterdir())
except PermissionError:
return {
"name": path.name,
"path": rel_prefix or ".",
"type": "directory",
"children": [],
}
for child in entries:
if child.name.startswith("."):
continue
if child.name in {"__pycache__", ".git", ".venv", "node_modules"}:
continue
if child.is_dir():
child_rel = str(child.relative_to(root)) if child != root else ""
children.append(_build(child, child_rel))
elif child.is_file():
ext = child.suffix.lower()
children.append(
{
"name": child.name,
"path": str(child.relative_to(root)),
"type": "file",
"size_bytes": child.stat().st_size,
"ext": ext,
"is_binary": ext in _BINARY_EXTENSIONS,
"is_image": ext
in {
".png",
".jpg",
".jpeg",
".gif",
".webp",
".svg",
".bmp",
},
"is_video": ext
in {".mp4", ".webm", ".ogg", ".mov", ".avi"},
"is_audio": ext
in {".mp3", ".wav", ".flac", ".aac", ".m4a", ".opus"},
"is_spreadsheet": ext in {".xlsx", ".xls", ".xlsm", ".ods"},
"is_document": ext in {".docx", ".doc"},
}
)
result: dict[str, Any] = {
"name": root.name if rel_prefix == "" else path.name,
"path": rel_prefix or ".",
"type": "directory",
"children": children,
}
if source_name == "research":
manifest_path = path / "manifest.json"
if manifest_path.is_file():
try:
manifest_raw: object = json.loads(
manifest_path.read_text(encoding="utf-8")
)
if isinstance(manifest_raw, dict):
manifest = cast(dict[str, object], manifest_raw)
created_at = manifest.get("created_at") or manifest.get(
"timestamp"
)
if created_at is not None:
result["created_at"] = str(created_at)
except (OSError, json.JSONDecodeError):
pass
return result
tree = _build(root)
tree["available"] = True
return tree
# ------------------------------------------------------------------
# Search
# ------------------------------------------------------------------
def search(self, source: str, query: str) -> list[dict[str, Any]]:
"""Search files by name or path."""
q = query.lower().strip()
if not q or len(q) < 2:
return []
try:
_obj, root, _ = self._resolve_source(source)
except FileManagerError:
return []
results: list[dict[str, Any]] = []
for f in sorted(root.rglob("*")):
if not f.is_file() or f.name.startswith("."):
continue
rel = str(f.relative_to(root))
if q in f.name.lower() or q in rel.lower():
ext = f.suffix.lower()
results.append(
{
"name": f.name,
"path": rel,
"size_bytes": f.stat().st_size,
"ext": ext,
"is_binary": ext in _BINARY_EXTENSIONS,
}
)
return results
# ------------------------------------------------------------------
# Statistics
# ------------------------------------------------------------------
def get_statistics(self) -> dict[str, Any]:
"""Return combined statistics for both sources."""
stats: dict[str, Any] = {}
for src_name in ("research", "docs"):
try:
_obj, root, _ = self._resolve_source(src_name)
total = 0
total_size = 0
for f in root.rglob("*"):
if f.is_file() and not f.name.startswith("."):
total += 1
total_size += f.stat().st_size
stats[src_name] = {
"available": True,
"total_files": total,
"total_size_bytes": total_size,
}
except FileManagerError:
stats[src_name] = {
"available": False,
"total_files": 0,
"total_size_bytes": 0,
}
return {"sources": stats}
# ------------------------------------------------------------------
# Content retrieval
# ------------------------------------------------------------------
def get_content(
self, source: str, file_path: str
) -> tuple[bytes | str, str | None, bool]:
"""Get file content.
Returns:
Tuple of (content, mime_type_or_None, is_binary).
For text files: (str, None, False).
For binary files: (bytes, mime_type, True).
"""
if not file_path or ".." in file_path:
raise PathTraversalError("Path traversal is not allowed.")
try:
_obj, root, _ = self._resolve_source(source)
except FileManagerError:
raise SourceNotAvailableError(f"Source '{source}' is not available.")
candidate = (root / file_path).resolve()
try:
candidate.relative_to(root)
except ValueError:
raise PathTraversalError("Path traversal detected.")
if not candidate.is_file():
raise FileNotFoundError(f"File not found: {file_path}")
ext = candidate.suffix.lower()
if ext in _BINARY_EXTENSIONS:
data = candidate.read_bytes()
return data, _mime_type(ext), True
content = candidate.read_text(encoding="utf-8", errors="replace")
return content, None, False
def _resolve_file(self, source: str, file_path: str) -> Path:
"""Resolve and validate a file path under a source root."""
if not file_path or ".." in file_path:
raise PathTraversalError("Path traversal is not allowed.")
try:
_obj, root, _ = self._resolve_source(source)
except FileManagerError:
raise SourceNotAvailableError(f"Source '{source}' is not available.")
candidate = (root / file_path).resolve()
try:
candidate.relative_to(root)
except ValueError:
raise PathTraversalError("Path traversal detected.")
return candidate
def save_content(
self, source: str, file_path: str, content: str, backup: bool = True
) -> dict[str, Any]:
"""Save text content to a file under the configured source root.
Args:
source: 'research' or 'docs'.
file_path: Relative path inside the source root.
content: New file content as UTF-8 text.
backup: If True, the existing file is renamed to `