import fs from 'fs'; import path from 'path'; import mime from 'mime-types'; // Storage root directory: defaults to /data on HuggingFace Space // Can be customized via DATA_DIR env var (e.g. for local testing) const DEFAULT_DATA_DIR = process.platform === 'win32' ? path.resolve(process.cwd(), 'data_local') : '/data'; export const DATA_ROOT = path.resolve(process.env.DATA_DIR || DEFAULT_DATA_DIR); // Ensure root storage directory exists in local/dev environment export function ensureDataRootExists(): string { if (!fs.existsSync(DATA_ROOT)) { try { fs.mkdirSync(DATA_ROOT, { recursive: true }); console.log(`[Storage] Created storage directory at: ${DATA_ROOT}`); } catch (err) { console.error(`[Storage Error] Could not create storage root ${DATA_ROOT}:`, err); } } return DATA_ROOT; } export interface FileItem { name: string; relativePath: string; isDirectory: boolean; size: number; formattedSize: string; modifiedAt: Date; extension: string; mimeType: string; category: 'folder' | 'image' | 'code' | 'document' | 'audio' | 'video' | 'archive' | 'data' | 'file'; } export interface Breadcrumb { name: string; path: string; } export function formatBytes(bytes: number, decimals: number = 2): string { if (bytes === 0) return '0 Bytes'; const k = 1024; const dm = decimals < 0 ? 0 : decimals; const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB', 'PB']; const i = Math.floor(Math.log(bytes) / Math.log(k)); return parseFloat((bytes / Math.pow(k, i)).toFixed(dm)) + ' ' + sizes[i]; } export function safeResolvePath(subPath: string = ''): { fullPath: string; relativePath: string } { ensureDataRootExists(); // Normalize and remove leading slashes const cleanSubPath = path.normalize(subPath).replace(/^(\.\.[\/\\])+/, '').replace(/^[/\\]+/, ''); const fullPath = path.resolve(DATA_ROOT, cleanSubPath); // Security check: path traversal prevention if (!fullPath.startsWith(DATA_ROOT)) { throw new Error('Access denied: Invalid path traversal target.'); } const relativePath = path.relative(DATA_ROOT, fullPath).replace(/\\/g, '/'); return { fullPath, relativePath }; } export function getBreadcrumbs(relativePath: string): Breadcrumb[] { const parts = relativePath.split('/').filter(Boolean); const crumbs: Breadcrumb[] = [{ name: 'data', path: '' }]; let accumulated = ''; for (const part of parts) { accumulated = accumulated ? `${accumulated}/${part}` : part; crumbs.push({ name: part, path: accumulated }); } return crumbs; } function getCategory(isDirectory: boolean, ext: string, mimeType: string): FileItem['category'] { if (isDirectory) return 'folder'; ext = ext.toLowerCase(); if (['jpg', 'jpeg', 'png', 'gif', 'webp', 'svg', 'bmp', 'ico'].includes(ext)) return 'image'; if (['txt', 'md', 'json', 'js', 'ts', 'html', 'css', 'py', 'sh', 'yaml', 'yml', 'xml', 'c', 'cpp', 'rs', 'go', 'java'].includes(ext)) return 'code'; if (['pdf', 'doc', 'docx', 'xls', 'xlsx', 'ppt', 'pptx'].includes(ext)) return 'document'; if (['mp3', 'wav', 'ogg', 'flac', 'm4a'].includes(ext)) return 'audio'; if (['mp4', 'webm', 'mkv', 'avi', 'mov'].includes(ext)) return 'video'; if (['zip', 'tar', 'gz', 'bz2', '7z', 'rar'].includes(ext)) return 'archive'; if (['csv', 'parquet', 'arrow', 'feather', 'h5', 'hdf5', 'npy', 'npz', 'pt', 'pth', 'bin', 'onnx', 'safetensors'].includes(ext)) return 'data'; return 'file'; } export function listDirectory(subPath: string = ''): { items: FileItem[]; relativePath: string; breadcrumbs: Breadcrumb[]; totalFiles: number; totalFolders: number; totalSize: number } { const { fullPath, relativePath } = safeResolvePath(subPath); if (!fs.existsSync(fullPath)) { throw new Error(`Directory does not exist: /data/${relativePath}`); } const stat = fs.statSync(fullPath); if (!stat.isDirectory()) { throw new Error(`Path /data/${relativePath} is a file, not a directory.`); } const filenames = fs.readdirSync(fullPath); const items: FileItem[] = []; let totalFiles = 0; let totalFolders = 0; let totalSize = 0; for (const name of filenames) { try { const itemFullPath = path.join(fullPath, name); const itemStat = fs.statSync(itemFullPath); const isDir = itemStat.isDirectory(); const ext = isDir ? '' : path.extname(name).slice(1); const mimeType = isDir ? '' : (mime.lookup(name) || 'application/octet-stream'); const itemRelPath = relativePath ? `${relativePath}/${name}` : name; const size = isDir ? 0 : itemStat.size; if (isDir) { totalFolders++; } else { totalFiles++; totalSize += size; } items.push({ name, relativePath: itemRelPath, isDirectory: isDir, size, formattedSize: isDir ? '--' : formatBytes(size), modifiedAt: itemStat.mtime, extension: ext, mimeType, category: getCategory(isDir, ext, mimeType), }); } catch (e) { console.warn(`Could not stat item ${name} in ${fullPath}`, e); } } // Sort folders first, then files alphabetically items.sort((a, b) => { if (a.isDirectory && !b.isDirectory) return -1; if (!a.isDirectory && b.isDirectory) return 1; return a.name.localeCompare(b.name, undefined, { sensitivity: 'base', numeric: true }); }); return { items, relativePath, breadcrumbs: getBreadcrumbs(relativePath), totalFiles, totalFolders, totalSize, }; } export function getFileDetails(subPath: string) { const { fullPath, relativePath } = safeResolvePath(subPath); if (!fs.existsSync(fullPath)) { throw new Error(`File does not exist: /data/${relativePath}`); } const stat = fs.statSync(fullPath); if (stat.isDirectory()) { throw new Error(`Path is a directory, not a file.`); } const name = path.basename(fullPath); const ext = path.extname(name).slice(1).toLowerCase(); const mimeType = mime.lookup(name) || 'application/octet-stream'; const category = getCategory(false, ext, mimeType); let textContent: string | null = null; const MAX_PREVIEW_SIZE = 1024 * 1024; // 1 MB limit for text preview if (category === 'code' || ['csv', 'txt', 'json', 'md', 'log', 'yaml', 'yml'].includes(ext)) { if (stat.size <= MAX_PREVIEW_SIZE) { try { textContent = fs.readFileSync(fullPath, 'utf-8'); } catch (err) { textContent = `[Unable to read text content: ${err}]`; } } else { // Read first 50KB for large files try { const fd = fs.openSync(fullPath, 'r'); const buffer = Buffer.alloc(50 * 1024); const bytesRead = fs.readSync(fd, buffer, 0, buffer.length, 0); fs.closeSync(fd); textContent = buffer.toString('utf-8', 0, bytesRead) + `\n\n... [Preview truncated. Total size: ${formatBytes(stat.size)}]`; } catch (err) { textContent = `[Error reading partial preview]`; } } } return { name, relativePath, fullPath, size: stat.size, formattedSize: formatBytes(stat.size), modifiedAt: stat.mtime, extension: ext, mimeType, category, textContent, breadcrumbs: getBreadcrumbs(relativePath), }; }