Natnael B. Haile
Cursor
feat: complete IR Studio with Git LFS tracking for all binary assets and NFCorpus index
fbad523 import React, { useState, useCallback, useMemo, useRef, useEffect } from 'react';
import {
Folder,
FolderOpen,
FileText,
FileCode,
Upload,
Copy,
Check,
X,
Trash2,
CheckCircle2,
AlertCircle,
Maximize2,
Minimize2,
Search,
WrapText,
AlignLeft,
Database,
ChevronDown,
ChevronUp,
ChevronRight,
ChevronsUpDown,
Sparkles,
Target,
Activity,
} from 'lucide-react';
import type { CorpusDataset, VirtualFile } from '../types/corpus';
import {
validateAndParseCorpus,
getSampleJsonlString,
getSampleTsvString,
MAX_CORPUS_DOCUMENTS,
} from '../utils/corpusValidator';
interface FileExplorerModalProps {
isOpen: boolean;
onClose: () => void;
datasets: CorpusDataset[];
activeDatasetId: string;
onSelectDataset: (id: string) => void;
onSaveCustomDataset: (dataset: CorpusDataset) => void;
onDeleteCustomDataset: (id: string) => void;
}
// Token type for JSON syntax highlighting
interface JsonToken {
type: 'key' | 'string' | 'number' | 'literal' | 'punct' | 'text';
val: string;
}
// Fast tokenizer for a single line of JSON
function tokenizeJsonLine(line: string): JsonToken[] {
const tokens: JsonToken[] = [];
const regex = /(\"(\\u[a-zA-Z0-9]{4}|\\[^u]|[^\\\"\n])*\")(?=\s*:)|(\"(\\u[a-zA-Z0-9]{4}|\\[^u]|[^\\\"\n])*\")|(\b(true|false|null)\b)|(-?\d+(?:\.\d*)?(?:[eE][+\-]?\d+)?)|([\{\}\[\],:])/g;
let lastIndex = 0;
let match: RegExpExecArray | null;
while ((match = regex.exec(line)) !== null) {
if (match.index > lastIndex) {
tokens.push({ type: 'text', val: line.slice(lastIndex, match.index) });
}
if (match[1]) tokens.push({ type: 'key', val: match[1] });
else if (match[3]) tokens.push({ type: 'string', val: match[3] });
else if (match[5]) tokens.push({ type: 'literal', val: match[5] });
else if (match[7]) tokens.push({ type: 'number', val: match[7] });
else if (match[8]) tokens.push({ type: 'punct', val: match[8] });
lastIndex = regex.lastIndex;
}
if (lastIndex < line.length) {
tokens.push({ type: 'text', val: line.slice(lastIndex) });
}
return tokens;
}
// Format raw JSON or JSONL into pretty indented string
function prettifyJsonContent(rawContent: string, isJsonl: boolean): string {
if (isJsonl) {
const lines = rawContent.trim().split(/\r?\n/);
const prettyObjects: string[] = [];
for (const line of lines) {
if (!line.trim()) continue;
try {
const obj = JSON.parse(line);
prettyObjects.push(JSON.stringify(obj, null, 2));
} catch {
prettyObjects.push(line);
}
}
return prettyObjects.join('\n\n');
}
try {
const parsed = JSON.parse(rawContent);
return JSON.stringify(parsed, null, 2);
} catch {
return rawContent;
}
}
// Split long passage text into clean, readable paragraphs
function formatTextIntoParagraphs(text: string): string[] {
if (!text) return [];
// Split on existing double newlines if present
if (text.includes('\n\n')) {
return text.split(/\n\n+/).filter((p) => p.trim().length > 0);
}
// If text is one long paragraph with multiple sentences (e.g. medical abstract or MS MARCO passage),
// split every 2-3 sentences to provide clean readability
const sentences = text.match(/[^.!?]+[.!?]+(\s+|$)/g);
if (!sentences || sentences.length <= 2) {
return [text];
}
const paragraphs: string[] = [];
let current = '';
sentences.forEach((sentence, idx) => {
current += sentence;
if ((idx + 1) % 2 === 0 || idx === sentences.length - 1) {
paragraphs.push(current.trim());
current = '';
}
});
if (current.trim()) {
paragraphs.push(current.trim());
}
return paragraphs;
}
export const FileExplorerModal: React.FC<FileExplorerModalProps> = ({
isOpen,
onClose,
datasets,
activeDatasetId,
onSelectDataset,
onSaveCustomDataset,
onDeleteCustomDataset,
}) => {
const [activeTab, setActiveTab] = useState<'browse' | 'upload'>('browse');
const [isFullscreen, setIsFullscreen] = useState<boolean>(false);
// Browse tab state
const [selectedFilePath, setSelectedFilePath] = useState<string>(() => {
const active = datasets.find((d) => d.id === activeDatasetId) || datasets[0];
return active?.files[0]?.path || '/collections/msmarco/collection_jsonl/docs00.json';
});
const [expandedFolders, setExpandedFolders] = useState<Record<string, boolean>>({
'/collections/msmarco': true,
'/collections/msmarco/collection_jsonl': true,
'/collections/nfcorpus': true,
'/collections/nfcorpus/qrels': true,
'/uploads/custom': true,
});
const [viewMode, setViewMode] = useState<'raw' | 'parsed'>('raw');
const [formatJson, setFormatJson] = useState<boolean>(true);
const [wrapLines, setWrapLines] = useState<boolean>(false);
const [copied, setCopied] = useState(false);
const [tableSearch, setTableSearch] = useState<string>('');
const [expandedRowDocId, setExpandedRowDocId] = useState<string | null>(null);
// Fast Hover Tooltip Popover state
const [tooltip, setTooltip] = useState<{
docId: string;
field: 'title' | 'content';
text: string;
top: number;
left: number;
} | null>(null);
const [tooltipCopied, setTooltipCopied] = useState(false);
const tooltipTimeoutRef = useRef<ReturnType<typeof setTimeout> | null>(null);
// Upload tab state
const [customName, setCustomName] = useState('My Custom Corpus');
const [customContent, setCustomContent] = useState('');
const [customFormat, setCustomFormat] = useState<'auto' | 'jsonl' | 'tsv'>('auto');
const [customQuery, setCustomQuery] = useState('');
const [selectedGoldDocIds, setSelectedGoldDocIds] = useState<string[]>([]);
const [goldRelevanceGrade, setGoldRelevanceGrade] = useState<number>(2);
const [isReadingFile, setIsReadingFile] = useState<boolean>(false);
// Upload validation
const validation = customContent.trim()
? validateAndParseCorpus(
customContent,
customFormat === 'auto' ? undefined : customFormat
)
: null;
// Auto-select first document as ground truth target when valid documents are parsed
useEffect(() => {
if (validation?.isValid && validation.parsedDocuments.length > 0) {
setSelectedGoldDocIds((prev) => {
if (prev.length === 0 || !validation.parsedDocuments.some((d) => d.id === prev[0])) {
return [validation.parsedDocuments[0].id];
}
return prev;
});
}
}, [validation?.isValid, validation?.parsedDocuments]);
const isUploadValid = Boolean(
validation &&
validation.isValid &&
validation.parsedDocuments.length > 0 &&
customQuery.trim().length > 0 &&
selectedGoldDocIds.length > 0
);
const handleSaveUpload = useCallback(() => {
if (!validation || !validation.isValid || validation.parsedDocuments.length === 0) return;
if (!customQuery.trim() || selectedGoldDocIds.length === 0) return;
const id = `custom-${Date.now().toString(36)}`;
const format = customFormat === 'auto' ? (customContent.trim().startsWith('{') ? 'jsonl' : 'tsv') : customFormat;
const virtualFile: VirtualFile = {
path: `/uploads/custom/${id}.${format}`,
name: `${customName.toLowerCase().replace(/\s+/g, '_')}.${format}`,
type: format,
sizeBytes: new TextEncoder().encode(customContent).length,
lineCount: customContent.trim().split(/\r?\n/).length,
datasetId: id,
rawContent: customContent,
};
const queryId = `q-${id}-1`;
const cleanQuery = customQuery.trim();
const goldSet = new Set(selectedGoldDocIds);
const newDataset: CorpusDataset = {
id,
name: customName.trim() || 'Custom Uploaded Corpus',
description: `User corpus (${validation.documentCount} documents). Evaluation query: "${cleanQuery}"`,
format,
documentCount: validation.documentCount,
documents: validation.parsedDocuments.map((d) => ({
...d,
goldRelevance: goldSet.has(d.id) ? goldRelevanceGrade : 0,
})),
queries: [
{
id: queryId,
text: cleanQuery,
datasetId: id,
relevantDocIds: selectedGoldDocIds,
},
],
qrels: selectedGoldDocIds.map((docId) => ({
queryId,
iteration: '0',
docId,
relevance: goldRelevanceGrade,
})),
files: [virtualFile],
isCustom: true,
createdAt: new Date().toISOString(),
};
onSaveCustomDataset(newDataset);
onSelectDataset(id);
setSelectedFilePath(virtualFile.path);
setActiveTab('browse');
setCustomContent('');
setCustomQuery('');
setSelectedGoldDocIds([]);
onClose();
}, [
validation,
customFormat,
customContent,
customName,
customQuery,
selectedGoldDocIds,
goldRelevanceGrade,
onSaveCustomDataset,
onSelectDataset,
onClose,
]);
// Clean up tooltip timeout
useEffect(() => {
return () => {
if (tooltipTimeoutRef.current) clearTimeout(tooltipTimeoutRef.current);
};
}, []);
// Collect all virtual files
const allFiles: VirtualFile[] = datasets.flatMap((d) => d.files || []);
const currentFile = allFiles.find((f) => f.path === selectedFilePath) || allFiles[0];
const currentDataset = datasets.find((d) => d.id === currentFile?.datasetId) || datasets.find((d) => d.id === activeDatasetId) || datasets[0];
const activeDataset = datasets.find((d) => d.id === activeDatasetId) || datasets[0];
// Group files by directory
const filesByDir: Record<string, VirtualFile[]> = {};
allFiles.forEach((file) => {
if (!file || !file.path) return;
const dir = file.path.substring(0, file.path.lastIndexOf('/')) || '/';
if (!filesByDir[dir]) {
filesByDir[dir] = [];
}
filesByDir[dir].push(file);
});
const toggleFolder = (dir: string) => {
setExpandedFolders((prev) => ({ ...prev, [dir]: !prev[dir] }));
};
const isAllExpanded = useMemo(() => {
const dirKeys = Object.keys(filesByDir);
if (dirKeys.length === 0) return true;
return dirKeys.every((dir) => expandedFolders[dir] !== false);
}, [filesByDir, expandedFolders]);
const toggleAllFolders = () => {
const nextState: Record<string, boolean> = {};
const targetState = !isAllExpanded;
Object.keys(filesByDir).forEach((dir) => {
nextState[dir] = targetState;
});
setExpandedFolders(nextState);
};
const handleCopyRaw = () => {
if (!currentFile) return;
navigator.clipboard.writeText(currentFile.rawContent);
setCopied(true);
setTimeout(() => setCopied(false), 2000);
};
const handleCopyTooltip = (text: string) => {
navigator.clipboard.writeText(text);
setTooltipCopied(true);
setTimeout(() => setTooltipCopied(false), 1500);
};
// Hover Tooltip Triggers
const handleCellMouseEnter = (
docId: string,
field: 'title' | 'content',
text: string,
e: React.MouseEvent<HTMLElement>
) => {
if (tooltipTimeoutRef.current) clearTimeout(tooltipTimeoutRef.current);
const rect = e.currentTarget.getBoundingClientRect();
const tooltipWidth = 460;
const tooltipHeight = 260;
let left = rect.left;
if (left + tooltipWidth > window.innerWidth - 16) {
left = window.innerWidth - tooltipWidth - 16;
}
if (left < 16) left = 16;
let top = rect.bottom + 8;
if (top + tooltipHeight > window.innerHeight - 16) {
top = Math.max(16, rect.top - tooltipHeight - 8);
}
setTooltip({
docId,
field,
text,
top,
left,
});
};
const handleCellMouseLeave = () => {
if (tooltipTimeoutRef.current) clearTimeout(tooltipTimeoutRef.current);
tooltipTimeoutRef.current = setTimeout(() => {
setTooltip(null);
}, 180);
};
const cancelTooltipHide = () => {
if (tooltipTimeoutRef.current) clearTimeout(tooltipTimeoutRef.current);
};
// Active corpus switch handler
const handleSwitchActiveCorpus = (newId: string) => {
onSelectDataset(newId);
const targetDataset = datasets.find((d) => d.id === newId);
if (targetDataset && targetDataset.files && targetDataset.files.length > 0) {
setSelectedFilePath(targetDataset.files[0].path);
// Auto expand folder
const dir = targetDataset.files[0].path.substring(0, targetDataset.files[0].path.lastIndexOf('/'));
setExpandedFolders((prev) => ({ ...prev, [dir]: true }));
}
};
// File type detection
const isJsonFile = currentFile
? currentFile.type === 'jsonl' ||
(currentFile.type as string) === 'json' ||
(currentFile.name && currentFile.name.endsWith('.json')) ||
(currentFile.name && currentFile.name.endsWith('.jsonl')) ||
(currentFile.rawContent && currentFile.rawContent.trim().startsWith('{'))
: false;
const isJsonlFormat = currentFile
? currentFile.type === 'jsonl' || Boolean(currentFile.name && currentFile.name.endsWith('.jsonl'))
: false;
// Compute displayed lines for Raw / JSON view
const formattedContent = useMemo(() => {
if (!currentFile || !currentFile.rawContent) return '';
if (isJsonFile && formatJson) {
return prettifyJsonContent(currentFile.rawContent, isJsonlFormat);
}
return currentFile.rawContent;
}, [currentFile, isJsonFile, formatJson, isJsonlFormat]);
const displayedLines = useMemo(() => {
return (formattedContent || '').split(/\r?\n/);
}, [formattedContent]);
// Filtered documents for Schema Table view
const filteredDocuments = useMemo(() => {
if (!currentDataset || !currentDataset.documents) return [];
if (!tableSearch.trim()) return currentDataset.documents;
const q = tableSearch.toLowerCase();
return currentDataset.documents.filter(
(d) =>
(d.id && d.id.toLowerCase().includes(q)) ||
(d.title && d.title.toLowerCase().includes(q)) ||
(d.text && d.text.toLowerCase().includes(q))
);
}, [currentDataset, tableSearch]);
const handleFileDrop = (e: React.DragEvent<HTMLDivElement>) => {
e.preventDefault();
const file = e.dataTransfer.files[0];
if (file) {
setIsReadingFile(true);
const reader = new FileReader();
reader.onload = (ev) => {
const text = ev.target?.result;
if (typeof text === 'string') {
setCustomContent(text);
setCustomName(file.name.replace(/\.[^/.]+$/, ''));
}
setIsReadingFile(false);
};
reader.onerror = () => setIsReadingFile(false);
reader.readAsText(file);
}
};
const handleFileInput = (e: React.ChangeEvent<HTMLInputElement>) => {
const file = e.target.files?.[0];
if (file) {
setIsReadingFile(true);
const reader = new FileReader();
reader.onload = (ev) => {
const text = ev.target?.result;
if (typeof text === 'string') {
setCustomContent(text);
setCustomName(file.name.replace(/\.[^/.]+$/, ''));
}
setIsReadingFile(false);
};
reader.onerror = () => setIsReadingFile(false);
reader.readAsText(file);
}
};
if (!isOpen) return null;
return (
<div
className={`fixed inset-0 z-50 bg-black/80 backdrop-blur-xs flex items-center justify-center ${
isFullscreen ? 'p-0' : 'p-3 sm:p-5'
}`}
>
<div
className={`bg-zinc-950 border border-zinc-800 flex flex-col shadow-2xl overflow-hidden font-sans ${
isFullscreen
? 'w-screen h-screen rounded-none border-none'
: 'w-full max-w-6xl h-[86vh] max-h-[880px] rounded-xl'
}`}
>
{/* Modal Top Bar */}
<div className="flex items-center justify-between px-4 py-2.5 border-b border-zinc-800 bg-zinc-900/70 shrink-0">
<div className="flex items-center space-x-2.5">
<div className="w-6 h-6 rounded bg-zinc-800 border border-zinc-700/60 flex items-center justify-center text-xs">
📁
</div>
<div className="flex items-center space-x-2">
<span className="text-sm font-semibold text-zinc-100">
Virtual File System
</span>
<span className="text-zinc-600">/</span>
<span className="text-xs text-zinc-400 font-mono hidden sm:inline">
Raw Collections & Schema Inspector
</span>
</div>
</div>
<div className="flex items-center space-x-2.5">
{/* View switcher */}
<div className="flex bg-zinc-900 p-0.5 rounded-lg border border-zinc-800 text-xs">
<button
onClick={() => setActiveTab('browse')}
className={`px-3 py-1 rounded-md transition font-medium ${
activeTab === 'browse'
? 'bg-zinc-800 text-white shadow-xs border border-zinc-700/60'
: 'text-zinc-400 hover:text-zinc-200'
}`}
>
Browse Files
</button>
<button
onClick={() => setActiveTab('upload')}
className={`px-3 py-1 rounded-md transition flex items-center space-x-1.5 font-medium ${
activeTab === 'upload'
? 'bg-zinc-800 text-white shadow-xs border border-zinc-700/60'
: 'text-zinc-400 hover:text-zinc-200'
}`}
>
<Upload className="w-3.5 h-3.5" />
<span>Upload Corpus</span>
</button>
</div>
{/* Fullscreen Toggle */}
<button
onClick={() => setIsFullscreen(!isFullscreen)}
className="p-1.5 rounded-md text-zinc-400 hover:text-white hover:bg-zinc-800 transition border border-transparent hover:border-zinc-700"
title={isFullscreen ? 'Exit Fullscreen' : 'Expand Fullscreen'}
>
{isFullscreen ? (
<Minimize2 className="w-4 h-4" />
) : (
<Maximize2 className="w-4 h-4" />
)}
</button>
{/* Close Button */}
<button
onClick={onClose}
className="p-1.5 rounded-md text-zinc-400 hover:text-white hover:bg-zinc-800 transition border border-transparent hover:border-zinc-700"
title="Close modal"
>
<X className="w-4 h-4" />
</button>
</div>
</div>
{/* Modal Content */}
{activeTab === 'browse' ? (
<div className="flex-1 flex overflow-hidden">
{/* Left Sidebar: Virtual File Tree & Active Corpus Switcher */}
<div className="w-64 sm:w-72 border-r border-zinc-800 bg-zinc-950 flex flex-col text-xs shrink-0 select-none">
<div className="px-3 py-2 border-b border-zinc-850 font-mono text-zinc-400 flex items-center justify-between text-[11px] bg-zinc-900/40">
<span className="flex items-center gap-1.5 font-semibold text-zinc-300">
<Folder className="w-3.5 h-3.5 text-amber-400/80" />
Virtual Workspace
</span>
<div className="flex items-center gap-2">
<span className="text-[10px] text-zinc-500 font-mono">{allFiles.length} files</span>
<button
onClick={toggleAllFolders}
className="p-1 rounded hover:bg-zinc-800/80 text-zinc-400 hover:text-zinc-200 transition cursor-pointer"
title={isAllExpanded ? "Collapse all folders" : "Expand all folders"}
>
<ChevronsUpDown className="w-3 h-3" />
</button>
</div>
</div>
{/* File Tree List */}
<div className="flex-1 overflow-y-auto p-2 space-y-1 font-mono">
{Object.entries(filesByDir).map(([dir, files]) => {
const isExpanded = expandedFolders[dir] ?? true;
const isCustomDir = dir.includes('custom');
const isDirActiveCorpus = files.some(
(f) => f.datasetId === activeDatasetId
);
return (
<div key={dir} className="space-y-0.5">
<button
onClick={() => toggleFolder(dir)}
className={`w-full group flex items-center justify-between px-2 py-1 rounded-md text-left transition cursor-pointer ${
isDirActiveCorpus
? 'text-zinc-200 bg-zinc-900/50 hover:bg-zinc-900 font-medium'
: 'text-zinc-400 hover:text-zinc-200 hover:bg-zinc-900/50'
}`}
title={isExpanded ? 'Click to collapse folder' : 'Click to expand folder'}
>
<div className="flex items-center space-x-1.5 truncate min-w-0">
<ChevronRight
className={`w-3 h-3 text-zinc-500 group-hover:text-zinc-300 transition-transform duration-150 shrink-0 ${
isExpanded ? 'rotate-90 text-zinc-300' : ''
}`}
/>
{isExpanded ? (
<FolderOpen className="w-3.5 h-3.5 text-amber-400/90 shrink-0" />
) : (
<Folder className="w-3.5 h-3.5 text-amber-400/70 shrink-0" />
)}
<span className="truncate text-[11px] font-mono tracking-tight">{dir}</span>
</div>
<div className="flex items-center space-x-1.5 shrink-0 ml-1.5">
<span className="text-[10px] text-zinc-500 font-mono px-1 py-0.2 rounded bg-zinc-900/80 border border-zinc-850">
{files.length}
</span>
{isDirActiveCorpus && (
<span
className="w-1.5 h-1.5 rounded-full bg-emerald-400 shrink-0"
title="Contains active mounted corpus"
/>
)}
</div>
</button>
{isExpanded && (
<div className="ml-2.5 pl-2 border-l border-zinc-850 space-y-0.5">
{files.map((file) => {
const isSelected = selectedFilePath === file.path;
const isDatasetActive = file.datasetId === activeDatasetId;
return (
<div
key={file.path}
className={`group flex items-center justify-between px-2 py-1 rounded-md cursor-pointer transition ${
isSelected
? 'bg-zinc-800 text-zinc-100 font-semibold border border-zinc-700/60 shadow-2xs'
: 'text-zinc-400 hover:text-zinc-200 hover:bg-zinc-900/60'
}`}
onClick={() => setSelectedFilePath(file.path)}
>
<div className="flex items-center space-x-1.5 truncate min-w-0">
{file.type === 'jsonl' || (file.type as string) === 'json' || file.name.endsWith('.json') ? (
<FileCode className={`w-3.5 h-3.5 shrink-0 ${isSelected ? 'text-sky-400' : 'text-zinc-500'}`} />
) : (
<FileText className={`w-3.5 h-3.5 shrink-0 ${isSelected ? 'text-amber-400' : 'text-zinc-500'}`} />
)}
<span className="truncate text-[11px]">{file.name}</span>
</div>
<div className="flex items-center space-x-1.5 shrink-0 ml-1.5">
{isDatasetActive ? (
<span className="text-[9px] font-mono px-1.5 py-0.2 rounded bg-emerald-950 text-emerald-300 border border-emerald-800/80">
active
</span>
) : (
<button
onClick={(e) => {
e.stopPropagation();
handleSwitchActiveCorpus(file.datasetId);
}}
className="opacity-0 group-hover:opacity-100 text-[9px] font-mono px-1 py-0.2 rounded bg-zinc-800 hover:bg-zinc-700 text-zinc-300 transition"
title="Mount this dataset as active"
>
mount
</button>
)}
{isCustomDir && (
<button
onClick={(e) => {
e.stopPropagation();
onDeleteCustomDataset(file.datasetId);
}}
className="opacity-0 group-hover:opacity-100 p-0.5 text-zinc-500 hover:text-rose-400 transition"
title="Delete custom dataset"
>
<Trash2 className="w-3 h-3" />
</button>
)}
</div>
</div>
);
})}
</div>
)}
</div>
);
})}
</div>
{/* Active Corpus Switcher (Compact, Refined, Modern) */}
<div className="p-2.5 border-t border-zinc-800 bg-zinc-900/40 flex flex-col gap-2 shrink-0">
<div className="flex items-center justify-between text-[11px]">
<div className="flex items-center gap-1.5 font-mono text-zinc-400 font-semibold tracking-wider text-[10px] uppercase">
<Database className="w-3.5 h-3.5 text-emerald-400 shrink-0" />
<span>Mounted Corpus</span>
</div>
<span className="inline-flex items-center gap-1 px-1.5 py-0.2 rounded-full bg-emerald-500/10 text-emerald-400 border border-emerald-500/20 text-[9px] font-mono font-medium">
<span className="w-1.5 h-1.5 rounded-full bg-emerald-400 animate-pulse" />
live
</span>
</div>
{/* Styled Dropdown with Custom Chevron */}
<div className="relative">
<select
value={activeDatasetId}
onChange={(e) => handleSwitchActiveCorpus(e.target.value)}
className="w-full appearance-none bg-zinc-900/90 border border-zinc-800 hover:border-zinc-700 focus:border-zinc-600 rounded-md pl-2.5 pr-7 py-1.5 text-xs text-zinc-100 font-medium cursor-pointer transition shadow-2xs focus:outline-none"
title="Switch active mounted corpus"
>
{datasets.map((d) => (
<option key={d.id} value={d.id} className="bg-zinc-900 text-zinc-100">
{d.name} ({d.documentCount} docs)
</option>
))}
</select>
<ChevronDown className="w-3.5 h-3.5 text-zinc-400 absolute right-2 top-1/2 -translate-y-1/2 pointer-events-none" />
</div>
{/* Compact Inline Stats Ribbon */}
<div className="flex items-center justify-between text-[10px] font-mono px-2 py-1 rounded-md bg-zinc-950/70 border border-zinc-850 text-zinc-400">
<span className="flex items-center gap-1">
<span className="text-zinc-500 text-[9px]">DOCS:</span>
<span className="text-zinc-200 font-semibold">{activeDataset?.documentCount}</span>
</span>
<span className="text-zinc-650">•</span>
<span className="flex items-center gap-1">
<span className="text-zinc-500 text-[9px]">FMT:</span>
<span className="text-zinc-200 font-semibold uppercase">{activeDataset?.format}</span>
</span>
<span className="text-zinc-650">•</span>
<span className="flex items-center gap-1">
<span className="text-zinc-500 text-[9px]">QRY:</span>
<span className="text-zinc-200 font-semibold">{activeDataset?.queries?.length ?? 1}</span>
</span>
</div>
</div>
</div>
{/* Right Column: File Content & Schema Inspector */}
<div className="flex-1 flex flex-col bg-zinc-950 overflow-hidden">
{currentFile ? (
<>
{/* File Metadata & Toolbar */}
<div className="px-4 py-2.5 border-b border-zinc-800 bg-zinc-900/40 flex flex-wrap items-center justify-between gap-2 text-xs shrink-0">
<div className="flex items-center space-x-3 text-zinc-400 font-mono">
<span className="text-zinc-100 font-semibold flex items-center gap-1.5">
<FileCode className="w-3.5 h-3.5 text-sky-400" />
{currentFile.path}
</span>
<span>•</span>
<span>{(currentFile.sizeBytes / 1024).toFixed(1)} KB</span>
<span>•</span>
<span>{currentFile.lineCount} lines</span>
</div>
<div className="flex items-center space-x-2">
{/* View mode switcher */}
<div className="flex bg-zinc-900 p-0.5 rounded-md border border-zinc-800 text-[11px]">
<button
onClick={() => setViewMode('raw')}
className={`px-2.5 py-1 rounded transition font-medium ${
viewMode === 'raw'
? 'bg-zinc-800 text-zinc-100 shadow-xs'
: 'text-zinc-400 hover:text-zinc-200'
}`}
>
Code & Raw
</button>
<button
onClick={() => setViewMode('parsed')}
className={`px-2.5 py-1 rounded transition font-medium ${
viewMode === 'parsed'
? 'bg-zinc-800 text-zinc-100 shadow-xs'
: 'text-zinc-400 hover:text-zinc-200'
}`}
>
Schema Table
</button>
</div>
{/* Raw Mode Controls: JSON Formatter & Line Wrap */}
{viewMode === 'raw' && (
<>
{isJsonFile && (
<button
onClick={() => setFormatJson(!formatJson)}
className={`px-2 py-1 rounded text-[11px] font-mono border transition flex items-center gap-1 ${
formatJson
? 'bg-sky-950/80 text-sky-300 border-sky-700/80 shadow-xs font-semibold'
: 'bg-zinc-900 text-zinc-400 border-zinc-800 hover:text-zinc-200'
}`}
title="Toggle pretty-printed JSON formatting"
>
<Sparkles className="w-3 h-3" />
<span>{formatJson ? 'Pretty JSON' : 'Compact JSON'}</span>
</button>
)}
<button
onClick={() => setWrapLines(!wrapLines)}
className={`px-2 py-1 rounded text-[11px] font-mono border transition flex items-center gap-1 ${
wrapLines
? 'bg-zinc-800 text-zinc-100 border-zinc-700'
: 'bg-zinc-900 text-zinc-400 border-zinc-800 hover:text-zinc-200'
}`}
title={wrapLines ? 'Disable line wrap (enable horizontal scroll)' : 'Enable line wrap'}
>
{wrapLines ? <WrapText className="w-3 h-3" /> : <AlignLeft className="w-3 h-3" />}
<span>{wrapLines ? 'Wrap' : 'Scroll-X'}</span>
</button>
</>
)}
{/* Copy File Button */}
<button
onClick={handleCopyRaw}
className="px-2.5 py-1 rounded bg-zinc-900 hover:bg-zinc-850 text-zinc-300 border border-zinc-800 text-xs flex items-center space-x-1.5 transition"
title="Copy entire raw file to clipboard"
>
{copied ? (
<>
<Check className="w-3 h-3 text-emerald-400" />
<span className="text-emerald-400 font-medium">Copied</span>
</>
) : (
<>
<Copy className="w-3 h-3 text-zinc-400" />
<span>Copy</span>
</>
)}
</button>
{/* Mount Button if file belongs to another dataset */}
{currentDataset && currentDataset.id !== activeDatasetId && (
<button
onClick={() => handleSwitchActiveCorpus(currentDataset.id)}
className="px-2.5 py-1 rounded bg-emerald-950 hover:bg-emerald-900 text-emerald-300 text-xs font-medium border border-emerald-700/80 transition flex items-center gap-1 shadow-xs"
>
<Database className="w-3 h-3" />
<span>Mount Corpus</span>
</button>
)}
</div>
</div>
{/* Viewer Content Area */}
<div className="flex-1 overflow-hidden flex flex-col">
{viewMode === 'raw' ? (
/* Code / Raw Lines Viewer */
<div className="flex-1 overflow-auto p-3 font-mono text-xs bg-zinc-950 select-text">
<table className="w-full border-collapse">
<tbody>
{displayedLines.map((line, idx) => {
const tokens = isJsonFile && formatJson ? tokenizeJsonLine(line) : null;
return (
<tr key={idx} className="hover:bg-zinc-900/40 group">
{/* Line Number Gutter */}
<td className="py-0.5 px-3 select-none text-zinc-600 group-hover:text-zinc-400 text-right w-12 border-r border-zinc-850 text-[11px] align-top">
{idx + 1}
</td>
{/* Code Content */}
<td
className={`py-0.5 px-3 leading-relaxed ${
wrapLines ? 'whitespace-pre-wrap break-words' : 'whitespace-pre'
}`}
>
{tokens ? (
<span>
{tokens.map((token, tIdx) => {
if (token.type === 'key') {
return (
<span key={tIdx} className="text-sky-300 font-semibold">
{token.val}
</span>
);
}
if (token.type === 'string') {
return (
<span key={tIdx} className="text-emerald-300/90">
{token.val}
</span>
);
}
if (token.type === 'number') {
return (
<span key={tIdx} className="text-amber-300 font-medium">
{token.val}
</span>
);
}
if (token.type === 'literal') {
return (
<span key={tIdx} className="text-purple-300 font-medium">
{token.val}
</span>
);
}
if (token.type === 'punct') {
return (
<span key={tIdx} className="text-zinc-500">
{token.val}
</span>
);
}
return (
<span key={tIdx} className="text-zinc-300">
{token.val}
</span>
);
})}
</span>
) : (
<span className="text-zinc-300">{line}</span>
)}
</td>
</tr>
);
})}
</tbody>
</table>
</div>
) : (
/* Enhanced Schema Table with Search, Expandable Rows, and Hover Tooltips */
<div className="flex-1 flex flex-col overflow-hidden">
{/* Table Search & Count Bar */}
<div className="p-3 border-b border-zinc-850 bg-zinc-900/30 flex items-center justify-between gap-3 text-xs shrink-0">
<div className="relative flex-1 max-w-sm">
<Search className="w-3.5 h-3.5 absolute left-2.5 top-2.5 text-zinc-500" />
<input
type="text"
value={tableSearch}
onChange={(e) => setTableSearch(e.target.value)}
placeholder="Search passages by docid, title, or keywords..."
className="w-full bg-zinc-950 border border-zinc-800 rounded-md pl-8 pr-3 py-1.5 text-xs text-zinc-200 placeholder-zinc-600 focus:outline-none focus:border-zinc-500 font-sans"
/>
{tableSearch && (
<button
onClick={() => setTableSearch('')}
className="absolute right-2.5 top-2.5 text-zinc-500 hover:text-zinc-200"
>
<X className="w-3.5 h-3.5" />
</button>
)}
</div>
<div className="flex items-center space-x-3 text-zinc-400 font-mono text-[11px]">
<span>
Showing {filteredDocuments.length} of {currentDataset?.documents.length || 0} passages
</span>
<span>•</span>
<span className="text-zinc-500 hidden sm:inline">
Hover cell for full formatted passage
</span>
</div>
</div>
{/* Horizontally Scrollable Table Container */}
<div className="flex-1 overflow-auto">
<table className="w-full text-left text-xs font-sans border-collapse min-w-[850px]">
<thead>
<tr className="sticky top-0 bg-zinc-900/95 backdrop-blur-xs z-10 border-b border-zinc-800 text-zinc-400 font-mono text-[11px]">
<th className="py-2.5 px-3 w-12 text-center">#</th>
<th className="py-2.5 px-3 w-32 min-w-[120px]">docid</th>
<th className="py-2.5 px-3 w-64 min-w-[240px]">title</th>
<th className="py-2.5 px-3 min-w-[460px]">passage content</th>
<th className="py-2.5 px-3 w-24 text-right pr-4">inspect</th>
</tr>
</thead>
<tbody className="divide-y divide-zinc-850">
{filteredDocuments.map((doc, idx) => {
const isExpanded = expandedRowDocId === doc.id;
return (
<React.Fragment key={doc.id}>
<tr className="hover:bg-zinc-900/50 transition group">
{/* Row Number */}
<td className="py-3 px-3 font-mono text-zinc-500 text-center select-none text-[11px] align-top">
{idx + 1}
</td>
{/* Doc ID */}
<td className="py-3 px-3 font-mono text-sky-300 font-semibold whitespace-nowrap align-top">
<div className="flex items-center space-x-1.5">
<span>{doc.id}</span>
{doc.goldRelevance !== undefined && doc.goldRelevance > 0 && (
<span
className={`text-[9px] px-1 rounded ${
doc.goldRelevance === 2
? 'bg-emerald-950 text-emerald-300 border border-emerald-800'
: 'bg-sky-950 text-sky-300 border border-sky-800'
}`}
>
rel={doc.goldRelevance}
</span>
)}
</div>
</td>
{/* Document Title (with Hover Tooltip) */}
<td
className="py-3 px-3 text-zinc-200 font-medium align-top cursor-pointer group-hover:text-white"
onMouseEnter={(e) =>
handleCellMouseEnter(doc.id, 'title', doc.title || doc.id, e)
}
onMouseLeave={handleCellMouseLeave}
>
<div className="line-clamp-2 leading-relaxed">
{doc.title || '-'}
</div>
</td>
{/* Passage Content (with Hover Tooltip and Full Readability) */}
<td
className="py-3 px-3 text-zinc-400 align-top cursor-pointer group-hover:text-zinc-200"
onMouseEnter={(e) =>
handleCellMouseEnter(doc.id, 'content', doc.text, e)
}
onMouseLeave={handleCellMouseLeave}
>
<div className="line-clamp-2 leading-relaxed">
{doc.text}
</div>
</td>
{/* Expand/Collapse Inline Row */}
<td className="py-3 px-3 text-right pr-4 align-top">
<button
onClick={() =>
setExpandedRowDocId(isExpanded ? null : doc.id)
}
className="text-[11px] font-mono text-zinc-400 hover:text-zinc-100 px-2 py-0.5 rounded bg-zinc-900 hover:bg-zinc-800 border border-zinc-800 transition inline-flex items-center gap-1"
>
<span>{isExpanded ? 'close' : 'view'}</span>
{isExpanded ? (
<ChevronUp className="w-3 h-3" />
) : (
<ChevronDown className="w-3 h-3" />
)}
</button>
</td>
</tr>
{/* Expanded Row Drawer */}
{isExpanded && (
<tr className="bg-zinc-900/60">
<td colSpan={5} className="p-4 border-b border-zinc-800">
<div className="space-y-3 bg-zinc-950 p-4 rounded-lg border border-zinc-800 text-xs font-sans">
<div className="flex items-center justify-between border-b border-zinc-850 pb-2">
<div className="flex items-center space-x-2">
<span className="font-mono text-sky-300 font-semibold text-xs">
[{doc.id}]
</span>
<h4 className="font-semibold text-zinc-100">
{doc.title || 'Untitled Passage'}
</h4>
</div>
<button
onClick={() => handleCopyTooltip(doc.text)}
className="flex items-center space-x-1 font-mono text-[11px] text-zinc-400 hover:text-zinc-100 px-2 py-1 rounded bg-zinc-900 hover:bg-zinc-800 border border-zinc-800 transition"
>
<Copy className="w-3 h-3" />
<span>Copy text</span>
</button>
</div>
<div className="space-y-2 text-zinc-300 leading-relaxed">
{formatTextIntoParagraphs(doc.text).map((p, pIdx) => (
<p key={pIdx}>{p}</p>
))}
</div>
</div>
</td>
</tr>
)}
</React.Fragment>
);
})}
</tbody>
</table>
</div>
</div>
)}
</div>
</>
) : (
<div className="flex-1 flex items-center justify-center text-zinc-500 text-xs font-mono">
Select a file from the sidebar.
</div>
)}
</div>
</div>
) : (
/* Tab 2: Upload Micro-Corpus */
<div className="flex-1 overflow-y-auto p-6 space-y-5 text-xs font-sans">
<div>
<h3 className="font-semibold text-sm text-zinc-100 flex items-center gap-2">
<Upload className="w-4 h-4 text-emerald-400" />
Upload Micro-Corpus to Virtual Workspace
</h3>
<p className="text-zinc-400 mt-1">
Paste or drop a JSONL (<code className="text-zinc-300 font-mono">id, contents</code>) or TSV (<code className="text-zinc-300 font-mono">id \t text</code>) file.
Corpora are capped at {MAX_CORPUS_DOCUMENTS} documents for instantaneous indexing and benchmark evaluation.
</p>
</div>
{/* Template Presets */}
<div className="flex items-center space-x-2">
<span className="text-zinc-500 text-[11px] font-mono">Format Templates:</span>
<button
type="button"
onClick={() => {
setCustomContent(getSampleJsonlString());
setCustomFormat('jsonl');
setCustomName('Sample Inverted Index Corpus');
setCustomQuery('BM25 ranking term saturation and length normalization');
setSelectedGoldDocIds(['doc-002']);
}}
className="px-2.5 py-1 rounded-md bg-zinc-900 hover:bg-zinc-800 text-zinc-200 border border-zinc-800 hover:border-zinc-700 text-[11px] font-mono transition flex items-center gap-1.5"
>
<FileCode className="w-3 h-3 text-sky-400" />
<span>+ JSONL Sample</span>
</button>
<button
type="button"
onClick={() => {
setCustomContent(getSampleTsvString());
setCustomFormat('tsv');
setCustomName('Sample Evaluation Corpus');
setCustomQuery('TREC evaluation metric ndcg and mrr calculation');
setSelectedGoldDocIds(['doc-004']);
}}
className="px-2.5 py-1 rounded-md bg-zinc-900 hover:bg-zinc-800 text-zinc-200 border border-zinc-800 hover:border-zinc-700 text-[11px] font-mono transition flex items-center gap-1.5"
>
<FileText className="w-3 h-3 text-amber-400" />
<span>+ TSV Sample</span>
</button>
</div>
{/* Inputs: Corpus Name & Format */}
<div className="grid grid-cols-1 md:grid-cols-3 gap-3.5">
<div className="md:col-span-2">
<label className="block text-zinc-400 mb-1 font-medium">Corpus Name</label>
<input
type="text"
value={customName}
onChange={(e) => setCustomName(e.target.value)}
placeholder="e.g. Biomed COVID Subcorpus"
className="w-full bg-zinc-900 border border-zinc-800 focus:border-zinc-600 rounded-md px-3 py-1.5 text-zinc-100 focus:outline-none text-xs"
/>
</div>
<div>
<label className="block text-zinc-400 mb-1 font-medium">Format</label>
<select
value={customFormat}
onChange={(e) => setCustomFormat(e.target.value as any)}
className="w-full bg-zinc-900 border border-zinc-800 focus:border-zinc-600 rounded-md px-2.5 py-1.5 text-zinc-100 focus:outline-none text-xs font-mono"
>
<option value="auto">Auto-detect format</option>
<option value="jsonl">JSONL (JSON Lines)</option>
<option value="tsv">TSV (Tab-separated)</option>
</select>
</div>
</div>
{/* Dropzone & Textarea */}
<div onDragOver={(e) => e.preventDefault()} onDrop={handleFileDrop} className="space-y-1.5">
<div className="flex items-center justify-between text-zinc-500 text-[11px] font-mono">
<span>Passage Data (Drop file or paste raw content):</span>
<label className="cursor-pointer text-zinc-400 hover:text-zinc-200 underline font-sans">
browse local file
<input
type="file"
accept=".jsonl,.tsv,.json,.txt"
onChange={handleFileInput}
className="hidden"
/>
</label>
</div>
{isReadingFile && (
<div className="flex items-center gap-2 text-xs font-mono text-sky-400 p-2 rounded bg-sky-950/40 border border-sky-800/40 animate-pulse">
<Activity className="w-3.5 h-3.5 animate-spin" />
<span>Reading and parsing corpus file from disk...</span>
</div>
)}
<textarea
rows={9}
value={customContent}
onChange={(e) => setCustomContent(e.target.value)}
placeholder={'{"id": "doc1", "contents": "..."}\n{"id": "doc2", "contents": "..."}'}
className="w-full bg-zinc-900 border border-zinc-800 rounded-lg p-3 font-mono text-xs text-zinc-200 placeholder-zinc-600 focus:outline-none focus:border-zinc-600 whitespace-pre"
/>
</div>
{/* Validation Feedback */}
{validation && (
<div
className={`p-3.5 rounded-lg border font-mono text-xs flex items-center space-x-2.5 ${
validation.isValid
? 'border-emerald-800/60 bg-emerald-950/30 text-emerald-300'
: 'border-rose-800/60 bg-rose-950/30 text-rose-300'
}`}
>
{validation.isValid ? (
<>
<CheckCircle2 className="w-4 h-4 text-emerald-400 shrink-0" />
<span>
Validated: {validation.documentCount} document{validation.documentCount === 1 ? '' : 's'} ready for indexing.
</span>
</>
) : (
<>
<AlertCircle className="w-4 h-4 text-rose-400 shrink-0" />
<span>
Failed validation: {validation.errors.join(', ')}
</span>
</>
)}
</div>
)}
{/* Step 2: Evaluation Query & Ground Truth Target (Required) */}
{validation?.isValid && (
<div className="p-4 rounded-xl border border-sky-900/40 bg-sky-950/20 space-y-3.5 shadow-xs">
<div className="flex flex-wrap items-center justify-between gap-2">
<div className="flex items-center gap-2">
<Target className="w-4 h-4 text-sky-400" />
<span className="text-xs font-semibold text-zinc-100">
Evaluation Query & Relevant Ground Truth Target
</span>
<span className="text-[10px] px-1.5 py-0.5 rounded bg-sky-500/20 text-sky-300 font-mono font-medium border border-sky-500/30">
Required
</span>
</div>
<span className="text-[11px] font-mono text-zinc-400">
Sets canonical test query & qrels for Arena evaluation
</span>
</div>
{/* Query Input */}
<div className="space-y-1">
<div className="flex items-center justify-between text-[11px]">
<label className="text-zinc-300 font-medium">
Evaluation Query (Default search query for this corpus):
</label>
{customQuery.trim() === '' ? (
<span className="text-amber-400 font-mono text-[10px] font-semibold">
* Required field
</span>
) : (
<span className="text-emerald-400 font-mono text-[10px]">
✓ {customQuery.trim().split(/\s+/).filter(Boolean).length} tokens
</span>
)}
</div>
<input
type="text"
value={customQuery}
onChange={(e) => setCustomQuery(e.target.value)}
placeholder="e.g. search query testing passage retrieval against this corpus..."
className="w-full bg-zinc-900 border border-zinc-750 focus:border-sky-500 rounded-lg px-3 py-2 text-xs text-zinc-100 placeholder-zinc-500 focus:outline-none font-medium"
/>
</div>
{/* Ground Truth Passage Selection & Relevance Grade */}
<div className="grid grid-cols-1 md:grid-cols-3 gap-3">
<div className="md:col-span-2 space-y-1">
<div className="flex items-center justify-between text-[11px]">
<label className="text-zinc-300 font-medium">
Relevant Ground Truth Passage (Gold Target):
</label>
{selectedGoldDocIds.length === 0 ? (
<span className="text-amber-400 font-mono text-[10px] font-semibold">
* Select at least 1 passage
</span>
) : (
<span className="text-emerald-400 font-mono text-[10px]">
✓ Target assigned
</span>
)}
</div>
<select
value={selectedGoldDocIds[0] || ''}
onChange={(e) => {
const val = e.target.value;
if (val) setSelectedGoldDocIds([val]);
}}
className="w-full bg-zinc-900 border border-zinc-750 focus:border-sky-500 rounded-lg px-2.5 py-2 text-xs text-zinc-100 font-mono focus:outline-none cursor-pointer"
>
{validation.parsedDocuments.map((doc) => (
<option key={doc.id} value={doc.id}>
[{doc.id}] {doc.title ? `${doc.title} - ` : ''}
{doc.text.slice(0, 55)}...
</option>
))}
</select>
</div>
<div className="space-y-1">
<label className="block text-[11px] text-zinc-300 font-medium">
Relevance Grade:
</label>
<select
value={goldRelevanceGrade}
onChange={(e) => setGoldRelevanceGrade(Number(e.target.value))}
className="w-full bg-zinc-900 border border-zinc-750 focus:border-sky-500 rounded-lg px-2.5 py-2 text-xs text-zinc-100 font-mono focus:outline-none cursor-pointer"
>
<option value={2}>rel = 2 (Highly Relevant / Gold)</option>
<option value={1}>rel = 1 (Relevant / Pertinent)</option>
</select>
</div>
</div>
{/* Preview Selected Ground Truth Passage */}
{selectedGoldDocIds.length > 0 && (
<div className="text-[11px] font-mono bg-zinc-900/90 border border-zinc-800 rounded-lg p-2.5 text-zinc-400">
<span className="text-sky-300 font-semibold">
Target Passage [{selectedGoldDocIds[0]}]:{' '}
</span>
<span className="text-zinc-200">
{validation.parsedDocuments.find((d) => d.id === selectedGoldDocIds[0])?.text.slice(0, 140)}...
</span>
</div>
)}
</div>
)}
{/* Submit Action */}
<div className="flex flex-wrap items-center justify-between gap-3 pt-2.5 border-t border-zinc-850">
<div className="text-[11px] font-mono text-zinc-400">
{validation?.isValid && (!customQuery.trim() || selectedGoldDocIds.length === 0) ? (
<span className="text-amber-400 font-semibold">
* Provide an evaluation query and select a relevant ground-truth passage to mount corpus.
</span>
) : isUploadValid ? (
<span className="text-emerald-400">
✓ Corpus ready to mount with custom query and ground truth target.
</span>
) : null}
</div>
<div className="flex items-center space-x-2.5">
<button
type="button"
onClick={() => setActiveTab('browse')}
className="px-3.5 py-1.5 rounded-md bg-zinc-900 hover:bg-zinc-800 text-zinc-400 text-xs transition cursor-pointer"
>
Cancel
</button>
<button
type="button"
onClick={handleSaveUpload}
disabled={!isUploadValid}
className={`px-4 py-1.5 rounded-md text-xs font-semibold transition flex items-center gap-1.5 ${
isUploadValid
? 'bg-zinc-100 hover:bg-white text-zinc-900 shadow-sm cursor-pointer'
: 'bg-zinc-900 text-zinc-600 cursor-not-allowed border border-zinc-800'
}`}
>
<Check className="w-3.5 h-3.5" />
<span>Save & Mount Corpus</span>
</button>
</div>
</div>
</div>
)}
</div>
{/* Fast Hover Tooltip Popover (Rendered at top level for clean clipping escape) */}
{tooltip && (
<div
style={{ top: `${tooltip.top}px`, left: `${tooltip.left}px` }}
className="fixed z-50 w-[460px] max-w-[92vw] bg-zinc-900/98 backdrop-blur-md border border-zinc-750 shadow-2xl rounded-xl p-4 text-xs font-sans pointer-events-auto transition-opacity duration-150 animate-in fade-in zoom-in-95"
onMouseEnter={cancelTooltipHide}
onMouseLeave={handleCellMouseLeave}
>
{/* Header */}
<div className="flex items-center justify-between pb-2.5 mb-2.5 border-b border-zinc-800">
<div className="flex items-center space-x-2">
<span className="font-mono text-[11px] px-2 py-0.5 rounded bg-zinc-800 text-sky-300 font-semibold border border-zinc-700/60">
[{tooltip.docId}]
</span>
<span className="text-[11px] font-semibold text-zinc-300 uppercase tracking-wider">
{tooltip.field === 'title' ? 'Document Title' : 'Passage Content'}
</span>
</div>
<div className="flex items-center space-x-2">
<span className="text-[10px] font-mono text-zinc-400">
{tooltip.text.split(/\s+/).filter(Boolean).length} words
</span>
<button
onClick={() => handleCopyTooltip(tooltip.text)}
className="p-1 rounded hover:bg-zinc-800 text-zinc-400 hover:text-zinc-100 transition"
title="Copy full text to clipboard"
>
{tooltipCopied ? (
<Check className="w-3.5 h-3.5 text-emerald-400" />
) : (
<Copy className="w-3.5 h-3.5" />
)}
</button>
</div>
</div>
{/* Formatted Paragraphs with Clean Spacing */}
<div className="max-h-64 overflow-y-auto pr-1 space-y-2.5 text-zinc-200 leading-relaxed text-xs font-normal select-text">
{formatTextIntoParagraphs(tooltip.text).map((para, i) => (
<p key={i} className="text-zinc-200">
{para}
</p>
))}
</div>
</div>
)}
</div>
);
};
import React, { useState, useCallback, useMemo, useRef, useEffect } from 'react';
import {
Folder,
FolderOpen,
FileText,
FileCode,
Upload,
Copy,
Check,
X,
Trash2,
CheckCircle2,
AlertCircle,
Maximize2,
Minimize2,
Search,
WrapText,
AlignLeft,
Database,
ChevronDown,
ChevronUp,
ChevronRight,
ChevronsUpDown,
Sparkles,
Target,
Activity,
} from 'lucide-react';
import type { CorpusDataset, VirtualFile } from '../types/corpus';
import {
validateAndParseCorpus,
getSampleJsonlString,
getSampleTsvString,
MAX_CORPUS_DOCUMENTS,
} from '../utils/corpusValidator';
interface FileExplorerModalProps {
isOpen: boolean;
onClose: () => void;
datasets: CorpusDataset[];
activeDatasetId: string;
onSelectDataset: (id: string) => void;
onSaveCustomDataset: (dataset: CorpusDataset) => void;
onDeleteCustomDataset: (id: string) => void;
}
// Token type for JSON syntax highlighting
interface JsonToken {
type: 'key' | 'string' | 'number' | 'literal' | 'punct' | 'text';
val: string;
}
// Fast tokenizer for a single line of JSON
function tokenizeJsonLine(line: string): JsonToken[] {
const tokens: JsonToken[] = [];
const regex = /(\"(\\u[a-zA-Z0-9]{4}|\\[^u]|[^\\\"\n])*\")(?=\s*:)|(\"(\\u[a-zA-Z0-9]{4}|\\[^u]|[^\\\"\n])*\")|(\b(true|false|null)\b)|(-?\d+(?:\.\d*)?(?:[eE][+\-]?\d+)?)|([\{\}\[\],:])/g;
let lastIndex = 0;
let match: RegExpExecArray | null;
while ((match = regex.exec(line)) !== null) {
if (match.index > lastIndex) {
tokens.push({ type: 'text', val: line.slice(lastIndex, match.index) });
}
if (match[1]) tokens.push({ type: 'key', val: match[1] });
else if (match[3]) tokens.push({ type: 'string', val: match[3] });
else if (match[5]) tokens.push({ type: 'literal', val: match[5] });
else if (match[7]) tokens.push({ type: 'number', val: match[7] });
else if (match[8]) tokens.push({ type: 'punct', val: match[8] });
lastIndex = regex.lastIndex;
}
if (lastIndex < line.length) {
tokens.push({ type: 'text', val: line.slice(lastIndex) });
}
return tokens;
}
// Format raw JSON or JSONL into pretty indented string
function prettifyJsonContent(rawContent: string, isJsonl: boolean): string {
if (isJsonl) {
const lines = rawContent.trim().split(/\r?\n/);
const prettyObjects: string[] = [];
for (const line of lines) {
if (!line.trim()) continue;
try {
const obj = JSON.parse(line);
prettyObjects.push(JSON.stringify(obj, null, 2));
} catch {
prettyObjects.push(line);
}
}
return prettyObjects.join('\n\n');
}
try {
const parsed = JSON.parse(rawContent);
return JSON.stringify(parsed, null, 2);
} catch {
return rawContent;
}
}