mackenzietechdocs's picture
Frozen Hack Week 25 snapshot from GitHub commit f2c77ec, with Space packaging
c3aa9b1 verified
Raw History Blame Contribute Delete
7.57 kB
import { LocalIndex } from 'vectra';
import { AIService } from './ai-service.js';
import path from 'path';
import fs from 'fs/promises';
import pLimit from 'p-limit';
export class VectorService {
constructor() {
this.indexPath = process.env.VECTOR_DB_PATH || './data/vectors';
this.index = new LocalIndex(this.indexPath);
this.aiService = new AIService();
this.initialized = false;
this.updateQueue = Promise.resolve();
// Limit concurrent embedding requests to avoid overwhelming Ollama
// Default to 2 for better stability with Ollama
this.embeddingConcurrency = parseInt(process.env.EMBEDDING_CONCURRENCY) || 2;
this.embeddingLimit = pLimit(this.embeddingConcurrency);
// Add delay between embedding requests (ms)
this.embeddingDelay = parseInt(process.env.EMBEDDING_DELAY) || 100;
}
async ensureInitialized() {
if (!this.initialized) {
try {
// Check if index directory exists
await fs.access(this.indexPath);
// Try to load existing index
if (await this.index.isIndexCreated()) {
await this.index.beginUpdate();
await this.index.endUpdate();
} else {
await this.index.createIndex();
}
} catch (error) {
// Create directory and index if it doesn't exist
await fs.mkdir(this.indexPath, { recursive: true });
await this.index.createIndex();
}
this.initialized = true;
}
}
async addDocument(doc) {
// Queue the update to serialize database operations
this.updateQueue = this.updateQueue.then(async () => {
await this.ensureInitialized();
try {
// Split content into chunks for better retrieval
// Reduced chunk size to 1500 to avoid Ollama connection issues with large texts
const chunkSize = parseInt(process.env.CHUNK_SIZE) || 1500;
const chunks = this.splitIntoChunks(doc.content, chunkSize);
await this.index.beginUpdate();
try {
// Generate embeddings with controlled concurrency to avoid overwhelming Ollama
const embeddingPromises = chunks.map(chunk =>
this.embeddingLimit(() => this.aiService.generateEmbedding(chunk))
);
const embeddings = await Promise.all(embeddingPromises);
// Insert all chunks at once
for (let i = 0; i < chunks.length; i++) {
const chunkId = `${doc.id}#chunk${i}`;
await this.index.insertItem({
id: chunkId,
metadata: {
...doc.metadata,
chunkIndex: i,
totalChunks: chunks.length,
originalDocId: doc.id,
},
vector: embeddings[i],
});
// Store chunk content separately
await this.storeChunkContent(chunkId, chunks[i]);
}
await this.index.endUpdate();
} catch (error) {
// Ensure endUpdate is called even if there's an error
try {
await this.index.endUpdate();
} catch (e) {
// Ignore endUpdate errors
}
throw error;
}
} catch (error) {
console.error(`Failed to add document ${doc.id}:`, error);
throw error;
}
});
// Wait for this operation to complete
await this.updateQueue;
}
async search(query, source = 'all', limit = 5) {
await this.ensureInitialized();
try {
// Generate query embedding
const queryEmbedding = await this.aiService.generateEmbedding(query);
// Search the index
const results = await this.index.queryItems(queryEmbedding, limit * 2);
// Filter by source if specified
const filteredResults = source === 'all'
? results
: results.filter((r) => r.item.metadata.source === source);
// Group chunks by original document and reconstruct content
const documentMap = new Map();
for (const result of filteredResults.slice(0, limit * 2)) {
const originalDocId = result.item.metadata.originalDocId;
if (!documentMap.has(originalDocId)) {
// Read the chunk content from the index
const content = await this.getChunkContent(result.item.id);
documentMap.set(originalDocId, {
content: content,
metadata: {
source: result.item.metadata.source,
url: result.item.metadata.url,
title: result.item.metadata.title,
score: result.score,
},
});
}
if (documentMap.size >= limit) {
break;
}
}
return Array.from(documentMap.values());
} catch (error) {
console.error('Search failed:', error);
return [];
}
}
sanitizeFilename(filename) {
// Replace invalid characters for file systems (Windows is most restrictive)
return filename
.replace(/[<>:"/\\|?*]/g, '_') // Replace invalid chars with underscore
.replace(/\s+/g, '_') // Replace spaces with underscore
.substring(0, 200); // Limit length to avoid path too long errors
}
async getChunkContent(chunkId) {
// In Vectra, we need to store content separately or in metadata
// For simplicity, we'll store it in a separate file structure
try {
const safeId = this.sanitizeFilename(chunkId);
const contentPath = path.join(this.indexPath, 'content', `${safeId}.txt`);
const content = await fs.readFile(contentPath, 'utf-8');
return content;
} catch (error) {
return '[Content not available]';
}
}
async storeChunkContent(chunkId, content) {
try {
const contentDir = path.join(this.indexPath, 'content');
await fs.mkdir(contentDir, { recursive: true });
const safeId = this.sanitizeFilename(chunkId);
const contentPath = path.join(contentDir, `${safeId}.txt`);
await fs.writeFile(contentPath, content, 'utf-8');
} catch (error) {
console.error(`Failed to store chunk content ${chunkId}:`, error);
}
}
splitIntoChunks(text, maxChunkSize) {
const chunks = [];
const paragraphs = text.split(/\n\n+/);
let currentChunk = '';
for (const paragraph of paragraphs) {
if (currentChunk.length + paragraph.length > maxChunkSize && currentChunk.length > 0) {
chunks.push(currentChunk.trim());
currentChunk = paragraph;
} else {
currentChunk += (currentChunk ? '\n\n' : '') + paragraph;
}
}
if (currentChunk.trim()) {
chunks.push(currentChunk.trim());
}
return chunks.length > 0 ? chunks : [text];
}
async deleteDocument(docId) {
await this.ensureInitialized();
try {
await this.index.beginUpdate();
// Find all chunks for this document
const allItems = await this.index.listItems();
const chunksToDelete = allItems.filter(
(item) => item.metadata.originalDocId === docId
);
for (const chunk of chunksToDelete) {
await this.index.deleteItem(chunk.id);
}
await this.index.endUpdate();
} catch (error) {
console.error(`Failed to delete document ${docId}:`, error);
throw error;
}
}
async clear() {
await this.ensureInitialized();
await this.index.beginUpdate();
const items = await this.index.listItems();
for (const item of items) {
await this.index.deleteItem(item.id);
}
await this.index.endUpdate();
}
}