File size: 3,157 Bytes
c3aa9b1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
#!/usr/bin/env node

/**
 * Index multiple documentation sources
 * Usage: node src/index-multi.js [sources...]
 * Example: node src/index-multi.js k3s rancher suse
 * Example: node src/index-multi.js all
 */

import { DocumentationService } from '../services/documentation-service.js';
import dotenv from 'dotenv';

dotenv.config();

const args = process.argv.slice(2);
let sources = args.length > 0 ? args : ['k3s'];

// If 'all' is specified, index all available sources
if (sources.includes('all')) {
  sources = ['suse', 'rancher', 'k3s', 'rke2', 'longhorn', 'harvester', 'neuvector', 'kubewarden'];
}

console.log(`\n🚀 Starting multi-source documentation indexing`);
console.log(`📚 Sources: ${sources.join(', ')}`);
console.log('=' .repeat(70));

const docService = new DocumentationService();
const results = [];
const startTime = Date.now();

for (const source of sources) {
  console.log(`\n📖 Indexing: ${source.toUpperCase()}`);
  console.log('-'.repeat(70));
  
  const sourceStartTime = Date.now();
  
  try {
    const result = await docService.indexDocumentation(source, false);
    const duration = ((Date.now() - sourceStartTime) / 1000).toFixed(2);
    
    results.push({
      source,
      success: result.success,
      duration,
      indexed: result.documentsIndexed,
      skipped: result.documentsSkipped || 0,
      cacheStats: result.cacheStats,
    });
    
    if (result.success) {
      console.log(`✅ ${source.toUpperCase()}: ${result.documentsIndexed} documents in ${duration}s`);
    } else {
      console.log(`❌ ${source.toUpperCase()}: Failed - ${result.error}`);
    }
  } catch (error) {
    const duration = ((Date.now() - sourceStartTime) / 1000).toFixed(2);
    console.log(`❌ ${source.toUpperCase()}: Error after ${duration}s - ${error.message}`);
    
    results.push({
      source,
      success: false,
      duration,
      indexed: 0,
      skipped: 0,
      error: error.message,
    });
  }
}

// Summary
const totalDuration = ((Date.now() - startTime) / 1000).toFixed(2);
const successCount = results.filter(r => r.success).length;
const totalIndexed = results.reduce((sum, r) => sum + r.indexed, 0);
const totalSkipped = results.reduce((sum, r) => sum + r.skipped, 0);

console.log('\n' + '='.repeat(70));
console.log('📊 INDEXING SUMMARY');
console.log('='.repeat(70));

for (const result of results) {
  const status = result.success ? '✅' : '❌';
  console.log(`${status} ${result.source.padEnd(10)} - ${result.indexed} indexed, ${result.skipped} skipped (${result.duration}s)`);
  
  if (result.cacheStats) {
    const pageHitRate = result.cacheStats.page.hitRate || 'N/A';
    const embeddingHitRate = result.cacheStats.embedding.hitRate || 'N/A';
    console.log(`   📄 Page Cache: ${pageHitRate}, 🧠 Embedding: ${embeddingHitRate}`);
  }
}

console.log('-'.repeat(70));
console.log(`📈 Total: ${totalIndexed} documents indexed, ${totalSkipped} skipped`);
console.log(`✅ Success: ${successCount}/${results.length} sources`);
console.log(`⏱️  Total Duration: ${totalDuration}s`);
console.log('=' .repeat(70));
console.log('\n✅ Multi-source indexing complete!\n');