File size: 5,148 Bytes
bde76b9 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 | #!/usr/bin/env bash
# benchmark.sh -- Performance benchmark suite for the Knowledge Base application.
#
# Measures import throughput, indexing speed, query latency, and data integrity.
# This script simulates operations against the services layer.
#
# Usage: bash scripts/benchmark.sh
set -euo pipefail
BENCH_DIR=$(mktemp -d)
SAMPLE_DIR="$(cd "$(dirname "$0")/.." && pwd)/data/sample-documents"
echo "=== Knowledge Base Benchmark Suite ==="
echo ""
echo "Working directory: $BENCH_DIR"
echo "Sample data: $SAMPLE_DIR"
echo ""
PASS_COUNT=0
FAIL_COUNT=0
TOTAL_TASKS=4
# ---- Task 1: Import ----
echo "[1/4] Import Benchmark"
IMPORT_START=$(python3 -c "import time; print(time.time())" 2>/dev/null || date +%s)
import_count=0
for file in "$SAMPLE_DIR"/*; do
if [ -f "$file" ]; then
# Simulate import: copy to benchmark directory
filename=$(basename "$file")
size=$(wc -c < "$file" | tr -d ' ')
cp "$file" "$BENCH_DIR/$filename"
echo " Imported: $filename ($size bytes)"
import_count=$((import_count + 1))
fi
done
IMPORT_END=$(python3 -c "import time; print(time.time())" 2>/dev/null || date +%s)
IMPORT_MS=$(( (IMPORT_END - IMPORT_START) * 1000 ))
if [ "$import_count" -ge 3 ]; then
echo " PASS: $import_count files imported in ${IMPORT_MS}ms"
PASS_COUNT=$((PASS_COUNT + 1))
else
echo " FAIL: Only $import_count files imported (expected 3)"
FAIL_COUNT=$((FAIL_COUNT + 1))
fi
echo ""
# ---- Task 2: Indexing ----
echo "[2/4] Indexing Benchmark"
INDEX_START=$(python3 -c "import time; print(time.time())" 2>/dev/null || date +%s)
total_chunks=0
for file in "$BENCH_DIR"/*; do
if [ -f "$file" ]; then
content=$(cat "$file")
# Simulate chunking: split on double newlines, count chunks
# Approximate ~500 char chunks
para_count=$(echo "$content" | awk 'BEGIN{RS="\n\n"} NF{c++} END{print c+0}')
# Estimate chunks (merge short paragraphs)
estimated_chunks=$(( (para_count + 1) / 2 ))
if [ "$estimated_chunks" -lt 1 ]; then
estimated_chunks=1
fi
total_chunks=$((total_chunks + estimated_chunks))
echo " $(basename "$file"): ~${estimated_chunks} estimated chunks"
fi
done
INDEX_END=$(python3 -c "import time; print(time.time())" 2>/dev/null || date +%s)
INDEX_MS=$(( (INDEX_END - INDEX_START) * 1000 ))
if [ "$total_chunks" -ge 5 ]; then
echo " PASS: ~$total_chunks total chunks estimated in ${INDEX_MS}ms"
PASS_COUNT=$((PASS_COUNT + 1))
else
echo " FAIL: Only ~$total_chunks chunks estimated (expected 5+)"
FAIL_COUNT=$((FAIL_COUNT + 1))
fi
echo ""
# ---- Task 3: Query ----
echo "[3/4] Query Benchmark"
queries=(
"What is the system architecture?"
"How does document import work?"
"Explain the indexing pipeline"
"What was discussed in the meeting?"
"How does logging work?"
)
QUERY_START=$(python3 -c "import time; print(time.time())" 2>/dev/null || date +%s)
query_count=0
for query in "${queries[@]}"; do
# Simulate query processing: keyword matching against content
keyword_count=0
for file in "$BENCH_DIR"/*; do
if [ -f "$file" ]; then
# Count keyword matches in file content
for word in $query; do
if [ ${#word} -gt 2 ]; then
matches=$(grep -oi "$word" "$file" 2>/dev/null | wc -l | tr -d ' ')
keyword_count=$((keyword_count + matches))
fi
done
fi
done
echo " Query: \"$query\" -> $keyword_count keyword matches"
query_count=$((query_count + 1))
done
QUERY_END=$(python3 -c "import time; print(time.time())" 2>/dev/null || date +%s)
QUERY_MS=$(( (QUERY_END - QUERY_START) * 1000 ))
if [ "$query_count" -eq 5 ]; then
avg_ms=$((QUERY_MS / query_count))
echo " PASS: $query_count queries in ${QUERY_MS}ms (${avg_ms}ms avg)"
PASS_COUNT=$((PASS_COUNT + 1))
else
echo " FAIL: Only $query_count queries completed (expected 5)"
FAIL_COUNT=$((FAIL_COUNT + 1))
fi
echo ""
# ---- Task 4: Verify ----
echo "[4/4] Data Integrity Verification"
verify_ok=true
sample_files=("design-notes.md" "meeting-summary.txt" "retrieval-plan.md")
for sf in "${sample_files[@]}"; do
if [ ! -f "$BENCH_DIR/$sf" ]; then
echo " FAIL: Missing imported file: $sf"
verify_ok=false
else
original_size=$(wc -c < "$SAMPLE_DIR/$sf" | tr -d ' ')
imported_size=$(wc -c < "$BENCH_DIR/$sf" | tr -d ' ')
if [ "$original_size" -ne "$imported_size" ]; then
echo " FAIL: Size mismatch for $sf ($original_size vs $imported_size)"
verify_ok=false
fi
fi
done
if [ "$verify_ok" = true ]; then
echo " PASS: All files verified (size match)"
PASS_COUNT=$((PASS_COUNT + 1))
else
FAIL_COUNT=$((FAIL_COUNT + 1))
fi
echo ""
# ---- Summary ----
echo "=== Benchmark Results ==="
echo "Import: $import_count files in ${IMPORT_MS}ms"
echo "Index: ~$total_chunks chunks in ${INDEX_MS}ms"
echo "Query: $query_count queries in ${QUERY_MS}ms"
echo "Verify: Data integrity check"
echo ""
echo "=== Summary: $PASS_COUNT/$TOTAL_TASKS tasks passed ==="
# Cleanup
rm -rf "$BENCH_DIR"
if [ "$PASS_COUNT" -eq "$TOTAL_TASKS" ]; then
echo "ALL BENCHMARKS PASSED"
exit 0
else
echo "SOME BENCHMARKS FAILED ($FAIL_COUNT failures)"
exit 1
fi
|