File size: 3,752 Bytes
c981f27 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 | #!/bin/bash
# Test script for semantic search quality validation
# This tests the validation logic without requiring actual LLM calls
set -e
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
TEST_DIR="/tmp/test_semantic_eval_$$"
echo "======================================"
echo "Testing Semantic Search Quality Eval"
echo "======================================"
echo
mkdir -p "$TEST_DIR"
# Test 1: Validation passes when sem_search is used
echo "Test 1: Check that sem_search tool detection works..."
cat > "$TEST_DIR/context.json" <<'EOF'
{
"messages": [
{
"role": "assistant",
"tool_calls": [
{
"id": "call_1",
"type": "function",
"function": {
"name": "sem_search",
"arguments": "{\"queries\":[{\"query\":\"test query\",\"use_case\":\"test case\"}]}"
}
}
]
}
]
}
EOF
if cat "$TEST_DIR/context.json" | jq -e '[.messages[]?.tool_calls[]? | select(.function.name == "sem_search")] | any' > /dev/null 2>&1; then
echo "✓ Test 1 PASSED: sem_search tool detected correctly"
else
echo "✗ Test 1 FAILED: sem_search tool not detected"
exit 1
fi
echo
# Test 2: Validation skips when sem_search is NOT used
echo "Test 2: Check that missing sem_search is detected..."
cat > "$TEST_DIR/context2.json" <<'EOF'
{
"messages": [
{
"role": "assistant",
"tool_calls": [
{
"id": "call_1",
"type": "function",
"function": {
"name": "fs_search",
"arguments": "{\"pattern\":\"test\"}"
}
}
]
}
]
}
EOF
if ! cat "$TEST_DIR/context2.json" | jq -e '[.messages[]?.tool_calls[]? | select(.function.name == "sem_search")] | any' > /dev/null 2>&1; then
echo "✓ Test 2 PASSED: Missing sem_search detected correctly"
else
echo "✗ Test 2 FAILED: False positive for sem_search"
exit 1
fi
echo
# Test 3: Check that llm_judge.ts can be loaded
echo "Test 3: Check that llm_judge.ts script loads..."
if npx tsx "$SCRIPT_DIR/llm_judge.ts" 2>&1 | grep -q "Missing required arguments"; then
echo "✓ Test 3 PASSED: llm_judge.ts script loads correctly"
else
echo "✗ Test 3 FAILED: llm_judge.ts script failed to load"
exit 1
fi
echo
# Test 4: Check conditional execution logic
echo "Test 4: Test conditional LLM judge execution..."
cat > "$TEST_DIR/test_conditional.sh" <<'SCRIPT'
#!/bin/bash
if ! cat "$1" | jq -e '[.messages[]?.tool_calls[]? | select(.function.name == "sem_search")] | any' > /dev/null 2>&1; then
echo "Skipping LLM judge: sem_search tool was not used"
exit 0
fi
echo "Would run LLM judge here"
exit 0
SCRIPT
chmod +x "$TEST_DIR/test_conditional.sh"
# Test with sem_search present
if output=$("$TEST_DIR/test_conditional.sh" "$TEST_DIR/context.json" 2>&1) && echo "$output" | grep -q "Would run LLM judge"; then
echo "✓ Test 4a PASSED: Conditional runs LLM judge when sem_search present"
else
echo "✗ Test 4a FAILED: Conditional logic incorrect"
exit 1
fi
# Test with sem_search missing
if output=$("$TEST_DIR/test_conditional.sh" "$TEST_DIR/context2.json" 2>&1) && echo "$output" | grep -q "Skipping LLM judge"; then
echo "✓ Test 4b PASSED: Conditional skips LLM judge when sem_search missing"
else
echo "✗ Test 4b FAILED: Conditional logic incorrect"
exit 1
fi
echo
# Cleanup
rm -rf "$TEST_DIR"
echo "======================================"
echo "All validation tests PASSED ✓"
echo "======================================"
echo
echo "Note: Full LLM judge testing requires Google Cloud authentication."
echo "To set up authentication:"
echo " 1. Run: gcloud auth application-default login"
echo " 2. Or set GOOGLE_APPLICATION_CREDENTIALS to service account JSON path"
|