Add loan-advisor-agent
Browse files- loan-advisor-agent/tests/eval/eval_config.json +21 -0
- loan-advisor-agent/tests/eval/evalsets/README.md +80 -0
- loan-advisor-agent/tests/eval/evalsets/basic.evalset.json +67 -0
- loan-advisor-agent/tests/integration/test_agent.py +57 -0
- loan-advisor-agent/tests/integration/test_agent_runtime_app.py +89 -0
- loan-advisor-agent/tests/unit/test_dummy.py +23 -0
loan-advisor-agent/tests/eval/eval_config.json
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"criteria": {
|
| 3 |
+
"rubric_based_final_response_quality_v1": {
|
| 4 |
+
"threshold": 0.8,
|
| 5 |
+
"judgeModelOptions": {
|
| 6 |
+
"judgeModel": "gemini-flash-latest",
|
| 7 |
+
"numSamples": 1
|
| 8 |
+
},
|
| 9 |
+
"rubrics": [
|
| 10 |
+
{
|
| 11 |
+
"rubricId": "relevance",
|
| 12 |
+
"rubricContent": { "textProperty": "The response directly addresses the user's query." }
|
| 13 |
+
},
|
| 14 |
+
{
|
| 15 |
+
"rubricId": "helpfulness",
|
| 16 |
+
"rubricContent": { "textProperty": "The response is helpful and provides useful information." }
|
| 17 |
+
}
|
| 18 |
+
]
|
| 19 |
+
}
|
| 20 |
+
}
|
| 21 |
+
}
|
loan-advisor-agent/tests/eval/evalsets/README.md
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Evaluation Sets
|
| 2 |
+
|
| 3 |
+
This directory contains evaluation sets for testing agent behavior using `adk eval`.
|
| 4 |
+
|
| 5 |
+
## Running Evaluations
|
| 6 |
+
|
| 7 |
+
```bash
|
| 8 |
+
# Run default evalset
|
| 9 |
+
agents-cli eval run
|
| 10 |
+
|
| 11 |
+
# Run specific evalset
|
| 12 |
+
agents-cli eval run --evalset tests/eval/evalsets/custom.evalset.json
|
| 13 |
+
|
| 14 |
+
# Run all evalsets
|
| 15 |
+
agents-cli eval run --all
|
| 16 |
+
```
|
| 17 |
+
|
| 18 |
+
## Evalset Format
|
| 19 |
+
|
| 20 |
+
Each `.evalset.json` follows the ADK evaluation format:
|
| 21 |
+
|
| 22 |
+
```json
|
| 23 |
+
{
|
| 24 |
+
"eval_set_id": "unique_id",
|
| 25 |
+
"name": "Human-readable name",
|
| 26 |
+
"description": "What this evalset tests",
|
| 27 |
+
"eval_cases": [
|
| 28 |
+
{
|
| 29 |
+
"eval_id": "case_id",
|
| 30 |
+
"conversation": [
|
| 31 |
+
{
|
| 32 |
+
"user_content": {
|
| 33 |
+
"parts": [{"text": "User message"}]
|
| 34 |
+
},
|
| 35 |
+
"intermediate_data": {
|
| 36 |
+
"tool_uses": [
|
| 37 |
+
{"name": "tool_name", "args": {"param": "value"}}
|
| 38 |
+
]
|
| 39 |
+
}
|
| 40 |
+
}
|
| 41 |
+
],
|
| 42 |
+
"session_input": {
|
| 43 |
+
"app_name": "app_name",
|
| 44 |
+
"user_id": "test_user",
|
| 45 |
+
"state": {}
|
| 46 |
+
}
|
| 47 |
+
}
|
| 48 |
+
]
|
| 49 |
+
}
|
| 50 |
+
```
|
| 51 |
+
|
| 52 |
+
## Key Fields
|
| 53 |
+
|
| 54 |
+
- `eval_cases`: Array of test scenarios
|
| 55 |
+
- `conversation`: Sequence of user messages
|
| 56 |
+
- `intermediate_data.tool_uses`: Expected tool calls (for trajectory matching)
|
| 57 |
+
- `session_input`: Initial session state
|
| 58 |
+
|
| 59 |
+
## Evaluation Metrics
|
| 60 |
+
|
| 61 |
+
ADK eval measures:
|
| 62 |
+
|
| 63 |
+
- **tool_trajectory_avg_score**: Are the correct tools called in the right order?
|
| 64 |
+
- **response_match_score**: How similar is the response to expected output?
|
| 65 |
+
|
| 66 |
+
## Creating Custom Evalsets
|
| 67 |
+
|
| 68 |
+
1. Copy `basic.evalset.json` as a template
|
| 69 |
+
2. Add cases based on your `DESIGN_SPEC.md` scenarios
|
| 70 |
+
3. Include expected tool calls for capability tests
|
| 71 |
+
4. Run `agents-cli eval run --evalset your_evalset.json`
|
| 72 |
+
|
| 73 |
+
## Tips
|
| 74 |
+
|
| 75 |
+
- Start with 3-5 representative cases
|
| 76 |
+
- Include both happy path and edge cases
|
| 77 |
+
- Test each core capability from DESIGN_SPEC.md
|
| 78 |
+
- Add cases when you find bugs in production
|
| 79 |
+
|
| 80 |
+
See [ADK documentation](https://google.github.io/adk-docs/) for advanced evaluation options.
|
loan-advisor-agent/tests/eval/evalsets/basic.evalset.json
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"eval_set_id": "loan_advisor_eval",
|
| 3 |
+
"name": "Loan Advisor Agent Evaluation",
|
| 4 |
+
"description": "Evaluation set for the loan advisor agent covering product recommendations, calculations, and comparisons.",
|
| 5 |
+
"eval_cases": [
|
| 6 |
+
{
|
| 7 |
+
"eval_id": "startup_loan_inquiry",
|
| 8 |
+
"conversation": [
|
| 9 |
+
{
|
| 10 |
+
"user_content": {
|
| 11 |
+
"parts": [{"text": "I'm a startup looking for €100,000 to expand my team. What loan options do you recommend?"}]
|
| 12 |
+
}
|
| 13 |
+
}
|
| 14 |
+
],
|
| 15 |
+
"session_input": {
|
| 16 |
+
"app_name": "app",
|
| 17 |
+
"user_id": "eval_user",
|
| 18 |
+
"state": {}
|
| 19 |
+
}
|
| 20 |
+
},
|
| 21 |
+
{
|
| 22 |
+
"eval_id": "loan_comparison_5yr_vs_10yr",
|
| 23 |
+
"conversation": [
|
| 24 |
+
{
|
| 25 |
+
"user_content": {
|
| 26 |
+
"parts": [{"text": "Compare a €150,000 loan over 5 years vs 10 years for my business expansion. What are the monthly costs?"}]
|
| 27 |
+
}
|
| 28 |
+
}
|
| 29 |
+
],
|
| 30 |
+
"session_input": {
|
| 31 |
+
"app_name": "app",
|
| 32 |
+
"user_id": "eval_user",
|
| 33 |
+
"state": {}
|
| 34 |
+
}
|
| 35 |
+
},
|
| 36 |
+
{
|
| 37 |
+
"eval_id": "equipment_financing_scenario",
|
| 38 |
+
"conversation": [
|
| 39 |
+
{
|
| 40 |
+
"user_content": {
|
| 41 |
+
"parts": [{"text": "We need €75,000 for equipment. Can you show me what loan products are available and calculate monthly payments for a 7-year term?"}]
|
| 42 |
+
}
|
| 43 |
+
}
|
| 44 |
+
],
|
| 45 |
+
"session_input": {
|
| 46 |
+
"app_name": "app",
|
| 47 |
+
"user_id": "eval_user",
|
| 48 |
+
"state": {}
|
| 49 |
+
}
|
| 50 |
+
},
|
| 51 |
+
{
|
| 52 |
+
"eval_id": "working_capital_quick_calculation",
|
| 53 |
+
"conversation": [
|
| 54 |
+
{
|
| 55 |
+
"user_content": {
|
| 56 |
+
"parts": [{"text": "What's my monthly payment if I borrow €50,000 at 7.1% interest for 3 years?"}]
|
| 57 |
+
}
|
| 58 |
+
}
|
| 59 |
+
],
|
| 60 |
+
"session_input": {
|
| 61 |
+
"app_name": "app",
|
| 62 |
+
"user_id": "eval_user",
|
| 63 |
+
"state": {}
|
| 64 |
+
}
|
| 65 |
+
}
|
| 66 |
+
]
|
| 67 |
+
}
|
loan-advisor-agent/tests/integration/test_agent.py
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2026 Google LLC
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# https://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
|
| 15 |
+
from google.adk.agents.run_config import RunConfig, StreamingMode
|
| 16 |
+
from google.adk.runners import Runner
|
| 17 |
+
from google.adk.sessions import InMemorySessionService
|
| 18 |
+
from google.genai import types
|
| 19 |
+
|
| 20 |
+
from app.agent import root_agent
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def test_agent_stream() -> None:
|
| 24 |
+
"""
|
| 25 |
+
Integration test for the agent stream functionality.
|
| 26 |
+
Tests that the agent returns valid streaming responses.
|
| 27 |
+
"""
|
| 28 |
+
|
| 29 |
+
session_service = InMemorySessionService()
|
| 30 |
+
|
| 31 |
+
session = session_service.create_session_sync(user_id="test_user", app_name="test")
|
| 32 |
+
runner = Runner(agent=root_agent, session_service=session_service, app_name="test")
|
| 33 |
+
|
| 34 |
+
message = types.Content(
|
| 35 |
+
role="user", parts=[types.Part.from_text(text="Why is the sky blue?")]
|
| 36 |
+
)
|
| 37 |
+
|
| 38 |
+
events = list(
|
| 39 |
+
runner.run(
|
| 40 |
+
new_message=message,
|
| 41 |
+
user_id="test_user",
|
| 42 |
+
session_id=session.id,
|
| 43 |
+
run_config=RunConfig(streaming_mode=StreamingMode.SSE),
|
| 44 |
+
)
|
| 45 |
+
)
|
| 46 |
+
assert len(events) > 0, "Expected at least one message"
|
| 47 |
+
|
| 48 |
+
has_text_content = False
|
| 49 |
+
for event in events:
|
| 50 |
+
if (
|
| 51 |
+
event.content
|
| 52 |
+
and event.content.parts
|
| 53 |
+
and any(part.text for part in event.content.parts)
|
| 54 |
+
):
|
| 55 |
+
has_text_content = True
|
| 56 |
+
break
|
| 57 |
+
assert has_text_content, "Expected at least one message with text content"
|
loan-advisor-agent/tests/integration/test_agent_runtime_app.py
ADDED
|
@@ -0,0 +1,89 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2026 Google LLC
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# https://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
|
| 15 |
+
import logging
|
| 16 |
+
|
| 17 |
+
import pytest
|
| 18 |
+
from google.adk.events.event import Event
|
| 19 |
+
|
| 20 |
+
from app.agent_runtime_app import AgentEngineApp
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
@pytest.fixture
|
| 24 |
+
def agent_app(monkeypatch: pytest.MonkeyPatch) -> AgentEngineApp:
|
| 25 |
+
"""Fixture to create and set up AgentEngineApp instance"""
|
| 26 |
+
# Set integration test flag to mock external services
|
| 27 |
+
monkeypatch.setenv("INTEGRATION_TEST", "TRUE")
|
| 28 |
+
|
| 29 |
+
from app.agent_runtime_app import agent_runtime
|
| 30 |
+
|
| 31 |
+
agent_runtime.set_up()
|
| 32 |
+
return agent_runtime
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
@pytest.mark.asyncio
|
| 36 |
+
async def test_agent_stream_query(agent_app: AgentEngineApp) -> None:
|
| 37 |
+
"""
|
| 38 |
+
Integration test for the agent stream query functionality.
|
| 39 |
+
Tests that the agent returns valid streaming responses.
|
| 40 |
+
"""
|
| 41 |
+
# Create message and events for the async_stream_query
|
| 42 |
+
message = "Hi!"
|
| 43 |
+
events = []
|
| 44 |
+
async for event in agent_app.async_stream_query(message=message, user_id="test"):
|
| 45 |
+
events.append(event)
|
| 46 |
+
assert len(events) > 0, "Expected at least one chunk in response"
|
| 47 |
+
|
| 48 |
+
# Check for valid content in the response
|
| 49 |
+
has_text_content = False
|
| 50 |
+
for event in events:
|
| 51 |
+
validated_event = Event.model_validate(event)
|
| 52 |
+
content = validated_event.content
|
| 53 |
+
if (
|
| 54 |
+
content is not None
|
| 55 |
+
and content.parts
|
| 56 |
+
and any(part.text for part in content.parts)
|
| 57 |
+
):
|
| 58 |
+
has_text_content = True
|
| 59 |
+
break
|
| 60 |
+
|
| 61 |
+
assert has_text_content, "Expected at least one event with text content"
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def test_agent_feedback(agent_app: AgentEngineApp) -> None:
|
| 65 |
+
"""
|
| 66 |
+
Integration test for the agent feedback functionality.
|
| 67 |
+
Tests that feedback can be registered successfully.
|
| 68 |
+
"""
|
| 69 |
+
feedback_data = {
|
| 70 |
+
"score": 5,
|
| 71 |
+
"text": "Great response!",
|
| 72 |
+
"user_id": "test-user-456",
|
| 73 |
+
"session_id": "test-session-456",
|
| 74 |
+
}
|
| 75 |
+
|
| 76 |
+
# Should not raise any exceptions
|
| 77 |
+
agent_app.register_feedback(feedback_data)
|
| 78 |
+
|
| 79 |
+
# Test invalid feedback
|
| 80 |
+
with pytest.raises(ValueError):
|
| 81 |
+
invalid_feedback = {
|
| 82 |
+
"score": "invalid", # Score must be numeric
|
| 83 |
+
"text": "Bad feedback",
|
| 84 |
+
"user_id": "test-user-789",
|
| 85 |
+
"session_id": "test-session-789",
|
| 86 |
+
}
|
| 87 |
+
agent_app.register_feedback(invalid_feedback)
|
| 88 |
+
|
| 89 |
+
logging.info("All assertions passed for agent feedback test")
|
loan-advisor-agent/tests/unit/test_dummy.py
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2026 Google LLC
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# https://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
"""
|
| 15 |
+
You can add your unit tests here.
|
| 16 |
+
This is where you test your business logic, including agent functionality,
|
| 17 |
+
data processing, and other core components of your application.
|
| 18 |
+
"""
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def test_dummy() -> None:
|
| 22 |
+
"""Placeholder - replace with real tests."""
|
| 23 |
+
assert 1 == 1
|