Amazon Bedrock AgentCore EvaluationsのOn-demand evaluationをStarter Toolkitで試してみた

# プロジェクトの作成と初期化
uv init strands-eval-on-demand
cd strands-eval-on-demand

# 必要なパッケージのインストール
uv add bedrock-agentcore strands-agents bedrock-agentcore-starter-toolkit aws-opentelemetry-distro

from bedrock_agentcore import BedrockAgentCoreApp
from strands import Agent, tool

app = BedrockAgentCoreApp()

# ツールの定義
@tool
def calculator(operation: str, a: float, b: float) -> str:
    """
    基本的な計算を実行する電卓ツール

    Args:
        operation: 実行する演算 (add, subtract, multiply, divide)
        a: 最初の数値
        b: 2番目の数値

    Returns:
        計算結果
    """
    operations = {
        "add": lambda x, y: x + y,
        "subtract": lambda x, y: x - y,
        "multiply": lambda x, y: x * y,
        "divide": lambda x, y: x / y if y != 0 else "Error: Division by zero",
    }

    if operation not in operations:
        return f"Error: Unknown operation '{operation}'. Use: add, subtract, multiply, divide"

    result = operations[operation](a, b)
    return f"{a} {operation} {b} = {result}"

@tool
def get_weather(city: str) -> str:
    """
    指定された都市の天気情報を取得する（デモ用のモックデータ）

    Args:
        city: 都市名

    Returns:
        天気情報
    """
    weather_data = {
        "tokyo": {"temp": 15, "condition": "Sunny", "humidity": 45},
        "osaka": {"temp": 17, "condition": "Cloudy", "humidity": 60},
        "new york": {"temp": 8, "condition": "Rainy", "humidity": 80},
        "london": {"temp": 10, "condition": "Cloudy", "humidity": 75},
    }

    city_lower = city.lower()
    if city_lower in weather_data:
        data = weather_data[city_lower]
        return f"Weather in {city}: {data['condition']}, Temperature: {data['temp']}C, Humidity: {data['humidity']}%"
    else:
        return f"Weather information for {city} is not available"

@tool
def search_knowledge(query: str) -> str:
    """
    ナレッジベースを検索する（デモ用のモックデータ）

    Args:
        query: 検索クエリ

    Returns:
        検索結果
    """
    knowledge_base = {
        "python": "Python is a versatile high-level programming language known for readability and simplicity.",
        "aws": "Amazon Web Services (AWS) is a cloud computing platform.",
        "ai": "Artificial Intelligence (AI) enables machines to perform human-like intelligent tasks.",
        "bedrock": "Amazon Bedrock is a fully managed service for building generative AI applications.",
    }

    query_lower = query.lower()
    results = []
    for key, value in knowledge_base.items():
        if key in query_lower or query_lower in key:
            results.append(value)

    if results:
        return "\n".join(results)
    else:
        return f"No information found for '{query}'"

# エージェントの作成
agent = Agent(
    system_prompt="""You are a helpful AI assistant.
Use the available tools to provide accurate and useful responses.

Available tools:
- calculator: Perform mathematical calculations
- get_weather: Get weather information for a city
- search_knowledge: Search the knowledge base for information

Respond clearly and concisely.""",
    tools=[calculator, get_weather, search_knowledge],
)

@app.entrypoint
def invoke(payload):
    """AgentCore Runtime entrypoint"""
    user_message = payload.get("prompt", "Hello! How can I help you?")
    result = agent(user_message)
    return {"result": result.message}

if __name__ == "__main__":
    app.run()

uv run agentcore configure -e sample_eval_test.py

uv run agentcore launch

# UUIDを作成（後で評価に使用するので記録しておく）
SESSION_ID=$(uuidgen) 
echo "セッションID: $SESSION_ID"

# リクエスト1: 東京の天気を質問
uv run agentcore invoke '{"prompt": "Tokyoの天気をおしえて"}' --session-id $SESSION_ID
Response:
{"result": {"role": "assistant", "content": [{"text": "東京の天気情報をお知らせします：\n\n- **天候**：晴れ\n- **気温**：15℃\n- **湿度**：45%\n\n今日は晴天で、気温も15℃と過ごしやすい陽気ですね。湿度も45%と適度な水準です。"}]}}
# リクエスト2: 淀屋橋（大阪）の天気を質問
uv run agentcore invoke '{"prompt": "淀屋橋の天気をおしえて"}' --session-id $SESSION_ID
Response:
{"result": {"role": "assistant", "content": [{"text": "淀屋橋がある大阪の天気情報をお知らせします：\n\n- **天候**：曇り\n- **気温**：17℃\n- **湿度**：60%\n\n淀屋橋周辺（大阪市内）は曇り空で、気温は17℃、湿度は60%となっています。東京より少し暖かく、湿度もやや高めですね。"}]}}
# リクエスト3: 福岡の天気を質問（モックデータに存在しない都市）
uv run agentcore invoke '{"prompt": "福岡の天気をおしえて"}' --session-id $SESSION_ID
Response:
{"result": {"role": "assistant", "content": [{"text": "申し訳ございませんが、福岡の天気情報は現在取得できませんでした。このデモ用の天気システムでは、福岡のデータが登録されていないようです。\n\n実際のシステムであれば、主要都市の天気情報が広くカバーされていると思いますが、このデモ環境では限定的なデータのみが利用可能のようです。"}]}}

uv run agentcore eval run --session-id $SESSION_ID \
  --evaluator "Builtin.Helpfulness" \
  --evaluator "Builtin.GoalSuccessRate" \
  --evaluator "Builtin.Coherence" \
  --evaluator "Builtin.Faithfulness" \
  --evaluator "Builtin.ToolSelectionAccuracy"

╭───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
│  Evaluator: Builtin.GoalSuccessRate                                                                                                       │
│                                                                                                                                           │
│  Score: 1.00                                                                                                                              │
│  Label: Yes                                                                                                                               │
│                                                                                                                                           │
│  Explanation:                                                                                                                             │
│  The conversation contains three user requests:                                                                                           │
│                                                                                                                                           │
│  1. **Tokyo weather request**: User asked for Tokyo's weather in Japanese. The assistant correctly used the get_weather tool with         │
│  'Tokyo' as parameter, received weather data (Sunny, 15C, 45% humidity), and provided an accurate, well-formatted response in Japanese.  │
│  Goal achieved.                                                                                                                           │
│                                                                                                                                           │
│  2. **Yodoyabashi weather request**: User asked for Yodoyabashi's weather. The assistant attempted to get weather for '淀屋橋' (failed),  │
│  then '大阪' (failed), then 'Osaka' (succeeded). It received weather data (Cloudy, 17C, 60% humidity) and provided a response stating    │
│  this was Osaka's weather where Yodoyabashi is located. The assistant provided weather information and explained the context             │
│  appropriately. Goal achieved.                                                                                                            │
│                                                                                                                                           │
│  3. **Fukuoka weather request**: User asked for Fukuoka's weather. The assistant tried both '福岡' and 'Fukuoka' parameters, but both    │
│  returned 'Weather information not available'. The assistant correctly informed the user that the weather information could not be       │
│  retrieved and explained it was a limitation of the demo system. While the user didn't get the weather data, the assistant properly      │
│  communicated the unavailability. Goal achieved - the assistant did what was possible given the tool limitations.                        │
╰───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯

╭───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
│  Evaluator: Builtin.Helpfulness                                                                                                           │
│                                                                                                                                           │
│  Score: 0.83                                                                                                                              │
│  Label: Very Helpful                                                                                                                      │
│                                                                                                                                           │
│  Explanation:                                                                                                                             │
│  The user's goal is clear: they want to know the weather in Tokyo. The assistant successfully retrieved weather data through a tool      │
│  call and received: Sunny, 15°C, 45% humidity.                                                                                            │
│                                                                                                                                           │
│  The assistant's response directly addresses the user's request by:                                                                       │
│  1. Presenting the weather information in Japanese (matching the user's language preference)                                              │
│  2. Clearly formatting the data with bullet points for easy reading                                                                       │
│  3. Translating all key information accurately                                                                                            │
│  4. Adding a brief contextual comment about the pleasant weather conditions                                                               │
╰───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯

╭───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
│  Evaluator: Builtin.Helpfulness                                                                                                           │
│                                                                                                                                           │
│  Score: 0.33                                                                                                                              │
│  Label: Somewhat Unhelpful                                                                                                                │
│                                                                                                                                           │
│  Explanation:                                                                                                                             │
│  The user's goal is clear: they want to know the weather in Fukuoka. The assistant attempted to retrieve this information by calling     │
│  the weather tool twice (with both Japanese '福岡' and English 'Fukuoka' parameters), but both attempts returned that the information    │
│  is not available.                                                                                                                        │
│                                                                                                                                           │
│  From the user's perspective, this response does NOT achieve their goal of getting Fukuoka's weather. However, it does provide a         │
│  clear, honest explanation of why the information cannot be provided.                                                                     │
╰───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯

╭───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
│  Evaluator: Builtin.Coherence                                                                                                             │
│                                                                                                                                           │
│  Score: 1.00                                                                                                                              │
│  Label: Completely Yes                                                                                                                    │
│                                                                                                                                           │
│  Explanation:                                                                                                                             │
│  The assistant's response demonstrates sound logical reasoning throughout. When asked about the weather in 淀屋橋 (Yodoyabashi), the     │
│  system attempted multiple tool calls: first trying '淀屋橋' directly, then '大阪' (Osaka in Japanese), and finally 'Osaka' in English.  │
│  The first two attempts returned no data, but the third succeeded.                                                                        │
│                                                                                                                                           │
│  The assistant correctly reasoned that:                                                                                                   │
│  1. Yodoyabashi is a location within Osaka city                                                                                           │
│  2. Since specific weather data for Yodoyabashi wasn't available, using Osaka's weather data is appropriate                               │
│  3. The response explicitly states '淀屋橋がある大阪' (Osaka where Yodoyabashi is located), clearly establishing the logical connection  │
╰───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯

import boto3

# クライアントの初期化
client = boto3.client('agentcore-evaluation-dataplane', region_name='us-west-2')

# トレースIDを指定して評価（Trace-level Evaluatorの場合）
response = client.evaluate(
    evaluatorId="Builtin.Helpfulness",
    evaluationInput={"sessionSpans": session_span_logs},
    evaluationTarget={"traceIds": ["trace-id-1", "trace-id-2"]}
)

print(response["evaluationResults"])

# スパンIDを指定して評価（Tool-level Evaluatorの場合）
response = client.evaluate(
    evaluatorId="Builtin.ToolSelectionAccuracy",
    evaluationInput={"sessionSpans": session_span_logs},
    evaluationTarget={"spanIds": ["span-id-1", "span-id-2"]}
)

print(response["evaluationResults"])