AWS DevOps Agent の調査完了イベントを使って、根本原因と緩和策だけを Slack に通知してみた

{
  "detail-type": "Investigation Completed",
  "source": "aws.aidevops",
  "detail": {
    "version": "1.0.0",
    "metadata": {
      "agent_space_id": "8f6187a7-0388-4926-8217-3a0fe32f757c",
      "task_id": "a1b2c3d4-5678-90ab-cdef-example11111",
      "execution_id": "b2c3d4e5-6789-01ab-cdef-example22222"
    },
    "data": {
      "task_type": "INVESTIGATION",
      "priority": "CRITICAL",
      "status": "COMPLETED",
      "created_at": "2026-03-12T18:00:00Z",
      "updated_at": "2026-03-12T18:10:00Z",
      "summary_record_id": "d4e5f6g7-6789-01ab-cdef-example44444"
    }
  }
}

import os

import boto3

dynamodb = boto3.resource("dynamodb")
TABLE_NAME = os.environ.get("TABLE_NAME", "devops-agent-verify-orders")

def lambda_handler(event, context):
    table = dynamodb.Table(TABLE_NAME)
    item = table.get_item(Key={"order_id": event.get("order_id", "0001")})
    return item.get("Item")

read -rs WEBHOOK_URL   # Webhook URL を貼り付けて Enter（画面には表示されません）
aws secretsmanager create-secret \
  --name devops-agent-verify/slack-webhook \
  --secret-string "$WEBHOOK_URL" \
  --region ap-northeast-1
unset WEBHOOK_URL

{
  "source": ["aws.aidevops"],
  "detail-type": [{"prefix": "Investigation"}, {"prefix": "Mitigation"}]
}

aws devops-agent create-backlog-task \
  --agent-space-id <AGENT_SPACE_ID> \
  --task-type INVESTIGATION \
  --priority HIGH \
  --title "検証: Lambda エラー増加" \
  --description "<関数名> でエラーが発生しています。原因を調査してください"

aws devops-agent list-journal-records \
  --agent-space-id <AGENT_SPACE_ID> \
  --execution-id <EXECUTION_ID> \
  --record-type investigation_summary_md

# Investigation Summary

## Symptoms

### Lambda 関数 devops-agent-verify-faulty でエラーバースト（エラー率100%）
**Description:** 2026-10-14 01:57 UTC(10:57 JST) に5回の呼び出しがすべて失敗（Errors=5/Invocations=5）。…

## Findings

### Root Cause: 存在しない DynamoDB テーブルへの get_item で未捕捉例外
**Description:** Lambda `devops-agent-verify-faulty` のハンドラ app.py 11行目で `table.get_item(Key={"order_id": ...})` を実行する際、参照先の DynamoDB テーブルが存在しないため `ResourceNotFoundException` が発生。…
**Cascades to:** symptom-lambda-errors

## Other Gaps

### Lambda コード本体を直接取得できず（lambda:GetFunction 権限不足）
…

aws devops-agent list-journal-records \
  --agent-space-id <AGENT_SPACE_ID> \
  --execution-id <EXECUTION_ID> \
  --record-type mitigation_summary_md

# Mitigation Summary

## Action
欠落している DynamoDB テーブル devops-agent-verify-orders を、パーティションキー order_id（String）・オンデマンド課金（PAY_PER_REQUEST）で ap-northeast-1 に作成し、Lambda 関数 devops-agent-verify-faulty の GetItem が ResourceNotFoundException を発生させない状態に復旧する。

## Reasoning
…

## Execution Plan
### Step 1: Prepare
…

import json
import os
import re
import urllib.request

import boto3

devops_agent = boto3.client("devops-agent")
_secrets = None

MAX_SECTION_CHARS = 1500
DRY_RUN = os.environ.get("DRY_RUN") == "true"
NOT_FOUND_TEXT = "（要約から見つかりませんでした）"

def list_journal_records(agent_space_id, execution_id, record_type):
    records = []
    kwargs = {
        "agentSpaceId": agent_space_id,
        "executionId": execution_id,
        "recordType": record_type,
    }
    while True:
        page = devops_agent.list_journal_records(**kwargs)
        records.extend(page.get("records", []))
        token = page.get("nextToken")
        if not token:
            return records
        kwargs["nextToken"] = token

def get_summary(agent_space_id, execution_id, summary_record_id):
    for record in list_journal_records(agent_space_id, execution_id, "investigation_summary_md"):
        if record["recordId"] == summary_record_id:
            return record["content"]
    return None

def get_mitigation_summary(agent_space_id, execution_id):
    records = list_journal_records(agent_space_id, execution_id, "mitigation_summary_md")
    if not records:
        return None
    return max(records, key=lambda r: r["createdAt"])["content"]

def get_task_title(agent_space_id, task_id):
    try:
        res = devops_agent.get_backlog_task(agentSpaceId=agent_space_id, taskId=task_id)
        return res.get("task", res).get("title")
    except Exception as e:
        print(f"タスクタイトルを取得できませんでした: {e}")
        return None

def extract_section(markdown, pattern):
    lines = markdown.splitlines()
    start = level = None
    for i, line in enumerate(lines):
        m = re.match(r"^(#{1,6})\s+(.*)$", line)
        if not m:
            continue
        if start is None:
            if re.search(pattern, m.group(2), re.IGNORECASE):
                start, level = i + 1, len(m.group(1))
        elif len(m.group(1)) <= level:
            return "\n".join(lines[start:i]).strip()
    if start is None:
        return None
    return "\n".join(lines[start:]).strip()

def extract_root_causes(markdown):
    lines = markdown.splitlines()
    causes = []
    i = 0
    while i < len(lines):
        m = re.match(r"^(#{1,6})\s+Root Cause:\s*(.*)$", lines[i], re.IGNORECASE)
        i += 1
        if not m:
            continue
        level, headline = len(m.group(1)), m.group(2).strip()
        body = []
        while i < len(lines):
            h = re.match(r"^(#{1,6})\s", lines[i])
            if h and len(h.group(1)) <= level:
                break
            body.append(lines[i])
            i += 1
        body = [re.sub(r"^\*\*Description:\*\*\s*", "", l) for l in body
                if not l.startswith("**Cascades to:**")]
        text = "\n".join(body).strip()
        causes.append(f"*{headline}*\n{text}" if text else f"*{headline}*")
    return "\n\n".join(causes)

def to_slack_mrkdwn(text):
    text = re.sub(r"^#{1,6}\s+(.*)$", r"*\1*", text, flags=re.MULTILINE)
    text = re.sub(r"\*\*(.+?)\*\*", r"*\1*", text)
    text = re.sub(r"\[([^\]]+)\]\(([^)]+)\)", r"<\2|\1>", text)
    if len(text) > MAX_SECTION_CHARS:
        text = text[:MAX_SECTION_CHARS] + "\n…（続きは調査画面をご確認ください）"
    return text

def post_to_slack(blocks, fallback_text):
    global _secrets
    payload = {"text": fallback_text, "blocks": blocks}
    if DRY_RUN:
        print(json.dumps(payload, ensure_ascii=False, indent=2))
        return None
    if _secrets is None:
        _secrets = boto3.client("secretsmanager")
    webhook_url = _secrets.get_secret_value(SecretId=os.environ["SLACK_WEBHOOK_SECRET_ID"])["SecretString"]
    body = json.dumps(payload).encode("utf-8")
    req = urllib.request.Request(
        webhook_url, data=body, headers={"Content-Type": "application/json"}
    )
    with urllib.request.urlopen(req, timeout=10) as res:
        return res.status

def lambda_handler(event, context):
    metadata = event["detail"]["metadata"]
    data = event["detail"]["data"]

    summary_record_id = data.get("summary_record_id")
    execution_id = metadata.get("execution_id")
    if not summary_record_id or not execution_id:
        print(f"要約がないため通知しません: {json.dumps(event['detail'])}")
        return

    summary = get_summary(metadata["agent_space_id"], execution_id, summary_record_id)
    if summary is None:
        print(f"要約レコードが見つかりません: {summary_record_id}")
        return

    agent_space_id = metadata["agent_space_id"]
    priority = data.get("priority", "-")
    task_title = get_task_title(agent_space_id, metadata["task_id"])
    title = f"[{priority}] 調査完了: {task_title}" if task_title else f"[{priority}] DevOps Agent の調査が完了しました"
    blocks = [{"type": "header", "text": {"type": "plain_text", "text": title[:150]}}]

    mitigation_md = get_mitigation_summary(agent_space_id, execution_id)
    sections = {
        "根本原因": extract_root_causes(summary),
        "緩和策": extract_section(mitigation_md, r"^action$") if mitigation_md else None,
    }
    for label, section in sections.items():
        blocks.append({
            "type": "section",
            "text": {"type": "mrkdwn", "text": f"*■ {label}*\n{to_slack_mrkdwn(section or NOT_FOUND_TEXT)}"},
        })

    blocks.append({
        "type": "context",
        "elements": [{"type": "mrkdwn", "text": f"Agent Space: `{agent_space_id}` / Task: `{metadata['task_id']}`"}],
    })

    post_to_slack(blocks, title)

{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Effect": "Allow",
      "Action": [
        "aidevops:ListJournalRecords",
        "aidevops:GetBacklogTask"
      ],
      "Resource": "arn:aws:aidevops:ap-northeast-1:<ACCOUNT_ID>:agentspace/<AGENT_SPACE_ID>"
    },
    {
      "Effect": "Allow",
      "Action": "secretsmanager:GetSecretValue",
      "Resource": "arn:aws:secretsmanager:ap-northeast-1:<ACCOUNT_ID>:secret:<SECRET_NAME>-*"
    }
  ]
}

{
  "source": ["aws.aidevops"],
  "detail-type": ["Investigation Completed"],
  "detail": {
    "metadata": {
      "agent_space_id": ["<AGENT_SPACE_ID>"]
    }
  }
}

0.1478 時間 × 3,600 秒 × 0.0083 USD ≒ 4.42 USD