AWS DevOps Agent で、EKS の Pod クラッシュを調査させてみた

# eksctl のインストール
curl --silent --location "https://github.com/weaveworks/eksctl/releases/latest/download/eksctl_$(uname -s)_amd64.tar.gz" | tar xz -C /tmp
sudo mv /tmp/eksctl /usr/local/bin/
eksctl version  # 以下のようにバージョンが表示されれば OK
# 0.224.0  # 例

# DevOps Agent CLI サービスモデルの追加
# サービス定義を手動登録することで `aws devopsagent` コマンドが使えるようになる。
curl -o devopsagent.json https://d1co8nkiwcta1g.cloudfront.net/devopsagent.json
aws configure add-model \
  --service-model "file://${PWD}/devopsagent.json" \
  --service-name devopsagent

eksctl create cluster \
  --name devops-agent-eks-demo \
  --region us-east-1 \
  --version 1.35 \
  --nodegroup-name demo-nodes \
  --node-type t3.medium \
  --nodes 1 \
  --nodes-min 1 \
  --nodes-max 1 \
  --managed

#出力結果例
2026-03-06 04:45:28 [ℹ]  eksctl version 0.224.0
2026-03-06 04:45:28 [ℹ]  using region us-east-1
2026-03-06 04:45:28 [ℹ]  setting availability zones to [us-east-1a us-east-1b]
2026-03-06 04:45:28 [ℹ]  subnets for us-east-1a - public:192.168.0.0/19 private:192.168.64.0/19
2026-03-06 04:45:28 [ℹ]  subnets for us-east-1b - public:192.168.32.0/19 private:192.168.96.0/19
2026-03-06 04:45:28 [ℹ]  nodegroup "demo-nodes" will use "" [AmazonLinux2023/1.35]
2026-03-06 04:45:28 [!]  Auto Mode will be enabled by default in an upcoming release of eksctl. This means managed node groups and managed networking add-ons will no longer be created by default. To maintain current behavior, explicitly set 'autoModeConfig.enabled: false' in your cluster configuration. Learn more: https://eksctl.io/usage/auto-mode/
2026-03-06 04:45:28 [ℹ]  using Kubernetes version 1.35
2026-03-06 04:45:28 [ℹ]  creating EKS cluster "devops-agent-eks-demo" in "us-east-1" region with managed nodes
2026-03-06 04:45:28 [ℹ]  will create 2 separate CloudFormation stacks for cluster itself and the initial managed nodegroup
2026-03-06 04:45:28 [ℹ]  if you encounter any issues, check CloudFormation console or try 'eksctl utils describe-stacks --region=us-east-1 --cluster=devops-agent-eks-demo'
2026-03-06 04:45:28 [ℹ]  Kubernetes API endpoint access will use default of {publicAccess=true, privateAccess=false} for cluster "devops-agent-eks-demo" in "us-east-1"
2026-03-06 04:45:28 [ℹ]  CloudWatch logging will not be enabled for cluster "devops-agent-eks-demo" in "us-east-1"
2026-03-06 04:45:28 [ℹ]  you can enable it with 'eksctl utils update-cluster-logging --enable-types={SPECIFY-YOUR-LOG-TYPES-HERE (e.g. all)} --region=us-east-1 --cluster=devops-agent-eks-demo'
2026-03-06 04:45:28 [ℹ]  default addons vpc-cni, kube-proxy, coredns, metrics-server were not specified, will install them as EKS addons
2026-03-06 04:45:28 [ℹ]  
2 sequential tasks: { create cluster control plane "devops-agent-eks-demo", 
    2 sequential sub-tasks: { 
        2 sequential sub-tasks: { 
            1 task: { create addons },
            wait for control plane to become ready,
        },
        create managed nodegroup "demo-nodes",
    } 
}
2026-03-06 04:45:28 [ℹ]  building cluster stack "eksctl-devops-agent-eks-demo-cluster"
2026-03-06 04:45:28 [ℹ]  deploying stack "eksctl-devops-agent-eks-demo-cluster"
...
2026-03-06 04:59:35 [✔]  EKS cluster "devops-agent-eks-demo" in "us-east-1" region is ready

aws eks update-kubeconfig \
  --region us-east-1 \
  --name devops-agent-eks-demo

Added new context arn:aws:eks:us-east-1:xxxxxxxxxxxx:cluster/devops-agent-eks-demo to /home/cloudshell-user/.kube/config

# ノードが Ready になっていることを確認
~ $ kubectl get nodes
NAME                            STATUS   ROLES    AGE   VERSION
ip-xxx-xxx-xx-xx.ec2.internal   Ready    <none>   10m   v1.35.0-eks-efcacff

ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
echo $ACCOUNT_ID  # 12桁の数字が表示されれば OK

# トラストポリシーファイルを作成
cat > ~/trust-policy.json << EOF
{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Effect": "Allow",
      "Principal": {
        "Service": "aidevops.amazonaws.com"
      },
      "Action": "sts:AssumeRole",
      "Condition": {
        "StringEquals": {
          "aws:SourceAccount": "${ACCOUNT_ID}"
        },
        "ArnLike": {
          "aws:SourceArn": "arn:aws:aidevops:us-east-1:${ACCOUNT_ID}:agentspace/*"
        }
      }
    }
  ]
}
EOF

# 確認
~ $ cat ~/trust-policy.json 
{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Effect": "Allow",
      "Principal": {
        "Service": "aidevops.amazonaws.com"
      },
      "Action": "sts:AssumeRole",
      "Condition": {
        "StringEquals": {
          "aws:SourceAccount": "xxxxxxxxxxxx"
        },
        "ArnLike": {
          "aws:SourceArn": "arn:aws:aidevops:us-east-1:xxxxxxxxxxxx:agentspace/*"
        }
      }
    }
  ]
}

# ロールを作成
aws iam create-role \
  --role-name DevOpsAgentRole-AgentSpace \
  --assume-role-policy-document file://~/trust-policy.json

# AWS 管理ポリシーをアタッチ（CloudWatch 等、基本的な権限）
aws iam attach-role-policy \
  --role-name DevOpsAgentRole-AgentSpace \
  --policy-arn arn:aws:iam::aws:policy/AIOpsAssistantPolicy

python3 -c "
import json, pathlib
policy = {
  'Version': '2012-10-17',
  'Statement': [
    {
      'Sid': 'AllowExpandedAIOpsAssistantPolicy',
      'Effect': 'Allow',
      'Action': [
        'eks:AccessKubernetesApi',
        'aidevops:*',
        'synthetics:GetCanaryRuns',
        'route53:GetHealthCheckStatus',
        'resource-explorer-2:Search'
      ],
      'Resource': ['*']
    }
  ]
}
pathlib.Path('/tmp/inline-policy.json').write_text(json.dumps(policy))
print('OK')
"

aws iam put-role-policy \
  --role-name DevOpsAgentRole-AgentSpace \
  --policy-name ExpandedAIOpsPolicy \
  --policy-document file:///tmp/inline-policy.json

# ロール ARN を変数に格納（以降の手順で使用）
ROLE_ARN=$(aws iam get-role \
  --role-name DevOpsAgentRole-AgentSpace \
  --query 'Role.Arn' \
  --output text)
echo $ROLE_ARN
arn:aws:iam::xxxxxxxxxxxx:role/DevOpsAgentRole-AgentSpace

ROLE_ARN=$(aws iam get-role --role-name DevOpsAgentRole-AgentSpace --query 'Role.Arn' --output text)
NODE_ROLE=$(aws eks describe-nodegroup --cluster-name devops-agent-eks-demo --nodegroup-name demo-nodes --region us-east-1 --query 'nodegroup.nodeRole' --output text)

# Access Entry を作成（IAM ロールに Kubernetes API アクセス権限を付与）
aws eks create-access-entry \
  --cluster-name devops-agent-eks-demo \
  --principal-arn $ROLE_ARN \
  --type STANDARD \
  --region us-east-1

  {
    "accessEntry": {
        "clusterName": "devops-agent-eks-demo",
        "principalArn": "arn:aws:iam::xxxxxxxxxxxx:role/DevOpsAgentRole-AgentSpace",
        "kubernetesGroups": [],
        "accessEntryArn": "arn:aws:eks:us-east-1:xxxxxxxxxxxx:access-entry/devops-agent-eks-demo/role/xxxxxxxxxxxx/DevOpsAgentRole-AgentSpace/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx",
        "createdAt": "2026-03-06T05:17:02.976000+00:00",
        "modifiedAt": "2026-03-06T05:17:02.976000+00:00",
        "tags": {},
        "username": "arn:aws:sts::xxxxxxxxxxxx:assumed-role/DevOpsAgentRole-AgentSpace/{{SessionName}}",
        "type": "STANDARD"
    }
}

# 最小権限の読み取り専用ポリシーを関連付け
aws eks associate-access-policy \
  --cluster-name devops-agent-eks-demo \
  --principal-arn $ROLE_ARN \
  --policy-arn arn:aws:eks::aws:cluster-access-policy/AmazonEKSAdminViewPolicy \
  --access-scope type=cluster \
  --region us-east-1

  {
    "clusterName": "devops-agent-eks-demo",
    "principalArn": "arn:aws:iam::xxxxxxxxxxxx:role/DevOpsAgentRole-AgentSpace",
    "associatedAccessPolicy": {
        "policyArn": "arn:aws:eks::aws:cluster-access-policy/AmazonEKSAdminViewPolicy",
        "accessScope": {
            "type": "cluster",
            "namespaces": []
        },
        "associatedAt": "2026-03-06T05:17:18.972000+00:00",
        "modifiedAt": "2026-03-06T05:17:18.972000+00:00"
    }
}

# Container Insights を有効化（Pod のメトリクスを CloudWatch に送信）
aws eks create-addon \
  --cluster-name devops-agent-eks-demo \
  --addon-name amazon-cloudwatch-observability \
  --region us-east-1

  {
    "addon": {
        "addonName": "amazon-cloudwatch-observability",
        "clusterName": "devops-agent-eks-demo",
        "status": "CREATING",
        "addonVersion": "v4.10.1-eksbuild.1",
        "health": {
            "issues": []
        },
        "addonArn": "arn:aws:eks:us-east-1:xxxxxxxxxxxx:addon/devops-agent-eks-demo/amazon-cloudwatch-observability/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx",
        "createdAt": "2026-03-06T05:20:40.126000+00:00",
        "modifiedAt": "2026-03-06T05:20:40.141000+00:00",
        "tags": {},
        "namespaceConfig": {
            "namespace": "amazon-cloudwatch"
        }
    }
}

# ノードグループの IAM ロール名を取得
NODE_ROLE=$(aws eks describe-nodegroup \
  --cluster-name devops-agent-eks-demo \
  --nodegroup-name demo-nodes \
  --region us-east-1 \
  --query 'nodegroup.nodeRole' \
  --output text)

# CloudWatchAgentServerPolicy をアタッチ
aws iam attach-role-policy \
  --role-name $(basename $NODE_ROLE) \
  --policy-arn arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy

# Container Insights の DaemonSet を再起動してメトリクス送信を開始
kubectl rollout restart daemonset -n amazon-cloudwatch

# OOMKill 検知用アラーム
# ※ 今回は ClusterName のみで絞ることで、クラスタ全体の再起動増加を検知  
# oom-demo 専用にしたい場合は --dimensions に PodName=oom-demo を追加する
aws cloudwatch put-metric-alarm \
  --alarm-name "EKS-OOMKill-Demo" \
  --alarm-description "OOMKill detection for DevOps Agent demo" \
  --namespace ContainerInsights \
  --metric-name pod_number_of_container_restarts \
  --dimensions Name=ClusterName,Value=devops-agent-eks-demo \
  --statistic Sum \
  --period 60 \
  --threshold 2 \
  --comparison-operator GreaterThanOrEqualToThreshold \
  --evaluation-periods 1 \
  --region us-east-1

# CloudShell 上でYAMLファイルを作成してデプロイ
cat > ~/oom-demo.yaml << 'EOF'
apiVersion: apps/v1
kind: Deployment
metadata:
  name: oom-demo
  labels:
    app: oom-demo
    devopsagent: "true"
spec:
  replicas: 1
  selector:
    matchLabels:
      app: oom-demo
  template:
    metadata:
      labels:
        app: oom-demo
    spec:
      containers:
      - name: oom-demo
        image: polinux/stress
        command: ["stress"]
        args: ["--vm", "1", "--vm-bytes", "200M", "--vm-hang", "1"]
        resources:
          requests:
            memory: "64Mi"
            cpu: "100m"
          limits:
            memory: "64Mi"
            cpu: "200m"
EOF

kubectl apply -f ~/oom-demo.yaml
deployment.apps/oom-demo created

# Pod の状態を確認
kubectl get pods -w
# STATUS が OOMKilled → CrashLoopBackOff に遷移する様子が確認できる
NAME                        READY   STATUS             RESTARTS     AGE
oom-demo-xxxxxxxxxx-xxxxx     0/1     OOMKilled          2 (15s ago)   19s
oom-demo-xxxxxxxxxx-xxxxx     0/1     CrashLoopBackOff   2 (28s ago)   47s
oom-demo-xxxxxxxxxx-xxxxx     0/1     OOMKilled          3 (29s ago)   48s
# Ctrl + C でキャンセル

# Pod の詳細を確認
kubectl describe pod -l app=oom-demo
# 以下のような出力が得られる:
# Last State: Terminated
#   Reason: OOMKilled
#   Exit Code: 137

# 例
~ $ kubectl describe pod -l app=oom-demo | grep OOMKilled -C5
...
    State:          Terminated
      Reason:       OOMKilled
      Exit Code:    137
      Started:      Fri, 06 Mar 2026 05:27:52 +0000
      Finished:     Fri, 06 Mar 2026 05:27:52 +0000
    Last State:     Terminated
      Reason:       OOMKilled
      Exit Code:    137
      Started:      Fri, 06 Mar 2026 05:27:02 +0000
      Finished:     Fri, 06 Mar 2026 05:27:02 +0000
    Ready:          False
    Restart Count:  4

aws cloudwatch describe-alarms \
  --alarm-names "EKS-OOMKill-Demo" \
  --region us-east-1 \
  --query 'MetricAlarms[0].StateValue' \
  --output text

# ALARM と表示されれば OK

  kubectl delete deployment oom-demo

  eksctl delete cluster \
    --name devops-agent-eks-demo \
    --region us-east-1

aws iam detach-role-policy \
   --role-name DevOpsAgentRole-AgentSpace \
   --policy-arn arn:aws:iam::aws:policy/AIOpsAssistantPolicy

  aws iam delete-role-policy \
    --role-name DevOpsAgentRole-AgentSpace \
    --policy-name ExpandedAIOpsPolicy

  aws iam delete-role --role-name DevOpsAgentRole-AgentSpace

aws cloudwatch delete-alarms \
  --alarm-names "EKS-OOMKill-Demo"