Status Check に失敗した EC2 を Lambda なしの Step Functions で自動復旧させてみた

AWSTemplateFormatVersion: '2010-09-09'
Description: 'EC2 Stop/Start Auto Recovery Test Environment'

Parameters:
  ProjectName:
    Type: String
    Default: ec2-recovery-test
  NotificationEmail:
    Type: String
    Default: alerts@example.com
  InstanceType:
    Type: String
    Default: t4g.nano
  LogRetentionDays:
    Type: Number
    Default: 7
  LatestAmiId:
    Type: AWS::SSM::Parameter::Value<AWS::EC2::Image::Id>
    Default: /aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-arm64

Resources:
  # ============================================================
  # EC2 Security Group (uses default VPC)
  # ============================================================
  EC2SecurityGroup:
    Type: AWS::EC2::SecurityGroup
    Properties:
      GroupDescription: !Sub ${ProjectName} - Outbound only for SSM
      Tags:
        - Key: Name
          Value: !Sub ${ProjectName}-sg

  # ============================================================
  # EC2 IAM Role and Instance Profile
  # ============================================================
  EC2Role:
    Type: AWS::IAM::Role
    Properties:
      RoleName: !Sub ${ProjectName}-ec2-role
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Principal:
              Service: ec2.amazonaws.com
            Action: sts:AssumeRole
      ManagedPolicyArns:
        - arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore

  EC2InstanceProfile:
    Type: AWS::IAM::InstanceProfile
    Properties:
      InstanceProfileName: !Sub ${ProjectName}-ec2-profile
      Roles:
        - !Ref EC2Role

  # ============================================================
  # EC2 Instance
  # ============================================================
  TestEC2Instance:
    Type: AWS::EC2::Instance
    Properties:
      InstanceType: !Ref InstanceType
      ImageId: !Ref LatestAmiId
      IamInstanceProfile: !Ref EC2InstanceProfile
      SecurityGroupIds:
        - !Ref EC2SecurityGroup
      Tags:
        - Key: Name
          Value: !Sub ${ProjectName}-test-ec2

  # ============================================================
  # SNS Topic and Subscription
  # ============================================================
  AlertTopic:
    Type: AWS::SNS::Topic
    Properties:
      TopicName: !Sub ${ProjectName}-alerts

  EmailSubscription:
    Type: AWS::SNS::Subscription
    Properties:
      TopicArn: !Ref AlertTopic
      Protocol: email
      Endpoint: !Ref NotificationEmail

  # ============================================================
  # CloudWatch Alarm
  # ============================================================
  StatusCheckAlarm:
    Type: AWS::CloudWatch::Alarm
    Properties:
      AlarmName: !Sub ${ProjectName}-StatusCheckFailed-Instance
      AlarmDescription: Alarm when StatusCheckFailed_Instance is >= 1
      Namespace: AWS/EC2
      MetricName: StatusCheckFailed_Instance
      Dimensions:
        - Name: InstanceId
          Value: !Ref TestEC2Instance
      Statistic: Maximum
      Period: 60
      EvaluationPeriods: 2
      Threshold: 1
      ComparisonOperator: GreaterThanOrEqualToThreshold
      TreatMissingData: missing

  # ============================================================
  # EventBridge IAM Role
  # ============================================================
  EventBridgeRole:
    Type: AWS::IAM::Role
    Properties:
      RoleName: !Sub ${ProjectName}-eventbridge-role
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Principal:
              Service: events.amazonaws.com
            Action: sts:AssumeRole
      Policies:
        - PolicyName: InvokeStepFunctions
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Effect: Allow
                Action: states:StartExecution
                Resource: !GetAtt RecoveryStateMachine.Arn

  # ============================================================
  # EventBridge Rule
  # ============================================================
  RecoveryTriggerRule:
    Type: AWS::Events::Rule
    Properties:
      Name: !Sub ${ProjectName}-recovery-trigger
      Description: Trigger Step Functions when StatusCheckFailed alarm goes to ALARM
      EventPattern:
        source:
          - aws.cloudwatch
        detail-type:
          - CloudWatch Alarm State Change
        detail:
          alarmName:
            - !Ref StatusCheckAlarm
          state:
            value:
              - ALARM
      State: ENABLED
      Targets:
        - Arn: !GetAtt RecoveryStateMachine.Arn
          RoleArn: !GetAtt EventBridgeRole.Arn
          Id: RecoveryStateMachineTarget

  # ============================================================
  # Step Functions IAM Role
  # ============================================================
  StepFunctionsRole:
    Type: AWS::IAM::Role
    Properties:
      RoleName: !Sub ${ProjectName}-stepfunctions-role
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Principal:
              Service: states.amazonaws.com
            Action: sts:AssumeRole
      Policies:
        - PolicyName: EC2Operations
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Effect: Allow
                Action:
                  - ec2:StopInstances
                  - ec2:StartInstances
                Resource: !Sub arn:aws:ec2:${AWS::Region}:${AWS::AccountId}:instance/${TestEC2Instance}
              - Effect: Allow
                Action:
                  - ec2:DescribeInstances
                  - ec2:DescribeInstanceStatus
                Resource: '*'
        - PolicyName: SNSPublish
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Effect: Allow
                Action: sns:Publish
                Resource: !Ref AlertTopic
        - PolicyName: CloudWatchLogs
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Effect: Allow
                Action:
                  - logs:CreateLogDelivery
                  - logs:GetLogDelivery
                  - logs:UpdateLogDelivery
                  - logs:DeleteLogDelivery
                  - logs:ListLogDeliveries
                  - logs:PutLogEvents
                  - logs:PutResourcePolicy
                  - logs:DescribeResourcePolicies
                  - logs:DescribeLogGroups
                Resource: '*'

  # ============================================================
  # Step Functions Log Group
  # ============================================================
  StepFunctionsLogGroup:
    Type: AWS::Logs::LogGroup
    Properties:
      LogGroupName: !Sub /aws/vendedlogs/states/${ProjectName}-ec2-recovery
      RetentionInDays: !Ref LogRetentionDays

  # ============================================================
  # Step Functions State Machine
  # ============================================================
  RecoveryStateMachine:
    Type: AWS::StepFunctions::StateMachine
    Properties:
      StateMachineName: !Sub ${ProjectName}-ec2-recovery
      StateMachineType: STANDARD
      RoleArn: !GetAtt StepFunctionsRole.Arn
      LoggingConfiguration:
        Level: ALL
        IncludeExecutionData: true
        Destinations:
          - CloudWatchLogsLogGroup:
              LogGroupArn: !GetAtt StepFunctionsLogGroup.Arn
      DefinitionSubstitutions:
        InstanceId: !Ref TestEC2Instance
        SnsTopicArn: !Ref AlertTopic
      DefinitionString: |
        {
          "Comment": "EC2 Stop/Start Auto Recovery Workflow",
          "QueryLanguage": "JSONata",
          "TimeoutSeconds": 2400,
          "StartAt": "InitializeVariables",
          "States": {
            "InitializeVariables": {
              "Type": "Pass",
              "Assign": {
                "stopCheckCount": 0,
                "startCheckCount": 0,
                "recoveryCheckCount": 0,
                "forceStopAttempted": false,
                "errorInfo": null
              },
              "Next": "GetEC2State"
            },
            "GetEC2State": {
              "Type": "Task",
              "Resource": "arn:aws:states:::aws-sdk:ec2:describeInstances",
              "Arguments": {
                "InstanceIds": ["${InstanceId}"]
              },
              "Assign": {
                "ec2State": "{% $states.result.Reservations[0].Instances[0].State.Name %}"
              },
              "Next": "GetEC2StatusCheck",
              "Catch": [
                {
                  "ErrorEquals": ["States.ALL"],
                  "Next": "NotifyUnexpectedError",
                  "Output": {
                    "errorInfo": "{% $states.errorOutput %}"
                  }
                }
              ]
            },
            "GetEC2StatusCheck": {
              "Type": "Task",
              "Resource": "arn:aws:states:::aws-sdk:ec2:describeInstanceStatus",
              "Arguments": {
                "InstanceIds": ["${InstanceId}"]
              },
              "Assign": {
                "instanceStatus": "{% $count($states.result.InstanceStatuses) > 0 ? $states.result.InstanceStatuses[0].InstanceStatus.Status : 'unknown' %}",
                "systemStatus": "{% $count($states.result.InstanceStatuses) > 0 ? $states.result.InstanceStatuses[0].SystemStatus.Status : 'unknown' %}"
              },
              "Next": "EvaluateState",
              "Catch": [
                {
                  "ErrorEquals": ["States.ALL"],
                  "Next": "NotifyUnexpectedError",
                  "Output": {
                    "errorInfo": "{% $states.errorOutput %}"
                  }
                }
              ]
            },
            "EvaluateState": {
              "Type": "Choice",
              "Choices": [
                {
                  "Condition": "{% $systemStatus = 'impaired' %}",
                  "Next": "EscalateSystem"
                },
                {
                  "Condition": "{% $instanceStatus = 'ok' and $systemStatus = 'ok' %}",
                  "Next": "AlreadyRecovered"
                },
                {
                  "Condition": "{% $ec2State = 'terminated' or $ec2State = 'shutting-down' %}",
                  "Next": "EscalateTerminated"
                },
                {
                  "Condition": "{% $ec2State = 'stopping' %}",
                  "Next": "WaitForStop"
                },
                {
                  "Condition": "{% $ec2State = 'stopped' %}",
                  "Next": "StartInstance"
                },
                {
                  "Condition": "{% $ec2State = 'pending' %}",
                  "Next": "WaitForStart"
                },
                {
                  "Condition": "{% $ec2State = 'running' and $instanceStatus = 'impaired' and $systemStatus = 'ok' %}",
                  "Next": "StopInstance"
                }
              ],
              "Default": "StopInstance"
            },
            "StopInstance": {
              "Type": "Task",
              "Resource": "arn:aws:states:::aws-sdk:ec2:stopInstances",
              "Arguments": {
                "InstanceIds": ["${InstanceId}"],
                "Force": false
              },
              "Assign": {
                "stopCheckCount": 0
              },
              "Next": "WaitForStop",
              "Catch": [
                {
                  "ErrorEquals": ["States.ALL"],
                  "Next": "NotifyUnexpectedError",
                  "Output": {
                    "errorInfo": "{% $states.errorOutput %}"
                  }
                }
              ]
            },
            "ForceStopInstance": {
              "Type": "Task",
              "Resource": "arn:aws:states:::aws-sdk:ec2:stopInstances",
              "Arguments": {
                "InstanceIds": ["${InstanceId}"],
                "Force": true
              },
              "Assign": {
                "forceStopAttempted": true,
                "stopCheckCount": 0
              },
              "Next": "WaitForStop",
              "Catch": [
                {
                  "ErrorEquals": ["States.ALL"],
                  "Next": "NotifyUnexpectedError",
                  "Output": {
                    "errorInfo": "{% $states.errorOutput %}"
                  }
                }
              ]
            },
            "WaitForStop": {
              "Type": "Wait",
              "Seconds": 15,
              "Next": "CheckStopStatus"
            },
            "CheckStopStatus": {
              "Type": "Task",
              "Resource": "arn:aws:states:::aws-sdk:ec2:describeInstances",
              "Arguments": {
                "InstanceIds": ["${InstanceId}"]
              },
              "Assign": {
                "ec2State": "{% $states.result.Reservations[0].Instances[0].State.Name %}",
                "stopCheckCount": "{% $stopCheckCount + 1 %}"
              },
              "Next": "EvaluateStopStatus",
              "Catch": [
                {
                  "ErrorEquals": ["States.ALL"],
                  "Next": "NotifyUnexpectedError",
                  "Output": {
                    "errorInfo": "{% $states.errorOutput %}"
                  }
                }
              ]
            },
            "EvaluateStopStatus": {
              "Type": "Choice",
              "Choices": [
                {
                  "Condition": "{% $ec2State = 'stopped' %}",
                  "Next": "StartInstance"
                },
                {
                  "Condition": "{% $stopCheckCount >= 40 and $forceStopAttempted = false %}",
                  "Next": "ForceStopInstance"
                },
                {
                  "Condition": "{% $stopCheckCount >= 40 and $forceStopAttempted = true %}",
                  "Next": "StopTimeout"
                }
              ],
              "Default": "WaitForStop"
            },
            "StartInstance": {
              "Type": "Task",
              "Resource": "arn:aws:states:::aws-sdk:ec2:startInstances",
              "Arguments": {
                "InstanceIds": ["${InstanceId}"]
              },
              "Assign": {
                "startCheckCount": 0
              },
              "Next": "WaitForStart",
              "Catch": [
                {
                  "ErrorEquals": ["States.ALL"],
                  "Next": "NotifyUnexpectedError",
                  "Output": {
                    "errorInfo": "{% $states.errorOutput %}"
                  }
                }
              ]
            },
            "WaitForStart": {
              "Type": "Wait",
              "Seconds": 15,
              "Next": "CheckStartStatus"
            },
            "CheckStartStatus": {
              "Type": "Task",
              "Resource": "arn:aws:states:::aws-sdk:ec2:describeInstances",
              "Arguments": {
                "InstanceIds": ["${InstanceId}"]
              },
              "Assign": {
                "ec2State": "{% $states.result.Reservations[0].Instances[0].State.Name %}",
                "startCheckCount": "{% $startCheckCount + 1 %}"
              },
              "Next": "EvaluateStartStatus",
              "Catch": [
                {
                  "ErrorEquals": ["States.ALL"],
                  "Next": "NotifyUnexpectedError",
                  "Output": {
                    "errorInfo": "{% $states.errorOutput %}"
                  }
                }
              ]
            },
            "EvaluateStartStatus": {
              "Type": "Choice",
              "Choices": [
                {
                  "Condition": "{% $ec2State = 'running' %}",
                  "Next": "WaitForRecovery"
                },
                {
                  "Condition": "{% $startCheckCount >= 20 %}",
                  "Next": "RecoveryFailed"
                }
              ],
              "Default": "WaitForStart"
            },
            "WaitForRecovery": {
              "Type": "Wait",
              "Seconds": 15,
              "Next": "CheckRecoveryStatus"
            },
            "CheckRecoveryStatus": {
              "Type": "Task",
              "Resource": "arn:aws:states:::aws-sdk:ec2:describeInstanceStatus",
              "Arguments": {
                "InstanceIds": ["${InstanceId}"]
              },
              "Assign": {
                "instanceStatus": "{% $count($states.result.InstanceStatuses) > 0 ? $states.result.InstanceStatuses[0].InstanceStatus.Status : 'unknown' %}",
                "systemStatus": "{% $count($states.result.InstanceStatuses) > 0 ? $states.result.InstanceStatuses[0].SystemStatus.Status : 'unknown' %}",
                "recoveryCheckCount": "{% $recoveryCheckCount + 1 %}"
              },
              "Next": "EvaluateRecoveryStatus",
              "Catch": [
                {
                  "ErrorEquals": ["States.ALL"],
                  "Next": "NotifyUnexpectedError",
                  "Output": {
                    "errorInfo": "{% $states.errorOutput %}"
                  }
                }
              ]
            },
            "EvaluateRecoveryStatus": {
              "Type": "Choice",
              "Choices": [
                {
                  "Condition": "{% $instanceStatus = 'ok' and $systemStatus = 'ok' %}",
                  "Next": "RecoverySuccess"
                },
                {
                  "Condition": "{% $recoveryCheckCount >= 40 %}",
                  "Next": "RecoveryFailed"
                }
              ],
              "Default": "WaitForRecovery"
            },
            "RecoverySuccess": {
              "Type": "Task",
              "Resource": "arn:aws:states:::aws-sdk:sns:publish",
              "Arguments": {
                "TopicArn": "${SnsTopicArn}",
                "Subject": "[TEST] EC2自動復旧成功",
                "Message": "{% 'EC2 Stop/Start による自動復旧が完了しました。\\n\\nInstanceId: ${InstanceId}\\nExecutionId: ' & $states.context.Execution.Id & '\\nFinal State: ' & $ec2State & '\\nInstance Status: ' & $instanceStatus & '\\nSystem Status: ' & $systemStatus %}"
              },
              "End": true
            },
            "RecoveryFailed": {
              "Type": "Task",
              "Resource": "arn:aws:states:::aws-sdk:sns:publish",
              "Arguments": {
                "TopicArn": "${SnsTopicArn}",
                "Subject": "[TEST] EC2自動復旧失敗",
                "Message": "{% 'EC2 Stop/Start による自動復旧がタイムアウトしました。手動確認が必要です。\\n\\nInstanceId: ${InstanceId}\\nExecutionId: ' & $states.context.Execution.Id & '\\nFinal State: ' & $ec2State & '\\nInstance Status: ' & $instanceStatus & '\\nSystem Status: ' & $systemStatus %}"
              },
              "End": true
            },
            "StopTimeout": {
              "Type": "Task",
              "Resource": "arn:aws:states:::aws-sdk:sns:publish",
              "Arguments": {
                "TopicArn": "${SnsTopicArn}",
                "Subject": "[TEST] EC2停止失敗",
                "Message": "{% 'EC2 インスタンスの停止がタイムアウトしました（Force停止後も失敗）。手動確認が必要です。\\n\\nInstanceId: ${InstanceId}\\nExecutionId: ' & $states.context.Execution.Id & '\\nFinal State: ' & $ec2State %}"
              },
              "End": true
            },
            "AlreadyRecovered": {
              "Type": "Task",
              "Resource": "arn:aws:states:::aws-sdk:sns:publish",
              "Arguments": {
                "TopicArn": "${SnsTopicArn}",
                "Subject": "[TEST] 自然復旧確認",
                "Message": "{% 'EC2 インスタンスは既に正常状態です（自然復旧）。\\n\\nInstanceId: ${InstanceId}\\nExecutionId: ' & $states.context.Execution.Id & '\\nInstance Status: ' & $instanceStatus & '\\nSystem Status: ' & $systemStatus %}"
              },
              "End": true
            },
            "EscalateSystem": {
              "Type": "Task",
              "Resource": "arn:aws:states:::aws-sdk:sns:publish",
              "Arguments": {
                "TopicArn": "${SnsTopicArn}",
                "Subject": "[TEST] System障害",
                "Message": "{% 'SystemStatus が impaired です。Stop/Start では復旧できない可能性があります。\\n\\nInstanceId: ${InstanceId}\\nExecutionId: ' & $states.context.Execution.Id & '\\nInstance Status: ' & $instanceStatus & '\\nSystem Status: ' & $systemStatus %}"
              },
              "End": true
            },
            "EscalateTerminated": {
              "Type": "Task",
              "Resource": "arn:aws:states:::aws-sdk:sns:publish",
              "Arguments": {
                "TopicArn": "${SnsTopicArn}",
                "Subject": "[TEST] インスタンス終了",
                "Message": "{% 'EC2 インスタンスが terminated または shutting-down 状態です。復旧不可能です。\\n\\nInstanceId: ${InstanceId}\\nExecutionId: ' & $states.context.Execution.Id & '\\nState: ' & $ec2State %}"
              },
              "End": true
            },
            "NotifyUnexpectedError": {
              "Type": "Task",
              "Resource": "arn:aws:states:::aws-sdk:sns:publish",
              "Arguments": {
                "TopicArn": "${SnsTopicArn}",
                "Subject": "[TEST] 予期せぬエラー",
                "Message": "{% 'Step Functions 実行中に予期せぬエラーが発生しました。\\n\\nInstanceId: ${InstanceId}\\nExecutionId: ' & $states.context.Execution.Id & '\\nError: ' & $string($errorInfo) %}"
              },
              "End": true
            }
          }
        }

  # ============================================================
  # FIS IAM Role
  # ============================================================
  FISRole:
    Type: AWS::IAM::Role
    Properties:
      RoleName: !Sub ${ProjectName}-fis-role
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Principal:
              Service: fis.amazonaws.com
            Action: sts:AssumeRole
      Policies:
        - PolicyName: FISSSMPolicy
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Effect: Allow
                Action:
                  - ssm:SendCommand
                  - ssm:GetCommandInvocation
                  - ssm:ListCommands
                  - ssm:ListCommandInvocations
                  - ssm:CancelCommand
                Resource: '*'
              - Effect: Allow
                Action:
                  - ec2:DescribeInstances
                  - ec2:DescribeInstanceStatus
                Resource: '*'

  # ============================================================
  # FIS Experiment Template
  # ============================================================
  NicDisableExperiment:
    Type: AWS::FIS::ExperimentTemplate
    Properties:
      Description: NIC disable to trigger StatusCheckFailed_Instance
      Actions:
        DisableNic:
          ActionId: aws:ssm:send-command
          Parameters:
            documentArn: !Sub arn:aws:ssm:${AWS::Region}::document/AWS-RunShellScript
            documentParameters: '{"commands":["sudo ip link set ens5 down"]}'
            duration: PT1M
          Targets:
            Instances: TargetEC2
      Targets:
        TargetEC2:
          ResourceType: aws:ec2:instance
          ResourceArns:
            - !Sub arn:aws:ec2:${AWS::Region}:${AWS::AccountId}:instance/${TestEC2Instance}
          SelectionMode: ALL
      StopConditions:
        - Source: none
      RoleArn: !GetAtt FISRole.Arn
      Tags:
        Name: !Sub ${ProjectName}-nic-disable

Outputs:
  EC2InstanceId:
    Description: EC2 Instance ID
    Value: !Ref TestEC2Instance
  EC2InstancePublicDnsName:
    Description: EC2 Instance Public DNS Name
    Value: !GetAtt TestEC2Instance.PublicDnsName
  SecurityGroupId:
    Description: Security Group ID
    Value: !Ref EC2SecurityGroup
  SNSTopicArn:
    Description: SNS Topic ARN
    Value: !Ref AlertTopic
  CloudWatchAlarmName:
    Description: CloudWatch Alarm Name
    Value: !Ref StatusCheckAlarm
  EventBridgeRuleName:
    Description: EventBridge Rule Name
    Value: !Ref RecoveryTriggerRule
  StepFunctionsStateMachineArn:
    Description: Step Functions State Machine ARN
    Value: !GetAtt RecoveryStateMachine.Arn
  StepFunctionsLogGroupName:
    Description: Step Functions Log Group Name
    Value: !Ref StepFunctionsLogGroup
  FISExperimentTemplateId:
    Description: FIS Experiment Template ID
    Value: !Ref NicDisableExperiment

# デプロイ
aws cloudformation create-stack \
  --stack-name ec2-recovery-test \
  --template-body file://ec2-recovery-test.yaml \
  --parameters ParameterKey=NotificationEmail,ParameterValue=<通知先メールアドレス> \
  --capabilities CAPABILITY_NAMED_IAM \
  --deployment-config '{"mode": "EXPRESS"}' \
  --region ap-northeast-1

aws cloudformation wait stack-create-complete \
  --stack-name ec2-recovery-test \
  --region ap-northeast-1

# FIS で NIC を無効化して StatusCheckFailed_Instance を発生させる
aws fis start-experiment \
  --experiment-template-id <FISExperimentTemplateId の出力値> \
  --region ap-northeast-1

# EventBridge から Step Functions までの連携だけを試す場合
aws cloudwatch set-alarm-state \
  --alarm-name ec2-recovery-test-StatusCheckFailed-Instance \
  --state-value ALARM \
  --state-reason "Manual test trigger" \
  --region ap-northeast-1

# 撤去
aws cloudformation delete-stack \
  --stack-name ec2-recovery-test \
  --region ap-northeast-1