EC2 アプリケーションステータスチェックと Step Functions で Nginx の2段階自動復旧を試してみた

{
  "commands": [
    "sudo systemctl stop nginx",
    "printf '%s\\n' 'import socket' 's=socket.socket()' 's.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1)' 's.bind((\"0.0.0.0\",80))' 's.listen(5)' 'while True:' '    c,a=s.accept()' | sudo tee /tmp/hog.py",
    "sudo setsid nohup python3 /tmp/hog.py >/tmp/hog.log 2>&1 < /dev/null &",
    "sleep 3",
    "sudo ss -ltnp | grep ':80' || echo 'NOT LISTENING'",
    "systemctl is-active nginx || true"
  ]
}

AWSTemplateFormatVersion: '2010-09-09'
Description: 'EC2 Nginx + Application Status Check + Step Functions 2-Stage Recovery'

Parameters:
  NotificationEmail:
    Type: String
  InstanceType:
    Type: String
    Default: t4g.nano
  LatestAmiId:
    Type: AWS::SSM::Parameter::Value<AWS::EC2::Image::Id>
    Default: /aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-arm64
  LogRetentionDays:
    Type: Number
    Default: 7
  VpcId:
    Type: AWS::EC2::VPC::Id
    Description: VPC ID
  SubnetId:
    Type: AWS::EC2::Subnet::Id
    Description: Subnet ID for EC2 and Application Status Check

Resources:
  EC2SecurityGroup:
    Type: AWS::EC2::SecurityGroup
    Properties:
      GroupDescription: !Sub ${AWS::StackName} - EC2 SG
      VpcId: !Ref VpcId
      Tags:
        - Key: Name
          Value: !Sub ${AWS::StackName}-ec2-sg

  AppStatusCheckSourceSG:
    Type: AWS::EC2::SecurityGroup
    Properties:
      GroupDescription: !Sub ${AWS::StackName} - Source SG for the application status check ENI
      VpcId: !Ref VpcId
      Tags:
        - Key: Name
          Value: !Sub ${AWS::StackName}-asc-sg

  EC2SecurityGroupIngress:
    Type: AWS::EC2::SecurityGroupIngress
    Properties:
      GroupId: !Ref EC2SecurityGroup
      IpProtocol: tcp
      FromPort: 80
      ToPort: 80
      SourceSecurityGroupId: !GetAtt AppStatusCheckSourceSG.GroupId
      Description: Application status check from the managed ENI

  EC2Role:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Principal:
              Service: ec2.amazonaws.com
            Action: sts:AssumeRole
      ManagedPolicyArns:
        - arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore

  EC2InstanceProfile:
    Type: AWS::IAM::InstanceProfile
    Properties:
      Roles:
        - !Ref EC2Role

  NginxEC2Instance:
    Type: AWS::EC2::Instance
    Properties:
      InstanceType: !Ref InstanceType
      ImageId: !Ref LatestAmiId
      IamInstanceProfile: !Ref EC2InstanceProfile
      SubnetId: !Ref SubnetId
      SecurityGroupIds:
        - !Ref EC2SecurityGroup
      UserData:
        Fn::Base64: |
          #!/bin/bash
          dnf install -y nginx
          echo ok > /usr/share/nginx/html/health
          systemctl enable nginx
          systemctl start nginx
      Tags:
        - Key: Name
          Value: !Sub ${AWS::StackName}-nginx

  ApplicationStatusCheck:
    Type: AWS::EC2::ApplicationStatusCheck
    Properties:
      Protocol: http
      Port: 80
      Path: /
      StatusCodeMatcher: '200'
      InitializationGracePeriodSeconds: 60
      HealthCheckPaths:
        - Source:
            SecurityGroupId: !GetAtt AppStatusCheckSourceSG.GroupId
            SubnetId: !Ref SubnetId
          Destinations:
            - SecurityGroupId: !GetAtt EC2SecurityGroup.GroupId
              SubnetId: !Ref SubnetId
      Tags:
        - Key: Name
          Value: !Sub ${AWS::StackName}-asc

  AppStatusCheckAlarm:
    Type: AWS::CloudWatch::Alarm
    Properties:
      AlarmName: !Sub ${AWS::StackName}-StatusCheckFailed-App
      Namespace: AWS/EC2
      MetricName: StatusCheckFailed_Application
      Dimensions:
        - Name: InstanceId
          Value: !Ref NginxEC2Instance
      Statistic: Maximum
      Period: 60
      EvaluationPeriods: 2
      Threshold: 1
      ComparisonOperator: GreaterThanOrEqualToThreshold
      TreatMissingData: missing

  AlertTopic:
    Type: AWS::SNS::Topic

  EmailSubscription:
    Type: AWS::SNS::Subscription
    Properties:
      TopicArn: !Ref AlertTopic
      Protocol: email
      Endpoint: !Ref NotificationEmail

  EventBridgeRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Principal:
              Service: events.amazonaws.com
            Action: sts:AssumeRole
      Policies:
        - PolicyName: InvokeStepFunctions
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Effect: Allow
                Action: states:StartExecution
                Resource: !GetAtt RecoveryStateMachine.Arn

  RecoveryTriggerRule:
    Type: AWS::Events::Rule
    Properties:
      EventPattern:
        source:
          - aws.cloudwatch
        detail-type:
          - CloudWatch Alarm State Change
        detail:
          alarmName:
            - !Ref AppStatusCheckAlarm
          state:
            value:
              - ALARM
      State: ENABLED
      Targets:
        - Arn: !GetAtt RecoveryStateMachine.Arn
          RoleArn: !GetAtt EventBridgeRole.Arn
          Id: RecoveryStateMachineTarget

  StepFunctionsRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Principal:
              Service: states.amazonaws.com
            Action: sts:AssumeRole
      Policies:
        - PolicyName: EC2AndSSM
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Effect: Allow
                Action:
                  - ec2:StopInstances
                  - ec2:StartInstances
                Resource: !Sub arn:aws:ec2:${AWS::Region}:${AWS::AccountId}:instance/${NginxEC2Instance}
              - Effect: Allow
                Action:
                  - ec2:DescribeInstances
                Resource: '*'
              - Effect: Allow
                Action: ssm:SendCommand
                Resource:
                  - !Sub arn:aws:ssm:${AWS::Region}::document/AWS-RunShellScript
                  - !Sub arn:aws:ec2:${AWS::Region}:${AWS::AccountId}:instance/${NginxEC2Instance}
              - Effect: Allow
                Action: ssm:GetCommandInvocation
                Resource: '*'
              - Effect: Allow
                Action: sns:Publish
                Resource: !Ref AlertTopic
        - PolicyName: CloudWatchLogs
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Effect: Allow
                Action:
                  - logs:CreateLogDelivery
                  - logs:GetLogDelivery
                  - logs:UpdateLogDelivery
                  - logs:DeleteLogDelivery
                  - logs:ListLogDeliveries
                  - logs:PutLogEvents
                  - logs:PutResourcePolicy
                  - logs:DescribeResourcePolicies
                  - logs:DescribeLogGroups
                Resource: '*'

  StepFunctionsLogGroup:
    Type: AWS::Logs::LogGroup
    Properties:
      LogGroupName: !Sub /aws/vendedlogs/states/${AWS::StackName}
      RetentionInDays: !Ref LogRetentionDays

  RecoveryStateMachine:
    Type: AWS::StepFunctions::StateMachine
    Properties:
      StateMachineType: STANDARD
      RoleArn: !GetAtt StepFunctionsRole.Arn
      LoggingConfiguration:
        Level: ALL
        IncludeExecutionData: true
        Destinations:
          - CloudWatchLogsLogGroup:
              LogGroupArn: !GetAtt StepFunctionsLogGroup.Arn
      DefinitionSubstitutions:
        InstanceId: !Ref NginxEC2Instance
        SnsTopicArn: !Ref AlertTopic
      DefinitionString: |
        {
          "Comment": "2-Stage Recovery: SSM restart -> EC2 Stop/Start",
          "QueryLanguage": "JSONata",
          "TimeoutSeconds": 1800,
          "StartAt": "Init",
          "States": {
            "Init": {
              "Type": "Pass",
              "Assign": {
                "stage1Retry": 0,
                "stopCheck": 0,
                "startCheck": 0,
                "recoveryCheck": 0,
                "forceStop": false
              },
              "Next": "CheckNginxInit"
            },
            "CheckNginxInit": {
              "Type": "Task",
              "Resource": "arn:aws:states:::aws-sdk:ssm:sendCommand",
              "Arguments": {
                "InstanceIds": ["${InstanceId}"],
                "DocumentName": "AWS-RunShellScript",
                "Parameters": {"commands": ["systemctl is-active nginx"]},
                "TimeoutSeconds": 30
              },
              "Assign": {"checkCmdId": "{% $states.result.Command.CommandId %}"},
              "Next": "WaitCheckInit",
              "Catch": [{"ErrorEquals": ["States.ALL"], "Next": "SSMRestart"}]
            },
            "WaitCheckInit": {
              "Type": "Wait",
              "Seconds": 5,
              "Next": "GetCheckInitResult"
            },
            "GetCheckInitResult": {
              "Type": "Task",
              "Resource": "arn:aws:states:::aws-sdk:ssm:getCommandInvocation",
              "Arguments": {"CommandId": "{% $checkCmdId %}", "InstanceId": "${InstanceId}"},
              "Assign": {
                "nginxStatus": "{% $states.result.StandardOutputContent %}",
                "checkExitCode": "{% $states.result.ResponseCode %}"
              },
              "Next": "EvalInitStatus",
              "Catch": [{"ErrorEquals": ["States.ALL"], "Next": "SSMRestart"}]
            },
            "EvalInitStatus": {
              "Type": "Choice",
              "Choices": [{"Condition": "{% $checkExitCode = 0 %}", "Next": "NotifyAlreadyOK"}],
              "Default": "SSMRestart"
            },
            "SSMRestart": {
              "Type": "Task",
              "Resource": "arn:aws:states:::aws-sdk:ssm:sendCommand",
              "Arguments": {
                "InstanceIds": ["${InstanceId}"],
                "DocumentName": "AWS-RunShellScript",
                "Parameters": {"commands": ["sudo systemctl restart nginx"]},
                "TimeoutSeconds": 60
              },
              "Assign": {"cmdId": "{% $states.result.Command.CommandId %}"},
              "Next": "WaitSSM",
              "Catch": [{"ErrorEquals": ["States.ALL"], "Next": "Stage2Stop"}]
            },
            "WaitSSM": {
              "Type": "Wait",
              "Seconds": 10,
              "Next": "CheckSSM"
            },
            "CheckSSM": {
              "Type": "Task",
              "Resource": "arn:aws:states:::aws-sdk:ssm:getCommandInvocation",
              "Arguments": {"CommandId": "{% $cmdId %}", "InstanceId": "${InstanceId}"},
              "Assign": {"ssmStatus": "{% $states.result.Status %}"},
              "Next": "EvalSSM",
              "Catch": [{"ErrorEquals": ["States.ALL"], "Next": "Stage2Stop"}]
            },
            "EvalSSM": {
              "Type": "Choice",
              "Choices": [
                {"Condition": "{% $ssmStatus = 'Success' %}", "Next": "WaitAfterRestart"},
                {"Condition": "{% $ssmStatus = 'Failed' or $ssmStatus = 'TimedOut' %}", "Next": "Stage2Stop"}
              ],
              "Default": "WaitSSM"
            },
            "WaitAfterRestart": {
              "Type": "Wait",
              "Seconds": 10,
              "Next": "CheckNginxAfterRestart"
            },
            "CheckNginxAfterRestart": {
              "Type": "Task",
              "Resource": "arn:aws:states:::aws-sdk:ssm:sendCommand",
              "Arguments": {
                "InstanceIds": ["${InstanceId}"],
                "DocumentName": "AWS-RunShellScript",
                "Parameters": {"commands": ["systemctl is-active nginx"]},
                "TimeoutSeconds": 30
              },
              "Assign": {"checkCmdId": "{% $states.result.Command.CommandId %}"},
              "Next": "WaitCheckAfterRestart",
              "Catch": [{"ErrorEquals": ["States.ALL"], "Next": "Stage2Stop"}]
            },
            "WaitCheckAfterRestart": {
              "Type": "Wait",
              "Seconds": 5,
              "Next": "GetCheckAfterRestartResult"
            },
            "GetCheckAfterRestartResult": {
              "Type": "Task",
              "Resource": "arn:aws:states:::aws-sdk:ssm:getCommandInvocation",
              "Arguments": {"CommandId": "{% $checkCmdId %}", "InstanceId": "${InstanceId}"},
              "Assign": {
                "checkExitCode": "{% $states.result.ResponseCode %}",
                "stage1Retry": "{% $stage1Retry + 1 %}"
              },
              "Next": "EvalStage1",
              "Catch": [{"ErrorEquals": ["States.ALL"], "Next": "Stage2Stop"}]
            },
            "EvalStage1": {
              "Type": "Choice",
              "Choices": [
                {"Condition": "{% $checkExitCode = 0 %}", "Next": "NotifyStage1OK"},
                {"Condition": "{% $stage1Retry >= 3 %}", "Next": "Stage2Stop"}
              ],
              "Default": "WaitAfterRestart"
            },
            "Stage2Stop": {
              "Type": "Task",
              "Resource": "arn:aws:states:::aws-sdk:ec2:stopInstances",
              "Arguments": {"InstanceIds": ["${InstanceId}"], "Force": false},
              "Assign": {"stopCheck": 0},
              "Next": "WaitStop",
              "Catch": [{"ErrorEquals": ["States.ALL"], "Next": "NotifyError"}]
            },
            "ForceStop": {
              "Type": "Task",
              "Resource": "arn:aws:states:::aws-sdk:ec2:stopInstances",
              "Arguments": {"InstanceIds": ["${InstanceId}"], "Force": true},
              "Assign": {"forceStop": true, "stopCheck": 0},
              "Next": "WaitStop",
              "Catch": [{"ErrorEquals": ["States.ALL"], "Next": "NotifyError"}]
            },
            "WaitStop": {
              "Type": "Wait",
              "Seconds": 15,
              "Next": "CheckStop"
            },
            "CheckStop": {
              "Type": "Task",
              "Resource": "arn:aws:states:::aws-sdk:ec2:describeInstances",
              "Arguments": {"InstanceIds": ["${InstanceId}"]},
              "Assign": {
                "ec2State": "{% $states.result.Reservations[0].Instances[0].State.Name %}",
                "stopCheck": "{% $stopCheck + 1 %}"
              },
              "Next": "EvalStop",
              "Catch": [{"ErrorEquals": ["States.ALL"], "Next": "NotifyError"}]
            },
            "EvalStop": {
              "Type": "Choice",
              "Choices": [
                {"Condition": "{% $ec2State = 'stopped' %}", "Next": "StartInstance"},
                {"Condition": "{% $stopCheck >= 40 and $forceStop = false %}", "Next": "ForceStop"},
                {"Condition": "{% $stopCheck >= 40 and $forceStop = true %}", "Next": "NotifyStopFail"}
              ],
              "Default": "WaitStop"
            },
            "StartInstance": {
              "Type": "Task",
              "Resource": "arn:aws:states:::aws-sdk:ec2:startInstances",
              "Arguments": {"InstanceIds": ["${InstanceId}"]},
              "Assign": {"startCheck": 0},
              "Next": "WaitStart",
              "Catch": [{"ErrorEquals": ["States.ALL"], "Next": "NotifyError"}]
            },
            "WaitStart": {
              "Type": "Wait",
              "Seconds": 15,
              "Next": "CheckStart"
            },
            "CheckStart": {
              "Type": "Task",
              "Resource": "arn:aws:states:::aws-sdk:ec2:describeInstances",
              "Arguments": {"InstanceIds": ["${InstanceId}"]},
              "Assign": {
                "ec2State": "{% $states.result.Reservations[0].Instances[0].State.Name %}",
                "startCheck": "{% $startCheck + 1 %}"
              },
              "Next": "EvalStart",
              "Catch": [{"ErrorEquals": ["States.ALL"], "Next": "NotifyError"}]
            },
            "EvalStart": {
              "Type": "Choice",
              "Choices": [
                {"Condition": "{% $ec2State = 'running' %}", "Next": "WaitRecovery"},
                {"Condition": "{% $startCheck >= 20 %}", "Next": "NotifyFail"}
              ],
              "Default": "WaitStart"
            },
            "WaitRecovery": {
              "Type": "Wait",
              "Seconds": 30,
              "Next": "CheckNginxRecovery"
            },
            "CheckNginxRecovery": {
              "Type": "Task",
              "Resource": "arn:aws:states:::aws-sdk:ssm:sendCommand",
              "Arguments": {
                "InstanceIds": ["${InstanceId}"],
                "DocumentName": "AWS-RunShellScript",
                "Parameters": {"commands": ["systemctl is-active nginx"]},
                "TimeoutSeconds": 30
              },
              "Assign": {"checkCmdId": "{% $states.result.Command.CommandId %}"},
              "Next": "WaitCheckRecovery",
              "Catch": [{"ErrorEquals": ["States.ALL"], "Next": "RetryRecoveryCheck"}]
            },
            "WaitCheckRecovery": {
              "Type": "Wait",
              "Seconds": 5,
              "Next": "GetCheckRecoveryResult"
            },
            "GetCheckRecoveryResult": {
              "Type": "Task",
              "Resource": "arn:aws:states:::aws-sdk:ssm:getCommandInvocation",
              "Arguments": {"CommandId": "{% $checkCmdId %}", "InstanceId": "${InstanceId}"},
              "Assign": {
                "checkExitCode": "{% $states.result.ResponseCode %}",
                "recoveryCheck": "{% $recoveryCheck + 1 %}"
              },
              "Next": "EvalRecovery",
              "Catch": [{"ErrorEquals": ["States.ALL"], "Next": "RetryRecoveryCheck"}]
            },
            "RetryRecoveryCheck": {
              "Type": "Pass",
              "Assign": {"recoveryCheck": "{% $recoveryCheck + 1 %}"},
              "Next": "EvalRecoveryRetry"
            },
            "EvalRecoveryRetry": {
              "Type": "Choice",
              "Choices": [{"Condition": "{% $recoveryCheck >= 10 %}", "Next": "NotifyFail"}],
              "Default": "WaitRecovery"
            },
            "EvalRecovery": {
              "Type": "Choice",
              "Choices": [
                {"Condition": "{% $checkExitCode = 0 %}", "Next": "NotifyStage2OK"},
                {"Condition": "{% $recoveryCheck >= 10 %}", "Next": "NotifyFail"}
              ],
              "Default": "WaitRecovery"
            },
            "NotifyStage1OK": {
              "Type": "Task",
              "Resource": "arn:aws:states:::aws-sdk:sns:publish",
              "Arguments": {
                "TopicArn": "${SnsTopicArn}",
                "Subject": "[OK] Nginx復旧 (SSM restart)",
                "Message": "{% 'SSM restart で復旧しました。\\nInstanceId: ${InstanceId}\\nRetry: ' & $string($stage1Retry) & '\\nExecutionId: ' & $states.context.Execution.Id %}"
              },
              "End": true
            },
            "NotifyStage2OK": {
              "Type": "Task",
              "Resource": "arn:aws:states:::aws-sdk:sns:publish",
              "Arguments": {
                "TopicArn": "${SnsTopicArn}",
                "Subject": "[OK] EC2復旧 (Stop/Start)",
                "Message": "{% 'EC2 Stop/Start で復旧しました。\\nInstanceId: ${InstanceId}\\nRecoveryCheck: ' & $string($recoveryCheck) & '\\nExecutionId: ' & $states.context.Execution.Id %}"
              },
              "End": true
            },
            "NotifyAlreadyOK": {
              "Type": "Task",
              "Resource": "arn:aws:states:::aws-sdk:sns:publish",
              "Arguments": {
                "TopicArn": "${SnsTopicArn}",
                "Subject": "[INFO] 自然復旧",
                "Message": "{% 'Nginxは既に正常です。\\nInstanceId: ${InstanceId}\\nExecutionId: ' & $states.context.Execution.Id %}"
              },
              "End": true
            },
            "NotifyFail": {
              "Type": "Task",
              "Resource": "arn:aws:states:::aws-sdk:sns:publish",
              "Arguments": {
                "TopicArn": "${SnsTopicArn}",
                "Subject": "[CRITICAL] 復旧失敗",
                "Message": "{% '自動復旧に失敗しました。手動確認が必要です。\\nInstanceId: ${InstanceId}\\nExecutionId: ' & $states.context.Execution.Id %}"
              },
              "End": true
            },
            "NotifyStopFail": {
              "Type": "Task",
              "Resource": "arn:aws:states:::aws-sdk:sns:publish",
              "Arguments": {
                "TopicArn": "${SnsTopicArn}",
                "Subject": "[CRITICAL] EC2停止失敗",
                "Message": "{% 'EC2の停止に失敗しました。\\nInstanceId: ${InstanceId}\\nExecutionId: ' & $states.context.Execution.Id %}"
              },
              "End": true
            },
            "NotifyError": {
              "Type": "Task",
              "Resource": "arn:aws:states:::aws-sdk:sns:publish",
              "Arguments": {
                "TopicArn": "${SnsTopicArn}",
                "Subject": "[ERROR] 予期せぬエラー",
                "Message": "{% 'エラーが発生しました。\\nInstanceId: ${InstanceId}\\nExecutionId: ' & $states.context.Execution.Id %}"
              },
              "End": true
            }
          }
        }

Outputs:
  InstanceId:
    Value: !Ref NginxEC2Instance
  ApplicationStatusCheckId:
    Value: !GetAtt ApplicationStatusCheck.ApplicationStatusCheckId
  AppStatusCheckSourceSGId:
    Value: !Ref AppStatusCheckSourceSG
  SNSTopicArn:
    Value: !Ref AlertTopic
  StateMachineArn:
    Value: !GetAtt RecoveryStateMachine.Arn
  StepFunctionsLogGroupName:
    Value: !Ref StepFunctionsLogGroup