AWS Batchの基礎が一通り学べるハンズオン「AWS Batch Deep Dive」やってみた

			
			cat <<EOF > s3policy.yaml
# Use as follow:
#   aws cloudformation create-stack --stack-name 'BatchWorkshop' \\
#                                   --template-body file://s3policy.yaml \\
#                                   --capabilities "CAPABILITY_IAM"
AWSTemplateFormatVersion: '2010-09-09'
Description: 'Create an S3 bucket and an ECS Task Role to access this bucket
  in order to write configuration data, read command arguments and write output data.'

Metadata:
  'AWS::CloudFormation::Interface':
    ParameterGroups:
    - Label:
        default: 'Parent Stacks'
      Parameters:
      - BucketName
Parameters:
    BucketName:
      Description: 'Optional name of the bucket.'
      Type: String
      Default: ''
Conditions:
  HasBucketName: !Not [!Equals [!Ref BucketName, '']]

Resources:
  # Set up a Role for Batch job execution.
  JobExecutionRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Principal:
              Service:
                - ecs-tasks.amazonaws.com
            Action:
              - sts:AssumeRole
      ManagedPolicyArns:
        - arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy

  # Set up an S3 Bucket, Policy and Role for tasks to read/write from/to the output bucket
  Bucket:
    Type: AWS::S3::Bucket
    Properties:
      BucketName: !If [HasBucketName, !Ref BucketName, !Ref 'AWS::NoValue']

  # Set up a Policy that allows access to the S3 Bucket and attach it to the Role.
  BucketPolicy:
    Type: AWS::IAM::Policy
    Properties:
      PolicyName: BucketPolicy
      PolicyDocument:
          Version: 2012-10-17
          Statement:
          - Effect: Allow
            Action:
              - 's3:AbortMultipartUpload'
              - 's3:GetBucketLocation'
              - 's3:GetObject'
              - 's3:ListBucket'
              - 's3:ListBucketMultipartUploads'
              - 's3:PutObject'
            Resource:
              - !Join [ "", [ "arn:aws:s3:::", !Ref 'Bucket' ] ]
              - !Join [ "", [ "arn:aws:s3:::", !Ref 'Bucket' , "/*" ] ]

      Roles:
        - !Ref 'JobExecutionRole'

Outputs:
  JobExecutionRole:
    Description: ECS Task Execution Role for AWS Batch Jobs
    Value: !Ref JobExecutionRole
  Bucket:
    Description: Bucket in which configuration and output will be written
    Value: !Ref Bucket
  BucketPolicy:
    Description: ECS Task Policy for S3 access to S3 Bucket
    Value: !Ref BucketPolicy
EOF

		

			
			cat > cfn-pre-requisites.sh << EOF
#!/bin/bash

STACK_NAME='BatchWorkshop'

# create the stack
echo "1/3 - Create Stack \$STACK_NAME"
aws cloudformation create-stack --stack-name \$STACK_NAME --template-body file://s3policy.yaml --capabilities "CAPABILITY_IAM"

echo "2/3 - Stack \$STACK_NAME being created"
# wait for the stack to be completed
aws cloudformation wait stack-create-complete --stack-name \$STACK_NAME

echo "3/3 - Stack \$STACK_NAME created"
# once done get the role and bucket
export EXECUTION_ROLE=\$(aws cloudformation describe-stacks --stack-name \$STACK_NAME --output text --query 'Stacks[0].Outputs[?OutputKey == \`JobExecutionRole\`].OutputValue')
export STRESS_BUCKET="s3://\$(aws cloudformation describe-stacks --stack-name \$STACK_NAME --output text --query 'Stacks[0].Outputs[?OutputKey == \`Bucket\`].OutputValue')"

echo "Use the following Job Execution Role ID with AWS Batch: \${EXECUTION_ROLE}"
echo "Use the following S3 Bucket for your AWS Batch jobs: \${STRESS_BUCKET}"
EOF
chmod +x cfn-pre-requisites.sh

		

			
			./cfn-pre-requisites.sh

		

			
			mkdir ~/dependency/leader
cd ~/dependency/leader

		

			
			cat <<EOF > Dockerfile

FROM public.ecr.aws/amazonlinux/amazonlinux:2023
RUN dnf install -y stress-ng awscli-2 

### Build mktests.sh
RUN echo \$'#!/bin/bash\n\\
FILE=/stress-tests.txt\n\\
rm \$FILE 2>/dev/null\n\\
COUNT=0\n\\
for II in \`stress-ng --cpu-method which 2>&1\`\n\\
do\n\\
    if [ \$COUNT -gt  5 ]; then\n\\
        echo "--cpu 1 -t 120s --times --cpu-method \$II" >> \$FILE\n\\
    fi\n\\
    COUNT=\`expr \$COUNT + 1\`\n\\
done' >> /mktests.sh
RUN chmod 0744 /mktests.sh

RUN echo \$'#!/bin/bash\n\\
/mktests.sh \n\\
aws s3 ls \${STRESS_BUCKET}/ \n\\
aws s3 cp /mktests.sh \${STRESS_BUCKET}/ --quiet \n\\
aws s3 cp /stress-tests.txt \${STRESS_BUCKET}/ --quiet \n\\
' >> /docker-entrypoint.sh
RUN chmod 0744 /docker-entrypoint.sh
ENTRYPOINT ["/docker-entrypoint.sh"]

EOF

		

			
			cat > ~/bin/create_repo.sh << EOF
#!/bin/bash
echo \${1}
aws ecr create-repository --repository-name \${1}
EOF
chmod +x ~/bin/create_repo.sh

		

			
			cat > ~/bin/build_container.sh << EOF
#!/bin/bash
echo \${1}
export AWS_ACCOUNT_ID=\$(aws sts get-caller-identity --output text --query Account)
if [ -n "\$AWS_REGION" ]; 
then 
        AWS_REGION=\$AWS_REGION
 else
        AWS_REGION=\$(aws configure get region)
fi
export AWS_REGION
aws ecr get-login-password --region \${AWS_REGION} | docker login --username AWS --password-stdin \${AWS_ACCOUNT_ID}.dkr.ecr.\${AWS_REGION}.amazonaws.com
docker build -t \${1} .
docker tag \${1}:latest \${AWS_ACCOUNT_ID}.dkr.ecr.\${AWS_REGION}.amazonaws.com/\${1}:latest
docker push \${AWS_ACCOUNT_ID}.dkr.ecr.\${AWS_REGION}.amazonaws.com/\${1}:latest
EOF
chmod +x ~/bin/build_container.sh

		

			
			~/bin/create_repo.sh stress-ng-leader

~/bin/build_container.sh stress-ng-leader

		

			
			export STACK_NAME=BatchWorkshop
export EXECUTION_ROLE="$(aws cloudformation describe-stacks --stack-name $STACK_NAME --output text --query 'Stacks[0].Outputs[?OutputKey == `JobExecutionRole`].OutputValue')"
export EXECUTION_ROLE_ARN=$(aws iam get-role --role-name $EXECUTION_ROLE | jq -r '.Role.Arn')
export LEADER_REPO=$(aws ecr describe-repositories --repository-names stress-ng-leader --output text --query 'repositories[0].[repositoryUri]')
cat > stress-ng-leader-job-definition.json << EOF
{
    "jobDefinitionName": "stress-ng-leader-job-definition",
    "type": "container",
    "containerProperties": {
        "image": "${LEADER_REPO}",
        "vcpus": 1,
        "memory": 1024,
        "jobRoleArn": "${EXECUTION_ROLE_ARN}",
        "executionRoleArn": "${EXECUTION_ROLE_ARN}"
    },
    "retryStrategy": {
        "attempts": 2
    }
}
EOF
aws batch register-job-definition --cli-input-json file://stress-ng-leader-job-definition.json

		

			
			export STACK_NAME=BatchWorkshop
export STRESS_BUCKET="s3://$(aws cloudformation describe-stacks --stack-name $STACK_NAME --output text --query 'Stacks[0].Outputs[?OutputKey == `Bucket`].OutputValue')"
cat <<EOF > ./stress-ng-leader-job.json
{
    "jobName": "stress-ng-leader",
    "jobQueue": "stress-ng-queue",
    "jobDefinition": "stress-ng-leader-job-definition",
    "containerOverrides": {
        "environment": [
        {
            "name": "STRESS_BUCKET",
            "value": "${STRESS_BUCKET}"
        }]
    }
}
EOF
aws batch submit-job --cli-input-json file://stress-ng-leader-job.json

		

			
			{
    "jobArn": "arn:aws:batch:ap-northeast-1:************:job/85a42e00-dadb-4353-a59f-f590e1e03c40",
    "jobName": "stress-ng-leader",
    "jobId": "85a42e00-dadb-4353-a59f-f590e1e03c40"
}

		

			
			mkdir ~/dependency/follower
cd ~/dependency/follower

		

			
			cat <<EOF > Dockerfile
FROM public.ecr.aws/amazonlinux/amazonlinux:2023
RUN dnf install -y stress-ng awscli-2

RUN echo $'#!/bin/bash\n\\
aws s3 ls \${STRESS_BUCKET}/ \n\\
aws s3 cp \${STRESS_BUCKET}/stress-tests.txt / --quiet \n\\
STRESS_ARGS=\`sed -n \$((AWS_BATCH_JOB_ARRAY_INDEX + 1))p /stress-tests.txt\` \n\\
echo "Passing the following arguments to stress-ng: \$STRESS_ARGS" \n\\
/usr/bin/stress-ng \${STRESS_ARGS} 2>&1 | aws s3 cp --quiet - \${STRESS_BUCKET}/\${AWS_BATCH_JOB_ID}_\${AWS_BATCH_JOB_ATTEMPT}_\${AWS_BATCH_JOB_ARRAY_INDEX}.txt \n\\
' >> /docker-entrypoint.sh
RUN chmod 0744 /docker-entrypoint.sh
RUN cat /docker-entrypoint.sh
ENTRYPOINT ["/docker-entrypoint.sh"]
EOF

		

			
			 ~/bin/create_repo.sh stress-ng-follower

 ~/bin/build_container.sh stress-ng-follower

		

			
			export STACK_NAME=BatchWorkshop
export EXECUTION_ROLE="$(aws cloudformation describe-stacks --stack-name $STACK_NAME --output text --query 'Stacks[0].Outputs[?OutputKey == `JobExecutionRole`].OutputValue')"
export EXECUTION_ROLE_ARN=$(aws iam get-role --role-name $EXECUTION_ROLE | jq -r '.Role.Arn')
export FOLLOWER_REPO=$(aws ecr describe-repositories --repository-names stress-ng-follower --output text --query 'repositories[0].[repositoryUri]')
cat > stress-ng-follower-job-definition.json << EOF
{
    "jobDefinitionName": "stress-ng-follower-job-definition",
    "type": "container",
    "containerProperties": {
        "image": "${FOLLOWER_REPO}",
        "vcpus": 1,
        "memory": 1024,
        "jobRoleArn": "${EXECUTION_ROLE_ARN}",
        "executionRoleArn": "${EXECUTION_ROLE_ARN}"
    },
    "retryStrategy": {
        "attempts": 2
    }
}
EOF
aws batch register-job-definition --cli-input-json file://stress-ng-follower-job-definition.json

		

			
			export STACK_NAME=BatchWorkshop
export STRESS_BUCKET="s3://$(aws cloudformation describe-stacks --stack-name $STACK_NAME --output text --query 'Stacks[0].Outputs[?OutputKey == `Bucket`].OutputValue')"
cat <<EOF > ./stress-ng-follower-job.json
{
    "jobName": "stress-ng-follower",
    "jobQueue": "stress-ng-queue",
    "arrayProperties": {
        "size": 2
    },
    "jobDefinition": "stress-ng-follower-job-definition",
    "containerOverrides": {
        "environment": [
        {
            "name": "STRESS_BUCKET",
            "value": "${STRESS_BUCKET}"
        }]
    }
}
EOF
aws batch submit-job --cli-input-json file://stress-ng-follower-job.json --array-properties size=7

		

			
			aws batch describe-jobs --jobs {FOLLOWER_JOBのID}
{
    "jobs": [
        {
            "jobArn": "arn:aws:batch:ap-northeast-1:***************:job/45454e3d-bc5d-4e99-9ef7-b2f45f9bb1de",
            "jobName": "stress-ng-follower",
            "jobId": "45454e3d-bc5d-4e99-9ef7-b2f45f9bb1de",
            "jobQueue": "arn:aws:batch:ap-northeast-1:***************:job-queue/stress-ng-queue",
            "status": "SUCCEEDED",
            "attempts": [],
            "createdAt": 1759207932821,
            "retryStrategy": {
                "attempts": 2,
                "evaluateOnExit": []
            },
            "dependsOn": [],
            "jobDefinition": "arn:aws:batch:ap-northeast-1:***************:job-definition/stress-ng-follower-job-definition:1",
            "parameters": {},
            "container": {
                "image": "***************.dkr.ecr.ap-northeast-1.amazonaws.com/stress-ng-follower",
                "vcpus": 1,
                "memory": 1024,
                "command": [],
                "jobRoleArn": "arn:aws:iam::***************:role/BatchWorkshop-JobExecutionRole-miu3kuD5NF4E",
                "executionRoleArn": "arn:aws:iam::***************:role/BatchWorkshop-JobExecutionRole-miu3kuD5NF4E",
                "volumes": [],
                "environment": [
                    {
                        "name": "STRESS_BUCKET",
                        "value": "s3://batchworkshop-bucket-imsrtso0qqx9"
                    }
                ],
                "mountPoints": [],
                "ulimits": [],
                "networkInterfaces": [],
                "resourceRequirements": [],
                "secrets": []
            },
            "arrayProperties": {
                "statusSummary": {
                    "STARTING": 0,
                    "FAILED": 0,
                    "RUNNING": 0,
                    "SUCCEEDED": 7,
                    "RUNNABLE": 0,
                    "SUBMITTED": 0,
                    "PENDING": 0
                },
                "size": 7
            },
            "tags": {},
            "platformCapabilities": [],
            "eksAttempts": []
        }
    ]
}


		

			
			### Submit the Leader job and determine its jobID.
cd ~/dependency
export LEADER_JOB=$(aws batch submit-job --cli-input-json file://leader/stress-ng-leader-job.json)
echo "${LEADER_JOB}"
export LEADER_JOB_ID=$(echo ${LEADER_JOB} | jq -r '.jobId')
echo "${LEADER_JOB_ID}"
### Submit the Follower array job with a dependency on the Leader jobID.
export FOLLOWER_JOB=$(aws batch submit-job --cli-input-json file://follower/stress-ng-follower-job.json --depends-on jobId="${LEADER_JOB_ID}" --array-properties size=12)
export FOLLOWER_JOB_ID=$(echo ${FOLLOWER_JOB} | jq -r '.jobId')
echo "${FOLLOWER_JOB_ID}"

		

			
			aws batch submit-job --cli-input-json file://follower/stress-ng-follower-job.json --depends-on jobId="${LEADER_JOB_ID}

		

			
			aws batch describe-jobs --jobs {FOLLOWER_JOBのID}

{
    "jobs": [
        {
            "jobArn": "arn:aws:batch:ap-northeast-1:***************:job/7e153307-b734-40c7-8ba1-7a4fe4cf099d",
            "jobName": "stress-ng-follower",
            "jobId": "7e153307-b734-40c7-8ba1-7a4fe4cf099d",
            "jobQueue": "arn:aws:batch:ap-northeast-1:***************:job-queue/stress-ng-queue",
            "status": "SUCCEEDED",
            "attempts": [],
            "createdAt": 1759209802446,
            "retryStrategy": {
                "attempts": 2,
                "evaluateOnExit": []
            },
            "dependsOn": [
                {
                    "jobId": "e1f23d09-e597-426f-9f0a-140571357523"
                }
            ],
            "jobDefinition": "arn:aws:batch:ap-northeast-1:***************:job-definition/stress-ng-follower-job-definition:1",
            "parameters": {},
            "container": {
                "image": "***************.dkr.ecr.ap-northeast-1.amazonaws.com/stress-ng-follower",
                "vcpus": 1,
                "memory": 1024,
                "command": [],
                "jobRoleArn": "arn:aws:iam::***************:role/BatchWorkshop-JobExecutionRole-miu3kuD5NF4E",
                "executionRoleArn": "arn:aws:iam::***************:role/BatchWorkshop-JobExecutionRole-miu3kuD5NF4E",
                "volumes": [],
                "environment": [
                    {
                        "name": "STRESS_BUCKET",
                        "value": "s3://batchworkshop-bucket-imsrtso0qqx9"
                    }
                ],
                "mountPoints": [],
                "ulimits": [],
                "networkInterfaces": [],
                "resourceRequirements": [],
                "secrets": []
            },
            "arrayProperties": {
                "statusSummary": {
                    "STARTING": 0,
                    "FAILED": 0,
                    "RUNNING": 0,
                    "SUCCEEDED": 12,
                    "RUNNABLE": 0,
                    "SUBMITTED": 0,
                    "PENDING": 0
                },
                "size": 12
            },
            "tags": {},
            "platformCapabilities": [],
            "eksAttempts": []
        }
    ]
}