Terraformを使ってS3とAWS Batchを連携するようAWSリソース作成してみた

.
├── main.tf
├── secrets.tfvars
└── variables.tf

terraform {
  required_providers {
    aws = {
      source  = "hashicorp/aws"
      version = "~> 4.16"
    }
  }
  required_version = ">= 1.2.0"
}

locals {
  prefix = "お好きな名前をご記入ください" # 自分が作成したAWSリソースを分かりやすくするために先頭文字列として付ける

  # S3
  bucket_name = "${local.prefix}-bucket"

  # AWS Batch
  job_name                            = "${local.prefix}-job"
  job_definition                      = "${local.prefix}-job-definition"
  compute_environment_name            = "${local.prefix}-compute-environment"
  job_queue                           = "${local.prefix}-job-queue"
  S3_for_instance_role                = "${local.prefix}-S3-for-instance-role"
  AmazonEC2ContainerServiceforEC2Role = "${local.prefix}-aws-batch-for-instance-role-AmazonEC2ContainerServiceforEC2Role"
  AWSBatchServiceRole                 = "${local.prefix}-AWSBatchServiceRole"
  aws_batch_service_role              = "${local.prefix}-aws-batch-service_role"
  instance_role_profile               = "${local.prefix}-terraform-iam-role-for-aws-batch-gpu"
  instance_role                       = "${local.prefix}-iam-role-for-aws-batch-gpu"

  # EventBridge
  event_rule               = "${local.prefix}-event-rule"
  eventbridge_service_role = "${local.prefix}_eventbridge_service_role"
  eventbridge_policy       = "${local.prefix}-eventbridge-policy"

  # VPC
  subnet_list = ["${var.subnet1}", "${var.subnet2}", "${var.subnet2}"]
}

# 現在使用しているアカウントIDを自動取得
data "aws_caller_identity" "current" {}

# 現在使用しているリージョンを自動取得
data "aws_region" "current" {}

# VPC
variable "subnet1" {}
variable "subnet2" {}
variable "subnet3" {}
variable "security_group_ids" {}

# AWS Batch
variable "container_name" {}

# VPC
subnet1            = "subnet-aaaaaaaa"
subnet2            = "subnet-bbbbbbbb"
subnet3            = "subnet-cccccccc"
security_group_ids = "sg-xxxxxxxx"

# AWS Batch
container_name = "コンテナ名を記載"

resource "aws_s3_bucket" "bucket" {
  bucket = local.bucket_name
}
resource "aws_s3_object" "input_folder" {
  bucket = aws_s3_bucket.bucket.id
  key    = "input-file/"
}

# EventBridgeへ通知を送るのに必要
resource "aws_s3_bucket_notification" "bucket_notification" {
  bucket      = aws_s3_bucket.bucket.bucket
  eventbridge = true
}

# ジョブ定義
resource "aws_batch_job_definition" "job_definition" {
  name                 = local.job_definition
  type                 = "container"
  container_properties = <<CONTAINER_PROP
  {
    "command":["python","app.py","--input_bucket_name","Ref::input_bucket_name","--input_s3_key","Ref::input_s3_key","--output_bucket_name","Ref::output_bucket_name","--output_s3_key","Ref::output_s3_key"],
    "image":"${data.aws_caller_identity.current.account_id}.dkr.ecr.${data.aws_region.current.name}.amazonaws.com/${var.container_name}:latest",
   "resourceRequirements": [
      {
        "value": "1",
        "type": "VCPU"
      },
      {
        "value": "2048",
        "type": "MEMORY"
      },
      {
        "value": "1",
        "type": "GPU"
      }
    ]
  }
  CONTAINER_PROP
}

# コンピューティング環境
resource "aws_batch_compute_environment" "compute_environment" {
  compute_environment_name = local.compute_environment_name
  compute_resources {
    instance_role      = aws_iam_instance_profile.iam-role-for-aws-batch-gpu.arn
    instance_type      = ["g4dn.xlarge"]
    max_vcpus          = 256
    subnets            = local.subnet_list
    security_group_ids = ["${var.security_group_ids}"]
    type               = "EC2"
    desired_vcpus      = 0
  }
  type         = "MANAGED"
  service_role = aws_iam_role.aws_batch_service_role.arn
  depends_on   = [aws_iam_role_policy_attachment.aws_batch_service_role_AWSBatchServiceRole]
}


# ジョブキュー
resource "aws_batch_job_queue" "job_queue" {
  name                 = local.job_queue
  state                = "ENABLED"
  priority             = 0
  compute_environments = [aws_batch_compute_environment.compute_environment.arn]
}

# インスタンスロール
resource "aws_iam_instance_profile" "iam-role-for-aws-batch-gpu" {
  name = local.instance_role_profile
  role = aws_iam_role.instance_role.name
}
data "aws_iam_policy" "S3_for_instance_role" {
  arn = "arn:aws:iam::aws:policy/AmazonS3FullAccess"
}
data "aws_iam_policy" "aws_batch_for_instance_role_AmazonEC2ContainerServiceforEC2Role" {
  arn = "arn:aws:iam::aws:policy/service-role/AmazonEC2ContainerServiceforEC2Role"
}
resource "aws_iam_policy" "S3_for_instance_role" {
  name   = local.S3_for_instance_role
  policy = data.aws_iam_policy.S3_for_instance_role.policy
}
resource "aws_iam_policy" "aws_batch_for_instance_role_AmazonEC2ContainerServiceforEC2Role" {
  name   = local.AmazonEC2ContainerServiceforEC2Role
  policy = data.aws_iam_policy.aws_batch_for_instance_role_AmazonEC2ContainerServiceforEC2Role.policy
}
resource "aws_iam_role_policy_attachment" "S3_for_instance_role" {
  role       = aws_iam_role.instance_role.name
  policy_arn = aws_iam_policy.S3_for_instance_role.arn
}
resource "aws_iam_role_policy_attachment" "aws_batch_for_instance_role_AmazonEC2ContainerServiceforEC2Role" {
  role       = aws_iam_role.instance_role.name
  policy_arn = aws_iam_policy.aws_batch_for_instance_role_AmazonEC2ContainerServiceforEC2Role.arn
}

data "aws_iam_policy_document" "assume_role_for_instance" {
  statement {
    effect = "Allow"

    principals {
      type        = "Service"
      identifiers = ["ec2.amazonaws.com"]
    }

    actions = ["sts:AssumeRole"]
  }
}

resource "aws_iam_role" "instance_role" {
  name               = local.instance_role
  assume_role_policy = data.aws_iam_policy_document.assume_role_for_instance.json
}


# サービスロール用のIAMロール
data "aws_iam_policy" "for_service_role_AWSBatchServiceRole" {
  arn = "arn:aws:iam::aws:policy/service-role/AWSBatchServiceRole"
}
resource "aws_iam_policy" "aws_batch_service_role_AWSBatchServiceRole" {
  name   = local.AWSBatchServiceRole
  policy = data.aws_iam_policy.for_service_role_AWSBatchServiceRole.policy
}
resource "aws_iam_role_policy_attachment" "aws_batch_service_role_AWSBatchServiceRole" {
  role       = aws_iam_role.aws_batch_service_role.name
  policy_arn = aws_iam_policy.aws_batch_service_role_AWSBatchServiceRole.arn
}
data "aws_iam_policy_document" "assume_role_for_service_role" {
  statement {
    effect = "Allow"

    principals {
      type        = "Service"
      identifiers = ["batch.amazonaws.com"]
    }

    actions = ["sts:AssumeRole"]
  }
}

resource "aws_iam_role" "aws_batch_service_role" {
  name               = local.aws_batch_service_role
  assume_role_policy = data.aws_iam_policy_document.assume_role_for_service_role.json
}

resource "aws_cloudwatch_event_rule" "eventbridge_rule" {
  name        = local.event_rule
  description = "S3の特定フォルダ配下にファイルアップロードしたときに、AWS Batchを起動するルールをIaC化"

  event_pattern = jsonencode({
    "detail-type" : ["Object Created"],
    "source" : ["aws.s3"],
    "detail" : {
      "bucket" : {
        "name" : ["${local.bucket_name}"]
      },
      "object" : {
        "key" : [{
          "prefix" : "input-file/"
        }]
      }
    }
  })
}

resource "aws_cloudwatch_event_target" "eventbridge_target" {
  rule = aws_cloudwatch_event_rule.eventbridge_rule.name

  arn = aws_batch_job_queue.job_queue.arn # ジョブキューのARN
  batch_target {
    job_definition = aws_batch_job_definition.job_definition.arn
    job_name       = local.job_name
  }
  input_transformer {
    input_paths = {
      "input_bucket_name" : "$.detail.bucket.name",
      "input_s3_key" : "$.detail.object.key"
    }
    input_template = <<TEMPLATE
{"Parameters": {"input_bucket_name":"<input_bucket_name>", "input_s3_key":"<input_s3_key>","output_bucket_name":"${local.bucket_name}", "output_s3_key":"output-eventbridge-test/output.csv"}}
    TEMPLATE
  }
  role_arn = aws_iam_role.eventbridge_service_role.arn
}

data "aws_iam_policy_document" "eventbridge_test" {
  statement {
    effect = "Allow"
    principals {
      type        = "Service"
      identifiers = ["events.amazonaws.com"]
    }
    actions = ["sts:AssumeRole"]
  }
}

resource "aws_iam_role" "eventbridge_service_role" {
  name               = local.eventbridge_service_role
  assume_role_policy = data.aws_iam_policy_document.eventbridge_test.json
  inline_policy {
    name = local.eventbridge_policy
    policy = jsonencode({
      "Version" : "2012-10-17"
      "Statement" : [
        {
          Effect = "Allow"
          Action = [
            "batch:SubmitJob"
          ],
          Resource = [
            "arn:aws:batch:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:job/${local.job_name}",
            "${aws_batch_job_definition.job_definition.arn}",
            "${aws_batch_job_queue.job_queue.arn}"
          ]
        }
      ]
    })
  }
}

Apply complete! Resources: 18 added, 0 changed, 18 destroyed.