Amazon EKSにデプロイしたGitLabのオブジェクトストレージをS3に変更する(EKS Pod Identity)

aws eks create-addon --cluster-name gitlab-cluster-blog --addon-name eks-pod-identity-agent

kubectl get pods -n kube-system | grep 'eks-pod-identity-agent'

eks-pod-identity-agent-lh22n                    1/1     Running   0          4m26s
eks-pod-identity-agent-vwshp                    1/1     Running   0          4m26s

kubectl create serviceaccount gitlab-global -n gitlab

terraform {
  required_version = "~> 1.10.0"

  required_providers {
    aws = {
      source  = "hashicorp/aws"
      version = ">= 5.82.0"
    }
  }
}

provider "aws" {
  region = var.region
}

variable "name_prefix" {}
variable "region" {
  default = "ap-northeast-1"
}

locals {
  buckets_names = [
    "registry",
    "lfs",
    "artifacts",
    "uploads",
    "packages",
    "backups",
    "tmp"
  ]
  oidc_provider = element(split("/id/", var.oidc_provider_arn), 1)
}

data "aws_kms_key" "aws_s3" {
  key_id = "alias/aws/s3"
}

resource "aws_s3_bucket" "gitlab" {
  for_each      = toset(local.buckets_names)
  bucket        = "${var.name_prefix}-${each.key}"
  force_destroy = true
}

resource "aws_s3_bucket_versioning" "gitlab" {
  for_each = aws_s3_bucket.gitlab

  bucket = each.value.id

  versioning_configuration {
    status = "Enabled"
  }
}

resource "aws_s3_bucket_server_side_encryption_configuration" "gitlab" {
  for_each = aws_s3_bucket.gitlab

  bucket = each.value.id

  rule {
    apply_server_side_encryption_by_default {
      sse_algorithm     = "aws:kms"
      kms_master_key_id = data.aws_kms_key.aws_s3.arn
    }
  }
}

resource "aws_s3_bucket_public_access_block" "gitlab" {
  for_each = aws_s3_bucket.gitlab

  bucket = each.value.id

  block_public_acls       = true
  block_public_policy     = true
  ignore_public_acls      = true
  restrict_public_buckets = true
}

## S3 access policy for all buckets except registry and backups (covered by separate policies)
resource "aws_iam_policy" "gitlab_s3_policy" {
  name = "${var.name_prefix}-s3-policy"

  policy = jsonencode({
    Version = "2012-10-17"
    Statement = [
      {
        Action = [
          "s3:PutObject",
          "s3:GetObject",
          "s3:DeleteObject",
        ]
        Effect   = "Allow"
        Resource = [for k, v in aws_s3_bucket.gitlab : "${v.arn}/*" if !contains(["registry", "backups"], k)]
      },
      # Backup Utility and Geo Replication
      {
        Action = [
          "s3:ListBucket",
        ]
        Effect   = "Allow"
        Resource = [for k, v in aws_s3_bucket.gitlab : v.arn if !contains(["registry", "backups"], k)]
      },
    ]
  })
}

resource "aws_iam_policy" "gitlab_s3_backups_policy" {
  name = "${var.name_prefix}-s3-backups-policy"

  policy = jsonencode({
    Version = "2012-10-17"
    Statement = [
      {
        Action = [
          "s3:PutObject",
          "s3:GetObject",
          "s3:DeleteObject",
        ]
        Effect   = "Allow"
        Resource = ["${aws_s3_bucket.gitlab["backups"].arn}/*"]
      }
    ]
  })
}

## S3 access policy for registry bucket
resource "aws_iam_policy" "gitlab_s3_registry_policy" {
  name = "${var.name_prefix}-s3-registry-policy"

  policy = jsonencode({
    Version = "2012-10-17"
    Statement = [
      # Docker Registry S3 bucket requires specific permissions
      # https://docs.docker.com/registry/storage-drivers/s3/#s3-permission-scopes
      {
        Action = [
          "s3:ListBucket",
          "s3:GetBucketLocation",
          "s3:ListBucketMultipartUploads",
        ]
        Effect   = "Allow"
        Resource = aws_s3_bucket.gitlab["registry"].arn
      },
      {
        Action = [
          "s3:PutObject",
          "s3:GetObject",
          "s3:DeleteObject",
          "s3:ListMultipartUploadParts",
          "s3:AbortMultipartUpload"
        ]
        Effect   = "Allow"
        Resource = "${aws_s3_bucket.gitlab["registry"].arn}/*"
      },
    ]
  })
}

resource "aws_iam_policy" "gitlab_s3_kms_policy" {
  name = "${var.name_prefix}-s3-kms-policy"

  policy = jsonencode({
    Version = "2012-10-17"
    Statement = [
      {
        Action = [
          "kms:Decrypt",
          "kms:GenerateDataKey"
        ]
        Effect = "Allow"
        Condition = {
          "StringLike" = {
            "kms:EncryptionContext:aws:s3:arn" = [
              for key, values in aws_s3_bucket.gitlab : values.arn
            ]
          }
        },
        Resource = data.aws_kms_key.aws_s3.arn
      }
    ]
  })
}

resource "aws_iam_role" "gitlab_eks_global" {
  name = "${var.prefix}-eks-global"

  assume_role_policy = jsonencode({
    Version = "2012-10-17"
    Statement = [
      {
        Action    = ["sts:AssumeRole", "sts:TagSession"]
        Effect    = "Allow"
        Principal = { "Service" = "pods.eks.amazonaws.com" }
      },
    ]
  })
}

resource "aws_iam_role_policy_attachment" "gitlab_s3_policy" {
  policy_arn = aws_iam_policy.gitlab_s3_policy.arn
  role       = aws_iam_role.gitlab_eks_global.name
}
resource "aws_iam_role_policy_attachment" "gitlab_s3_backups_policy" {
  policy_arn = aws_iam_policy.gitlab_s3_backups_policy.arn
  role       = aws_iam_role.gitlab_eks_global.name
}
resource "aws_iam_role_policy_attachment" "gitlab_s3_registry_policy" {
  policy_arn = aws_iam_policy.gitlab_s3_registry_policy.arn
  role       = aws_iam_role.gitlab_eks_global.name
}
resource "aws_iam_role_policy_attachment" "gitlab_s3_kms_policy" {
  policy_arn = aws_iam_policy.gitlab_s3_kms_policy.arn
  role       = aws_iam_role.gitlab_eks_global.name
}

output "gitlab_eks_global_role" {
  value = aws_iam_role.gitlab_eks_global.arn
}

name_prefix = "" # Example: example-gitlab-eks
region = "" # Example: us-east-1

terraform init
terraform plan
terraform apply

aws eks create-pod-identity-association \
    --cluster-name gitlab-cluster-blog --role-arn <IAMロールARN> \
    --namespace gitlab --service-account gitlab-global

kubectl describe pod gitlab-webservice-default-57d7598f89-ffjxq -n gitlab | grep AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE:

AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE:  /var/run/secrets/pods.eks.amazonaws.com/serviceaccount/eks-pod-identity-token

# Deploymentの再起動
kubectl rollout restart deployment -n gitlab

# StatefulSetの再起動
kubectl rollout restart statefulset -n gitlab