EKSにデプロイしたGitLabのオブジェクトストレージをS3に変更する

terraform {
  required_version = "~> 1.10.0"

  required_providers {
    aws = {
      source  = "hashicorp/aws"
      version = ">= 5.82.0"
    }
  }
}

provider "aws" {
  region = var.region
}

variable "name_prefix" {}
variable "region" {
  default = "ap-northeast-1"
}
variable "oidc_provider_arn" {}

locals {
  buckets_names = [
    "registry",
    "lfs",
    "artifacts",
    "uploads",
    "packages",
    "backups",
    "tmp"
  ]
  oidc_provider = element(split("/id/", var.oidc_provider_arn), 1)
}

data "aws_kms_key" "aws_s3" {
  key_id = "alias/aws/s3"
}

resource "aws_s3_bucket" "gitlab" {
  for_each      = toset(local.buckets_names)
  bucket        = "${var.name_prefix}-${each.key}"
  force_destroy = true
}

resource "aws_s3_bucket_versioning" "gitlab" {
  for_each = aws_s3_bucket.gitlab

  bucket = each.value.id

  versioning_configuration {
    status = "Enabled"
  }
}

resource "aws_s3_bucket_server_side_encryption_configuration" "gitlab" {
  for_each = aws_s3_bucket.gitlab

  bucket = each.value.id

  rule {
    apply_server_side_encryption_by_default {
      sse_algorithm     = "aws:kms"
      kms_master_key_id = data.aws_kms_key.aws_s3.arn
    }
  }
}

resource "aws_s3_bucket_public_access_block" "gitlab" {
  for_each = aws_s3_bucket.gitlab

  bucket = each.value.id

  block_public_acls       = true
  block_public_policy     = true
  ignore_public_acls      = true
  restrict_public_buckets = true
}

## S3 access policy for all buckets except registry and backups (covered by separate policies)
resource "aws_iam_policy" "gitlab_s3_policy" {
  name = "${var.name_prefix}-s3-policy"

  policy = jsonencode({
    Version = "2012-10-17"
    Statement = [
      {
        Action = [
          "s3:PutObject",
          "s3:GetObject",
          "s3:DeleteObject",
        ]
        Effect   = "Allow"
        Resource = [for k, v in aws_s3_bucket.gitlab : "${v.arn}/*" if !contains(["registry", "backups"], k)]
      },
      # Backup Utility and Geo Replication
      {
        Action = [
          "s3:ListBucket",
        ]
        Effect   = "Allow"
        Resource = [for k, v in aws_s3_bucket.gitlab : v.arn if !contains(["registry", "backups"], k)]
      },
    ]
  })
}

resource "aws_iam_policy" "gitlab_s3_backups_policy" {
  name = "${var.name_prefix}-s3-backups-policy"

  policy = jsonencode({
    Version = "2012-10-17"
    Statement = [
      {
        Action = [
          "s3:PutObject",
          "s3:GetObject",
          "s3:DeleteObject",
        ]
        Effect   = "Allow"
        Resource = ["${aws_s3_bucket.gitlab["backups"].arn}/*"]
      }
    ]
  })
}

## S3 access policy for registry bucket
resource "aws_iam_policy" "gitlab_s3_registry_policy" {
  name = "${var.name_prefix}-s3-registry-policy"

  policy = jsonencode({
    Version = "2012-10-17"
    Statement = [
      # Docker Registry S3 bucket requires specific permissions
      # https://docs.docker.com/registry/storage-drivers/s3/#s3-permission-scopes
      {
        Action = [
          "s3:ListBucket",
          "s3:GetBucketLocation",
          "s3:ListBucketMultipartUploads",
        ]
        Effect   = "Allow"
        Resource = aws_s3_bucket.gitlab["registry"].arn
      },
      {
        Action = [
          "s3:PutObject",
          "s3:GetObject",
          "s3:DeleteObject",
          "s3:ListMultipartUploadParts",
          "s3:AbortMultipartUpload"
        ]
        Effect   = "Allow"
        Resource = "${aws_s3_bucket.gitlab["registry"].arn}/*"
      },
    ]
  })
}

resource "aws_iam_policy" "gitlab_s3_kms_policy" {
  name = "${var.name_prefix}-s3-kms-policy"

  policy = jsonencode({
    Version = "2012-10-17"
    Statement = [
      {
        Action = [
          "kms:Decrypt",
          "kms:GenerateDataKey"
        ]
        Effect = "Allow"
        Condition = {
          "StringLike" = {
            "kms:EncryptionContext:aws:s3:arn" = [
              for key, values in aws_s3_bucket.gitlab : values.arn
            ]
          }
        },
        Resource = data.aws_kms_key.aws_s3.arn
      }
    ]
  })
}

resource "aws_iam_role" "gitlab_eks_global" {
  name = "${var.name_prefix}-eks-global"

  assume_role_policy = jsonencode({
    Version = "2012-10-17"
    Statement = [
      {
        Action    = "sts:AssumeRoleWithWebIdentity"
        Effect    = "Allow"
        Principal = { Federated = var.oidc_provider_arn }
        Condition = {
          "StringEquals" = {
            "${local.oidc_provider}:aud" = "sts.amazonaws.com",
            "${local.oidc_provider}:sub" = "system:serviceaccount:gitlab:gitlab-global"
          }
        }
      },
    ]
  })
}

resource "aws_iam_role_policy_attachment" "gitlab_s3_policy" {
  policy_arn = aws_iam_policy.gitlab_s3_policy.arn
  role       = aws_iam_role.gitlab_eks_global.name
}
resource "aws_iam_role_policy_attachment" "gitlab_s3_backups_policy" {
  policy_arn = aws_iam_policy.gitlab_s3_backups_policy.arn
  role       = aws_iam_role.gitlab_eks_global.name
}
resource "aws_iam_role_policy_attachment" "gitlab_s3_registry_policy" {
  policy_arn = aws_iam_policy.gitlab_s3_registry_policy.arn
  role       = aws_iam_role.gitlab_eks_global.name
}
resource "aws_iam_role_policy_attachment" "gitlab_s3_kms_policy" {
  policy_arn = aws_iam_policy.gitlab_s3_kms_policy.arn
  role       = aws_iam_role.gitlab_eks_global.name
}

output "gitlab_eks_global_role" {
  value = aws_iam_role.gitlab_eks_global.arn
}

name_prefix = "" # Example: example-gitlab-eks
region = "" # Example: us-east-1
oidc_provider_arn = "" # Example: arn:aws:iam::0123456890:oidc-provider/oidc.eks.us-east-1.amazonaws.com/id/HOGEHOGEHOGE

# 1. クラスターのOIDC IDを取得
CLUSTER_ID=$(aws eks describe-cluster --name <クラスター名> \
  --query "cluster.identity.oidc.issuer" \
  --output text --region us-east-2 | cut -d'/' -f5)

# 2. OIDC IDを使ってARNをフィルタリング
aws iam list-open-id-connect-providers \
  --query "OpenIDConnectProviderList[?contains(Arn, '${CLUSTER_ID}')].Arn" \
  --output text

terraform init
terraform plan
terraform apply

kubectl create serviceaccount gitlab-global

kubectl annotate serviceaccount -n gitlab gitlab-global eks.amazonaws.com/role-arn=<IAMロールARN>

provider: AWS
use_iam_profile: true
region: <リージョン名> # Example: us-east-1

[default]
bucket_location = <リージョン名> # Example: us-east-1

s3:
  bucket: <バケット名> # Example: example-gitlab-eks-registry
  region:  <リージョン名> # Example: us-east-1
  v4auth: true

kubectl create secret generic gitlab-rails-storage \
    --from-file=connection=rails.yaml
kubectl create secret generic s3cmd-config \
    --from-file=config=s3cmd.conf
kubectl create secret generic registry-storage \
    --from-file=config=registry.yaml

global:
  serviceAccount:
    # グローバルServiceAccount設定
    # このSAは以下のコンポーネントで使用されます：
    # - webservice
    # - sidekiq
    # - その他のGitLabコアコンポーネント
    #
    # 以下のコンポーネントは独自のSAを使用します：
    # - certmanager (gitlab-certmanager)
    # - redis (gitlab-redis)
    # - prometheus (gitlab-prometheus-server)
    enabled: true
    create: false # name指定でtrueはできないため、事前にsaを作成する
    name: gitlab-global
  minio:
    enabled: false
  registry:
    bucket: example-gitlab-eks-registry
  appConfig:
    lfs:
      bucket: example-gitlab-eks-lfs
      connection:
        secret: gitlab-rails-storage
        key: connection
    artifacts:
      bucket: example-gitlab-eks-artifacts
      connection:
        secret: gitlab-rails-storage
        key: connection
    uploads:
      bucket: example-gitlab-eks-uploads
      connection:
        secret: gitlab-rails-storage
        key: connection
    packages:
      bucket: example-gitlab-eks-packages
      connection:
        secret: gitlab-rails-storage
        key: connection
    backups:
      bucket: example-gitlab-eks-backups
      tmpBucket: example-gitlab-eks-tmp
  toolbox:
    backups:
      objectStorage:
        config:
          secret: s3cmd-config
          key: config
registry:
  enabled: true
  service:
    type: NodePort
  storage:
    secret: registry-storage
    key: config

helm upgrade gitlab gitlab/gitlab -n gitlab -f values.yaml