Google Distributed Cloudで始めるおうちGKE

terraform {
  required_version = ">= 1.14.0"

  required_providers {
    proxmox = {
      source  = "bpg/proxmox"
      version = ">= 0.90.0"
    }
  }
}

provider "proxmox" {
  endpoint  = var.pm_api_url
  api_token = "${var.pm_api_token_id}=${var.pm_api_token_secret}"
  insecure  = true

  ssh {
    agent    = false
    username = var.pm_ssh_username
    password = var.pm_ssh_password
  }
}

locals {
  cloud_init_templates = {
    cluster-node = "${path.module}/config/cloud-init/cluster-node.yaml.tftpl"
  }

  cloud_init_vars = {
    username      = var.vm_user
    password_hash = var.vm_password_hash
  }
}

# ------------------------------------------------------------------------------
# Cloud-Init Configuration (Snippets)
# Role-based cloud-init configuration using external template files
# ------------------------------------------------------------------------------

resource "proxmox_virtual_environment_file" "cloud_init_user_data" {
  for_each = var.vms

  content_type = "snippets"
  datastore_id = "local"
  node_name    = each.value.node_name

  source_raw {
    data = templatefile(
      local.cloud_init_templates[each.value.role],
      merge(local.cloud_init_vars, {
        hostname = each.value.hostname
      })
    )

    file_name = "cloud-init-${each.key}.yaml"
  }
}

# ------------------------------------------------------------------------------
# Cloud Image Download
# Downloads Ubuntu cloud image to each node for VM creation
# ------------------------------------------------------------------------------

resource "proxmox_virtual_environment_download_file" "ubuntu_cloud_image" {
  for_each = toset([for vm in var.vms : vm.node_name])

  content_type = "iso"
  datastore_id = "local"
  node_name    = each.value

  url       = var.ubuntu_cloud_image_url
  file_name = "ubuntu-24.04-cloudimg-amd64.img"

  overwrite = false
}

# ------------------------------------------------------------------------------
# Cloud-Init VM
# Ubuntu VM with Cloud-Init configuration for GDC
# ------------------------------------------------------------------------------

resource "proxmox_virtual_environment_vm" "this" {
  for_each = var.vms

  node_name = each.value.node_name
  vm_id     = each.value.vm_id
  name      = each.value.hostname

  description = "GDC ${each.value.role} - ${each.value.hostname}"
  tags        = ["gdc", each.value.role, "terraform"]

  on_boot = each.value.started
  started = each.value.started

  agent {
    enabled = true
  }

  cpu {
    cores   = each.value.cpu_cores
    sockets = 1
    type    = "host"
  }

  memory {
    dedicated = each.value.memory
  }

  disk {
    datastore_id = "local-lvm"
    file_id      = proxmox_virtual_environment_download_file.ubuntu_cloud_image[each.value.node_name].id
    interface    = "scsi0"
    size         = each.value.disk_size
    discard      = "on"
    iothread     = true
  }

  scsi_hardware = "virtio-scsi-single"

  network_device {
    bridge = "vmbr0"
    model  = "virtio"
    mtu    = 1450
  }

  vga {
    type = "serial0"
  }

  serial_device {}

  initialization {
    # Cloud-Init設定（カスタムsnippets使用）
    user_data_file_id = proxmox_virtual_environment_file.cloud_init_user_data[each.key].id
    datastore_id = "local-lvm"
    ip_config {
      ipv4 {
        address = each.value.ip_address
        gateway = each.value.gateway
      }
    }

    dns {
      servers = ["8.8.8.8", "8.8.4.4"]
    }
  }
}

# ------------------------------------------------------------------------------
# Outputs
# ------------------------------------------------------------------------------

output "vm_ip_addresses" {
  description = "IP addresses of created VMs"
  value = {
    for k, v in proxmox_virtual_environment_vm.this : k => v.ipv4_addresses
  }
}

output "vm_roles" {
  description = "VM roles mapping"
  value = {
    for k, v in var.vms : k => {
      hostname = v.hostname
      role     = v.role
      ip       = v.ip_address
    }
  }
}

# ------------------------------------------------------------------------------
# Proxmox Nodes Configuration
# ------------------------------------------------------------------------------
variable "pm_api_url" {
  description = "Proxmox API URL"
  type        = string
}

variable "pm_api_token_id" {
  description = "Proxmox API token ID"
  type        = string
}

variable "pm_api_token_secret" {
  description = "Proxmox API token secret"
  type        = string
  sensitive   = true
}

variable "pm_ssh_username" {
  description = "SSH username for Proxmox nodes"
  type        = string
  default     = "root"
}

variable "pm_ssh_password" {
  description = "SSH password for Proxmox nodes"
  type        = string
  sensitive   = true
}

# ------------------------------------------------------------------------------
# VM Configuration
# ------------------------------------------------------------------------------

variable "ubuntu_cloud_image_url" {
  description = "URL for Ubuntu cloud image download"
  type        = string
  default     = "https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img"
}

variable "vm_user" {
  description = "Default user name for VMs"
  type        = string
  default     = "ubuntu"
}

variable "vm_password_hash" {
  description = "Hashed password for VM user (generate with: openssl passwd -6 'yourpassword')"
  type        = string
  sensitive   = true
}

variable "vms" {
  description = "VM definitions with Cloud-Init for GDC"
  type = map(object({
    node_name  = optional(string, "pve-01")
    hostname   = string
    vm_id      = number
    ip_address = string
    gateway    = string
    role       = string
    cpu_cores  = optional(number, 2)
    memory     = optional(number, 2048)
    disk_size  = optional(number, 20)
    started    = optional(bool, true)
  }))
}

pm_api_token_id     = "user01@pam!user01-token"
pm_api_token_secret = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
pm_api_url          = "https://192.168.10.155:8006/api2/json"
pm_ssh_password     = "password"
vm_password_hash    = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"

vms = {
  "gdc-node-1" = {
    node_name  = "pve-01"
    hostname   = "gdc-node-1"
    vm_id      = 210
    ip_address = "192.168.10.210/24"
    gateway    = "192.168.10.1"
    role       = "cluster-node"
    cpu_cores  = 4
    memory     = 8192
    disk_size  = 256
  }
  "gdc-node-2" = {
    node_name  = "pve-02"
    hostname   = "gdc-node-2"
    vm_id      = 211
    ip_address = "192.168.10.211/24"
    gateway    = "192.168.10.1"
    role       = "cluster-node"
    cpu_cores  = 4
    memory     = 8192
    disk_size  = 256
  }
  "gdc-node-3" = {
    node_name  = "pve-03"
    hostname   = "gdc-node-3"
    vm_id      = 212
    ip_address = "192.168.10.212/24"
    gateway    = "192.168.10.1"
    role       = "cluster-node"
    cpu_cores  = 4
    memory     = 8192
    disk_size  = 256
  }
}

#cloud-config
# Cluster Node configuration for Google Distributed Cloud
# Used for Control Plane and Worker nodes
# Requirements: NTP (chrony), kernel tuning, UFW disabled, working package manager
# Note: containerd is installed by bmctl during cluster creation

hostname: ${hostname}
manage_etc_hosts: true
timezone: Asia/Tokyo

users:
  - name: ${username}
    groups: [adm, sudo]
    shell: /bin/bash
    sudo: ALL=(ALL) NOPASSWD:ALL
    lock_passwd: false
    passwd: ${password_hash}

ssh_pwauth: true

write_files:
  # Enable SSH root login and password authentication
  - path: /etc/ssh/sshd_config.d/99-gdc.conf
    content: |
      PermitRootLogin yes
      PasswordAuthentication yes
    permissions: "0644"

  # Kernel parameters for GDC
  - path: /etc/sysctl.d/99-gdc-node.conf
    content: |
      # inotify limits for Kubernetes (required for Ubuntu 22.04+)
      fs.inotify.max_user_instances = 8192
      fs.inotify.max_user_watches = 524288
    permissions: "0644"

  # Load kernel modules for networking
  - path: /etc/modules-load.d/gdc.conf
    content: |
      overlay
      br_netfilter
      nf_tables
    permissions: "0644"

package_update: true
packages:
  - qemu-guest-agent
  - curl
  - wget
  - gnupg
  - ca-certificates
  - apt-transport-https
  - chrony
  - socat
  - conntrack
  - ipset
  - ethtool

runcmd:
  # Enable qemu-guest-agent
  - systemctl enable qemu-guest-agent
  - systemctl start qemu-guest-agent

  # Load kernel modules
  - modprobe overlay
  - modprobe br_netfilter
  - modprobe nf_tables

  # Apply sysctl settings
  - sysctl --system

  # Disable UFW (GDC requirement)
  - systemctl stop ufw || true
  - systemctl disable ufw || true

  # Enable and start chrony for NTP (GDC requirement)
  - systemctl enable chrony
  - systemctl start chrony

  # Restart SSH to apply configuration
  - systemctl restart ssh

# Google Cloud認証
gcloud auth login
gcloud auth application-default login

gcloud projects list

# プロジェクト設定
export CLOUD_PROJECT_ID=<your-project-id>
gcloud config set project $CLOUD_PROJECT_ID

mkdir ~/baremetal && cd ~/baremetal
export PATH="$HOME/baremetal:$PATH"

# bmctlのバージョンを指定
export BMCTL_VERSION=1.34.0-gke.566

# ダウンロード
gcloud storage cp gs://anthos-baremetal-release/bmctl/${BMCTL_VERSION}/linux-amd64/bmctl .
chmod +x ./bmctl
bmctl version

bmctl version: 1.34.0-gke.566, git commit: 544e9f784aaf86427d5f51e84878dc7bc33f880a, build date: 2025-12-05 11:15:57 PST , metadata image digest: sha256:ff741325230ff5c40a3fa9ad2c935c6c85d9f2c2bfd4f7c3cfb2379ecfc038ad

# 鍵を生成
ssh-keygen -t rsa -b 4096 -f ~/.ssh/gdc-key -N ""

# 全クラスタノードに公開鍵をコピー
ssh-copy-id -i ~/.ssh/gdc-key.pub ubuntu@192.168.10.210
ssh-copy-id -i ~/.ssh/gdc-key.pub ubuntu@192.168.10.211
ssh-copy-id -i ~/.ssh/gdc-key.pub ubuntu@192.168.10.212

bmctl create config -c edge-ha \
  --enable-apis \
  --create-service-accounts \
  --project-id=$CLOUD_PROJECT_ID

[2026-01-02 05:29:49+0900] Enabling APIs for GCP project <your-project-id>:
[2026-01-02 05:29:49+0900] Enabling the following APIs for GCP project <your-project-id>:
[2026-01-02 05:29:49+0900] - anthos.googleapis.com
[2026-01-02 05:29:49+0900] - anthosaudit.googleapis.com
[2026-01-02 05:29:49+0900] - anthosgke.googleapis.com
[2026-01-02 05:29:49+0900] - cloudresourcemanager.googleapis.com
[2026-01-02 05:29:49+0900] - compute.googleapis.com
[2026-01-02 05:29:49+0900] - connectgateway.googleapis.com
[2026-01-02 05:29:49+0900] - container.googleapis.com
[2026-01-02 05:29:49+0900] - gkeconnect.googleapis.com
[2026-01-02 05:29:49+0900] - gkehub.googleapis.com
[2026-01-02 05:29:49+0900] - gkeonprem.googleapis.com
[2026-01-02 05:29:49+0900] - iam.googleapis.com
[2026-01-02 05:29:49+0900] - kubernetesmetadata.googleapis.com
[2026-01-02 05:29:49+0900] - logging.googleapis.com
[2026-01-02 05:29:49+0900] - monitoring.googleapis.com
[2026-01-02 05:29:49+0900] - opsconfigmonitoring.googleapis.com
[2026-01-02 05:29:49+0900] - serviceusage.googleapis.com
[2026-01-02 05:29:49+0900] - stackdriver.googleapis.com
[2026-01-02 05:29:49+0900] - storage.googleapis.com
[2026-01-02 05:29:54+0900] Creating service accounts with keys for GCP project <your-project-id>

[2026-01-02 05:29:58+0900] Service account keys stored at directory bmctl-workspace/.sa-keys

[2026-01-02 05:29:58+0900] Created config: bmctl-workspace/edge-ha/edge-ha.yaml

.
├── bmctl
└── bmctl-workspace
    └── edge-ha
        └── edge-ha.yaml

gcrKeyPath: bmctl-workspace/.sa-keys/<your-project-id>-anthos-baremetal-gcr.json
sshPrivateKeyPath: /home/<os-user-name>/.ssh/gdc-key # ノードへログインする際に利用するSSH鍵のファイルパスを指定
gkeConnectAgentServiceAccountKeyPath: bmctl-workspace/.sa-keys/<your-project-id>-anthos-baremetal-connect.json
gkeConnectRegisterServiceAccountKeyPath: bmctl-workspace/.sa-keys/<your-project-id>-anthos-baremetal-register.json
cloudOperationsServiceAccountKeyPath: bmctl-workspace/.sa-keys/<your-project-id>-anthos-baremetal-cloud-ops.json
---
apiVersion: v1
kind: Namespace
metadata:
  name: cluster-edge-ha
---
apiVersion: baremetal.cluster.gke.io/v1
kind: Cluster
metadata:
  name: edge-ha
  namespace: cluster-edge-ha
spec:
  type: standalone
  profile: edge
  anthosBareMetalVersion: 1.34.0-gke.566
  gkeConnect:
    projectID: <your-project-id>
  # VMのIPを指定
  controlPlane:
    nodePoolSpec:
      nodes:
      - address: 192.168.10.210
      - address: 192.168.10.211
      - address: 192.168.10.212
  # 物理ネットワークと競合しない範囲を指定
  clusterNetwork:
    pods:
      cidrBlocks:
      - 10.244.0.0/16
    services:
      cidrBlocks:
      - 10.96.0.0/20
  loadBalancer:
    mode: bundled
    ports:
      controlPlaneLBPort: 443
    vips:
      # ノードと同じサブネット内であり、尚且つaddressPoolsの範囲外を指定
      controlPlaneVIP: 192.168.10.220
      # addressPoolsの範囲内を指定
      ingressVIP: 192.168.10.230
    # LoadBalancer型Serviceに割り当てられるIPの範囲を指定
    addressPools:
    - name: pool1
      addresses:
      - 192.168.10.230-192.168.10.254
  clusterOperations:
    projectID: <your-project-id>
    location: asia-northeast1
  storage:
    lvpNodeMounts:
      path: /mnt/localpv-disk
      storageClassName: local-disks
    lvpShare:
      path: /mnt/localpv-share
      storageClassName: local-shared
      numPVUnderSharedPath: 5
  # ノードアクセス設定（設定しない場合はrootユーザーとなる）
  nodeAccess:
    loginUser: ubuntu
  nodeConfig:
    podDensity:
      maxPodsPerNode: 110

bmctl create cluster -c edge-ha

[2026-01-02 07:57:30+0900] Running command: bmctl create cluster -c edge-ha
Please check the logs at bmctl-workspace/edge-ha/log/create-cluster-20260102-075730/create-cluster.log
2026/01/02 07:57:39 well-defined vars that were never replaced: DISTRIBUTION,KUBE_RBAC_PROXY_IMAGE_TAG
[2026-01-02 07:57:39+0900] Creating bootstrap cluster... ⠇ 2026/01/02 07:59:40 well-defined vars that were never replaced: IPV6_DUALSTACK
[2026-01-02 07:57:39+0900] Creating bootstrap cluster... OK
[2026-01-02 08:00:30+0900] Installing dependency components... ⠧ W0102 08:01:17.062653 1570660 schema.go:148] unexpected field validation directive: validator, skipping validation
[2026-01-02 08:00:30+0900] Installing dependency components... ⠏ I0102 08:01:18.243517 1570660 warnings.go:110] "Warning: spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`."
[2026-01-02 08:00:30+0900] Installing dependency components... ⠏ I0102 08:01:23.293357 1570660 warnings.go:110] "Warning: metadata.name: this is used in Pod names and hostnames, which can result in surprising behavior; a DNS label is recommended: [must not contain dots]"
[2026-01-02 08:00:30+0900] Installing dependency components... ⠹ I0102 08:01:23.622053 1570660 warnings.go:110] "Warning: metadata.name: this is used in Pod names and hostnames, which can result in surprising behavior; a DNS label is recommended: [must not contain dots]"
[2026-01-02 08:00:30+0900] Installing dependency components... ⠦ I0102 08:01:23.982776 1570660 warnings.go:110] "Warning: metadata.name: this is used in Pod names and hostnames, which can result in surprising behavior; a DNS label is recommended: [must not contain dots]"
[2026-01-02 08:00:30+0900] Installing dependency components... ⠏ I0102 08:01:24.258648 1570660 warnings.go:110] "Warning: metadata.name: this is used in Pod names and hostnames, which can result in surprising behavior; a DNS label is recommended: [must not contain dots]"
[2026-01-02 08:00:30+0900] Installing dependency components... ⠙ I0102 08:01:24.524826 1570660 warnings.go:110] "Warning: spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`."
[2026-01-02 08:00:30+0900] Installing dependency components... ⠸ 2026/01/02 08:01:24 well-defined vars that were never replaced: DISTRIBUTION,KUBE_RBAC_PROXY_IMAGE_TAG
[2026-01-02 08:00:30+0900] Installing dependency components... ⠋ I0102 08:01:27.442661 1570660 warnings.go:110] "Warning: spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`."
[2026-01-02 08:00:30+0900] Installing dependency components... ⠇ 

$ docker ps -a

CONTAINER ID   IMAGE                                                                          COMMAND                   CREATED         STATUS         PORTS                       NAMES
f835bda2d96d   gcr.io/anthos-baremetal-release/kindest/node:v0.24.0-gke.132-v1.32.9-gke.700   "/usr/local/bin/entr…"   2 minutes ago   Up 2 minutes   127.0.0.1:43047->6443/tcp   bmctl-control-plane

export KUBECONFIG=~/baremetal/bmctl-workspace/.kindkubeconfig

$ kubectl get node

NAME                  STATUS   ROLES           AGE   VERSION
bmctl-control-plane   Ready    control-plane   10m   v1.32.9-gke.700

$ kubectl get pod -A

NAMESPACE                       NAME                                                              READY   STATUS      RESTARTS   AGE
capi-kubeadm-bootstrap-system   capi-kubeadm-bootstrap-controller-manager-1.34.0-gke.566-d86xqv   2/2     Running     0          9m32s
capi-system                     capi-controller-manager-1.34.0-gke.566-9bcc4c448-rpn5h            2/2     Running     0          9m32s
cert-manager                    cert-manager-5d7f6fbcb8-8f4vm                                     1/1     Running     0          9m40s
cert-manager                    cert-manager-cainjector-6bcc886fd5-dn574                          1/1     Running     0          9m40s
cert-manager                    cert-manager-webhook-f79db77f9-sbv4f                              1/1     Running     0          9m40s
cluster-edge-ha                 bm-system-check-kernel-edge-ha-34jxc-k5d8j                        0/1     Completed   0          2m39s

[2026-01-02 08:02:54+0900] Waiting for preflight check operator to show up... OK
[2026-01-02 08:04:07+0900] Waiting for preflight check job to finish... OK
[2026-01-02 08:07:27+0900] - Validation Category: machines and network
[2026-01-02 08:07:27+0900]      - [PASSED] 192.168.10.211
[2026-01-02 08:07:27+0900]      - [PASSED] 192.168.10.211-gcp
[2026-01-02 08:07:27+0900]      - [PASSED] 192.168.10.212
[2026-01-02 08:07:27+0900]      - [PASSED] 192.168.10.212-gcp
[2026-01-02 08:07:27+0900]      - [PASSED] gcp
[2026-01-02 08:07:27+0900]      - [PASSED] pod-cidr
[2026-01-02 08:07:27+0900]      - [PASSED] 192.168.10.210
[2026-01-02 08:07:27+0900]      - [PASSED] 192.168.10.210-gcp
[2026-01-02 08:07:27+0900]      - [PASSED] node-network
[2026-01-02 08:07:27+0900] Flushing logs... OK
[2026-01-02 08:07:28+0900] Applying resources for new cluster
[2026-01-02 08:07:28+0900] Waiting for cluster kubeconfig to become ready OK

[2026-01-02 08:15:38+0900] Writing kubeconfig file
[2026-01-02 08:15:38+0900] The kubeconfig of the cluster being created is present at bmctl-workspace/edge-ha/edge-ha-kubeconfig
[2026-01-02 08:15:38+0900] Please restrict access to this file, as it contains the authentication credentials of your cluster.
Waiting for cluster to become ready: Requeue after 10s, Control Plane NodePool, System Services Deployment, GKE Connect Dependent System Services Deployment, GKE Connect⠋

[2026-01-02 08:52:48+0900] Please run the following command to get the cluster nodes' status:
[2026-01-02 08:52:48+0900] kubectl --kubeconfig bmctl-workspace/edge-ha/edge-ha-kubeconfig get nodes
[2026-01-02 08:52:48+0900] Waiting for node pools to become ready OK
[2026-01-02 08:53:08+0900] Waiting for metrics to become ready in GCP OK
[2026-01-02 08:53:38+0900] Waiting for cluster API provider to install in the created admin cluster OK
[2026-01-02 08:53:49+0900] Moving admin cluster resources to the created admin cluster
[2026-01-02 08:53:56+0900] Flushing logs... OK
[2026-01-02 08:53:56+0900] Deleting bootstrap cluster... OK

.
├── bmctl
└── bmctl-workspace
    ├── config.json
    ├── config.toml
    └── edge-ha
        ├── edge-ha-kubeconfig
        ├── edge-ha.yaml
        └── log
            ├── create-cluster-20260102-075730
            │   ├── 192.168.10.210
            │   ├── 192.168.10.211
            │   ├── 192.168.10.212
            │   ├── bootstrap-cluster
            │   ├── check-kernel
            │   └── create-cluster.log
            └── preflight-20260102-075730

export KUBECONFIG=~/baremetal/bmctl-workspace/edge-ha/edge-ha-kubeconfig
kubectl get nodes

NAME         STATUS   ROLES           AGE   VERSION
gdc-node-1   Ready    control-plane   22m   v1.34.1-gke.2900
gdc-node-2   Ready    control-plane   10m   v1.34.1-gke.2900
gdc-node-3   Ready    control-plane   38m   v1.34.1-gke.2900

kubectl get pod -A

NAMESPACE                       NAME                                                              READY   STATUS      RESTARTS       AGE
anthos-identity-service         ais-fbxfh                                                         3/3     Running     0              6m32s
anthos-identity-service         ais-mmzq5                                                         3/3     Running     0              5m26s
anthos-identity-service         ais-zhdmz                                                         3/3     Running     0              5m59s
capi-kubeadm-bootstrap-system   capi-kubeadm-bootstrap-controller-manager-1.34.0-gke.566-ddxbbh   2/2     Running     0              2m45s
capi-system                     capi-controller-manager-1.34.0-gke.566-9bcc4c448-wkkmt            2/2     Running     0              2m46s
cert-manager                    cert-manager-5d7f6fbcb8-szx4w                                     1/1     Running     0              20m
cert-manager                    cert-manager-cainjector-6bcc886fd5-tf6q2                          1/1     Running     0              20m
cert-manager                    cert-manager-webhook-f79db77f9-bdwdx                              1/1     Running     0              20m
cluster-edge-ha                 bm-system-network-health-check-edge-ha-5xj95-sfqzn                0/1     Completed   0              89s
cluster-edge-ha                 cluster-profile-deployer-1.34.0-gke.566-8c9sj                     0/1     Completed   0              53s

# ClusterRoleの作成
cat <<EOF > cloud-console-reader-role.yaml
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: cloud-console-reader-role
rules:
- apiGroups: [""]
  resources: ["nodes", "persistentvolumes", "pods"]
  verbs: ["get", "list", "watch"]
- apiGroups: ["storage.k8s.io"]
  resources: ["storageclasses"]
  verbs: ["get", "list", "watch"]
EOF
kubectl apply -f cloud-console-reader-role.yaml

# ServiceAccountの作成
KSA_NAME=cloud-console-reader-ksa
kubectl create serviceaccount ${KSA_NAME}
kubectl create clusterrolebinding cloud-console-reader-ksa-binding \
   --clusterrole view --serviceaccount default:${KSA_NAME}
kubectl create clusterrolebinding cloud-console-reader-role-binding \
   --clusterrole cloud-console-reader-role --serviceaccount default:${KSA_NAME}

# Tokenを作成
SECRET_NAME=cloud-console-reader-ksa-token
kubectl apply -f - << __EOF__
apiVersion: v1
kind: Secret
metadata:
  name: "${SECRET_NAME}"
  annotations:
    kubernetes.io/service-account.name: "${KSA_NAME}"
type: kubernetes.io/service-account-token
__EOF__

# トークンを表示
kubectl get secret ${SECRET_NAME} -o jsonpath='{$.data.token}' | base64 --decode

kubectl get pods -n kube-system | grep metallb

NAMESPACE                       NAME                                                              READY   STATUS      RESTARTS      AGE
kube-system                     pod/metallb-controller-867db85799-tksqc                               1/1     Running     0             33m
kube-system                     pod/metallb-speaker-jfs98                                             1/1     Running     0             33m
kube-system                     pod/metallb-speaker-m76qp                                             1/1     Running     0             24m
kube-system                     pod/metallb-speaker-sqhql                                             1/1     Running     0             33m

NAMESPACE                 NAME                        DESIRED   CURRENT   READY   UP-TO-DATE   AVAILABLE   NODE SELECTOR                                                 AGE
kube-system               daemonset.apps/metallb-speaker             3         3         3       3            3           baremetal.cluster.gke.io/lbnode=true,kubernetes.io/os=linux   33m

NAMESPACE                       NAME                                                       READY   UP-TO-DATE   AVAILABLE   AGE
kube-system                     deployment.apps/metallb-controller                                         1/1     1            1           33m

NAMESPACE                       NAME                                                                 DESIRED   CURRENT   READY   AGE
kube-system                     replicaset.apps/metallb-controller-867db85799                                        1         1         1       33m

cat <<'EOF' > nginx-sample.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: nginx-sample
  labels:
    app: nginx-sample
spec:
  replicas: 3
  selector:
    matchLabels:
      app: nginx-sample
  template:
    metadata:
      labels:
        app: nginx-sample
    spec:
      containers:
      - name: nginx
        image: nginx
        ports:
        - containerPort: 80
---
apiVersion: v1
kind: Service
metadata:
  name: nginx-sample
spec:
  type: LoadBalancer
  selector:
    app: nginx-sample
  ports:
  - port: 80
    targetPort: 80
EOF

kubectl apply -f nginx-sample.yaml

NAME                           READY   UP-TO-DATE   AVAILABLE   AGE
deployment.apps/nginx-sample   3/3     3            3           53s

NAME                   TYPE           CLUSTER-IP    EXTERNAL-IP      PORT(S)        AGE
service/kubernetes     ClusterIP      10.96.0.1     <none>           443/TCP        49m
service/nginx-sample   LoadBalancer   10.96.2.205   192.168.10.231   80:32092/TCP   53s

# LoadBalancer経由でアクセス
curl http://192.168.10.231/

# IngressVIP経由でアクセス
curl http://192.168.10.230/

<!DOCTYPE html>
<html>
<head>
<title>Welcome to nginx!</title>
<style>
html { color-scheme: light dark; }
body { width: 35em; margin: 0 auto;
font-family: Tahoma, Verdana, Arial, sans-serif; }
</style>
</head>
<body>
<h1>Welcome to nginx!</h1>
<p>If you see this page, the nginx web server is successfully installed and
working. Further configuration is required.</p>

<p>For online documentation and support please refer to
<a href="http://nginx.org/">nginx.org</a>.<br/>
Commercial support is available at
<a href="http://nginx.com/">nginx.com</a>.</p>

<p><em>Thank you for using nginx.</em></p>
</body>
</html>

# GPGキーの追加
sudo mkdir -p --mode=0755 /usr/share/keyrings
curl -fsSL https://pkg.cloudflare.com/cloudflare-main.gpg | sudo tee /usr/share/keyrings/cloudflare-main.gpg >/dev/null

# リポジトリの追加
echo 'deb [signed-by=/usr/share/keyrings/cloudflare-main.gpg] https://pkg.cloudflare.com/cloudflared jammy main' | sudo tee /etc/apt/sources.list.d/cloudflared.list

# インストール
sudo apt-get update && sudo apt-get install cloudflared

# バージョン確認
cloudflared --version

cloudflared version 2025.11.1 (built 2025-11-07-16:59 UTC)

# Cloudflareにログイン
cloudflared tunnel login

# Tunnelを作成
cloudflared tunnel create gdc-tunnel

# DNSレコードを設定
cloudflared tunnel route dns gdc-tunnel gdc.your-domain.com

# Namespaceとクレデンシャルの作成
kubectl create namespace cloudflare-tunnel
kubectl create secret generic tunnel-credentials \
  --from-file=credentials.json=$HOME/.cloudflared/<TUNNEL_ID>.json \
  -n cloudflare-tunnel

# cloudflaredのデプロイ
cat <<'EOF' > cloudflared.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: cloudflared-config
  namespace: cloudflare-tunnel
data:
  config.yaml: |
    tunnel: <TUNNEL_ID>
    credentials-file: /etc/cloudflared/creds/credentials.json
    metrics: 0.0.0.0:2000
    no-autoupdate: true
    ingress:
      - hostname: gdc.your-domain.com
        service: http://192.168.10.231:80
      - service: http_status:404
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: cloudflared
  namespace: cloudflare-tunnel
spec:
  replicas: 3
  selector:
    matchLabels:
      app: cloudflared
  template:
    metadata:
      labels:
        app: cloudflared
    spec:
      containers:
      - name: cloudflared
        image: cloudflare/cloudflared:latest
        args:
        - tunnel
        - --config
        - /etc/cloudflared/config/config.yaml
        - run
        volumeMounts:
        - name: config
          mountPath: /etc/cloudflared/config
          readOnly: true
        - name: creds
          mountPath: /etc/cloudflared/creds
          readOnly: true
      volumes:
      - name: config
        configMap:
          name: cloudflared-config
      - name: creds
        secret:
          secretName: tunnel-credentials
EOF

kubectl apply -f cloudflared.yaml

kubectl get pods -n cloudflare-tunnel

NAME                           READY   STATUS    RESTARTS   AGE
cloudflared-5fccc9cfb8-2lnn9   1/1     Running   0          30s
cloudflared-5fccc9cfb8-nl95r   1/1     Running   0          30s
cloudflared-5fccc9cfb8-s2n8t   1/1     Running   0          30s

kubectl logs -n cloudflare-tunnel -l app=cloudflared

2026-01-02T00:18:26Z INF Registered tunnel connection connIndex=0 connection=52f24c5f-1fea-4c91-80ee-c88621dd7b32 event=0 ip=198.41.200.23 location=nrt15 protocol=quic
2026-01-02T00:18:26Z INF Tunnel connection curve preferences: [X25519MLKEM768 CurveP256] connIndex=1 event=0 ip=198.41.192.227
2026-01-02T00:18:26Z INF Registered tunnel connection connIndex=1 connection=6a55ecfc-1bb3-44f3-a363-fca95476a5c6 event=0 ip=198.41.192.227 location=nrt09 protocol=quic
2026-01-02T00:18:27Z INF Tunnel connection curve preferences: [X25519MLKEM768 CurveP256] connIndex=2 event=0 ip=198.41.200.193
2026-01-02T00:18:27Z INF Registered tunnel connection connIndex=2 connection=0c9da0b4-42f4-445c-b63e-1e7224df266c event=0 ip=198.41.200.193 location=nrt07 protocol=quic
2026-01-02T00:18:28Z INF Tunnel connection curve preferences: [X25519MLKEM768 CurveP256] connIndex=3 event=0 ip=198.41.192.7
2026-01-02T00:18:28Z INF Registered tunnel connection connIndex=3 connection=e3451d10-45eb-4471-a653-cb03f0949616 event=0 ip=198.41.192.7 location=nrt01 protocol=quic

kubectl --namespace kube-system patch stackdriver stackdriver --type=merge -p '
spec:
  enableCloudLoggingForApplications: true
'

cat <<'EOF' > my-app.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: "monitoring-example"
  namespace: "default"
  labels:
    app: "monitoring-example"
spec:
  replicas: 1
  selector:
    matchLabels:
      app: "monitoring-example"
  template:
    metadata:
      labels:
        app: "monitoring-example"
    spec:
      containers:
      - image: gcr.io/google-samples/prometheus-dummy-exporter:latest
        name: prometheus-example-exporter
        imagePullPolicy: Always
        command:
        - /bin/sh
        - -c
        - ./prometheus-dummy-exporter --metric-name=example_monitoring_up --metric-value=1 --port=9090
        resources:
          requests:
            cpu: 100m
EOF

kubectl apply -f my-app.yaml

kubectl --namespace kube-system edit stackdriver stackdriver

spec:
  enableCloudLoggingForApplications: true
  appLogFilter:
    keepLogRules:
    - namespaces:
      - default
      contentRegexes:
      - ".*(ERROR|WARN).*"
      ruleName: keep-important-logs
    dropLogRules:
    - podLabelSelectors:
      - disableGCPLogging=yes
      ruleName: drop-excluded-logs

kubectl run pod1 \
    --image gcr.io/cloud-marketplace-containers/google/debian10:latest \
    --restart Never --command -- \
    /bin/sh -c "while true; do echo 'ERROR is 404\\nINFO is not 404' && sleep 1; done"

export PROJECT_ID=${PROJECT_ID}
export REGION=asia-northeast1

# Artifact Registry APIを有効化
gcloud services enable artifactregistry.googleapis.com

# リポジトリを作成
gcloud artifacts repositories create gdc-images \
  --repository-format=docker \
  --location=$REGION \
  --description="GDC private container images"

# サービスアカウント作成
gcloud iam service-accounts create gdc-registry-reader \
  --display-name="GDC Registry Reader"

# 権限付与
gcloud artifacts repositories add-iam-policy-binding gdc-images \
  --location=$REGION \
  --member="serviceAccount:gdc-registry-reader@${PROJECT_ID}.iam.gserviceaccount.com" \
  --role="roles/artifactregistry.reader"

# サービスアカウントキーを作成
gcloud iam service-accounts keys create ~/baremetal/ar-key.json \
  --iam-account=gdc-registry-reader@${PROJECT_ID}.iam.gserviceaccount.com

# Secretの作成
kubectl create secret docker-registry ar-pull-secret \
  --docker-server=${REGION}-docker.pkg.dev \
  --docker-username=_json_key \
  --docker-password="$(cat ~/baremetal/ar-key.json)" \
  -n default

# default ServiceAccountに追加
kubectl patch serviceaccount default -n default \
  -p '{"imagePullSecrets": [{"name": "ar-pull-secret"}]}'

# テスト用イメージをプッシュ
docker pull nginx:latest
docker tag nginx:latest ${REGION}-docker.pkg.dev/${PROJECT_ID}/gdc-images/nginx:latest
gcloud auth configure-docker ${REGION}-docker.pkg.dev
docker push ${REGION}-docker.pkg.dev/${PROJECT_ID}/gdc-images/nginx:latest

# プライベートレジストリからPodをデプロイ
cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Pod
metadata:
  name: ar-test-pod
spec:
  containers:
  - name: nginx
    image: ${REGION}-docker.pkg.dev/${PROJECT_ID}/gdc-images/nginx:latest
    ports:
    - containerPort: 80
EOF

$ kubectl get pod ar-test-pod
NAME          READY   STATUS    RESTARTS   AGE
ar-test-pod   1/1     Running   0          26s

$ kubectl describe pod ar-test-pod | grep -A 5 "Events"
Events:
  Type    Reason     Age   From               Message
  ----    ------     ----  ----               -------
  Normal  Scheduled  39s   default-scheduler  Successfully assigned default/ar-test-pod to gdc-node-1
  Normal  Pulling    39s   kubelet            Pulling image "asia-northeast1-docker.pkg.dev/<your-project-id>/gdc-images/nginx:latest"
  Normal  Pulled     36s   kubelet            Successfully pulled image "asia-northeast1-docker.pkg.dev/<your-project-id>/gdc-images/nginx:latest" in 3.063s (3.063s including waiting). Image size: 59797235 bytes.