NVIDIA GPU Operatorは何をしてくれるのか

カーネルバージョン
    └─→ ドライババージョン
           └─→ CUDAバージョン
                  └─→ cuDNN/TensorRTバージョン
                         └─→ フレームワーク（PyTorch/TensorFlow）

管理者が定義: 「GPUノードにはドライバv550とDevice Pluginが必要」
     ↓
ClusterPolicy CRDとして宣言
     ↓
Operatorが現在の状態を監視
     ↓
差分があれば自動で収束（Reconcile）

# NFDが付与するラベル例
feature.node.kubernetes.io/pci-10de.present=true  # NVIDIA GPU検出
nvidia.com/gpu.product=Tesla-T4                    # GPUモデル
nvidia.com/cuda.driver.major=550                   # ドライババージョン

curl -sfL https://get.k3s.io | sh - 

% sudo k3s kubectl get node
NAME              STATUS   ROLES                  AGE   VERSION
ip-172-20-1-199   Ready    control-plane,master   5s    v1.33.6+k3s1

$ sudo kubectl version
Client Version: v1.33.6+k3s1
Kustomize Version: v5.6.0
Server Version: v1.33.6+k3s1

$ tee -a ~/gpu-test.yaml <<EOF
apiVersion: v1
kind: Pod
metadata:
  name: gpu-test
spec:
  restartPolicy: OnFailure
  runtimeClassName: nvidia
  containers:
  - name: vectoradd
    image: nvcr.io/nvidia/k8s/cuda-sample:vectoradd-cuda12.5.0
    resources:
      limits:
        nvidia.com/gpu: 1
EOF

$ sudo kubectl apply -f ~/gpu-test.yaml
pod/gpu-test created
$ sudo kubectl get pod
NAME       READY   STATUS    RESTARTS   AGE
gpu-test   0/1     Pending   0          2m25s
$ sudo kubectl get event
LAST SEEN   TYPE      REASON                           OBJECT                 MESSAGE
89s         Warning   FailedScheduling                 pod/gpu-test           0/1 nodes are available: 1 Insufficient nvidia.com/gpu. preemption: 0/1 nodes are available: 1 No preemption victims found for incoming pod.

# PCIユーティリティの更新
$ sudo apt-get update && sudo apt-get install -y pciutils

# PCIバス上のNVIDIAデバイスを検査
$ lspci | grep -i nvidia
00:1e.0 3D controller: NVIDIA Corporation GA102GL [A10G] (rev a1)

# 特定のAWSカーネルバージョン用のヘッダーをインストール
$ sudo apt-get install -y linux-headers-$(uname -r) build-essential pkg-config libglvnd-dev

cd ~

# 1. Ubuntu 24.04用のキーリングをダウンロード（G5向けにx86_64に焦点を当てる）
wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/cuda-keyring_1.1-1_all.deb

# 2. キーリングをインストール
sudo dpkg -i cuda-keyring_1.1-1_all.deb

# 3. パッケージインデックスを更新
sudo apt-get update

# ヘッドレスサーバー用ドライバーとユーティリティをインストール
sudo apt-get install -y nvidia-driver-550-server nvidia-utils-550-server

sudo reboot

$ nvidia-smi
Thu Dec 18 08:52:40 2025
+-----------------------------------------------------------------------------------------+
| NVIDIA-SMI 570.195.03             Driver Version: 570.195.03     CUDA Version: 12.8     |
|-----------------------------------------+------------------------+----------------------+
| GPU  Name                 Persistence-M | Bus-Id          Disp.A | Volatile Uncorr. ECC |
| Fan  Temp   Perf          Pwr:Usage/Cap |           Memory-Usage | GPU-Util  Compute M. |
|                                         |                        |               MIG M. |
|=========================================+========================+======================|
|   0  NVIDIA A10G                    Off |   00000000:00:1E.0 Off |                    0 |
|  0%   31C    P0             61W /  300W |       0MiB /  23028MiB |     18%      Default |
|                                         |                        |                  N/A |
+-----------------------------------------+------------------------+----------------------+

+-----------------------------------------------------------------------------------------+
| Processes:                                                                              |
|  GPU   GI   CI              PID   Type   Process name                        GPU Memory |
|        ID   ID                                                               Usage      |
|=========================================================================================|
|  No running processes found                                                             |
+-----------------------------------------------------------------------------------------+

# リポジトリのセットアップ
cd ~
curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg

curl -s -L https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list | sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' | sed 's#$(ARCH)#amd64#g' | sudo tee /etc/apt/sources.list.d/nvidia-container-toolkit.list

sudo apt-get update

# インストール
sudo apt-get install -y nvidia-container-toolkit

# コンテナデバイスインターフェース（CDI）の構成
sudo nvidia-ctk cdi generate --output=/etc/cdi/nvidia.yaml

$ sudo tee -a /var/lib/rancher/k3s/agent/etc/containerd/config-v3.toml.tmpl <<EOF
{{ template "base" . }}

[plugins."io.containerd.grpc.v1.cri".containerd.runtimes."nvidia"]
  runtime_type = "io.containerd.runc.v2"

[plugins."io.containerd.grpc.v1.cri".containerd.runtimes."nvidia".options]
  BinaryName = "/usr/bin/nvidia-container-runtime"
  SystemdCgroup = true
EOF

% sudo systemctl restart k3s

$ sudo crictl info | grep -A 5 "runtimes"

$ tee -a ~/nvidia-runtime.yaml <<EOF
apiVersion: node.k8s.io/v1
kind: RuntimeClass
metadata:
  name: nvidia
handler: nvidia
EOF

# 適用
$ sudo kubectl apply -f ~/nvidia-runtime.yaml

# Device PluginのDaemonSetのデプロイ
$ sudo kubectl apply -f https://raw.githubusercontent.com/NVIDIA/k8s-device-plugin/v0.18.0/deployments/static/nvidia-device-plugin.yml

$ sudo kubectl set env  daemonset/nvidia-device-plugin-daemonset -n kube-system DEVICE_DISCOVERY_STRATEGY=nvml

$ sudo kubectl patch daemonset nvidia-device-plugin-daemonset -n kube-system -p '{"spec":{"template":{"spec":{"runtimeClassName":"nvidia"}}}}'

$ sudo kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.allocatable.nvidia\.com/gpu}{"\n"}{end}'
ip-172-20-1-50  1 # GPUコア数=1

$ sudo kubectl logs gpu-test
[Vector addition of 50000 elements]
Copy input data from the host memory to the CUDA device
CUDA kernel launch with 196 blocks of 256 threads
Copy output data from the CUDA device to the host memory
Test PASSED
Done

$ cd ~/
$ curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-4
$ chmod 700 get_helm.sh
$ ./get_helm.sh
[WARNING] Could not find git. It is required for plugin installation.
Downloading https://get.helm.sh/helm-v4.0.4-linux-amd64.tar.gz
Verifying checksum... Done.
Preparing to install helm into /usr/local/bin
helm installed into /usr/local/bin/helm

$ helm version
version.BuildInfo{Version:"v4.0.4", GitCommit:"8650e1dad9e6ae38b41f60b712af9218a0d8cc11", GitTreeState:"clean", GoVersion:"go1.25.5", KubeClientVersion:"v1.34"}

# NVIDIAのHelmリポジトリを追加
sudo helm repo add nvidia https://helm.ngc.nvidia.com/nvidia
sudo helm repo update

# valuesファイルを作成
$ tee -a ~/gpu-operator-values.yaml <<EOF
toolkit:
  env:
    - name: CONTAINERD_CONFIG
      value: /var/lib/rancher/k3s/agent/etc/containerd/config.toml
    - name: CONTAINERD_SOCKET
      value: /run/k3s/containerd/containerd.sock
    - name: CONTAINERD_RUNTIME_CLASS
      value: nvidia
    - name: CONTAINERD_SET_AS_DEFAULT
      value: "true"
EOF

# gpu-operatorネームスペースにインストール
sudo KUBECONFIG=/etc/rancher/k3s/k3s.yaml \
helm install --wait gpu-operator \
    -n gpu-operator --create-namespace \
    nvidia/gpu-operator \
    -f ~/gpu-operator-values.yaml

# Podの状態を確認
$ sudo kubectl get pod -n gpu-operator
NAME                                                         READY   STATUS      RESTARTS      AGE
gpu-feature-discovery-8nm2n                                  1/1     Running     0             4m18s
gpu-operator-6b7854bc77-ddr9f                                1/1     Running     1 (96s ago)   4m46s
gpu-operator-node-feature-discovery-gc-74dd579c7f-vsf8v      1/1     Running     0             4m46s
gpu-operator-node-feature-discovery-master-948c8bb7c-lt2rq   1/1     Running     1 (97s ago)   4m46s
gpu-operator-node-feature-discovery-worker-q495w             1/1     Running     1 (98s ago)   4m46s
nvidia-container-toolkit-daemonset-xkzwd                     1/1     Running     0             4m18s
nvidia-cuda-validator-j7pnf                                  0/1     Completed   0             90s
nvidia-dcgm-exporter-qsxmt                                   1/1     Running     0             4m18s
nvidia-device-plugin-daemonset-5jlv4                         1/1     Running     0             4m18s
nvidia-driver-daemonset-th9jx                                1/1     Running     0             4m24s
nvidia-operator-validator-dgts2                              1/1     Running     0             4m18s

$ sudo kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.allocatable.nvidia\.com/gpu}{"\n"}{end}'
ip-172-20-1-199 1 # GPUコア数=1

$ sudo kubectl logs gpu-test
[Vector addition of 50000 elements]
Copy input data from the host memory to the CUDA device
CUDA kernel launch with 196 blocks of 256 threads
Copy output data from the CUDA device to the host memory
Test PASSED
Done