[SageMaker HyperPod] sample-physical-ai-scaffolding-kit ベースで FSx for Lustre 込みの最小クラスタを試してみました

# (1) worker (ml.g5.2xlarge × 1) 用のクラスタ枠を確認 — 1 以上必要
aws service-quotas list-service-quotas \
  --service-code sagemaker \
  --region ap-northeast-1 \
  --query 'Quotas[?QuotaName==`ml.g5.2xlarge for cluster usage`].[QuotaName,Value,QuotaCode]' \
  --output table
----------------------------------------------------------
|                    ListServiceQuotas                   |
+----------------------------------+------+--------------+
|  ml.g5.2xlarge for cluster usage |  1.0 |  L-596C3331  |
+----------------------------------+------+--------------+

# (2) controller (ml.c5.xlarge × 1) 用のクラスタ枠を確認 — 1 以上必要
aws service-quotas list-service-quotas \
  --service-code sagemaker \
  --region ap-northeast-1 \
  --query 'Quotas[?QuotaName==`ml.c5.xlarge for cluster usage`].[QuotaName,Value,QuotaCode]' \
  --output table
---------------------------------------------------------
|                   ListServiceQuotas                   |
+---------------------------------+-------+--------------+
|  ml.c5.xlarge for cluster usage |  30.0 |  L-AD3B35FF  |
+---------------------------------+-------+--------------+

# (3) FSx for Lustre のストレージ枠(SSD)を確認 — 1.2 TiB(1228 GiB)以上必要
#     既存の FSx 利用量との合算で枠が足りるかを判断する
aws service-quotas list-service-quotas \
  --service-code fsx \
  --region ap-northeast-1 \
  --query 'Quotas[?contains(QuotaName, `SSD storage capacity`)].[QuotaName,Value,QuotaCode]' \
  --output table

# (4) ローカル必須ツール(無いものは事前に導入)
which aws pnpm node git jq python3 session-manager-plugin

# 上限緩和したいインスタンスタイプの QuotaName を指定
QUOTA_NAME="ml.g5.2xlarge for cluster usage"

# (a) QuotaCode を取得
QUOTA_CODE=$(aws service-quotas list-service-quotas \
  --service-code sagemaker \
  --region ap-northeast-1 \
  --query 'Quotas[?QuotaName==`'"${QUOTA_NAME}"'`].QuotaCode' \
  --output text)
echo "${QUOTA_CODE}"

# (b) 1 に引き上げる申請を出す
aws service-quotas request-service-quota-increase \
  --service-code sagemaker \
  --quota-code "${QUOTA_CODE}" \
  --desired-value 1 \
  --region ap-northeast-1

# (c) 申請状況の確認(Status が CASE_CLOSED + DesiredValue 達成で完了)
aws service-quotas list-requested-service-quota-change-history \
  --service-code sagemaker \
  --region ap-northeast-1 \
  --query 'RequestedQuotas[?QuotaCode==`'"${QUOTA_CODE}"'`].[QuotaName,Status,DesiredValue,LastUpdated]' \
  --output table

./scripts/sync-lifecycle.sh

cd cdk
pnpm install
pnpm cdk bootstrap   # 初回のみ
pnpm cdk deploy

./scripts/create-fsx.sh

$ ./scripts/create-fsx.sh
[create-fsx] 作成要求送信: fs-xxxxxxxxxxxxxxxxx
[create-fsx] FSx Status: CREATING
[create-fsx] FSx Status: CREATING
...
[create-fsx] FSx Status: AVAILABLE
[create-fsx] FSx 完了: fs-xxxxxxxxxxxxxxxxx (AVAILABLE)
[create-fsx] DRA 作成要求送信: dra-xxxxxxxxxxxxxxxxx
[create-fsx] DRA Status: CREATING
[create-fsx] DRA Status: CREATING
...
[create-fsx] DRA Status: AVAILABLE
[create-fsx] DRA 完了: dra-xxxxxxxxxxxxxxxxx (AVAILABLE)
[create-fsx] 次は ./scripts/create.sh でクラスタを作成してください

./scripts/create.sh

aws sagemaker describe-cluster \
  --cluster-name aws-hyperpod-slurm-hello-world \
  --region ap-northeast-1 \
  --query 'ClusterStatus'
"Creating" <= この表示が、InService になったら完了

./scripts/connect.sh

hostname               # controller のホスト名確認
sinfo                  # クラスタ状態
srun -N1 hostname      # worker のホスト名確認
srun -N1 nvidia-smi    # GPU 認識確認

# controller ノード上で(自ノードのマウント確認)
mount | grep fsx
# 期待: <FSX_DNS>@tcp:/<MOUNT> on /fsx type lustre (rw,...)

# worker ノードで同じ確認(controller から srun 経由)
srun -N1 bash -c 'mount | grep fsx'

# 共有ストレージとして動作することの確認(controller で書き、worker で読む)
mkdir -p /fsx/jobs
echo "hello from controller" > /fsx/jobs/hello.txt
srun -N1 cat /fsx/jobs/hello.txt
# 期待: hello from controller

import os
import socket
import subprocess

def main() -> None:
    print(f"hostname: {socket.gethostname()}")

    if os.path.exists("/fsx"):
        entries = os.listdir("/fsx")
        print(f"/fsx mounted: {entries}")
    else:
        print("/fsx not mounted")

    try:
        result = subprocess.run(
            ["nvidia-smi", "--query-gpu=name,memory.total", "--format=csv,noheader"],
            capture_output=True,
            text=True,
            check=True,
        )
        print(f"gpu: {result.stdout.strip()}")
    except (FileNotFoundError, subprocess.CalledProcessError) as e:
        print(f"gpu: not available ({e})")

if __name__ == "__main__":
    main()

#!/bin/bash
#SBATCH --job-name=hello
#SBATCH --output=/fsx/jobs/hello.%j.out
#SBATCH --nodes=1
#SBATCH --ntasks=1

mkdir -p /fsx/jobs
cd /fsx/jobs
python3 hello.py

# /fsx/jobs/ に hello.py / hello.sh を配置済みの前提
cd /fsx/jobs
sbatch hello.sh
squeue
ls /fsx/jobs/
cat /fsx/jobs/hello.*.out

$ ./scripts/delete-cluster.sh
{
    "ClusterArn": "arn:aws:sagemaker:ap-northeast-1:<ACCOUNT_ID>:cluster/xxxxxxxxxxxx"
}
[delete-cluster] クラスタ削除中...
[delete-cluster] クラスタ削除中...
...
[delete-cluster] クラスタ削除完了

$ ./scripts/delete-fsx.sh
[delete-fsx] DRA 削除要求送信: dra-xxxxxxxxxxxxxxxxx
[delete-fsx] DRA dra-xxxxxxxxxxxxxxxxx Status: AVAILABLE
[delete-fsx] DRA dra-xxxxxxxxxxxxxxxxx Status: DELETING
[delete-fsx] DRA dra-xxxxxxxxxxxxxxxxx Status: DELETING
...
[delete-fsx] DRA dra-xxxxxxxxxxxxxxxxx Status: DELETING
[delete-fsx] DRA dra-xxxxxxxxxxxxxxxxx Status: None
[delete-fsx] DRA 削除完了: dra-xxxxxxxxxxxxxxxxx
[delete-fsx] FSx 削除要求送信: fs-xxxxxxxxxxxxxxxxx
[delete-fsx] FSx 削除中...
[delete-fsx] FSx 削除中...
...
[delete-fsx] FSx 削除中...
[delete-fsx] 完了

cd cdk
pnpm cdk destroy --all --force

./scripts/teardown.sh

# HyperPod クラスタ残存チェック
aws sagemaker list-clusters --region ap-northeast-1

# FSx for Lustre 残存チェック
aws fsx describe-file-systems --region ap-northeast-1 \
  --query 'FileSystems[?Lifecycle!=`DELETED`].[FileSystemId,Lifecycle,Tags[?Key==`Name`].Value|[0]]' \
  --output table

# 利用可能状態の EBS（使われていないボリュームが残っていないか）
aws ec2 describe-volumes --region ap-northeast-1 \
  --filters Name=status,Values=available

# NAT Gateway 残存チェック
aws ec2 describe-nat-gateways --region ap-northeast-1 \
  --filter "Name=state,Values=available"

# 未関連付け EIP
aws ec2 describe-addresses --region ap-northeast-1 \
  --query 'Addresses[?AssociationId==null]'

# CloudFormation スタック
aws cloudformation describe-stacks \
  --stack-name AwsHyperpodSlurmHelloWorldStack \
  --region ap-northeast-1 2>&1 | head -3
# → does not exist 等のエラーが返れば OK

# 残留 ENI 確認（課金はないが、VPC 削除を阻害していないか）
aws ec2 describe-network-interfaces \
  --filters Name=description,Values="*hyperpod*" \
  --region ap-northeast-1 \
  --query 'NetworkInterfaces[].NetworkInterfaceId'
# → 空配列 [] であれば OK