EC2 の Nitro Sandboxes を Firecracker で試してみた

aws ec2 create-volume --availability-zone <親と同じ AZ> --size 1 --volume-type gp3
aws ec2 attach-volume --volume-id <ボリューム ID> --instance-id <親のインスタンス ID> --device /dev/sdf

set -e
cd /opt/sbx
chmod +x sandbox-init
cp /boot/vmlinuz-$(uname -r) vmlinuz
dracut --force --no-hostonly --reproducible --no-early-microcode \
    --modules bash \
    --add-drivers "virtio_pci virtio_net vmw_vsock_virtio_transport nsm virtio_blk ext4" \
    --install "mount umount modprobe ip sleep mkdir socat cat ls uname dd grep head tail od curl ping ps dmesg awk sed free df wc sync date id lsblk rm cut tr hostname touch strings" \
    --include "$PWD/sandbox-init" /init \
    --kver $(uname -r) initrd.img
ls -l vmlinuz initrd.img
file vmlinuz 2>/dev/null || true

#!/bin/bash
export PATH=/usr/local/bin:/usr/sbin:/usr/bin
mount -t proc proc /proc
mount -t sysfs sysfs /sys
mount -t devtmpfs devtmpfs /dev
mkdir -p /tmp /run /data /dev/shm
mount -t tmpfs tmpfs /tmp
mount -t tmpfs tmpfs /dev/shm
modprobe -a virtio_pci virtio_net vmw_vsock_virtio_transport nsm virtio_blk ext4

# カーネルコマンドラインの sbx_ip / sbx_gw を読む（台ごとに IP を変えるため）
for p in $(cat /proc/cmdline); do
  case $p in sbx_*=*) export "$p";; esac
done

# eth0 が現れるまで待つ
for i in 1 2 3 4 5 6 7 8 9 10; do
  [ -e /sys/class/net/eth0 ] && break
  sleep 0.5
done

# eth0 に静的アドレスと、親インスタンス側の TAP をデフォルトゲートウェイとして設定する
ip link set lo up
ip addr add "$sbx_ip" dev eth0
ip link set eth0 up
ip route add default via "$sbx_gw"
echo "nameserver 169.254.169.253" > /etc/resolv.conf

# ドライブがあればマウントする
if [ -b /dev/vda ]; then
  mount /dev/vda /data
fi

# 親へ vsock で逆接続して bash を渡す（CID 2 = 親、ポート 5000）
while true; do
  socat VSOCK-CONNECT:2:5000 EXEC:/bin/bash,stderr
  sleep 1
done

set -e
# TAP 2 本（sandbox a / b 用）
ip tuntap add dev tap0 mode tap
ip addr add 192.0.2.1/24 dev tap0
ip link set tap0 up
ip tuntap add dev tap1 mode tap
ip addr add 198.51.100.1/24 dev tap1
ip link set tap1 up
sysctl -w net.ipv4.ip_forward=1

# NAT（この時点では IMDS の reject なし）
IFACE=$(ip route show default | awk '{print $5; exit}')
nft -f - <<NFT
table ip sandbox {
    chain postrouting {
        type nat hook postrouting priority srcnat; policy accept;
        iifname { "tap0", "tap1" } oifname "$IFACE" masquerade
    }
}
NFT
nft list ruleset

# sandbox a: 追加 EBS（/dev/nvme1n1）に ext4 を作る。親ではマウントしない
mkfs.ext4 -q /dev/nvme1n1
lsblk -f /dev/nvme1n1

# sandbox b: 64 MiB の raw イメージファイル
truncate -s 64M /opt/sbx/b-data.img
mkfs.ext4 -q /opt/sbx/b-data.img
ls -l /opt/sbx/b-data.img
mkdir -p /run/sandbox /run/sbx

#!/bin/bash
# usage: start-sandbox.sh <name> <tap> <guest-ip/prefix> <gateway> <confidential|direct> <drive-path|none>
set -e
N=$1; TAP=$2; IP=$3; GW=$4; MODE=$5; DRIVE=$6
mkdir -p /run/sandbox /run/sbx
rm -f /run/sandbox/$N.sock*

if [ "$MODE" = direct ]; then
  ENC='{"cpu_overcommit": true, "direct_mode": true}'
else
  ENC='{"cpu_overcommit": true}'
fi
if [ "$DRIVE" = none ]; then
  DR='[]'
else
  DR="[{\"drive_id\":\"data\",\"path_on_host\":\"$DRIVE\",\"is_root_device\":false,\"is_read_only\":false}]"
fi

cat > /opt/sbx/sandbox-$N.json <<JSON
{
    "boot-source": {
        "kernel_image_path": "/opt/sbx/vmlinuz",
        "initrd_path": "/opt/sbx/initrd.img",
        "boot_args": "panic=1 initcall_blacklist=i8042_init sbx_ip=$IP sbx_gw=$GW"
    },
    "machine-config": {
        "vcpu_count": 2,
        "mem_size_mib": 1024
    },
    "drives": $DR,
    "network-interfaces": [
        {
            "iface_id": "eth0",
            "host_dev_name": "$TAP"
        }
    ],
    "vsock": {
        "guest_cid": 3,
        "uds_path": "/run/sandbox/$N.sock"
    },
    "enclave": $ENC
}
JSON

# ゲストが逆接続してくる vsock のリスナー（親側）。入力は .in に追記、出力は .out に溜まる
: > /run/sbx/$N.in
: > /run/sbx/$N.out
setsid nohup bash -c "while true; do tail -n0 -f /run/sbx/$N.in | socat UNIX-LISTEN:/run/sandbox/$N.sock_5000,unlink-early STDIO >> /run/sbx/$N.out 2>>/run/sbx/$N.listener.err; sleep 1; done" >/dev/null 2>&1 &

setsid nohup firecracker --no-api --config-file /opt/sbx/sandbox-$N.json > /run/sbx/$N.fc.log 2>&1 &
echo $! > /run/sbx/$N.pid
sleep 1
echo "started $N mode=$MODE pid=$(cat /run/sbx/$N.pid)"

#!/bin/bash
# usage: gexec <name> < script    ゲストの bash でスクリプトを実行し、出力を返す
N=$1
SCRIPT=$(cat)
ID=$RANDOM$RANDOM
START=$(stat -c %s /run/sbx/$N.out)
printf '{\n%s\n} 2>&1\necho "__END_%s__ rc=$?"\n' "$SCRIPT" "$ID" >> /run/sbx/$N.in
for i in $(seq 1 240); do
  if tail -c +$((START+1)) /run/sbx/$N.out | grep -q "__END_${ID}__"; then break; fi
  sleep 0.5
done
tail -c +$((START+1)) /run/sbx/$N.out | sed "s/^__END_${ID}__ rc=/[guest rc=/; /^\[guest rc=/s/\$/]/"

#!/usr/bin/env python3
"""Firecracker プロセスにマップされたゲストメモリ領域（fc.log の "CMA alloc: ... host=0x... size=N MiB"）を
/proc/<pid>/mem から読み、パターンを数える。
usage: memscan.py <name>   (name = a|b。/run/sbx/<name>.pid と <name>.fc.log を使う)
"""
import re, sys

name = sys.argv[1]
pid = int(open(f"/run/sbx/{name}.pid").read())
log = open(f"/run/sbx/{name}.fc.log").read()
m = re.search(r"CMA alloc: gpa=(0x[0-9a-f]+) host=(0x[0-9a-f]+) size=(\d+) MiB", log)
gpa, host, size_mib = int(m.group(1), 16), int(m.group(2), 16), int(m.group(3))
size = size_mib * 1024 * 1024
import os
if os.environ.get("LIMIT_MIB"):
    size = min(size, int(os.environ["LIMIT_MIB"]) * 1024 * 1024)
patterns = {
    "kernel banner 'Linux version 6.18'": b"Linux version 6.18",
    "initrd marker 'sbx_ip='": b"sbx_ip=",
    "guest tmpfs marker A": b"SBX-MEM-MARKER-A-c04e55",
    "guest tmpfs marker B": b"SBX-MEM-MARKER-B-91aa07",
}
print(f"sandbox={name} pid={pid} region host=0x{host:x} size={size_mib} MiB scanning={size // (1024 * 1024)} MiB")
MAXLEN = max(len(p) for p in patterns.values())
counts = {k: 0 for k in patterns}
ok = err = nonzero = 0
CH = 1024 * 1024
posf = open(f"/run/sbx/{name}.scanpos", "w", buffering=1)
tail = b""
with open(f"/proc/{pid}/mem", "rb", buffering=0) as f:
    off = 0
    while off < size:
        n = min(CH, size - off)
        posf.seek(0); posf.truncate(); posf.write(f"about to read offset={off // CH} MiB of {size_mib} MiB\n")
        try:
            f.seek(host + off)
            buf = f.read(n)
        except OSError as e:
            err += n
            off += n
            tail = b""
            first_err = getattr(sys.modules[__name__], "first_err", None)
            if first_err is None:
                first_err = str(e)
                setattr(sys.modules[__name__], "first_err", first_err)
            continue
        if not buf:
            err += n
            off += n
            continue
        ok += len(buf)
        nonzero += len(buf) - buf.count(0)
        data = tail + buf
        for k, p in patterns.items():
            counts[k] += data.count(p)
        tail = buf[-(MAXLEN - 1):]
        off += n
print(f"bytes read ok={ok} error={err} nonzero_bytes={nonzero}")
if err:
    print("first error:", getattr(sys.modules[__name__], "first_err", ""))
for k, v in counts.items():
    print(f"  {k}: {v} hit(s)")

/opt/sbx/gexec a <<'G'
echo "SBX-MEM-MARKER-A-c04e55-SBX-MEM-MARKER-A-c04e55" > /dev/shm/memmarker
G
/opt/sbx/gexec b <<'G'
echo "SBX-MEM-MARKER-B-91aa07-SBX-MEM-MARKER-B-91aa07" > /dev/shm/memmarker
cat /dev/shm/memmarker
G