Trying Out EC2 Nitro Sandboxes with Firecracker

aws ec2 create-volume --availability-zone <親と同じ AZ> --size 1 --volume-type gp3
aws ec2 attach-volume --volume-id <ボリューム ID> --instance-id <親のインスタンス ID> --device /dev/sdf

set -e
cd /opt/sbx
chmod +x sandbox-init
cp /boot/vmlinuz-$(uname -r) vmlinuz
dracut --force --no-hostonly --reproducible --no-early-microcode \
    --modules bash \
    --add-drivers "virtio_pci virtio_net vmw_vsock_virtio_transport nsm virtio_blk ext4" \
    --install "mount umount modprobe ip sleep mkdir socat cat ls uname dd grep head tail od curl ping ps dmesg awk sed free df wc sync date id lsblk rm cut tr hostname touch strings" \
    --include "$PWD/sandbox-init" /init \
    --kver $(uname -r) initrd.img
ls -l vmlinuz initrd.img
file vmlinuz 2>/dev/null || true

#!/bin/bash
export PATH=/usr/local/bin:/usr/sbin:/usr/bin
mount -t proc proc /proc
mount -t sysfs sysfs /sys
mount -t devtmpfs devtmpfs /dev
mkdir -p /tmp /run /data /dev/shm
mount -t tmpfs tmpfs /tmp
mount -t tmpfs tmpfs /dev/shm
modprobe -a virtio_pci virtio_net vmw_vsock_virtio_transport nsm virtio_blk ext4

# Read sbx_ip / sbx_gw from the kernel command line (to change the IP per sandbox)
for p in $(cat /proc/cmdline); do
  case $p in sbx_*=*) export "$p";; esac
done

# Wait for eth0 to appear
for i in 1 2 3 4 5 6 7 8 9 10; do
  [ -e /sys/class/net/eth0 ] && break
  sleep 0.5
done

# Set a static address on eth0 and the parent instance's TAP as the default gateway
ip link set lo up
ip addr add "$sbx_ip" dev eth0
ip link set eth0 up
ip route add default via "$sbx_gw"
echo "nameserver 169.254.169.253" > /etc/resolv.conf

# Mount the drive if present
if [ -b /dev/vda ]; then
  mount /dev/vda /data
fi

# Connect back to the parent over vsock and hand over bash (CID 2 = parent, port 5000)
while true; do
  socat VSOCK-CONNECT:2:5000 EXEC:/bin/bash,stderr
  sleep 1
done

set -e
# Two TAPs (for sandbox a / b)
ip tuntap add dev tap0 mode tap
ip addr add 192.0.2.1/24 dev tap0
ip link set tap0 up
ip tuntap add dev tap1 mode tap
ip addr add 198.51.100.1/24 dev tap1
ip link set tap1 up
sysctl -w net.ipv4.ip_forward=1

# NAT（この時点では IMDS の reject なし）
IFACE=$(ip route show default | awk '{print $5; exit}')
nft -f - <<NFT
table ip sandbox {
    chain postrouting {
        type nat hook postrouting priority srcnat; policy accept;
        iifname { "tap0", "tap1" } oifname "$IFACE" masquerade
    }
}
NFT
nft list ruleset

# sandbox a: 追加 EBS（/dev/nvme1n1）に ext4 を作る。親ではマウントしない
mkfs.ext4 -q /dev/nvme1n1
lsblk -f /dev/nvme1n1

# sandbox b: 64 MiB の raw イメージファイル
truncate -s 64M /opt/sbx/b-data.img
mkfs.ext4 -q /opt/sbx/b-data.img
ls -l /opt/sbx/b-data.img
mkdir -p /run/sandbox /run/sbx

#!/bin/bash
# usage: start-sandbox.sh <name> <tap> <guest-ip/prefix> <gateway> <confidential|direct> <drive-path|none>
set -e
N=$1; TAP=$2; IP=$3; GW=$4; MODE=$5; DRIVE=$6
mkdir -p /run/sandbox /run/sbx
rm -f /run/sandbox/$N.sock*

if [ "$MODE" = direct ]; then
  ENC='{"cpu_overcommit": true, "direct_mode": true}'
else
  ENC='{"cpu_overcommit": true}'
fi
if [ "$DRIVE" = none ]; then
  DR='[]'
else
  DR="[{\"drive_id\":\"data\",\"path_on_host\":\"$DRIVE\",\"is_root_device\":false,\"is_read_only\":false}]"
fi

cat > /opt/sbx/sandbox-$N.json <<JSON
{
    "boot-source": {
        "kernel_image_path": "/opt/sbx/vmlinuz",
        "initrd_path": "/opt/sbx/initrd.img",
        "boot_args": "panic=1 initcall_blacklist=i8042_init sbx_ip=$IP sbx_gw=$GW"
    },
    "machine-config": {
        "vcpu_count": 2,
        "mem_size_mib": 1024
    },
    "drives": $DR,
    "network-interfaces": [
        {
            "iface_id": "eth0",
            "host_dev_name": "$TAP"
        }
    ],
    "vsock": {
        "guest_cid": 3,
        "uds_path": "/run/sandbox/$N.sock"
    },
    "enclave": $ENC
}
JSON

# ゲストが逆接続してくる vsock のリスナー（親側）。入力は .in に追記、出力は .out に溜まる
: > /run/sbx/$N.in
: > /run/sbx/$N.out
setsid nohup bash -c "while true; do tail -n0 -f /run/sbx/$N.in | socat UNIX-LISTEN:/run/sandbox/$N.sock_5000,unlink-early STDIO >> /run/sbx/$N.out 2>>/run/sbx/$N.listener.err; sleep 1; done" >/dev/null 2>&1 &

setsid nohup firecracker --no-api --config-file /opt/sbx/sandbox-$N.json > /run/sbx/$N.fc.log 2>&1 &
echo $! > /run/sbx/$N.pid
sleep 1
echo "started $N mode=$MODE pid=$(cat /run/sbx/$N.pid)"

#!/bin/bash
# usage: gexec <name> < script    ゲストの bash でスクリプトを実行し、出力を返す
N=$1
SCRIPT=$(cat)
ID=$RANDOM$RANDOM
START=$(stat -c %s /run/sbx/$N.out)
printf '{\n%s\n} 2>&1\necho "__END_%s__ rc=$?"\n' "$SCRIPT" "$ID" >> /run/sbx/$N.in
for i in $(seq 1 240); do
  if tail -c +$((START+1)) /run/sbx/$N.out | grep -q "__END_${ID}__"; then break; fi
  sleep 0.5
done
tail -c +$((START+1)) /run/sbx/$N.out | sed "s/^__END_${ID}__ rc=/[guest rc=/; /^\[guest rc=/s/\$/]/"

#!/usr/bin/env python3
"""Firecracker プロセスにマップされたゲストメモリ領域（fc.log の "CMA alloc: ... host=0x... size=N MiB"）を
/proc/<pid>/mem から読み、パターンを数える。
usage: memscan.py <name>   (name = a|b。/run/sbx/<name>.pid と <name>.fc.log を使う)
"""
import re, sys

name = sys.argv[1]
pid = int(open(f"/run/sbx/{name}.pid").read())
log = open(f"/run/sbx/{name}.fc.log").read()
m = re.search(r"CMA alloc: gpa=(0x[0-9a-f]+) host=(0x[0-9a-f]+) size=(\d+) MiB", log)
gpa, host, size_mib = int(m.group(1), 16), int(m.group(2), 16), int(m.group(3))
size = size_mib * 1024 * 1024
import os
if os.environ.get("LIMIT_MIB"):
    size = min(size, int(os.environ["LIMIT_MIB"]) * 1024 * 1024)
patterns = {
    "kernel banner 'Linux version 6.18'": b"Linux version 6.18",
    "initrd marker 'sbx_ip='": b"sbx_ip=",
    "guest tmpfs marker A": b"SBX-MEM-MARKER-A-c04e55",
    "guest tmpfs marker B": b"SBX-MEM-MARKER-B-91aa07",
}
print(f"sandbox={name} pid={pid} region host=0x{host:x} size={size_mib} MiB scanning={size // (1024 * 1024)} MiB")
MAXLEN = max(len(p) for p in patterns.values())
counts = {k: 0 for k in patterns}
ok = err = nonzero = 0
CH = 1024 * 1024
posf = open(f"/run/sbx/{name}.scanpos", "w", buffering=1)
tail = b""
with open(f"/proc/{pid}/mem", "rb", buffering=0) as f:
    off = 0
    while off < size:
        n = min(CH, size - off)
        posf.seek(0); posf.truncate(); posf.write(f"about to read offset={off // CH} MiB of {size_mib} MiB\n")
        try:
            f.seek(host + off)
            buf = f.read(n)
        except OSError as e:
            err += n
            off += n
            tail = b""
            first_err = getattr(sys.modules[__name__], "first_err", None)
            if first_err is None:
                first_err = str(e)
                setattr(sys.modules[__name__], "first_err", first_err)
            continue
        if not buf:
            err += n
            off += n
            continue
        ok += len(buf)
        nonzero += len(buf) - buf.count(0)
        data = tail + buf
        for k, p in patterns.items():
            counts[k] += data.count(p)
        tail = buf[-(MAXLEN - 1):]
        off += n
print(f"bytes read ok={ok} error={err} nonzero_bytes={nonzero}")
if err:
    print("first error:", getattr(sys.modules[__name__], "first_err", ""))
for k, v in counts.items():
    print(f"  {k}: {v} hit(s)")

/opt/sbx/gexec a <<'G'
echo "SBX-MEM-MARKER-A-c04e55-SBX-MEM-MARKER-A-c04e55" > /dev/shm/memmarker
G
/opt/sbx/gexec b <<'G'
echo "SBX-MEM-MARKER-B-91aa07-SBX-MEM-MARKER-B-91aa07" > /dev/shm/memmarker
cat /dev/shm/memmarker
G