Kiro CLI が Claude Opus 4.8 に対応。Opus モデル別の違いを確認してみた

AWSTemplateFormatVersion: '2010-09-09'
Description: Sample stack for analysis (intentionally contains ~10 planted issues)

Resources:
  DataBucket:
    Type: AWS::S3::Bucket
    Properties:
      BucketName: my-app-data-bucket
      AccessControl: PublicRead

  AppSecurityGroup:
    Type: AWS::EC2::SecurityGroup
    Properties:
      GroupDescription: app sg
      VpcId: vpc-0abc1234
      SecurityGroupIngress:
        - IpProtocol: tcp
          FromPort: 22
          ToPort: 22
          CidrIp: 0.0.0.0/0
        - IpProtocol: tcp
          FromPort: 3306
          ToPort: 3306
          CidrIp: 0.0.0.0/0

  AppRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Principal: { Service: ec2.amazonaws.com }
            Action: sts:AssumeRole
      Policies:
        - PolicyName: full-access
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Effect: Allow
                Action: '*'
                Resource: '*'

  AppInstanceProfile:
    Type: AWS::IAM::InstanceProfile
    Properties:
      Roles: [ !Ref AppRole ]

  AppServer:
    Type: AWS::EC2::Instance
    Properties:
      ImageId: ami-0123456789abcdef0
      InstanceType: t3.large
      IamInstanceProfile: !Ref AppInstanceProfile
      SecurityGroupIds: [ !Ref AppSecurityGroup ]

  AppDatabase:
    Type: AWS::RDS::DBInstance
    Properties:
      Engine: mysql
      DBInstanceClass: db.t3.medium
      AllocatedStorage: '20'
      MasterUsername: admin
      MasterUserPassword: Password123!
      PubliclyAccessible: true
      StorageEncrypted: false
      VPCSecurityGroups: [ !Ref AppSecurityGroup ]

Outputs:
  DBEndpoint:
    Value: !GetAtt AppDatabase.Endpoint.Address

#!/usr/bin/env bash
# 同一プロンプトを条件ごとにREPS回くり返し実行し、生出力とインデックスを残す。
# 使い方: ./bench.sh [REPS] [MODE]
#   REPS : 各条件のくり返し回数 (default 5)
#   MODE : effort | models | both   (default effort)
set -uo pipefail

DIR="$(cd "$(dirname "$0")" && pwd)"
REPS="${1:-5}"
MODE="${2:-effort}"
MODEL="claude-opus-4.8"
EFFORTS=(low medium high xhigh max default)
MODELS=(claude-opus-4.8 claude-opus-4.7 claude-opus-4.6)

OUT="$DIR/bench/out"; ERR="$DIR/bench/err"; IDX="$DIR/bench/runs.tsv"
mkdir -p "$OUT" "$ERR"
[ -f "$IDX" ] || printf "cond\trun\twall_sec\toutfile\terrfile\n" > "$IDX"

TPL=$(cat "$DIR/eval_template.yaml")
PROMPT="次のCloudFormationテンプレートを逆解析し、(1)アーキテクチャ概要 と (2)セキュリティ/ベストプラクティス上の問題点 を箇条書きで漏れなく指摘してください。

\`\`\`yaml
$TPL
\`\`\`"

set_effort() {
  if [ "$1" = "default" ]; then kiro-cli settings --delete chat.modelDefaults >/dev/null 2>&1
  else kiro-cli settings chat.modelDefaults "{\"$MODEL\":{\"output_config\":{\"effort\":\"$1\"}}}" >/dev/null; fi
}

one_run() {
  local cond="$1" model="$2" r o e s en wall
  for r in $(seq 1 "$REPS"); do
    o="$OUT/${cond}_${r}.txt"; e="$ERR/${cond}_${r}.txt"
    s=$(date +%s.%N)
    timeout 600 kiro-cli chat --no-interactive --trust-tools=read --model "$model" "$PROMPT" > "$o" 2> "$e"
    en=$(date +%s.%N); wall=$(echo "$en - $s" | bc)
    printf "%s\t%s\t%s\t%s\t%s\n" "$cond" "$r" "$wall" "$o" "$e" >> "$IDX"
    echo "[$(date +%T)] $cond run $r/$REPS  wall=${wall}s"
  done
}

if [ "$MODE" = "effort" ] || [ "$MODE" = "both" ]; then
  for ef in "${EFFORTS[@]}"; do set_effort "$ef"; one_run "effort_${ef}" "$MODEL"; done
fi
if [ "$MODE" = "models" ] || [ "$MODE" = "both" ]; then
  kiro-cli settings --delete chat.modelDefaults >/dev/null 2>&1
  for m in "${MODELS[@]}"; do one_run "model_${m}" "$m"; done
fi

kiro-cli settings --delete chat.modelDefaults >/dev/null 2>&1
echo "[$(date +%T)] DONE. 解析: python3 analyze.py"

#!/usr/bin/env python3
"""bench.sh の出力を解析。条件ごとに平均・中央値・外れ値を出力。"""
import re, os, statistics as st

DIR = os.path.dirname(os.path.abspath(__file__))
IDX = os.path.join(DIR, "bench", "runs.tsv")
ANSI = re.compile(r'\x1b\[[0-9;?]*[a-zA-Z]')

CHECKS = {
    "S3公開":            ["publicread", "パブリック", "公開"],
    "S3無暗号":          ["bucketencryption", "暗号化されていない", "暗号化されない",
                          "暗号化未設定", "暗号化が未", "暗号化が無", "暗号化無",
                          "暗号化なし", "sse-", "暗号化を有効", "保存データが平文"],
    "S3ブロック無":      ["publicaccessblock", "パブリックアクセスブロック"],
    "22全開放":          ["22", "ssh"],
    "3306全開放":        ["3306"],
    "IAM全権限":         ["action: '*'", "ワイルドカード", "全権限", "'*'", "管理者", "フルアクセス"],
    "DBパス平文":        ["平文", "ハードコード", "password123", "secrets manager", "直書き"],
    "RDS公開":           ["publiclyaccessible", "インターネット", "公開され", "パブリックip"],
    "RDS無暗号":         ["storageencrypted", "保管時", "ストレージ暗号", "保存データが暗号化"],
    "削除/バックアップ無": ["deletionpolicy", "削除保護", "スナップショット",
                           "バックアップ", "multiaz", "multi-az"],
}

def strip(p):
    return ANSI.sub("", open(p, encoding="utf-8", errors="replace").read())

def parse_run(outf, errf):
    body = strip(outf); lo = body.lower()
    detect = sum(any(w in lo for w in ks) for ks in CHECKS.values())
    chars = len(re.sub(r"\s", "", body))
    err = strip(errf) if os.path.exists(errf) else ""
    m = re.search(r'Credits:\s*([\d.]+).*?Time:\s*(\d+)\s*([hms])', err)
    credits = float(m.group(1)) if m else None
    app_t = None
    if m:
        v = int(m.group(2))
        app_t = v*60 if m.group(3) == "m" else (v*3600 if m.group(3) == "h" else v)
    return credits, app_t, detect, chars

def iqr_outliers(vals):
    xs = sorted(v for v in vals if v is not None)
    if len(xs) < 4:
        return set()
    q1 = st.quantiles(xs, n=4)[0]; q3 = st.quantiles(xs, n=4)[2]; iqr = q3 - q1
    lo, hi = q1 - 1.5*iqr, q3 + 1.5*iqr
    return {v for v in xs if v < lo or v > hi}

def stats(vals):
    xs = [v for v in vals if v is not None]
    if not xs:
        return None
    return dict(n=len(xs), mean=st.mean(xs), median=st.median(xs),
                stdev=(st.pstdev(xs) if len(xs) > 1 else 0.0),
                min=min(xs), max=max(xs))

runs = {}
if not os.path.exists(IDX):
    raise SystemExit(f"not found: {IDX}  (先に ./bench.sh を実行)")
for line in open(IDX):
    if line.startswith("cond\t"):
        continue
    cond, r, wall, outf, errf = line.rstrip("\n").split("\t")
    cr, at, det, ch = parse_run(outf, errf)
    runs.setdefault(cond, []).append(
        dict(run=int(r), wall=float(wall), credits=cr, app_t=at, detect=det, chars=ch))

metrics = [("credits", "cr"), ("app_t", "s"), ("wall", "s"), ("detect", "/10"), ("chars", "")]
for cond in sorted(runs):
    rs = runs[cond]; n = len(rs)
    print(f"\n===== {cond}  (n={n}) =====")
    print(f"{'metric':9}{'mean':>9}{'median':>9}{'stdev':>9}{'min':>8}{'max':>8}{'  外れ値(run#)'}")
    for key, unit in metrics:
        s = stats([x[key] for x in rs])
        if not s:
            print(f"{key:9}{'n/a':>9}"); continue
        outs = iqr_outliers([x[key] for x in rs])
        orun = [f"#{x['run']}({x[key]})" for x in rs
                if x[key] in outs and x[key] is not None]
        print(f"{key:9}{s['mean']:>9.2f}{s['median']:>9.2f}{s['stdev']:>9.2f}"
              f"{s['min']:>8.2f}{s['max']:>8.2f}  {', '.join(orun) if orun else '-'}")