AWS IoT Jobsで100台にファームウェア更新を配り、失敗・拒否・応答なしを混ぜたときの動きを確かめてみた

{
  "Version": "2012-10-17",
  "Statement": [
    { "Effect": "Allow", "Action": "iot:Connect",
      "Resource": "arn:aws:iot:ap-northeast-1:<アカウントID>:client/${iot:Connection.Thing.ThingName}" },
    { "Effect": "Allow", "Action": ["iot:Publish", "iot:Receive"],
      "Resource": "arn:aws:iot:ap-northeast-1:<アカウントID>:topic/$aws/things/${iot:Connection.Thing.ThingName}/jobs/*" },
    { "Effect": "Allow", "Action": "iot:Subscribe",
      "Resource": "arn:aws:iot:ap-northeast-1:<アカウントID>:topicfilter/$aws/things/${iot:Connection.Thing.ThingName}/jobs/*" }
  ]
}

const base = `$aws/things/${thing}/jobs`;

client.on("connect", () => {
  client.subscribe(
    [`${base}/notify-next`, `${base}/start-next/accepted`, `${base}/+/update/accepted`],
    { qos: 1 },
    () => startNext(), // 接続前から待っているジョブを拾う
  );
});

client.on("message", (topic, payload) => {
  const msg = JSON.parse(payload.toString());
  if (topic.endsWith("/notify-next") && msg.execution) startNext();
  else if (topic.endsWith("/start-next/accepted")) onStarted(msg.execution);
});

function onStarted(exec) {
  if (!exec) return; // 待っているジョブが無い
  const target = exec.jobDocument.version; // "1.1.0"
  setTimeout(() => {
    dev.version = target;
    publish(`${base}/${exec.jobId}/update`, {
      status: "SUCCEEDED",
      statusDetails: { firmwareVersion: dev.version },
      expectedVersion: exec.versionNumber,
    });
  }, workMs());
}

InvalidStateTransition: Only queued jobs can be rejected. Job was in status IN_PROGRESS

// 拒否する台は IN_PROGRESS にせず、次のジョブ実行の中身だけを読む
const startNext = () =>
  behavior === "reject" ? publish(`${base}/$next/get`, {}) : publish(`${base}/start-next`, {});

function onDescribed(exec) { // $next/get/accepted で受け取る
  if (!exec) return; // 待っているジョブが無い
  publish(`${base}/${exec.jobId}/update`, {
    status: "REJECTED",
    statusDetails: { reason: "unsupported hardware revision" },
    expectedVersion: exec.versionNumber,
  });
}

{
  "operation": "firmwareUpdate",
  "version": "1.1.0",
  "releaseNotes": "センサー読み取り間隔の修正"
}

aws s3 mb s3://<バケット名> --region ap-northeast-1
aws s3 cp jobs/document.json s3://<バケット名>/document.json

{
  "topic": "$aws/events/job/fw-110-a-console/completed",
  "status": "COMPLETED",
  "jobProcessDetails": {
    "numberOfCanceledThings": 0,
    "numberOfRejectedThings": 0,
    "numberOfFailedThings": 0,
    "numberOfRemovedThings": 0,
    "numberOfSucceededThings": 100,
    "numberOfTimedOutThings": 0
  }
}

# 通知が届いた時刻の最初と最後（ジョブが無いという通知は除く）
jq -r 'select(.event == "notify-next" and .jobId != null) | .at' runs/a/devices.jsonl | sort | sed -n '1p;$p'
# 成功を報告した時刻の最初と最後
jq -r 'select(.event == "report") | .at' runs/a/devices.jsonl | sort | sed -n '1p;$p'
# ジョブを作った時刻と、COMPLETED になった時刻
jq -r 'select(.operation == "completed") | "\(.createdAt / 1000 | todate)  \(.completedAt / 1000 | todate)"' runs/a/events.jsonl

2026-10-28T05:46:07.780Z
2026-10-28T05:46:14.093Z
2026-10-28T05:46:11.185Z
2026-10-28T05:46:19.408Z
2026-10-28T05:46:07Z  2026-10-28T05:46:31Z

aws iot create-job \
  --cli-input-json file://jobs/b-no-abort.json \
  --job-id fw-110-b-<日時> \
  --targets arn:aws:iot:ap-northeast-1:<アカウントID>:thinggroup/jobs-fleet \
  --target-selection SNAPSHOT \
  --document file://jobs/document.json

{
  "description": "B: 一部失敗・中止設定なし"
}

node scripts/sim.js --run b2 --fail 67-79 --reject 80-86 --hang 87-93 --offline 94-100

aws iot describe-job --job-id fw-110-b-11041658 \
  --query '{status: job.status, reasonCode: job.reasonCode, comment: job.comment, jobProcessDetails: job.jobProcessDetails}'

{
  "status": "IN_PROGRESS",
  "reasonCode": null,
  "comment": null,
  "jobProcessDetails": {
    "numberOfSucceededThings": 66,
    "numberOfFailedThings": 13,
    "numberOfRejectedThings": 7,
    "numberOfQueuedThings": 7,
    "numberOfInProgressThings": 7
  }
}

aws iot cancel-job --force --job-id fw-110-b-11041658

{
  "status": "CANCELED",
  "reasonCode": null,
  "comment": null,
  "jobProcessDetails": {
    "numberOfCanceledThings": 14,
    "numberOfSucceededThings": 66,
    "numberOfFailedThings": 13,
    "numberOfRejectedThings": 7
  }
}

{
  "description": "C: 中止設定あり・1分あたり10台ずつ配る",
  "jobExecutionsRolloutConfig": {
    "maximumPerMinute": 10
  },
  "abortConfig": {
    "criteriaList": [
      {
        "failureType": "FAILED",
        "action": "CANCEL",
        "thresholdPercentage": 20,
        "minNumberOfExecutedThings": 30
      }
    ]
  }
}

node scripts/sim.js --run c --fail 3-99/3

{
  "status": "CANCELED",
  "reasonCode": "ABORT_CANCEL",
  "comment": "Abort criteria for failure type FAILED with threshold 20.00% met. Total completed executions: 30, total FAILED executions: 11",
  "jobProcessDetails": {
    "numberOfCanceledThings": 10,
    "numberOfSucceededThings": 19,
    "numberOfFailedThings": 11
  }
}

aws iot list-job-executions-for-job --job-id fw-110-c-11041631 > executions.json
jq -r '.executionSummaries | sort_by(.jobExecutionSummary.queuedAt) | .[]
  | "\(.jobExecutionSummary.queuedAt[11:19])  \(.thingArn | split("/") | last)  \(.jobExecutionSummary.status)  \(.jobExecutionSummary.startedAt[11:19] // "-")"' executions.json

16:31:16  jobs-dev-090  FAILED  16:31:18
16:31:22  jobs-dev-024  FAILED  16:31:23
16:31:28  jobs-dev-026  SUCCEEDED  16:31:29
（中略）
16:34:09  jobs-dev-077  SUCCEEDED  16:34:11
16:34:15  jobs-dev-039  FAILED  16:34:16
16:34:21  jobs-dev-003  CANCELED  -
16:34:27  jobs-dev-017  CANCELED  -
（中略）
16:35:09  jobs-dev-097  CANCELED  -
16:35:15  jobs-dev-025  CANCELED  -

jq -r 'select(.event == "notify-next" and .jobId != null) | .thing' runs/c/devices.jsonl | sort -u | wc -l

      30

{
  "description": "C2: 中止設定あり・配る速度の制限なし（C との比較）",
  "abortConfig": {
    "criteriaList": [
      {
        "failureType": "FAILED",
        "action": "CANCEL",
        "thresholdPercentage": 20,
        "minNumberOfExecutedThings": 30
      }
    ]
  }
}

{
  "status": "CANCELED",
  "reasonCode": "ABORT_CANCEL",
  "comment": "Abort criteria for failure type FAILED with threshold 20.00% met. Total completed executions: 58, total FAILED executions: 23",
  "jobProcessDetails": {
    "numberOfSucceededThings": 67,
    "numberOfFailedThings": 33
  }
}

{
  "description": "D: タイムアウト1分・再試行あり",
  "timeoutConfig": { "inProgressTimeoutInMinutes": 1 },
  "jobExecutionsRetryConfig": {
    "criteriaList": [
      { "failureType": "FAILED", "numberOfRetries": 2 },
      { "failureType": "TIMED_OUT", "numberOfRetries": 1 }
    ]
  }
}

node scripts/sim.js --run d --fail 67-79 --reject 80-86 --hang 87-93 --offline 94-100

aws iot list-job-executions-for-job --job-id fw-110-d-11041644 > executions.json
jq -r '.executionSummaries[] | .jobExecutionSummary.status' executions.json | sort | uniq -c

  39 FAILED
   7 QUEUED
   7 REJECTED
  66 SUCCEEDED
  14 TIMED_OUT

jq -r 'select(.thing == "jobs-dev-067" and (.event == "started" or .event == "report"))
  | "\(.at[11:23])  \(.event)  \(.executionNumber // "")\(.status // "")"' runs/d/devices.jsonl

07:44:58.147  started  1
07:45:02.748  report  FAILED
07:45:04.839  started  2
07:45:10.293  report  FAILED
07:45:11.703  started  3
07:45:17.481  report  FAILED

# 1回目の実行が始まった時刻
jq -r 'select(.event == "started" and .executionNumber == 1 and .thing >= "jobs-dev-087" and .thing <= "jobs-dev-093")
  | "\(.at[11:23])  \(.thing)"' runs/d/devices.jsonl
# TIMED_OUT になった時刻
jq -r 'select(.operation == "timed_out") | "\(.at[11:23])  \(.thingArn | split("/") | last)  exec#\(.executionNumber)"' runs/d/events.jsonl | sort -k2