AWS DevOps Agent にエラーログ・コードスニペット・Backlog 課題を自動送信する Lambda を Claude Codeと対話して実装してみた

AWSTemplateFormatVersion: "2010-09-09"
Description: "DevOps Agent Incident Handler - Recommended Stack Name: DevOpsAgentLambdaStack"

Metadata:
  AWS::CloudFormation::Interface:
    ParameterGroups:
      - Label:
          default: "SSM Parameter Store Settings (Recommended Stack Name: DevOpsAgentLambdaStack)"
        Parameters:
          - SsmParameterPrefix
    ParameterLabels:
      SsmParameterPrefix:
        default: SSM Parameter Prefix

Parameters:
  SsmParameterPrefix:
    Type: String
    Default: /incident-handler
    Description: |
      SSM Parameter Store prefix. Create ALL parameters before deployment:
      [Required - SecureString]
        {prefix}/github-token, {prefix}/devops-agent-secret, {prefix}/backlog-api-key
      [Required - String]
        {prefix}/config/sns-topic-arn, {prefix}/config/webhook-url
        {prefix}/config/github-owner, {prefix}/config/github-repo
        {prefix}/config/backlog-space-id, {prefix}/config/backlog-project-id
      [Optional - String]
        {prefix}/config/github-branch (default: main)
        {prefix}/config/github-code-lines-before (default: 8)
        {prefix}/config/github-code-lines-after (default: 7)
        {prefix}/config/backlog-domain (default: backlog.jp)
        {prefix}/config/backlog-search-hours (default: 24)
    AllowedPattern: "^/[a-zA-Z0-9/_-]+$"
    ConstraintDescription: Must start with / and contain only alphanumeric, /, _, -

Resources:
  # DevOps Agent Incident Handler IAM Role
  DevOpsAgentIncidentHandlerRole:
    Type: AWS::IAM::Role
    Properties:
      RoleName: devops-agent-incident-handler-role
      AssumeRolePolicyDocument:
        Version: "2012-10-17"
        Statement:
          - Effect: Allow
            Principal:
              Service: lambda.amazonaws.com
            Action: sts:AssumeRole
      ManagedPolicyArns:
        - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole
      Policies:
        - PolicyName: CloudWatchLogsPolicy
          PolicyDocument:
            Version: "2012-10-17"
            Statement:
              - Effect: Allow
                Action:
                  - logs:DescribeMetricFilters
                  - logs:FilterLogEvents
                Resource: "*"
        - PolicyName: SSMParameterStorePolicy
          PolicyDocument:
            Version: "2012-10-17"
            Statement:
              - Effect: Allow
                Action:
                  - ssm:GetParameter
                  - ssm:GetParameters
                Resource:
                  - !Sub "arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:parameter${SsmParameterPrefix}/*"

  # DevOps Agent Incident Handler Lambda Function
  DevOpsAgentIncidentHandlerFunction:
    Type: AWS::Lambda::Function
    Properties:
      FunctionName: devops-agent-incident-handler
      Description: CloudWatch Alarm to DevOps Agent (with code snippets and Backlog issues) - SSM version
      Runtime: nodejs22.x
      Handler: index.handler
      Role: !GetAtt DevOpsAgentIncidentHandlerRole.Arn
      Timeout: 120
      MemorySize: 256
      Environment:
        Variables:
          SSM_PARAMETER_PREFIX: !Ref SsmParameterPrefix
          AWS_ACCOUNT_ID: !Ref AWS::AccountId
      Code:
        ZipFile: |
          const { createHmac } = require("node:crypto");
          const https = require("node:https");
          const { URL } = require("node:url");
          const { CloudWatchLogsClient, DescribeMetricFiltersCommand, FilterLogEventsCommand } = require("@aws-sdk/client-cloudwatch-logs");
          const { SSMClient, GetParametersCommand } = require("@aws-sdk/client-ssm");

          const logsClient = new CloudWatchLogsClient({});
          const ssmClient = new SSMClient({});

          // Cache for SSM parameters (to avoid repeated API calls)
          let cachedConfig = null;

          async function getConfig() {
            if (cachedConfig) {
              return cachedConfig;
            }

            const prefix = process.env.SSM_PARAMETER_PREFIX || "/incident-handler";

            // Define parameter names
            const secretParams = [
              `${prefix}/github-token`,
              `${prefix}/devops-agent-secret`,
              `${prefix}/backlog-api-key`,
            ];

            const configParams = [
              `${prefix}/config/webhook-url`,
              `${prefix}/config/github-owner`,
              `${prefix}/config/github-repo`,
              `${prefix}/config/github-branch`,
              `${prefix}/config/github-code-lines-before`,
              `${prefix}/config/github-code-lines-after`,
              `${prefix}/config/backlog-space-id`,
              `${prefix}/config/backlog-domain`,
              `${prefix}/config/backlog-project-id`,
              `${prefix}/config/backlog-search-hours`,
            ];

            // Fetch secrets (with decryption)
            const secretsResult = await ssmClient.send(new GetParametersCommand({
              Names: secretParams,
              WithDecryption: true,
            }));

            // Fetch config (no decryption needed for String type)
            const configResult = await ssmClient.send(new GetParametersCommand({
              Names: configParams,
              WithDecryption: false,
            }));

            const getValue = (params, suffix) => {
              const param = params.find(p => p.Name?.endsWith(suffix));
              return param?.Value || "";
            };

            const secretParamsResult = secretsResult.Parameters || [];
            const configParamsResult = configResult.Parameters || [];

            cachedConfig = {
              // Secrets
              gitHubToken: getValue(secretParamsResult, "/github-token"),
              eventAiSecret: getValue(secretParamsResult, "/devops-agent-secret"),
              backlogApiKey: getValue(secretParamsResult, "/backlog-api-key"),
              // Config
              eventAiWebhookUrl: getValue(configParamsResult, "/config/webhook-url"),
              gitHubOwner: getValue(configParamsResult, "/config/github-owner"),
              gitHubRepo: getValue(configParamsResult, "/config/github-repo"),
              gitHubDefaultBranch: getValue(configParamsResult, "/config/github-branch") || "main",
              gitHubCodeLinesBefore: parseInt(getValue(configParamsResult, "/config/github-code-lines-before") || "8", 10),
              gitHubCodeLinesAfter: parseInt(getValue(configParamsResult, "/config/github-code-lines-after") || "7", 10),
              backlogSpaceId: getValue(configParamsResult, "/config/backlog-space-id"),
              backlogDomain: getValue(configParamsResult, "/config/backlog-domain") || "backlog.jp",
              backlogProjectId: getValue(configParamsResult, "/config/backlog-project-id"),
              backlogSearchPeriodHours: parseInt(getValue(configParamsResult, "/config/backlog-search-hours") || "24", 10),
              // Static
              awsAccountId: process.env.AWS_ACCOUNT_ID || "unknown",
              awsRegion: process.env.AWS_REGION || "ap-northeast-1",
            };

            // Log missing required parameters
            const missing = [];
            if (!cachedConfig.eventAiWebhookUrl) missing.push("webhook-url");
            if (!cachedConfig.eventAiSecret) missing.push("devops-agent-secret");
            if (!cachedConfig.gitHubOwner) missing.push("github-owner");
            if (!cachedConfig.gitHubRepo) missing.push("github-repo");
            if (!cachedConfig.gitHubToken) missing.push("github-token");
            if (!cachedConfig.backlogSpaceId) missing.push("backlog-space-id");
            if (!cachedConfig.backlogProjectId) missing.push("backlog-project-id");
            if (!cachedConfig.backlogApiKey) missing.push("backlog-api-key");

            if (missing.length > 0) {
              console.warn("Missing SSM parameters:", missing.join(", "));
            }

            // Log loaded configuration (with secrets masked)
            const maskSecret = (value) => value ? "***" : "(not set)";
            console.log("Loaded SSM configuration:", JSON.stringify({
              // Secrets (masked)
              gitHubToken: maskSecret(cachedConfig.gitHubToken),
              eventAiSecret: maskSecret(cachedConfig.eventAiSecret),
              backlogApiKey: maskSecret(cachedConfig.backlogApiKey),
              // Config (visible)
              eventAiWebhookUrl: cachedConfig.eventAiWebhookUrl || "(not set)",
              gitHubOwner: cachedConfig.gitHubOwner || "(not set)",
              gitHubRepo: cachedConfig.gitHubRepo || "(not set)",
              gitHubDefaultBranch: cachedConfig.gitHubDefaultBranch,
              gitHubCodeLinesBefore: cachedConfig.gitHubCodeLinesBefore,
              gitHubCodeLinesAfter: cachedConfig.gitHubCodeLinesAfter,
              backlogSpaceId: cachedConfig.backlogSpaceId || "(not set)",
              backlogDomain: cachedConfig.backlogDomain,
              backlogProjectId: cachedConfig.backlogProjectId || "(not set)",
              backlogSearchPeriodHours: cachedConfig.backlogSearchPeriodHours,
              // Static
              awsAccountId: cachedConfig.awsAccountId,
              awsRegion: cachedConfig.awsRegion,
            }, null, 2));

            return cachedConfig;
          }

          // Settings
          const OUTPUT_LIMIT = 20;
          const TIME_FROM_MIN = 3;
          const TIME_ERROR_AROUND = 30;
          const BACKLOG_MAX_ISSUES = 5;

          exports.handler = async (event) => {
            console.log("Received event:", JSON.stringify(event, null, 2));

            // Get config from SSM Parameter Store
            const config = await getConfig();

            for (const record of event.Records) {
              try {
                const message = JSON.parse(record.Sns.Message);
                console.log("Parsed message:", JSON.stringify(message, null, 2));

                // Skip OK state notifications (auto-resolve disabled)
                if (message.NewStateValue === "OK") {
                  console.log("Alarm recovered to OK state, skipping notification (auto-resolve disabled)");
                  continue;
                }

                // Get metric filters and error logs
                const filter = await getMetricFilters(message);
                let errorLogs = [];

                if (filter) {
                  const triggerLogs = await getTriggerLogEvents(message, filter);
                  if (triggerLogs.length > 0) {
                    const triggerLogMessage = triggerLogs[0].message || "";
                    if (isLambdaPowertools(triggerLogMessage)) {
                      console.log("Lambda Powertools format detected, fetching context logs");
                      errorLogs = await getContextLogs(triggerLogMessage, filter);
                    } else {
                      errorLogs = triggerLogs;
                    }
                  }
                }

                // Get code snippets from GitHub
                let codeSnippets = "";
                if (config.gitHubToken && config.gitHubOwner && config.gitHubRepo && errorLogs.length > 0) {
                  try {
                    codeSnippets = await getCodeSnippets(errorLogs, config);
                    console.log("Code snippets retrieved:", codeSnippets.length > 0 ? "Yes" : "No");
                  } catch (error) {
                    console.log("GitHub code fetch failed:", error.message);
                  }
                }

                // Search related Backlog issues
                let backlogIssues = [];
                if (config.backlogApiKey && config.backlogSpaceId && config.backlogProjectId) {
                  try {
                    backlogIssues = await searchBacklogIssues(message, config);
                    console.log("Backlog issues found:", backlogIssues.length);
                  } catch (error) {
                    console.log("Backlog search failed:", error.message);
                  }
                }

                // Send to DevOps Agent
                const devOpsAgentResult = await sendToDevOpsAgentWithRetry(message, errorLogs, codeSnippets, backlogIssues, config);
                console.log("DevOps Agent result:", devOpsAgentResult);

                if (devOpsAgentResult.statusCode < 200 || devOpsAgentResult.statusCode >= 300) {
                  throw new Error(`DevOps Agent failed: ${devOpsAgentResult.statusCode} - ${devOpsAgentResult.body}`);
                }
              } catch (error) {
                console.error("Error processing record:", error);
                throw error;
              }
            }

            return { statusCode: 200, body: "OK" };
          };

          // ========== CloudWatch Logs Functions ==========

          async function getMetricFilters(message) {
            try {
              const command = new DescribeMetricFiltersCommand({
                metricName: message.Trigger?.MetricName,
                metricNamespace: message.Trigger?.Namespace,
              });
              const response = await logsClient.send(command);

              if (!response.metricFilters?.[0]) {
                console.log("MetricFilters not found");
                return null;
              }
              return response.metricFilters[0];
            } catch (error) {
              console.warn("Failed to get metric filters:", error.message);
              return null;
            }
          }

          async function getTriggerLogEvents(message, filter) {
            try {
              const stateChangeTime = message.StateChangeTime
                ? new Date(message.StateChangeTime)
                : new Date();

              const endTime = stateChangeTime.getTime() + 60000;
              const startTime = endTime - (TIME_FROM_MIN * 60 * 1000);

              const command = new FilterLogEventsCommand({
                logGroupName: filter.logGroupName,
                filterPattern: filter.filterPattern,
                startTime,
                endTime,
                limit: OUTPUT_LIMIT,
              });

              const response = await logsClient.send(command);
              return (response.events || []).map(e => ({
                timestamp: new Date(e.timestamp).toISOString(),
                message: e.message || "",
              }));
            } catch (error) {
              console.warn("Failed to fetch trigger logs:", error.message);
              return [];
            }
          }

          function isLambdaPowertools(logMessage) {
            try {
              const json = JSON.parse(logMessage);
              return Object.prototype.hasOwnProperty.call(json, "function_request_id");
            } catch {
              return false;
            }
          }

          async function getContextLogs(triggerLogMessage, filter) {
            try {
              const logJson = JSON.parse(triggerLogMessage);
              const errorTime = new Date(logJson.timestamp).getTime();

              const command = new FilterLogEventsCommand({
                logGroupName: filter.logGroupName,
                filterPattern: `{$.function_request_id = "${logJson.function_request_id}"}`,
                startTime: errorTime - (TIME_ERROR_AROUND * 1000),
                endTime: errorTime + (TIME_ERROR_AROUND * 1000),
                limit: OUTPUT_LIMIT,
              });

              const response = await logsClient.send(command);
              return (response.events || []).map(e => ({
                timestamp: new Date(e.timestamp).toISOString(),
                message: e.message || "",
              }));
            } catch (error) {
              console.warn("Failed to fetch context logs:", error.message);
              return [];
            }
          }

          function formatErrorLogs(errorLogs) {
            if (errorLogs.length === 0) return "";
            return errorLogs.map((log, i) => {
              let formattedMessage = log.message;

              // JSON形式のメッセージを整形
              try {
                const parsed = JSON.parse(log.message);
                // スタックトレースを行ごとに分割
                if (parsed.error?.stack) {
                  parsed.error.stack = parsed.error.stack
                    .split("\\n")
                    .map(line => line.trim())
                    .join("\n      ");
                }
                formattedMessage = JSON.stringify(parsed, null, 2);
              } catch {
                // JSON でない場合はそのまま
              }

              return `[${i + 1}] ${log.timestamp}\n${formattedMessage}`;
            }).join("\n\n");
          }

          // ========== GitHub Functions ==========

          function extractFilePathsFromLogs(errorLogs) {
            const filePaths = new Set();
            const patterns = [
              /at\s+(?:\S+\s+\()?(?:\/var\/task\/)?([^:)]+):(\d+)/g,
              /File\s+"([^"]+)",\s+line\s+(\d+)/g,
              /([a-zA-Z0-9_/.-]+\.[tj]sx?):(\d+)/g,
            ];

            for (const log of errorLogs) {
              const message = typeof log === "string" ? log : log.message || "";
              for (const pattern of patterns) {
                pattern.lastIndex = 0;
                let match;
                while ((match = pattern.exec(message)) !== null) {
                  let filePath = match[1];
                  if (filePath.includes("node_modules") || filePath.startsWith("node:")) continue;
                  filePath = filePath.replace(/^\/var\/task\//, "").replace(/^\/+/, "");
                  const lineNum = parseInt(match[2], 10);
                  if (filePath && lineNum > 0) {
                    filePaths.add(JSON.stringify({ path: filePath, line: lineNum }));
                  }
                }
              }
            }

            return [...filePaths].map(JSON.parse).slice(0, 3);
          }

          async function fetchGitHubCode(filePath, config) {
            const encodedPath = filePath.split("/").map(segment => encodeURIComponent(segment)).join("/");
            return new Promise((resolve, reject) => {
              const options = {
                hostname: "api.github.com",
                path: `/repos/${config.gitHubOwner}/${config.gitHubRepo}/contents/${encodedPath}?ref=${config.gitHubDefaultBranch}`,
                method: "GET",
                headers: {
                  "Authorization": `Bearer ${config.gitHubToken}`,
                  "Accept": "application/vnd.github.raw+json",
                  "X-GitHub-Api-Version": "2022-11-28",
                  "User-Agent": "DevOpsAgentIncidentHandler/1.0",
                },
              };

              const req = https.request(options, (res) => {
                let data = "";
                res.on("data", chunk => data += chunk);
                res.on("end", () => {
                  if (res.statusCode === 200) {
                    resolve(data);
                  } else {
                    reject(new Error(`GitHub API error: ${res.statusCode}`));
                  }
                });
              });

              req.on("error", reject);
              req.setTimeout(10000, () => {
                req.destroy();
                reject(new Error("GitHub request timeout"));
              });
              req.end();
            });
          }

          async function getCodeSnippets(errorLogs, config) {
            const filePaths = extractFilePathsFromLogs(errorLogs);
            if (filePaths.length === 0) {
              console.log("No file paths extracted from logs");
              return "";
            }

            console.log("Extracted file paths:", JSON.stringify(filePaths));
            const snippets = [];

            for (const { path, line } of filePaths) {
              try {
                const content = await fetchGitHubCode(path, config);
                const lines = content.split("\n");
                const start = Math.max(0, line - config.gitHubCodeLinesBefore);
                const end = Math.min(lines.length, line + config.gitHubCodeLinesAfter);
                const snippet = lines.slice(start, end)
                  .map((l, i) => {
                    const lineNum = start + i + 1;
                    const marker = lineNum === line ? ">" : " ";
                    return `${String(lineNum).padStart(4)}${marker}| ${l}`;
                  })
                  .join("\n");
                const snippetBlock = `=== ${path}:${line} ===\n${snippet}`;
                console.log(`Code snippet from GitHub:\n${snippetBlock}`);
                snippets.push(snippetBlock);
              } catch (error) {
                console.log(`Failed to fetch ${path}: ${error.message}`);
              }
            }

            return snippets.join("\n\n");
          }

          // ========== Backlog Functions ==========

          async function searchBacklogIssues(alarmMessage, config) {
            const alarmTime = alarmMessage.StateChangeTime
              ? new Date(alarmMessage.StateChangeTime)
              : new Date();

            const createdSince = new Date(alarmTime.getTime() - (config.backlogSearchPeriodHours * 60 * 60 * 1000));
            const createdUntil = new Date(alarmTime.getTime() + (60 * 60 * 1000));

            const formatDate = (d) => d.toISOString().split("T")[0];

            const params = new URLSearchParams();
            params.append("projectId[]", config.backlogProjectId);
            params.append("statusId[]", "1");
            params.append("statusId[]", "2");
            params.append("statusId[]", "3");
            params.append("createdSince", formatDate(createdSince));
            params.append("createdUntil", formatDate(createdUntil));
            params.append("sort", "created");
            params.append("order", "desc");
            params.append("count", String(BACKLOG_MAX_ISSUES));

            const backlogHost = `${config.backlogSpaceId}.${config.backlogDomain}`;
            console.log("Backlog API host:", backlogHost);

            return new Promise((resolve, reject) => {
              const options = {
                hostname: backlogHost,
                path: `/api/v2/issues?apiKey=${config.backlogApiKey}&${params.toString()}`,
                method: "GET",
                headers: {
                  "Accept": "application/json",
                },
              };

              const req = https.request(options, (res) => {
                let data = "";
                res.on("data", chunk => data += chunk);
                res.on("end", () => {
                  if (res.statusCode === 200) {
                    try {
                      const issues = JSON.parse(data);
                      Promise.all(issues.slice(0, 3).map(issue => fetchIssueWithComments(issue, config)))
                        .then(resolve)
                        .catch(() => resolve(issues.map(formatIssue)));
                    } catch (e) {
                      reject(new Error(`Backlog parse error: ${e.message}`));
                    }
                  } else {
                    reject(new Error(`Backlog API error: ${res.statusCode} - ${data.substring(0, 200)}`));
                  }
                });
              });

              req.on("error", reject);
              req.setTimeout(15000, () => {
                req.destroy();
                reject(new Error("Backlog request timeout"));
              });
              req.end();
            });
          }

          async function fetchIssueWithComments(issue, config) {
            try {
              const comments = await fetchIssueComments(issue.issueKey, config);
              return {
                ...formatIssue(issue),
                comments: comments.slice(0, 3),
              };
            } catch {
              return formatIssue(issue);
            }
          }

          async function fetchIssueComments(issueKey, config) {
            const backlogHost = `${config.backlogSpaceId}.${config.backlogDomain}`;
            return new Promise((resolve, reject) => {
              const options = {
                hostname: backlogHost,
                path: `/api/v2/issues/${issueKey}/comments?apiKey=${config.backlogApiKey}&count=5&order=desc`,
                method: "GET",
                headers: { "Accept": "application/json" },
              };

              const req = https.request(options, (res) => {
                let data = "";
                res.on("data", chunk => data += chunk);
                res.on("end", () => {
                  if (res.statusCode === 200) {
                    try {
                      const comments = JSON.parse(data);
                      resolve(comments.map(c => ({
                        content: c.content || "",
                        createdUser: c.createdUser?.name || "Unknown",
                        created: c.created || "",
                      })));
                    } catch {
                      resolve([]);
                    }
                  } else {
                    resolve([]);
                  }
                });
              });

              req.on("error", () => resolve([]));
              req.setTimeout(10000, () => {
                req.destroy();
                resolve([]);
              });
              req.end();
            });
          }

          function formatIssue(issue) {
            return {
              issueKey: issue.issueKey || "",
              summary: issue.summary || "",
              description: issue.description || "",
              status: issue.status?.name || "",
              priority: issue.priority?.name || "",
              createdUser: issue.createdUser?.name || "",
              created: issue.created || "",
              updated: issue.updated || "",
            };
          }

          function formatBacklogIssues(issues, config) {
            if (issues.length === 0) return "";

            return issues.map((issue, i) => {
              const lines = [
                `[${i + 1}] ${issue.issueKey}: ${issue.summary}`,
                `    Status: ${issue.status} | Priority: ${issue.priority}`,
                `    Created: ${issue.created} by ${issue.createdUser}`,
              ];

              if (issue.description) {
                const desc = issue.description.length > 300
                  ? issue.description.substring(0, 300) + "..."
                  : issue.description;
                lines.push(`    Description: ${desc.replace(/\n/g, " ")}`);
              }

              if (issue.comments && issue.comments.length > 0) {
                lines.push(`    Recent Comments:`);
                issue.comments.forEach((c, j) => {
                  const content = c.content.length > 150
                    ? c.content.substring(0, 150) + "..."
                    : c.content;
                  lines.push(`      [${j + 1}] ${c.createdUser} (${c.created}): ${content.replace(/\n/g, " ")}`);
                });
              }

              return lines.join("\n");
            }).join("\n\n");
          }

          // ========== DevOps Agent Functions ==========

          async function sendToDevOpsAgentWithRetry(alarmMessage, errorLogs, codeSnippets, backlogIssues, config) {
            if (alarmMessage.NewStateValue === "OK") {
              console.log("Skipping DevOps Agent for OK state");
              return { statusCode: 200, body: "Skipped for OK state" };
            }

            const dimensions = alarmMessage.Trigger?.Dimensions || [];
            const functionDim = dimensions.find(d => d.name === "FunctionName");

            const descriptionLines = [];
            descriptionLines.push("Please respond in Japanese.");
            descriptionLines.push("");
            descriptionLines.push(`Status: ${alarmMessage.NewStateValue || "ALARM"}`);
            descriptionLines.push("");

            if (alarmMessage.NewStateReason) {
              descriptionLines.push(`Reason: ${alarmMessage.NewStateReason}`);
              descriptionLines.push("");
            }

            if (functionDim?.value) {
              descriptionLines.push(`Lambda Function: ${functionDim.value}`);
              descriptionLines.push("");
            }

            if (alarmMessage.Trigger?.Namespace && alarmMessage.Trigger?.MetricName) {
              descriptionLines.push(`Metric: ${alarmMessage.Trigger.Namespace}/${alarmMessage.Trigger.MetricName}`);
              descriptionLines.push("");
            }

            if (alarmMessage.AlarmName) {
              descriptionLines.push(`Alarm Name: ${alarmMessage.AlarmName}`);
              descriptionLines.push("");
            }

            if (alarmMessage.Region) {
              descriptionLines.push(`Region: ${alarmMessage.Region}`);
              descriptionLines.push("");
            }

            if (alarmMessage.AWSAccountId) {
              descriptionLines.push(`Account ID: ${alarmMessage.AWSAccountId}`);
              descriptionLines.push("");
            }

            if (alarmMessage.StateChangeTime) {
              descriptionLines.push(`Incident Time: ${alarmMessage.StateChangeTime}`);
              descriptionLines.push("");
            }

            if (alarmMessage.AlarmArn) {
              const consoleUrl = `https://${alarmMessage.Region}.console.aws.amazon.com/cloudwatch/home?region=${alarmMessage.Region}#alarmsV2:alarm/${encodeURIComponent(alarmMessage.AlarmName)}`;
              descriptionLines.push(`CloudWatch Console: ${consoleUrl}`);
              descriptionLines.push("");
            }

            if (errorLogs.length > 0) {
              descriptionLines.push("");
              descriptionLines.push("=== Error Logs (CloudWatch Logs) ===");
              descriptionLines.push("");
              descriptionLines.push(formatErrorLogs(errorLogs));
            }

            if (codeSnippets) {
              descriptionLines.push("");
              descriptionLines.push("=== Related Code (GitHub) ===");
              descriptionLines.push("");
              descriptionLines.push(codeSnippets);
            }

            if (backlogIssues && backlogIssues.length > 0) {
              descriptionLines.push("");
              descriptionLines.push(`=== Related Customer Inquiries (Backlog - within ${config.backlogSearchPeriodHours}h) ===`);
              descriptionLines.push("");
              descriptionLines.push(formatBacklogIssues(backlogIssues, config));
            }

            const payload = {
              eventType: "incident",
              incidentId: `cloudwatch:${alarmMessage.AlarmArn || Date.now()}`,
              action: "created",
              priority: "HIGH",
              title: `[${alarmMessage.NewStateValue}] ${alarmMessage.AlarmName}: ${alarmMessage.AlarmDescription || "CloudWatch Alarm triggered"}`,
              description: descriptionLines.join("\n"),
              timestamp: alarmMessage.StateChangeTime || new Date().toISOString(),
              service: "cloudwatch",
              affectedResources: alarmMessage.AlarmArn ? [alarmMessage.AlarmArn] : [],
            };

            console.log("Sending to DevOps Agent - Code snippets:", codeSnippets ? "Yes" : "No", "- Backlog issues:", backlogIssues?.length || 0);
            console.log("DevOps Agent payload summary:", JSON.stringify({
              eventType: payload.eventType,
              incidentId: payload.incidentId,
              priority: payload.priority,
              title: payload.title,
              descriptionLength: payload.description.length,
              hasCodeSnippets: !!codeSnippets,
              backlogIssuesCount: backlogIssues?.length || 0,
            }, null, 2));

            return sendToEventAiWithRetry(payload, config);
          }

          async function sendToEventAiWithRetry(payload, config) {
            const maxRetries = 3;
            const retryDelayMs = 1000;
            let lastError;

            for (let attempt = 1; attempt <= maxRetries; attempt++) {
              try {
                const result = await sendToEventAi(payload, config);

                if (result.statusCode >= 200 && result.statusCode < 300) {
                  return result;
                }

                if (result.statusCode >= 400 && result.statusCode < 500) {
                  console.error("Client error, not retrying:", result.statusCode);
                  return result;
                }

                lastError = new Error(`HTTP ${result.statusCode}: ${result.body}`);
              } catch (error) {
                lastError = error;
                console.error(`Request failed (attempt ${attempt}):`, error.message);
              }

              if (attempt < maxRetries) {
                const delay = retryDelayMs * attempt;
                await new Promise(resolve => setTimeout(resolve, delay));
              }
            }

            throw lastError || new Error("Unknown error");
          }

          async function sendToEventAi(payload, config) {
            const payloadString = JSON.stringify(payload);
            const tsHeader = new Date().toISOString();
            const hmac = createHmac("sha256", config.eventAiSecret);
            hmac.update(`${tsHeader}:${payloadString}`, "utf8");
            const signature = hmac.digest("base64");

            const url = new URL(config.eventAiWebhookUrl);
            const options = {
              hostname: url.hostname,
              path: url.pathname + url.search,
              method: "POST",
              port: url.port || 443,
              headers: {
                "Content-Type": "application/json",
                "x-amzn-event-timestamp": tsHeader,
                "x-amzn-event-signature": signature,
                "Content-Length": Buffer.byteLength(payloadString),
              },
            };

            return new Promise((resolve, reject) => {
              const req = https.request(options, (res) => {
                let data = "";
                res.on("data", chunk => data += chunk);
                res.on("end", () => {
                  console.log(`DevOps Agent response: ${res.statusCode} - ${data}`);
                  resolve({ statusCode: res.statusCode || 200, body: data });
                });
              });
              req.on("error", reject);
              req.write(payloadString);
              req.end();
            });
          }

  # SNS -> Lambda Permission
  DevOpsAgentIncidentHandlerPermission:
    Type: AWS::Lambda::Permission
    Properties:
      FunctionName: !Ref DevOpsAgentIncidentHandlerFunction
      Action: lambda:InvokeFunction
      Principal: sns.amazonaws.com
      SourceArn: !Sub "{{resolve:ssm:${SsmParameterPrefix}/config/sns-topic-arn}}"

  # SNS Subscription
  DevOpsAgentIncidentHandlerSubscription:
    Type: AWS::SNS::Subscription
    Properties:
      TopicArn: !Sub "{{resolve:ssm:${SsmParameterPrefix}/config/sns-topic-arn}}"
      Protocol: lambda
      Endpoint: !GetAtt DevOpsAgentIncidentHandlerFunction.Arn

  # Log Group
  DevOpsAgentIncidentHandlerLogGroup:
    Type: AWS::Logs::LogGroup
    Properties:
      LogGroupName: /aws/lambda/devops-agent-incident-handler
      RetentionInDays: 30

Outputs:
  DevOpsAgentIncidentHandlerArn:
    Description: DevOps Agent Incident Handler Lambda ARN
    Value: !GetAtt DevOpsAgentIncidentHandlerFunction.Arn

  DevOpsAgentIncidentHandlerLogGroup:
    Description: CloudWatch Logs log group name
    Value: !Ref DevOpsAgentIncidentHandlerLogGroup

  SsmParameterPaths:
    Description: Required SSM Parameter Store paths (must be created before deployment)
    Value: !Sub |
      [Secrets - SecureString]
      ${SsmParameterPrefix}/github-token
      ${SsmParameterPrefix}/devops-agent-secret
      ${SsmParameterPrefix}/backlog-api-key

      [Config - String (Required)]
      ${SsmParameterPrefix}/config/sns-topic-arn
      ${SsmParameterPrefix}/config/webhook-url
      ${SsmParameterPrefix}/config/github-owner
      ${SsmParameterPrefix}/config/github-repo
      ${SsmParameterPrefix}/config/backlog-space-id
      ${SsmParameterPrefix}/config/backlog-project-id

      [Config - String (Optional)]
      ${SsmParameterPrefix}/config/github-branch (default: main)
      ${SsmParameterPrefix}/config/backlog-domain (default: backlog.jp)
      ${SsmParameterPrefix}/config/backlog-search-hours (default: 24)

  IntegrationSummary:
    Description: Integration summary
    Value: "CloudWatch Alarm -> DevOps Agent (error logs + GitHub code + Backlog issues) - All config from SSM Parameter Store"