CloudWatch エージェントが journald に対応したので Amazon Linux 2023 で試してみた

  InstanceRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Principal:
              Service: ec2.amazonaws.com
            Action: sts:AssumeRole
      ManagedPolicyArns:
        - arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore
      Policies:
        - PolicyName: journald-logs-minimal
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Effect: Allow
                Action:
                  - logs:CreateLogGroup
                  - logs:CreateLogStream
                  - logs:PutLogEvents
                  - logs:PutRetentionPolicy
                Resource: !Sub arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:journald-*

aws cloudformation create-stack \
  --stack-name journald-demo \
  --template-body file://journald-demo.yaml \
  --capabilities CAPABILITY_IAM \
  --deployment-config Mode=EXPRESS \
  --parameters ParameterKey=VpcId,ParameterValue=vpc-xxxxxxxxxxxxxxxxx \
               ParameterKey=SubnetId,ParameterValue=subnet-xxxxxxxxxxxxxxxxx

{
  "agent": {
    "run_as_user": "root"
  },
  "logs": {
    "logs_collected": {
      "journald": {
        "collect_list": [
          {
            "log_group_name": "journald-units",
            "log_stream_name": "{instance_id}",
            "units": ["sshd", "systemd-logind", "chronyd", "amazon-ssm-agent"],
            "retention_in_days": 1
          },
          {
            "log_group_name": "journald-kernel",
            "log_stream_name": "{instance_id}",
            "matches": [{"_TRANSPORT": "kernel"}],
            "retention_in_days": 1
          },
          {
            "log_group_name": "journald-audit",
            "log_stream_name": "{instance_id}",
            "matches": [{"_TRANSPORT": "audit"}],
            "priority": "debug",
            "retention_in_days": 1
          },
          {
            "log_group_name": "journald-docker",
            "log_stream_name": "{instance_id}",
            "units": ["docker"],
            "retention_in_days": 1
          }
        ]
      }
    }
  }
}

sudo /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c file:/opt/aws/amazon-cloudwatch-agent/etc/cw-journald.json

{
    "events": [
        {
            "timestamp": 1787982970748,
            "message": "{\"body\":{\"MESSAGE\":\"pam_unix(sudo:session): session closed for user root\",\"PRIORITY\":\"6\",\"SYSLOG_FACILITY\":\"10\",\"SYSLOG_IDENTIFIER\":\"sudo\",\"_CMDLINE\":\"sudo docker run --rm public.ecr.aws/docker/library/alpine:3.20 true\",\"_COMM\":\"sudo\",\"_EXE\":\"/usr/bin/sudo\",\"_HOSTNAME\":\"ip-10-0-x-x.ap-northeast-1.compute.internal\",\"_PID\":\"29152\",\"_SYSTEMD_UNIT\":\"amazon-ssm-agent.service\",\"_TRANSPORT\":\"syslog\",\"_UID\":\"0\",\"__CURSOR\":\"s=xxxxxxxx;i=e09;b=xxxxxxxx;m=2155d6a2;t=65a2939d5ff89;x=xxxxxxxx\"}}",
            "ingestionTime": 1787982974133
        }
    ]
}

{
 "body": {
  "MESSAGE": "Accepted publickey for ec2-user from 127.0.0.1 port 54478 ssh2: ED25519 SHA256:xxxxxxxx",
  "PRIORITY": "6",
  "SYSLOG_FACILITY": "10",
  "SYSLOG_IDENTIFIER": "sshd-session",
  "SYSLOG_PID": "29125",
  "_COMM": "sshd-session",
  "_EXE": "/usr/libexec/openssh/sshd-session",
  "_HOSTNAME": "ip-10-0-x-x.ap-northeast-1.compute.internal",
  "_PID": "29125",
  "_SELINUX_CONTEXT": "system_u:system_r:sshd_t:s0-s0:c0.c1023",
  "_SYSTEMD_CGROUP": "/system.slice/sshd.service",
  "_SYSTEMD_UNIT": "sshd.service",
  "_TRANSPORT": "syslog",
  "_UID": "0"
 }
}

{
    "events": [
        {
            "timestamp": 1787982734525,
            "message": "{\"body\":{\"MESSAGE\":\"veth97da191 (unregistering): left allmulticast mode\",\"PRIORITY\":\"6\",\"SYSLOG_FACILITY\":\"0\",\"SYSLOG_IDENTIFIER\":\"kernel\",\"_BOOT_ID\":\"xxxxxxxx\",\"_HOSTNAME\":\"ip-10-0-x-x.ap-northeast-1.compute.internal\",\"_TRANSPORT\":\"kernel\",\"__CURSOR\":\"s=xxxxxxxx;i=9bc;b=xxxxxxxx;m=13415a86;t=65a292bc1836e;x=xxxxxxxx\"}}",
            "ingestionTime": 1787982739614
        }
    ]
}

{
    "events": [
        {
            "timestamp": 1787983006400,
            "message": "{\"body\":{\"AUDIT_FIELD_EXE\":\"/usr/lib/systemd/systemd\",\"AUDIT_FIELD_RES\":\"success\",\"AUDIT_FIELD_UNIT\":\"user@0\",\"MESSAGE\":\"SERVICE_STOP pid=1 uid=0 auid=4294967295 ses=4294967295 subj=system_u:system_r:init_t:s0 msg='unit=user@0 comm=\\\"systemd\\\" exe=\\\"/usr/lib/systemd/systemd\\\" hostname=? addr=? terminal=? res=success'\",\"SYSLOG_FACILITY\":\"4\",\"SYSLOG_IDENTIFIER\":\"audit\",\"_AUDIT_ID\":\"1433\",\"_AUDIT_TYPE\":\"1131\",\"_AUDIT_TYPE_NAME\":\"SERVICE_STOP\",\"_HOSTNAME\":\"ip-10-0-x-x.ap-northeast-1.compute.internal\",\"_PID\":\"1\",\"_TRANSPORT\":\"audit\",\"_UID\":\"0\"}}",
            "ingestionTime": 1787983009138
        }
    ]
}

{
    "events": [
        {
            "timestamp": 1787982718503,
            "message": "{\"body\":{\"MESSAGE\":\"time=\\\"2026-08-29T05:51:58.502767587Z\\\" level=info msg=\\\"API listen on /run/docker.sock\\\"\",\"PRIORITY\":\"6\",\"SYSLOG_FACILITY\":\"3\",\"SYSLOG_IDENTIFIER\":\"dockerd\",\"_CMDLINE\":\"/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock --default-ulimit nofile=32768:65536\",\"_COMM\":\"dockerd\",\"_EXE\":\"/usr/bin/dockerd\",\"_HOSTNAME\":\"ip-10-0-x-x.ap-northeast-1.compute.internal\",\"_PID\":\"7416\",\"_SYSTEMD_UNIT\":\"docker.service\",\"_TRANSPORT\":\"stdout\",\"_UID\":\"0\"}}",
            "ingestionTime": 1787982724803
        }
    ]
}

stats count(*) as cnt by body._SYSTEMD_UNIT | sort cnt desc

stats count(*) as cnt by @log, body.PRIORITY | sort @log, body.PRIORITY

stats count(*) as cnt by body._AUDIT_TYPE_NAME | sort cnt desc | limit 8

filter body._SYSTEMD_UNIT = "sshd.service" and body.MESSAGE like /Invalid user/
| fields @timestamp, body.MESSAGE | sort @timestamp desc

#!/bin/bash
set -xeuo pipefail

# systemd 管理下のアプリ例として Docker を導入
dnf install -y docker
systemctl enable --now docker

# AL2023 の dnf リポジトリ版は journald 非対応バージョンのため S3 の最新 RPM を使う
cd /tmp
curl -sS -O https://amazoncloudwatch-agent.s3.amazonaws.com/amazon_linux/amd64/latest/amazon-cloudwatch-agent.rpm
dnf install -y ./amazon-cloudwatch-agent.rpm

cat > /opt/aws/amazon-cloudwatch-agent/etc/cw-journald.json <<'CONFIG'
{
  "agent": { "run_as_user": "root" },
  "logs": {
    "logs_collected": {
      "journald": {
        "collect_list": [
          {
            "log_group_name": "journald-userdata-units",
            "log_stream_name": "{instance_id}",
            "units": ["sshd", "systemd-logind", "chronyd", "amazon-ssm-agent"],
            "retention_in_days": 1
          },
          {
            "log_group_name": "journald-userdata-kernel",
            "log_stream_name": "{instance_id}",
            "matches": [{"_TRANSPORT": "kernel"}],
            "retention_in_days": 1
          },
          {
            "log_group_name": "journald-userdata-audit",
            "log_stream_name": "{instance_id}",
            "matches": [{"_TRANSPORT": "audit"}],
            "priority": "debug",
            "retention_in_days": 1
          },
          {
            "log_group_name": "journald-userdata-docker",
            "log_stream_name": "{instance_id}",
            "units": ["docker"],
            "retention_in_days": 1
          }
        ]
      }
    }
  }
}
CONFIG

/opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl \
  -a fetch-config -m ec2 -s \
  -c file:/opt/aws/amazon-cloudwatch-agent/etc/cw-journald.json