AWS FireLens (AWS for Fluent Bit) で16KB以上のログを扱ってみた

[FILTER]
  name                  multiline
  match                 *
  multiline.key_content log
  mode                  partial_message

app.get("/large-log", (req: Request, res: Response) => {
  // 16KB を超える大きなログを生成
  // FluentBitの16KB制限をテストするため
  const largeData = {
    message: "Testing 16KB+ log handling",
    timestamp: new Date().toISOString(),
    data: "X".repeat(20000),
  };

  req.log.error(largeData, "Large log entry generated");

  res.status(200).json({
    message: "Large log generated successfully",
    logSize: JSON.stringify(largeData).length,
    timestamp: new Date().toISOString(),
  });
});

> curl http://EcsNat-AlbCo-gB01lRgFgX9v-1376474775.us-east-1.elb.amazonaws.com/app/large-log/
{"error":"Large log generated successfully","logSize":20089,"timestamp":"2026-01-17T08:17:23.126Z"}%

{
    "partial_ordinal": "1",
    "partial_last": "false",
    "container_id": "16e9de31cf944b9f86e68b2f3f8a36d4-0527074092",
    "container_name": "app",
    "source": "stderr",
    "log": "{\"level\":\"error\",\"time\":\"2026-01-17T08:27:27.388Z\",\"pid\":50,\"hostname\":\"ip-10-10-10-102.ec2.internal\",\"req\":{\"id\":16,\"method\":\"GET\",\"url\":\"/large-log/\",\"query\":{},\"params\":{},\"headers\":{\"host\":\"ecsnat-albco-gb01lrgfgx9v-1376474775.us-east-1.elb.amazonaws.com\",\"x-real-ip\":\"10.10.10.11\",\"x-forwarded-for\":\"<送信元IPアドレス> 10.10.10.11\",\"x-forwarded-proto\":\"http\",\"connection\":\"close\",\"x-forwarded-port\":\"80\",\"x-amzn-trace-id\":\"Root=1-696b47ef-5967e86740233a79392a7665\",\"accept\":\"*/*\",\"user-agent\":\"curl/8.7.1\"},\"remoteAddress\":\"127.0.0.1\",\"remotePort\":42364},\"message\":\"Testing 16KB+ log handling\",\"timestamp\":\"2026-01-17T08:27:27.388Z\",\"data\":\"XXXXX..(中略)..XXXXXX",
    "partial_message": "true",
    "partial_id": "56571121fe67f58e10e882d7812d3e9de3e7c22145bb37ce793f4b06bce2878c",
    "ecs_cluster": "EcsNativeBlueGreenStack-EcsConstructCluster14AE103B-0UD5SYuupUOg",
    "ecs_task_arn": "arn:aws:ecs:us-east-1:<AWSアカウントID>:task/EcsNativeBlueGreenStack-EcsConstructCluster14AE103B-0UD5SYuupUOg/16e9de31cf944b9f86e68b2f3f8a36d4",
    "ecs_task_definition": "EcsNativeBlueGreenStackEcsConstructTaskDefinitionF683F4B2:47"
}
{
    "partial_last": "true",
    "container_id": "16e9de31cf944b9f86e68b2f3f8a36d4-0527074092",
    "container_name": "app",
    "source": "stderr",
    "log": "XXXX..(中略)..XXX\",\"msg\":\"Large log entry generated\"}",
    "partial_message": "true",
    "partial_id": "56571121fe67f58e10e882d7812d3e9de3e7c22145bb37ce793f4b06bce2878c",
    "partial_ordinal": "2",
    "ecs_cluster": "EcsNativeBlueGreenStack-EcsConstructCluster14AE103B-0UD5SYuupUOg",
    "ecs_task_arn": "arn:aws:ecs:us-east-1:<AWSアカウントID>:task/EcsNativeBlueGreenStack-EcsConstructCluster14AE103B-0UD5SYuupUOg/16e9de31cf944b9f86e68b2f3f8a36d4",
    "ecs_task_definition": "EcsNativeBlueGreenStackEcsConstructTaskDefinitionF683F4B2:47"
}

# ref : 詳解 FireLens – Amazon ECS タスクで高度なログルーティングを実現する機能を深く知る | Amazon Web Services ブログ https://aws.amazon.com/jp/blogs/news/under-the-hood-firelens-for-amazon-ecs-tasks/
# ref : aws-for-fluent-bit/use_cases/init-process-for-fluent-bit/README.md at mainline · aws/aws-for-fluent-bit https://github.com/aws/aws-for-fluent-bit/blob/mainline/use_cases/init-process-for-fluent-bit/README.md
[SERVICE]
    Flush                 1
    Grace                 30
    Parsers_File          /fluent-bit/parsers/parsers.conf

# 16KB制限により分割されたログレコードを再結合
# partial_message モードで元の1つのログレコードに再結合
# ref : Multiline | Fluent Bit: Official Manual https://docs.fluentbit.io/manual/data-pipeline/filters/multiline-stacktrace
[FILTER]
    Name                  multiline
    Match                 *-firelens-*
    multiline.key_content log
    mode                  partial_message
    flush_ms              2000

# appコンテナのログにマルチラインパーサーを適用
# Loggerとしてpinoを使用して構造化ログとして出力させているため、今回は効果を発揮しない
# ref : Multiline parsing | Fluent Bit: Official Manual https://docs.fluentbit.io/manual/data-pipeline/parsers/multiline-parsing
# ref : 複数行またはスタックトレースの Amazon ECS ログメッセージの連結 - Amazon Elastic Container Service https://docs.aws.amazon.com/ja_jp/AmazonECS/latest/developerguide/firelens-concatanate-multiline.html
[FILTER]
    Name                  multiline
    Match                 app-firelens*
    multiline.key_content log
    multiline.parser      nodejs_stacktrace

# appコンテナのログをJSONとしてパース
# タイムスタンプはISO 8601形式でそのまま保持（json_without_timeパーサーを使用）
# そのままデフォルトのパーサーを使用すると、以下のようにパースに失敗する
#  [2026/01/13 01:47:55.17286147] [error] [parser] cannot parse '2026-01-13T01:47:53.124Z'
#  [2026/01/13 01:47:55.17313453] [ warn] [parser:json] invalid time format %d/%b/%Y:%H:%M:%S %z for '2026-01-13T01:47:53.124Z'
# ref : JSON format | Fluent Bit: Official Manual https://docs.fluentbit.io/manual/data-pipeline/parsers/json
[FILTER]
    Name                  parser
    Match                 app-firelens-*
    Key_Name              log
    Parser                json_without_time
    Reserve_Data          On

# webコンテナのログをパース
# Nginxを使用しているためNginxのパーサーを使用して、構造化ログに
[FILTER]
    Name                  parser
    Match                 web-firelens-*
    Key_Name              log
    Parser                nginx
    Reserve_Data          On

# webコンテナのログからELBヘルスチェック時のログ除外
# User agentで判定
[FILTER]
    Name                  grep
    Match                 web-firelens-*
    Exclude               agent ELB-HealthChecker/2\.0

# エラーログにタグを付与
# ステータスコードが5xxまたはstderrのログに付与
# マルチラインパーサーの無限ループ防止のため、すでに5xxまたはstderrのタグが付いているタグを除外
[FILTER]
    Name                  rewrite_tag
    Match_Regex           ^(?!5xx\.)(?!stderr\.).*-firelens-.*
    Rule                  $code ^5\d{2}$ 5xx.$TAG false
    Rule                  $source ^stderr$ stderr.$TAG false
    Emitter_Name          re_emitted

# 全てのログをFirehoseへ出力
# Data Firehose側での動的パーティショニングを行うため圧縮はしない
# Data Firehose側で解凍処理も可能だが、追加料金がかかる and データソースがCloudWatch Logsのみ
# ref : Amazon Kinesis Data Firehose | Fluent Bit: Official Manual https://docs.fluentbit.io/manual/data-pipeline/outputs/firehose
# ref : CloudWatch Logs を解凍する - Amazon Data Firehose https://docs.aws.amazon.com/ja_jp/firehose/latest/dev/writing-with-cloudwatch-logs-decompression.html
[OUTPUT]
    Name                  kinesis_firehose
    Match                 *-firelens-*
    delivery_stream       ${FIREHOSE_DELIVERY_STREAM_NAME}
    region                ${AWS_REGION}
    time_key              datetime
    time_key_format       %Y-%m-%dT%H:%M:%S.%3NZ

# 先頭が 5xx もしくは stderr のタグが付与されているログのみCloudWatch Logsへ出力
# ref : Amazon CloudWatch | Fluent Bit: Official Manual https://docs.fluentbit.io/manual/data-pipeline/outputs/cloudwatch
[OUTPUT]
    Name                  cloudwatch_logs
    Match_Regex           ^(5xx\.|stderr\.).*
    region                ${AWS_REGION}
    log_group_name        ${LOG_GROUP_NAME}
    log_stream_prefix     error/

> curl http://EcsNat-AlbCo-gB01lRgFgX9v-1376474775.us-east-1.elb.amazonaws.com/app/large-log/
{"message":"Large log generated successfully","logSize":20089,"timestamp":"2026-01-17T08:41:25.539Z"}%

{
    "level": "error",
    "time": "2026-01-17T08:41:25.536Z",
    "pid": 51,
    "hostname": "ip-10-10-10-72.ec2.internal",
    "req": {
        "id": 7,
        "method": "GET",
        "url": "/large-log/",
        "query": {},
        "params": {},
        "headers": {
            "host": "ecsnat-albco-gb01lrgfgx9v-1376474775.us-east-1.elb.amazonaws.com",
            "x-real-ip": "10.10.10.11",
            "x-forwarded-for": "<送信元IPアドレス> 10.10.10.11",
            "x-forwarded-proto": "http",
            "connection": "close",
            "x-forwarded-port": "80",
            "x-amzn-trace-id": "Root=1-696b4b35-64996db075d1717b53df48e4",
            "accept": "*/*",
            "user-agent": "curl/8.7.1"
        },
        "remoteAddress": "127.0.0.1",
        "remotePort": 50042
    },
    "message": "Testing 16KB+ log handling",
    "timestamp": "2026-01-17T08:41:25.536Z",
    "data": "XXXXX..(中略)..XXXXX",
    "msg": "Large log entry generated",
    "container_id": "4e8d9e60ba3a410cbe0d8901b14811d8-0527074092",
    "container_name": "app",
    "source": "stderr",
    "ecs_cluster": "EcsNativeBlueGreenStack-EcsConstructCluster14AE103B-0UD5SYuupUOg",
    "ecs_task_arn": "arn:aws:ecs:us-east-1:<AWSアカウントID>:task/EcsNativeBlueGreenStack-EcsConstructCluster14AE103B-0UD5SYuupUOg/4e8d9e60ba3a410cbe0d8901b14811d8",
    "ecs_task_definition": "EcsNativeBlueGreenStackEcsConstructTaskDefinitionF683F4B2:47"
}