[初心者向け]シンプルなものを触りながら理解するFluent Bit

[2025/1/15 15:00] FROM:Aさん TO:Bさん MESSAGE:こんにちは1
[2025/1/15 15:01] FROM:Aさん TO:Cさん MESSAGE:こんにちは2
[2025/1/15 15:02] FROM:Bさん TO:Cさん MESSAGE:こんにちは3

{
    "日時": "2025/1/15 15:00",
    "送信者": "Aさん",
    "受信者": "Bさん",
    "メッセージ": "こんにちは"
}

[2025/1/15 15:00] FROM:Aさん TO:Bさん MESSAGE:こんにちは1
[2025/1/15 15:01] FROM:Aさん TO:Cさん MESSAGE:こんにちは2
[2025/1/15 15:02] FROM:Bさん TO:Cさん MESSAGE:こんにちは3

# 出力例
[2025/1/15 15:00] FROM:Aさん TO:Bさん MESSAGE:こんにちは1
[2025/1/15 15:01] FROM:Aさん TO:Cさん MESSAGE:こんにちは2

{
    "datetime": "2025/1/15 15:00",
    "method": "GET",
    "ResponseCode": 200,
    "Path": "/index.html",
    "client": "192.168.0.2"
},
{
    "datetime": "2025/1/15 15:02",
    "method": "GET",
    "ResponseCode": 404,
    "Path": "/not_found.html",
    "client": "192.168.0.3"
},
{
    "datetime": "2025/1/15 15:05",
    "method": "GET",
    "ResponseCode": 200,
    "Path": "/hello_world.html",
    "client": "192.168.0.5"
}

{
    "datetime": "2025/1/15 15:00",
    "method": "GET",
    "ResponseCode": 200,
    "Path": "/index.html",
    "client": "192.168.0.2"
},
{
    "tag": "error",
    "datetime": "2025/1/15 15:02",
    "method": "GET",
    "ResponseCode": 404,
    "Path": "/not_found.html",
    "client": "192.168.0.3"
},
{
    "datetime": "2025/1/15 15:05",
    "method": "GET",
    "ResponseCode": 200,
    "Path": "/hello_world.html",
    "client": "192.168.0.5"
}

[SERVICE]
    parser_file /opt/homebrew/etc/fluent-bit/parsers.conf

/opt/homebrew/etc/fluent-bit/parsers.conf 

[PARSER]
    Name   nginx
    Format regex
    Regex ^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")
    Time_Key time
    Time_Format %d/%b/%Y:%H:%M:%S %z

127.0.0.1 - - [16/Jan/2025:11:35:27 +0900] "GET / HTTP/1.1" 200 615 "-" "curl/8.7.1"

{
    "remote": "127.0.0.1",
    "host": "-",
    "user": "-",
    "time": "16/Jan/2025:11:35:27 +0900",
    "method": "GET",
    "path": "/",
    "code": "200",
    "size": "615",
    "referer": "-",
    "agent": "curl/8.7.1"
}

[INPUT]
    name tail
    path <Webサーバーログファイルへのパス>
    Parser nginx
    Tag nginx.logs

[INPUT]
    name tail           # 取得方式としてtailを指定
    path <ログへのパス>   # 指定したパスをtailで取得
    Parser nginx        # nginxという名前のパーサーを使用
    Tag nginx.logs      # nginx.logsというタグを付与

[FILTER]
    Name rewrite_tag
    Match nginx.logs
    Rule $code ^404$ nginx.logs.not_found false

[FILTER]
    Name rewrite_tag    # タグの書き換え操作
    Match nginx.logs    # nginx.logsタグがついているログを対象
    Rule $code ^404$ nginx.logs.not_found false # codeが404の場合、タグをnginx.logs.not_foundに書き換えて、元のデータを破棄する

[OUTPUT]
    Name file
    Match nginx.logs.not_found
    Path <ログ出力先ディレクトリへのパス>

[OUTPUT]
    Name stdout
    Match *

[SERVICE]
    parsers_file /opt/homebrew/etc/fluent-bit/parsers.conf

[INPUT]
    name tail
    path <ログへのパス>
    Parser nginx
    Tag nginx.logs

[FILTER]
    Name rewrite_tag
    Match nginx.logs
    Rule $code ^404$ nginx.logs.not_found false

[OUTPUT]
    Name file
    Match nginx.logs.not_found
    Path <ログ出力先ディレクトリへのパス>

[OUTPUT]
    Name stdout
    Match *

fluent-bit -c <設定ファイル>

Fluent Bit v3.2.4
* Copyright (C) 2015-2024 The Fluent Bit Authors
* Fluent Bit is a CNCF sub-project under the umbrella of Fluentd
* https://fluentbit.io

______ _                  _    ______ _ _           _____  _____ 
|  ___| |                | |   | ___ (_) |         |____ |/ __  \
| |_  | |_   _  ___ _ __ | |_  | |_/ /_| |_  __   __   / /`' / /'
|  _| | | | | |/ _ \ '_ \| __| | ___ \ | __| \ \ / /   \ \  / /  
| |   | | |_| |  __/ | | | |_  | |_/ / | |_   \ V /.___/ /./ /___
\_|   |_|\__,_|\___|_| |_|\__| \____/|_|\__|   \_/ \____(_)_____/

[2025/01/16 14:59:54] [ info] [fluent bit] version=3.2.4, commit=, pid=77226
[2025/01/16 14:59:54] [ info] [storage] ver=1.5.2, type=memory, sync=normal, checksum=off, max_chunks_up=128
[2025/01/16 14:59:54] [ info] [simd    ] disabled
[2025/01/16 14:59:54] [ info] [cmetrics] version=0.9.9
[2025/01/16 14:59:54] [ info] [ctraces ] version=0.5.7
[2025/01/16 14:59:54] [ info] [input:tail:tail.0] initializing
[2025/01/16 14:59:54] [ info] [input:tail:tail.0] storage_strategy='memory' (memory only)
[2025/01/16 14:59:54] [ info] [input:emitter:emitter_for_rewrite_tag.0] initializing
[2025/01/16 14:59:54] [ info] [input:emitter:emitter_for_rewrite_tag.0] storage_strategy='memory' (memory only)
[2025/01/16 14:59:54] [ info] [output:file:file.0] worker #0 started
[2025/01/16 14:59:54] [ info] [sp] stream processor started
[2025/01/16 14:59:54] [ info] [output:stdout:stdout.1] worker #0 started

curl http://localhost:8080

[0] nginx.logs: [[1737007276.000000000, {}], {"remote"=>"127.0.0.1", "host"=>"-", "user"=>"-", "method"=>"GET", "path"=>"/", "code"=>"200", "size"=>"615", "referer"=>"-", "agent"=>"curl/8.7.1"}]

curl http://localhost:8080/notfound

[0] nginx.logs.not_found: [[1737007362.000000000, {}], {"remote"=>"127.0.0.1", "host"=>"-", "user"=>"-", "method"=>"GET", "path"=>"/notfound", "code"=>"404", "size"=>"153", "referer"=>"-", "agent"=>"curl/8.7.1"}]

xxxxxxxxxxxxxx@xxxxxxxx errors % ls -l 
total 8
-rw-r--r--  1 xxxxxxxxxxxxxx  staff  186  1 16 15:03 nginx.logs.not_found

xxxxxxxxxxxxxx@xxxxxxxx errors % cat nginx.logs.not_found 
nginx.logs.not_found: [1737007362.000000000, {"remote":"127.0.0.1","host":"-","user":"-","method":"GET","path":"/notfound","code":"404","size":"153","referer":"-","agent":"curl/8.7.1"}]

[2025/1/15 15:00] FROM:Aさん TO:Bさん MESSAGE:こんにちは1

[PARSER]
    Name sample-parser
    Format regex
    Regex ^\[(?<datetime>[^\]]*)\] FROM\:(?<sender>[^ ]*) TO\:(?<destination>[^ ]*) MESSAGE\:(?<message>[^ ]*)

[SERVICE]
    parsers_file <ファイルパス>

[INPUT]
    name tail
    path <ログへのパス>
    Parser sample-parser