CloudWatch が PromQL 対応したので、Amazon Managed Grafana から接続してみた

"""Demo app for CloudWatch OTel metrics, logs, and traces.

Metrics: OpenTelemetry SDK → CloudWatch agent OTLP (4318) → CloudWatch OTel Metrics.
Traces: Application Signals auto-instrumentation
(instrumentation.opentelemetry.io/inject-python). A local MeterProvider is used so
ADOT inject cannot replace app metrics.
Logs include the current trace_id when the injected OpenTelemetry SDK is present.
"""

from __future__ import annotations

import asyncio
import json
import logging
import os
import random
import time

from fastapi import FastAPI, Request
from fastapi.responses import JSONResponse, PlainTextResponse
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
from opentelemetry.sdk.resources import Resource

_OTLP_METRICS_ENDPOINT = os.environ.get(
    "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT",
    "http://cloudwatch-agent.amazon-cloudwatch:4318/v1/metrics",
)
_EXPORT_INTERVAL_MS = int(os.environ.get("OTEL_METRIC_EXPORT_INTERVAL", "15000"))

# Keep the provider at module scope so the periodic reader is not garbage-collected.
# Do not call metrics.set_meter_provider(): ADOT inject may already own the global.
_meter_provider = MeterProvider(
    resource=Resource.create(
        {
            "service.name": os.environ.get("OTEL_SERVICE_NAME", "demo-app"),
            "service.namespace": "demo-app",
        }
    ),
    metric_readers=[
        PeriodicExportingMetricReader(
            OTLPMetricExporter(endpoint=_OTLP_METRICS_ENDPOINT),
            export_interval_millis=_EXPORT_INTERVAL_MS,
        )
    ],
)
_meter = _meter_provider.get_meter("demo-app")
REQUESTS = _meter.create_counter(
    "demo_http_requests_total",
    description="HTTP requests",
    unit="{request}",
)
LATENCY = _meter.create_histogram(
    "demo_http_request_duration_seconds",
    description="HTTP request duration in seconds",
    unit="s",
)

def _trace_fields() -> dict[str, str]:
    try:
        from opentelemetry import trace

        ctx = trace.get_current_span().get_span_context()
        if ctx is None or not ctx.is_valid:
            return {}
        return {
            "trace_id": format(ctx.trace_id, "032x"),
            "span_id": format(ctx.span_id, "016x"),
        }
    except Exception:
        return {}

class JsonFormatter(logging.Formatter):
    def format(self, record: logging.LogRecord) -> str:
        payload = {
            "level": record.levelname,
            "logger": record.name,
            "message": record.getMessage(),
            **_trace_fields(),
        }
        if record.exc_info:
            payload["exception"] = self.formatException(record.exc_info)
        return json.dumps(payload, ensure_ascii=False)

def configure_logging() -> None:
    handler = logging.StreamHandler()
    handler.setFormatter(JsonFormatter())
    root = logging.getLogger()
    root.handlers.clear()
    root.addHandler(handler)
    root.setLevel(logging.INFO)
    logging.getLogger("uvicorn.access").handlers.clear()

configure_logging()
log = logging.getLogger("demo-app")

app = FastAPI(title="demo-app", docs_url=None, redoc_url=None)

@app.middleware("http")
async def record_metrics(request: Request, call_next):
    started = time.perf_counter()
    response = await call_next(request)
    path = request.url.path
    method = request.method
    status = str(response.status_code)
    REQUESTS.add(1, {"method": method, "path": path, "status": status})
    LATENCY.record(time.perf_counter() - started, {"method": method, "path": path})
    return response

@app.get("/healthz", response_class=PlainTextResponse)
async def healthz() -> str:
    return "ok\n"

@app.get("/")
async def root() -> dict[str, str]:
    log.info("handled GET /")
    return {"service": "demo-app", "status": "ok"}

@app.get("/work")
async def work() -> dict[str, object]:
    delay = random.uniform(0.05, 0.25)
    try:
        from opentelemetry import trace

        tracer = trace.get_tracer("demo-app")
        with tracer.start_as_current_span("do-work") as span:
            span.set_attribute("work.delay_seconds", delay)
            await asyncio.sleep(delay)
    except ImportError:
        await asyncio.sleep(delay)
    log.info("completed work delay=%.3f", delay)
    return {"ok": True, "delay": round(delay, 3)}

@app.get("/error")
async def error() -> JSONResponse:
    log.error("simulated error")
    return JSONResponse({"error": "boom"}, status_code=500)

{
  "annotations": {
    "list": [
      {
        "builtIn": 1,
        "datasource": {
          "type": "grafana",
          "uid": "-- Grafana --"
        },
        "enable": true,
        "hide": true,
        "iconColor": "rgba(0, 211, 255, 1)",
        "name": "Annotations & Alerts",
        "type": "dashboard"
      }
    ]
  },
  "editable": true,
  "fiscalYearStartMonth": 0,
  "graphTooltip": 1,
  "id": null,
  "links": [],
  "liveNow": false,
  "panels": [
    {
      "datasource": {
        "type": "cloudwatch",
        "uid": "${datasource}"
      },
      "fieldConfig": {
        "defaults": {
          "unit": "reqps"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 0
      },
      "id": 1,
      "title": "HTTP requests / sec (CloudWatch PromQL)",
      "type": "timeseries",
      "targets": [
        {
          "datasource": {
            "type": "cloudwatch",
            "uid": "${datasource}"
          },
          "queryMode": "Metrics",
          "metricQueryType": 2,
          "metricEditorMode": 1,
          "promqlExpression": "sum(rate(demo_http_requests_total[5m]))",
          "range": true,
          "instant": false,
          "legendFormat": "requests",
          "region": "default",
          "refId": "A"
        }
      ]
    },
    {
      "datasource": {
        "type": "cloudwatch",
        "uid": "${datasource}"
      },
      "fieldConfig": {
        "defaults": {
          "unit": "s"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 0
      },
      "id": 2,
      "title": "Request latency (CloudWatch PromQL)",
      "type": "timeseries",
      "targets": [
        {
          "datasource": {
            "type": "cloudwatch",
            "uid": "${datasource}"
          },
          "queryMode": "Metrics",
          "metricQueryType": 2,
          "metricEditorMode": 1,
          "promqlExpression": "histogram_quantile(0.95, {\"demo_http_request_duration_seconds\", \"@resource.service.name\"=\"demo-app\"})",
          "range": true,
          "instant": false,
          "legendFormat": "p95",
          "region": "default",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "cloudwatch",
            "uid": "${datasource}"
          },
          "queryMode": "Metrics",
          "metricQueryType": 2,
          "metricEditorMode": 1,
          "promqlExpression": "histogram_avg({\"demo_http_request_duration_seconds\", \"@resource.service.name\"=\"demo-app\"})",
          "range": true,
          "instant": false,
          "legendFormat": "mean",
          "region": "default",
          "refId": "B"
        }
      ]
    },
    {
      "datasource": {
        "type": "cloudwatch",
        "uid": "${datasource}"
      },
      "fieldConfig": {
        "defaults": {
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 8
      },
      "id": 3,
      "title": "Container CPU (OTel Container Insights)",
      "type": "timeseries",
      "targets": [
        {
          "datasource": {
            "type": "cloudwatch",
            "uid": "${datasource}"
          },
          "queryMode": "Metrics",
          "metricQueryType": 2,
          "metricEditorMode": 1,
          "promqlExpression": "sum(rate(container_cpu_usage_seconds_total[5m]))",
          "range": true,
          "instant": false,
          "legendFormat": "cpu",
          "region": "default",
          "refId": "A"
        }
      ]
    },
    {
      "datasource": {
        "type": "grafana-x-ray-datasource",
        "uid": "${xray}"
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 8
      },
      "id": 4,
      "title": "X-Ray traces (demo-app)",
      "type": "table",
      "targets": [
        {
          "datasource": {
            "type": "grafana-x-ray-datasource",
            "uid": "${xray}"
          },
          "queryType": "getTraceSummaries",
          "query": "service(\"demo-app\")",
          "region": "default",
          "refId": "A"
        }
      ]
    },
    {
      "datasource": {
        "type": "cloudwatch",
        "uid": "${datasource}"
      },
      "gridPos": {
        "h": 12,
        "w": 24,
        "x": 0,
        "y": 16
      },
      "id": 5,
      "title": "Application logs (filter by trace_id in Explore if needed)",
      "type": "logs",
      "targets": [
        {
          "datasource": {
            "type": "cloudwatch",
            "uid": "${datasource}"
          },
          "queryMode": "Logs",
          "expression": "fields @timestamp, @message | filter kubernetes.namespace_name = \"demo-app\" or @logStream like /demo-app/ | sort @timestamp desc | limit 50",
          "logGroupNames": ["/aws/containerinsights/$cluster/application"],
          "refId": "A",
          "region": "default"
        }
      ]
    }
  ],
  "refresh": "30s",
  "schemaVersion": 41,
  "tags": ["eks", "demo-app", "cloudwatch", "otel", "promql", "xray"],
  "templating": {
    "list": [
      {
        "hide": 0,
        "includeAll": false,
        "label": "CloudWatch (PromQL / logs)",
        "name": "datasource",
        "options": [],
        "query": "cloudwatch",
        "refresh": 1,
        "regex": "",
        "type": "datasource"
      },
      {
        "hide": 0,
        "includeAll": false,
        "label": "X-Ray",
        "name": "xray",
        "options": [],
        "query": "grafana-x-ray-datasource",
        "refresh": 1,
        "regex": "",
        "type": "datasource"
      },
      {
        "current": {
          "text": "eks-obs-demo",
          "value": "eks-obs-demo"
        },
        "hide": 0,
        "label": "Cluster",
        "name": "cluster",
        "query": "eks-obs-demo",
        "type": "textbox"
      }
    ]
  },
  "time": {
    "from": "now-1h",
    "to": "now"
  },
  "timepicker": {},
  "timezone": "browser",
  "title": "EKS demo-app (OTel metrics / logs / traces)",
  "uid": "eks-demo-app-observability",
  "version": 3,
  "weekStart": ""
}