opentelemetry collectorでHost metrixをPrometheusに送る

otelcolでトレースを収集するついでに、hostmetricsも移行することにした。

設定はこんな感じ。ポイントはremote writeでpushで送るってのと、mackerel checks exporterを利用することで既存のmackerelのチェック監視やmetricsを吸い出せるようにしてある。

receivers:
  otlp:
    protocols:
      http:
      grpc:
  hostmetrics:
    collection_interval: 1m
    scrapers:
      cpu:
        metrics:
          system.cpu.utilization:
            enabled: true
      disk:
      load:
        metrics:
          system.cpu.load_average.1m:
            enabled: true
          system.cpu.load_average.5m:
            enabled: true
          system.cpu.load_average.15m:
            enabled: true
      filesystem:
      memory:
      network:
      paging:
      processes:
      process:
        metrics:
          process.cpu.time:
            enabled: true
          process.disk.io:
            enabled: true
          process.memory.virtual:
            enabled: true
          process.memory.usage:
            enabled: true
  prometheus:
    config:
      scrape_configs:
      - job_name: 'mackerel-checks-exporter'
        metrics_path: /metrics
        scrape_interval: 1m
        static_configs:
          - targets:
            - "127.0.0.1:9111"
exporters:
  otlp:
    endpoint: <%= @tempo_endpoint %>
    tls:
      insecure: true
  prometheusremotewrite:
    endpoint: <%= @prometheus_endpoint %>
    add_metric_suffixes: true
    external_labels:
      instance: <%= @hostname %>
      host_name: <%= @hostname %>
      os_type: "linux"
    tls:
      insecure: true
service:
  pipelines:
    traces:
      receivers: [otlp]
      exporters: [otlp]
    metrics:
      receivers: [hostmetrics, prometheus]
      exporters: [prometheusremotewrite]