Prometheus+Grafanaでk8s上のGoアプリケーションのメトリクスを回収して可視化

eksctl create cluster --name sample-cluster \
  --version 1.14 \
  --nodegroup-name sample-cluster-worker \
  --node-type t3.small \
  --nodes 3 \
  --nodes-min 1 \
  --nodes-max 5 \
  --managed

go get github.com/prometheus/client_golang

import (
	"net/http"
	"github.com/prometheus/client_golang/prometheus/promhttp"
)

func main() {
	http.Handle("/metrics", promhttp.Handler())
	http.ListenAndServe(":2112", nil)
}

func main() {
  // 設定とかで、メトリクス公開設定とか入れとく
	if conf.COLLECT_METRICS {
		http.Handle("/metrics", promhttp.Handler())
		go http.ListenAndServe(":2112", nil)
	}
  // メインの処理
  // ...
}

$ go run main.go
$ curl localhost:2112/metrics
## HELP go_gc_duration_seconds A summary of the GC invocation durations.
## TYPE go_gc_duration_seconds summary
go_gc_duration_seconds{quantile="0"} 0
go_gc_duration_seconds{quantile="0.25"} 0
go_gc_duration_seconds{quantile="0.5"} 0
go_gc_duration_seconds{quantile="0.75"} 0
go_gc_duration_seconds{quantile="1"} 0
go_gc_duration_seconds_sum 0
go_gc_duration_seconds_count 0
...

apiVersion: apps/v1
kind: Deployment
metadata:
  name: sample-go-app
spec:
  selector:
    matchLabels:
      app: prometheus-export-sample
  replicas: 4
  template:
    metadata:
      labels:
        app: prometheus-export-sample
    spec:
      containers:
        - name: prometheus-export-sample
          image: esaka/prometheus-export-sample
          ports:
            - containerPort: 2112

      - job_name: 'go-metrics'
        kubernetes_sd_configs:
        - role: pod
        relabel_configs:
        - source_labels:
          - __meta_kubernetes_namespace
          - __meta_kubernetes_pod_container_port_number
          regex: default;2112
          action: keep
        - source_labels:
          - __meta_kubernetes_pod_name
          target_label: job
        - source_labels:
          - __meta_kubernetes_pod_name
          target_label: pod

$ kubectl get svc -n monitoring grafana
NAME      TYPE           CLUSTER-IP       EXTERNAL-IP                                                                    PORT(S)          AGE
grafana   LoadBalancer   10.100.191.149   aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.ap-northeast-1.elb.amazonaws.com   3000:32017/TCP   7m46s