im_server/server/metrics/metrics.go

package metrics

import (
	"expvar"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/prometheus/client_golang/prometheus"
)

var (
	Registry = prometheus.NewRegistry()

	HTTPRequestsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: "http_requests_total",
		Help: "Total number of HTTP requests by method, normalized path, and status.",
	}, []string{"method", "path", "status"})

	HTTPRequestDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
		Name:    "http_request_duration_seconds",
		Help:    "HTTP request duration by method, normalized path, and status.",
		Buckets: []float64{0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10},
	}, []string{"method", "path", "status"})

	DBQueryDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
		Name:    "db_query_duration_seconds",
		Help:    "Database operation duration by driver, operation, and status.",
		Buckets: []float64{0.001, 0.003, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5},
	}, []string{"driver", "operation", "status"})

	RealtimeFanoutEventsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: "realtime_fanout_events_total",
		Help: "Total number of realtime fanout events by stage, kind, target type, and status.",
	}, []string{"stage", "kind", "target_type", "status"})

	RealtimeFanoutDeliveryDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
		Name:    "realtime_fanout_delivery_duration_seconds",
		Help:    "Realtime fanout delivery age measured from message timestamp to local dispatch.",
		Buckets: []float64{0.001, 0.003, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10},
	}, []string{"kind", "target_type"})

	WSConnectionsActive = prometheus.NewGaugeFunc(prometheus.GaugeOpts{
		Name: "ws_connections_active",
		Help: "Current active WebSocket connections.",
	}, func() float64 {
		return float64(expvarIntValue("LiveSessions"))
	})
	IMWSConnectionsActive = prometheus.NewGaugeFunc(prometheus.GaugeOpts{
		Name:        "im_ws_connections_active",
		Help:        "Current active WebSocket connections, using the IM metric namespace.",
		ConstLabels: prometheus.Labels{"pod": metricPodName()},
	}, func() float64 {
		return float64(expvarIntValue("LiveSessions"))
	})
	IMClusterLivePeers = prometheus.NewGaugeFunc(prometheus.GaugeOpts{
		Name:        "im_cluster_live_peers",
		Help:        "Current connected cluster peers.",
		ConstLabels: prometheus.Labels{"pod": metricPodName()},
	}, func() float64 {
		return float64(expvarIntValue("LiveClusterNodes"))
	})
	IMClusterExpectedPeers = prometheus.NewGaugeFunc(prometheus.GaugeOpts{
		Name:        "im_cluster_expected_peers",
		Help:        "Expected cluster peers derived from configured total nodes.",
		ConstLabels: prometheus.Labels{"pod": metricPodName()},
	}, func() float64 {
		total := expvarIntValue("TotalClusterNodes")
		if total > 0 {
			total--
		}
		return float64(total)
	})

	ClusterP2MQueueLength = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: "im_cluster_p2m_queue_length",
		Help: "Current proxy-to-master queue depth by pod and peer.",
	}, []string{"pod", "peer"})
	ClusterP2MQueueCapacity = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: "im_cluster_p2m_queue_capacity",
		Help: "Configured proxy-to-master queue capacity by pod and peer.",
	}, []string{"pod", "peer"})
	ClusterP2MEnqueueTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: "im_cluster_p2m_enqueue_total",
		Help: "Proxy-to-master enqueue attempts by pod, peer, and result.",
	}, []string{"pod", "peer", "result"})
	ClusterRPCInflight = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: "im_cluster_rpc_inflight",
		Help: "Current synchronous cluster RPC calls by pod, peer, and method.",
	}, []string{"pod", "peer", "method"})
	ClusterRPCDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
		Name:    "im_cluster_rpc_duration_seconds",
		Help:    "Cluster RPC duration by pod, peer, method, and result.",
		Buckets: []float64{0.001, 0.003, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5},
	}, []string{"pod", "peer", "method", "result"})
	ClusterRouteTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: "im_cluster_route_total",
		Help: "Cluster route attempts by pod, peer, and result.",
	}, []string{"pod", "peer", "result"})
	ClusterProxyResponseTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: "im_cluster_proxy_response_total",
		Help: "Cluster proxy responses by pod and result.",
	}, []string{"pod", "result"})
	MessagePersistDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
		Name:    "im_message_persist_duration_seconds",
		Help:    "Kafka message persistence duration by message kind and result.",
		Buckets: []float64{0.001, 0.003, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10},
	}, []string{"kind", "result"})
	MessagePublishStageDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
		Name:    "im_message_publish_stage_duration_seconds",
		Help:    "Message publish hot-path duration by message kind, stage, and result.",
		Buckets: []float64{0.001, 0.003, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10},
	}, []string{"kind", "stage", "result"})
	KafkaMessagePersistTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: "im_kafka_message_persist_total",
		Help: "Kafka message persistence outcomes by message kind and result.",
	}, []string{"kind", "result"})
	KafkaConsumerLag = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: "im_kafka_consumer_lag",
		Help: "Kafka reader lag reported by kafka-go by pod, topic, and consumer group.",
	}, []string{"pod", "topic", "group"})
	KafkaBacklogOldestAge = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: "im_kafka_backlog_oldest_age_seconds",
		Help: "Age of the latest fetched Kafka message by pod, topic, and consumer group.",
	}, []string{"pod", "topic", "group"})
	KafkaDeadLettersTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: "im_kafka_dead_letters_total",
		Help: "Kafka messages moved to dead letter by message kind and reason.",
	}, []string{"kind", "reason"})
)

func init() {
	Registry.MustRegister(WSConnectionsActive, IMWSConnectionsActive, IMClusterLivePeers, IMClusterExpectedPeers,
		HTTPRequestsTotal, HTTPRequestDuration, DBQueryDuration,
		RealtimeFanoutEventsTotal, RealtimeFanoutDeliveryDuration, ClusterP2MQueueLength,
		ClusterP2MQueueCapacity, ClusterP2MEnqueueTotal, ClusterRPCInflight, ClusterRPCDuration,
		ClusterRouteTotal, ClusterProxyResponseTotal, MessagePersistDuration, MessagePublishStageDuration,
		KafkaMessagePersistTotal)
	Registry.MustRegister(KafkaConsumerLag, KafkaBacklogOldestAge, KafkaDeadLettersTotal)
}

func metricPodName() string {
	if value := strings.TrimSpace(os.Getenv("HOSTNAME")); value != "" {
		return value
	}
	return "standalone"
}

func ObserveDB(driverName, operation string, started time.Time, err error) {
	status := "ok"
	if err != nil {
		status = "error"
	}
	DBQueryDuration.WithLabelValues(driverName, operation, status).Observe(time.Since(started).Seconds())
}

func ObserveMessagePublishStage(kind, stage string, started time.Time, err error) {
	status := "ok"
	if err != nil {
		status = "error"
	}
	MessagePublishStageDuration.WithLabelValues(normalizeLabel(kind), normalizeLabel(stage), status).
		Observe(time.Since(started).Seconds())
}

func IncRealtimeFanout(stage, kind, targetType, status string) {
	RealtimeFanoutEventsTotal.WithLabelValues(normalizeLabel(stage), normalizeLabel(kind),
		normalizeLabel(targetType), normalizeLabel(status)).Inc()
}

func ObserveRealtimeFanoutDelivery(kind, targetType string, timestamp time.Time) {
	if timestamp.IsZero() {
		return
	}
	RealtimeFanoutDeliveryDuration.WithLabelValues(normalizeLabel(kind), normalizeLabel(targetType)).
		Observe(time.Since(timestamp).Seconds())
}

func normalizeLabel(value string) string {
	if value == "" {
		return "unknown"
	}
	return value
}

func expvarIntValue(name string) int64 {
	value := expvar.Get(name)
	if value == nil {
		return 0
	}
	n, err := strconv.ParseInt(value.String(), 10, 64)
	if err != nil {
		return 0
	}
	return n
}