im_server/server/monitoring_http_test.go

package main

import (
	"errors"
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"
	"time"

	"github.com/golang/mock/gomock"
	immetrics "github.com/tinode/chat/server/metrics"
	"github.com/tinode/chat/server/store"
	"github.com/tinode/chat/server/store/mock_store"
)

func TestServeHealthzOK(t *testing.T) {
	ctrl := gomock.NewController(t)
	mockStore := mock_store.NewMockPersistentStorageInterface(ctrl)
	oldStore := store.Store
	store.Store = mockStore
	t.Cleanup(func() { store.Store = oldStore })
	mockStore.EXPECT().HealthCheck(gomock.Any()).Return(nil)

	wrt := httptest.NewRecorder()
	serveHealthz(wrt, httptest.NewRequest(http.MethodGet, "/healthz", nil))

	if wrt.Code != http.StatusOK {
		t.Fatalf("unexpected status %d body=%s", wrt.Code, wrt.Body.String())
	}
	if body := wrt.Body.String(); !strings.Contains(body, `"status":"ok"`) || !strings.Contains(body, `"mysql":"ok"`) {
		t.Fatalf("unexpected body %s", body)
	}
}

func TestServeHealthzError(t *testing.T) {
	ctrl := gomock.NewController(t)
	mockStore := mock_store.NewMockPersistentStorageInterface(ctrl)
	oldStore := store.Store
	store.Store = mockStore
	t.Cleanup(func() { store.Store = oldStore })
	mockStore.EXPECT().HealthCheck(gomock.Any()).Return(errors.New("dial timeout"))

	wrt := httptest.NewRecorder()
	serveHealthz(wrt, httptest.NewRequest(http.MethodGet, "/healthz", nil))

	if wrt.Code != http.StatusServiceUnavailable {
		t.Fatalf("unexpected status %d body=%s", wrt.Code, wrt.Body.String())
	}
	if body := wrt.Body.String(); !strings.Contains(body, `"status":"error"`) || !strings.Contains(body, `"mysql":"dial timeout"`) {
		t.Fatalf("unexpected body %s", body)
	}
}

func TestServeLivez(t *testing.T) {
	wrt := httptest.NewRecorder()
	serveLivez(wrt, httptest.NewRequest(http.MethodGet, "/livez", nil))
	if wrt.Code != http.StatusOK || wrt.Body.String() != "ok\n" {
		t.Fatalf("unexpected livez response: code=%d body=%q", wrt.Code, wrt.Body.String())
	}
}

func TestClusterQueueHealthRequiresSustainedHighWatermark(t *testing.T) {
	clusterQueueHealthMu.Lock()
	clusterQueueHighSince = make(map[string]time.Time)
	clusterQueueHealthMu.Unlock()
	queue := make(chan *ClusterReq, 10)
	for i := 0; i < 9; i++ {
		queue <- &ClusterReq{}
	}
	cluster := &Cluster{nodes: map[string]*ClusterNode{"peer": {p2mSenders: []chan *ClusterReq{queue}}}}
	now := time.Now()
	if healthy, _ := clusterQueueHealthy(cluster, now); !healthy {
		t.Fatal("instant high watermark must not fail readiness")
	}
	if healthy, _ := clusterQueueHealthy(cluster, now.Add(31*time.Second)); healthy {
		t.Fatal("sustained high watermark must fail readiness")
	}
	<-queue
	if healthy, _ := clusterQueueHealthy(cluster, now.Add(32*time.Second)); !healthy {
		t.Fatal("queue recovery must restore readiness")
	}
}

func TestServeMetricsIncludesWSConnections(t *testing.T) {
	immetrics.ClusterP2MQueueLength.WithLabelValues("test", "peer").Set(0)
	immetrics.ClusterP2MEnqueueTotal.WithLabelValues("test", "peer", "ok").Add(0)
	immetrics.ClusterRPCDuration.WithLabelValues("test", "peer", "Cluster.Route", "ok").Observe(0)
	immetrics.ClusterRouteTotal.WithLabelValues("test", "peer", "ok").Add(0)
	immetrics.KafkaMessagePersistTotal.WithLabelValues("p2p", "batch_ok").Add(0)
	immetrics.KafkaConsumerLag.WithLabelValues("test", "topic", "group").Set(0)
	immetrics.KafkaDeadLettersTotal.WithLabelValues("p2p", "test").Add(0)
	immetrics.ObserveInternalQueueEnqueue("test", 0, 1, true)
	wrt := httptest.NewRecorder()
	serveMetrics(wrt, httptest.NewRequest(http.MethodGet, "/metrics", nil))

	if wrt.Code != http.StatusOK {
		t.Fatalf("unexpected status %d body=%s", wrt.Code, wrt.Body.String())
	}
	body := wrt.Body.String()
	if !strings.Contains(body, "# TYPE ws_connections_active gauge") || !strings.Contains(body, "ws_connections_active ") {
		t.Fatalf("unexpected body %s", body)
	}
	for _, metric := range []string{"im_ws_connections_active", "im_cluster_p2m_queue_length",
		"im_cluster_p2m_enqueue_total", "im_cluster_rpc_duration_seconds", "im_cluster_route_total",
		"im_internal_queue_enqueue_total", "im_internal_queue_depth", "im_internal_queue_capacity",
		"im_kafka_message_persist_total", "im_kafka_consumer_lag", "im_kafka_dead_letters_total"} {
		if !strings.Contains(body, "# HELP "+metric+" ") {
			t.Fatalf("metric %s was not exported", metric)
		}
	}
}

func TestMonitorHTTPRecordsRequestMetrics(t *testing.T) {
	handler := monitorHTTP(http.HandlerFunc(func(wrt http.ResponseWriter, req *http.Request) {
		wrt.WriteHeader(http.StatusCreated)
	}))

	handler.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodPost, "/v1/im/users/1234567890123456789/profile", nil))
	wrt := httptest.NewRecorder()
	serveMetrics(wrt, httptest.NewRequest(http.MethodGet, "/metrics", nil))

	body := wrt.Body.String()
	if !strings.Contains(body, `http_requests_total{method="POST",path="/v1/im/users/{id}/profile",status="201"} 1`) {
		t.Fatalf("request counter was not exported: %s", body)
	}
	if !strings.Contains(body, `http_request_duration_seconds_bucket{method="POST",path="/v1/im/users/{id}/profile",status="201"`) {
		t.Fatalf("request duration was not exported: %s", body)
	}
}

func TestServeMetricsIncludesDBQueryDuration(t *testing.T) {
	immetrics.ObserveDB("mysql", "query", time.Now().Add(-time.Millisecond), nil)

	wrt := httptest.NewRecorder()
	serveMetrics(wrt, httptest.NewRequest(http.MethodGet, "/metrics", nil))

	body := wrt.Body.String()
	if !strings.Contains(body, `db_query_duration_seconds_bucket{driver="mysql",operation="query",status="ok"`) {
		t.Fatalf("db query duration was not exported: %s", body)
	}
}