package main import ( "errors" "net/http" "net/http/httptest" "strings" "testing" "time" "github.com/golang/mock/gomock" immetrics "github.com/tinode/chat/server/metrics" "github.com/tinode/chat/server/store" "github.com/tinode/chat/server/store/mock_store" ) func TestServeHealthzOK(t *testing.T) { ctrl := gomock.NewController(t) mockStore := mock_store.NewMockPersistentStorageInterface(ctrl) oldStore := store.Store store.Store = mockStore t.Cleanup(func() { store.Store = oldStore }) mockStore.EXPECT().HealthCheck(gomock.Any()).Return(nil) wrt := httptest.NewRecorder() serveHealthz(wrt, httptest.NewRequest(http.MethodGet, "/healthz", nil)) if wrt.Code != http.StatusOK { t.Fatalf("unexpected status %d body=%s", wrt.Code, wrt.Body.String()) } if body := wrt.Body.String(); !strings.Contains(body, `"status":"ok"`) || !strings.Contains(body, `"mysql":"ok"`) { t.Fatalf("unexpected body %s", body) } } func TestServeHealthzError(t *testing.T) { ctrl := gomock.NewController(t) mockStore := mock_store.NewMockPersistentStorageInterface(ctrl) oldStore := store.Store store.Store = mockStore t.Cleanup(func() { store.Store = oldStore }) mockStore.EXPECT().HealthCheck(gomock.Any()).Return(errors.New("dial timeout")) wrt := httptest.NewRecorder() serveHealthz(wrt, httptest.NewRequest(http.MethodGet, "/healthz", nil)) if wrt.Code != http.StatusServiceUnavailable { t.Fatalf("unexpected status %d body=%s", wrt.Code, wrt.Body.String()) } if body := wrt.Body.String(); !strings.Contains(body, `"status":"error"`) || !strings.Contains(body, `"mysql":"dial timeout"`) { t.Fatalf("unexpected body %s", body) } } func TestServeLivez(t *testing.T) { wrt := httptest.NewRecorder() serveLivez(wrt, httptest.NewRequest(http.MethodGet, "/livez", nil)) if wrt.Code != http.StatusOK || wrt.Body.String() != "ok\n" { t.Fatalf("unexpected livez response: code=%d body=%q", wrt.Code, wrt.Body.String()) } } func TestClusterQueueHealthRequiresSustainedHighWatermark(t *testing.T) { clusterQueueHealthMu.Lock() clusterQueueHighSince = make(map[string]time.Time) clusterQueueHealthMu.Unlock() queue := make(chan *ClusterReq, 10) for i := 0; i < 9; i++ { queue <- &ClusterReq{} } cluster := &Cluster{nodes: map[string]*ClusterNode{"peer": {p2mSenders: []chan *ClusterReq{queue}}}} now := time.Now() if healthy, _ := clusterQueueHealthy(cluster, now); !healthy { t.Fatal("instant high watermark must not fail readiness") } if healthy, _ := clusterQueueHealthy(cluster, now.Add(31*time.Second)); healthy { t.Fatal("sustained high watermark must fail readiness") } <-queue if healthy, _ := clusterQueueHealthy(cluster, now.Add(32*time.Second)); !healthy { t.Fatal("queue recovery must restore readiness") } } func TestServeMetricsIncludesWSConnections(t *testing.T) { immetrics.ClusterP2MQueueLength.WithLabelValues("test", "peer").Set(0) immetrics.ClusterP2MEnqueueTotal.WithLabelValues("test", "peer", "ok").Add(0) immetrics.ClusterRPCDuration.WithLabelValues("test", "peer", "Cluster.Route", "ok").Observe(0) immetrics.ClusterRouteTotal.WithLabelValues("test", "peer", "ok").Add(0) immetrics.KafkaMessagePersistTotal.WithLabelValues("p2p", "batch_ok").Add(0) immetrics.KafkaConsumerLag.WithLabelValues("test", "topic", "group").Set(0) immetrics.KafkaDeadLettersTotal.WithLabelValues("p2p", "test").Add(0) immetrics.ObserveInternalQueueEnqueue("test", 0, 1, true) immetrics.SetInternalQueueCurrent("test", 0) immetrics.ObserveWSWrite("text", "ok", "none", time.Now(), 10) immetrics.IncWSDisconnect("close_normal") immetrics.ObserveWSSendQueueEnqueue(1, 160, true) immetrics.ObserveWSSendQueueDequeue(time.Now(), false) immetrics.ObserveTopicQueueWait("group", time.Now()) immetrics.ObservePushDispatchEnqueue(1, true) immetrics.ObservePushDispatch(time.Now(), 1, time.Now(), nil) wrt := httptest.NewRecorder() serveMetrics(wrt, httptest.NewRequest(http.MethodGet, "/metrics", nil)) if wrt.Code != http.StatusOK { t.Fatalf("unexpected status %d body=%s", wrt.Code, wrt.Body.String()) } body := wrt.Body.String() if !strings.Contains(body, "# TYPE ws_connections_active gauge") || !strings.Contains(body, "ws_connections_active ") { t.Fatalf("unexpected body %s", body) } for _, metric := range []string{"im_ws_connections_active", "im_cluster_p2m_queue_length", "im_ws_write_total", "im_ws_write_duration_seconds", "im_ws_write_bytes_total", "im_ws_disconnect_total", "im_ws_send_queue_current", "im_ws_send_queue_capacity", "im_ws_send_queue_enqueue_total", "im_ws_send_queue_depth", "im_ws_send_queue_wait_duration_seconds", "im_cluster_p2m_enqueue_total", "im_cluster_rpc_duration_seconds", "im_cluster_route_total", "im_internal_queue_enqueue_total", "im_internal_queue_depth", "im_internal_queue_capacity", "im_internal_queue_current", "im_topic_queue_wait_duration_seconds", "im_push_dispatch_queue_enqueue_total", "im_push_dispatch_queue_depth", "im_push_dispatch_queue_wait_duration_seconds", "im_push_dispatch_duration_seconds", "im_push_recipient_count", "im_kafka_message_persist_total", "im_kafka_consumer_lag", "im_kafka_dead_letters_total"} { if !strings.Contains(body, "# HELP "+metric+" ") { t.Fatalf("metric %s was not exported", metric) } } } func TestMonitorHTTPRecordsRequestMetrics(t *testing.T) { handler := monitorHTTP(http.HandlerFunc(func(wrt http.ResponseWriter, req *http.Request) { wrt.WriteHeader(http.StatusCreated) })) handler.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodPost, "/v1/im/users/1234567890123456789/profile", nil)) wrt := httptest.NewRecorder() serveMetrics(wrt, httptest.NewRequest(http.MethodGet, "/metrics", nil)) body := wrt.Body.String() if !strings.Contains(body, `http_requests_total{method="POST",path="/v1/im/users/{id}/profile",status="201"} 1`) { t.Fatalf("request counter was not exported: %s", body) } if !strings.Contains(body, `http_request_duration_seconds_bucket{method="POST",path="/v1/im/users/{id}/profile",status="201"`) { t.Fatalf("request duration was not exported: %s", body) } } func TestServeMetricsIncludesDBQueryDuration(t *testing.T) { immetrics.ObserveDB("mysql", "query", time.Now().Add(-time.Millisecond), nil) wrt := httptest.NewRecorder() serveMetrics(wrt, httptest.NewRequest(http.MethodGet, "/metrics", nil)) body := wrt.Body.String() if !strings.Contains(body, `db_query_duration_seconds_bucket{driver="mysql",operation="query",status="ok"`) { t.Fatalf("db query duration was not exported: %s", body) } }