fix: harden multi-pod delivery and load verification

server/monitoring_http_test.go

@@ -52,7 +52,44 @@ func TestServeHealthzError(t *testing.T) {
 	}
 }
 
+func TestServeLivez(t *testing.T) {
+	wrt := httptest.NewRecorder()
+	serveLivez(wrt, httptest.NewRequest(http.MethodGet, "/livez", nil))
+	if wrt.Code != http.StatusOK || wrt.Body.String() != "ok\n" {
+		t.Fatalf("unexpected livez response: code=%d body=%q", wrt.Code, wrt.Body.String())
+	}
+}
+
+func TestClusterQueueHealthRequiresSustainedHighWatermark(t *testing.T) {
+	clusterQueueHealthMu.Lock()
+	clusterQueueHighSince = make(map[string]time.Time)
+	clusterQueueHealthMu.Unlock()
+	queue := make(chan *ClusterReq, 10)
+	for i := 0; i < 9; i++ {
+		queue <- &ClusterReq{}
+	}
+	cluster := &Cluster{nodes: map[string]*ClusterNode{"peer": {p2mSenders: []chan *ClusterReq{queue}}}}
+	now := time.Now()
+	if healthy, _ := clusterQueueHealthy(cluster, now); !healthy {
+		t.Fatal("instant high watermark must not fail readiness")
+	}
+	if healthy, _ := clusterQueueHealthy(cluster, now.Add(31*time.Second)); healthy {
+		t.Fatal("sustained high watermark must fail readiness")
+	}
+	<-queue
+	if healthy, _ := clusterQueueHealthy(cluster, now.Add(32*time.Second)); !healthy {
+		t.Fatal("queue recovery must restore readiness")
+	}
+}
+
 func TestServeMetricsIncludesWSConnections(t *testing.T) {
+	immetrics.ClusterP2MQueueLength.WithLabelValues("test", "peer").Set(0)
+	immetrics.ClusterP2MEnqueueTotal.WithLabelValues("test", "peer", "ok").Add(0)
+	immetrics.ClusterRPCDuration.WithLabelValues("test", "peer", "Cluster.Route", "ok").Observe(0)
+	immetrics.ClusterRouteTotal.WithLabelValues("test", "peer", "ok").Add(0)
+	immetrics.KafkaMessagePersistTotal.WithLabelValues("p2p", "batch_ok").Add(0)
+	immetrics.KafkaConsumerLag.WithLabelValues("test", "topic", "group").Set(0)
+	immetrics.KafkaDeadLettersTotal.WithLabelValues("p2p", "test").Add(0)
 	wrt := httptest.NewRecorder()
 	serveMetrics(wrt, httptest.NewRequest(http.MethodGet, "/metrics", nil))
 
@@ -63,6 +100,13 @@ func TestServeMetricsIncludesWSConnections(t *testing.T) {
 	if !strings.Contains(body, "# TYPE ws_connections_active gauge") || !strings.Contains(body, "ws_connections_active ") {
 		t.Fatalf("unexpected body %s", body)
 	}
+	for _, metric := range []string{"im_ws_connections_active", "im_cluster_p2m_queue_length",
+		"im_cluster_p2m_enqueue_total", "im_cluster_rpc_duration_seconds", "im_cluster_route_total",
+		"im_kafka_message_persist_total", "im_kafka_consumer_lag", "im_kafka_dead_letters_total"} {
+		if !strings.Contains(body, "# HELP "+metric+" ") {
+			t.Fatalf("metric %s was not exported", metric)
+		}
+	}
 }
 
 func TestMonitorHTTPRecordsRequestMetrics(t *testing.T) {