fix: harden multi-pod delivery and load verification

server/monitoring_http.go

@@ -4,12 +4,14 @@ import (
 	"bufio"
 	"context"
 	"encoding/json"
+	"fmt"
 	"io"
 	"net"
 	"net/http"
 	"regexp"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus/promhttp"
@@ -20,6 +22,8 @@ import (
 var (
 	metricsHandler        = promhttp.HandlerFor(immetrics.Registry, promhttp.HandlerOpts{})
 	uuidLikeMetricSegment = regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F-]{13,}$`)
+	clusterQueueHighSince = make(map[string]time.Time)
+	clusterQueueHealthMu  sync.Mutex
 )
 
 type healthzResponse struct {
@@ -45,6 +49,28 @@ func serveHealthz(wrt http.ResponseWriter, req *http.Request) {
 	} else {
 		checks["mysql"] = "ok"
 	}
+	if globals.cluster != nil {
+		expected := len(globals.cluster.nodes)
+		live := 0
+		for _, node := range globals.cluster.nodes {
+			node.lock.Lock()
+			connected := node.connected
+			node.lock.Unlock()
+			if connected {
+				live++
+			}
+		}
+		checks["cluster"] = fmt.Sprintf("%d/%d peers connected", live, expected)
+		if live != expected {
+			status = http.StatusServiceUnavailable
+		}
+		if healthy, detail := clusterQueueHealthy(globals.cluster, time.Now()); !healthy {
+			checks["cluster_queue"] = detail
+			status = http.StatusServiceUnavailable
+		} else {
+			checks["cluster_queue"] = detail
+		}
+	}
 
 	body := healthzResponse{Status: "ok", Checks: checks}
 	if status != http.StatusOK {
@@ -56,6 +82,47 @@ func serveHealthz(wrt http.ResponseWriter, req *http.Request) {
 	}
 }
 
+func clusterQueueHealthy(cluster *Cluster, now time.Time) (bool, string) {
+	clusterQueueHealthMu.Lock()
+	defer clusterQueueHealthMu.Unlock()
+	healthy := true
+	detail := "ok"
+	for name, node := range cluster.nodes {
+		depth, capacity := 0, 0
+		for _, sender := range node.p2mSenders {
+			depth += len(sender)
+			capacity += cap(sender)
+		}
+		if capacity == 0 || depth*100 < capacity*90 {
+			delete(clusterQueueHighSince, name)
+			continue
+		}
+		since, exists := clusterQueueHighSince[name]
+		if !exists {
+			clusterQueueHighSince[name] = now
+			continue
+		}
+		if now.Sub(since) >= 30*time.Second {
+			healthy = false
+			detail = fmt.Sprintf("peer %s queue %d/%d above 90%% for %s", name, depth, capacity,
+				now.Sub(since).Round(time.Second))
+		}
+	}
+	return healthy, detail
+}
+
+func serveLivez(wrt http.ResponseWriter, req *http.Request) {
+	if req.Method != http.MethodGet && req.Method != http.MethodHead {
+		wrt.Header().Set("Allow", http.MethodGet+", "+http.MethodHead)
+		http.Error(wrt, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	wrt.WriteHeader(http.StatusOK)
+	if req.Method != http.MethodHead {
+		_, _ = io.WriteString(wrt, "ok\n")
+	}
+}
+
 func serveMetrics(wrt http.ResponseWriter, req *http.Request) {
 	if req.Method != http.MethodGet && req.Method != http.MethodHead {
 		wrt.Header().Set("Allow", http.MethodGet+", "+http.MethodHead)