fix: harden single-machine load handling

2026-09-29 17:52:13 +08:00
parent ab73b69870
commit 4b00b5ae37
24 changed files with 1030 additions and 144 deletions
@@ -2,8 +2,11 @@ package main

 import (
 	"encoding/json"
+	"errors"
 	"fmt"
+	"io"
 	"math/bits"
+	"net"
 	"sort"
 	"strings"
 	"sync"
@@ -11,6 +14,8 @@ import (
 	"time"
 )

+const observationShardCount = 64
+
 type observation struct {
 	topic         string
 	seq           int64
@@ -21,6 +26,17 @@ type observation struct {
 	bits          []uint64
 }

+type observationShard struct {
+	mu       sync.RWMutex
+	observed map[string]*observation
+	history  map[string]bool
+}
+
+type sequenceState struct {
+	mu   sync.Mutex
+	last map[string]int64
+}
+
 type deliveryRecorder struct {
 	machine      string
 	localUsers   int
@@ -32,26 +48,35 @@ type deliveryRecorder struct {
 	runID        atomic.Value
 	historyMode  atomic.Bool

-	mu                    sync.Mutex
-	observed              map[string]*observation
-	lastSeq               []map[string]int64
-	duplicates            int64
-	outOfOrder            int64
-	ordering              []OrderingViolation
-	malformed             int64
-	crossClockSkewSamples int64
-	history               map[string]bool
+	shards                []observationShard
+	sequences             []sequenceState
+	duplicates            atomic.Int64
+	outOfOrder            atomic.Int64
+	malformed             atomic.Int64
+	crossClockSkewSamples atomic.Int64
+
+	orderingMu sync.Mutex
+	ordering   []OrderingViolation
+
+	disconnectMu      sync.Mutex
+	disconnectReasons map[string]int64
+	disconnectSamples []DisconnectSample
 }

 func newDeliveryRecorder(runID, machine string, localUsers int) *deliveryRecorder {
-	lastSeq := make([]map[string]int64, localUsers)
-	for i := range lastSeq {
-		lastSeq[i] = make(map[string]int64)
+	shards := make([]observationShard, observationShardCount)
+	for i := range shards {
+		shards[i].observed = make(map[string]*observation)
+		shards[i].history = make(map[string]bool)
+	}
+	sequences := make([]sequenceState, localUsers)
+	for i := range sequences {
+		sequences[i].last = make(map[string]int64)
 	}
 	recorder := &deliveryRecorder{
 		machine: machine, localUsers: localUsers, latency: NewDurationHistogram(),
 		localLatency: NewDurationHistogram(), crossLatency: NewDurationHistogram(),
-		observed: make(map[string]*observation), lastSeq: lastSeq, history: make(map[string]bool),
+		shards: shards, sequences: sequences, disconnectReasons: make(map[string]int64),
 	}
 	recorder.runID.Store(runID)
 	return recorder
@@ -59,7 +84,13 @@ func newDeliveryRecorder(runID, machine string, localUsers int) *deliveryRecorde

 func (r *deliveryRecorder) setRunID(runID string) { r.runID.Store(runID) }

-func (r *deliveryRecorder) resetDisconnects() { r.disconnects.Store(0) }
+func (r *deliveryRecorder) resetDisconnects() {
+	r.disconnectMu.Lock()
+	r.disconnects.Store(0)
+	r.disconnectReasons = make(map[string]int64)
+	r.disconnectSamples = nil
+	r.disconnectMu.Unlock()
+}

 func (r *deliveryRecorder) onData(account AccountRecord, data dataPacket) {
 	runID, _ := r.runID.Load().(string)
@@ -75,40 +106,50 @@ func (r *deliveryRecorder) onData(account AccountRecord, data dataPacket) {
 		SentAt        int64  `json:"sent_at_unix_nano"`
 	}
 	if json.Unmarshal(data.Content, &content) != nil || content.RunID != runID {
-		r.mu.Lock()
-		r.malformed++
-		r.mu.Unlock()
+		r.malformed.Add(1)
 		return
 	}
 	if r.historyMode.Load() {
-		r.mu.Lock()
-		r.history[mid] = true
-		r.mu.Unlock()
+		shard := r.shard(mid)
+		shard.mu.Lock()
+		shard.history[mid] = true
+		shard.mu.Unlock()
 		return
 	}
 	index := account.Index - 1
 	if index < 0 || index >= r.localUsers {
 		return
 	}
-	r.mu.Lock()
-	defer r.mu.Unlock()
-	obs := r.observed[mid]
+	shard := r.shard(mid)
+	shard.mu.Lock()
+	obs := shard.observed[mid]
 	if obs == nil {
 		obs = &observation{topic: data.Topic, seq: data.Seq, senderMachine: content.SenderMachine,
 			senderIndex: content.SenderIndex, scenario: content.Scenario, sentAt: content.SentAt,
 			bits: make([]uint64, (r.localUsers+63)/64)}
-		r.observed[mid] = obs
+		shard.observed[mid] = obs
 	}
 	word, bit := index/64, uint(index%64)
 	mask := uint64(1) << bit
 	if obs.bits[word]&mask != 0 {
-		r.duplicates++
+		shard.mu.Unlock()
+		r.duplicates.Add(1)
 		return
 	}
 	obs.bits[word] |= mask
+	shard.mu.Unlock()
 	r.received.Add(1)
-	if previous := r.lastSeq[index][data.Topic]; previous > 0 && data.Seq < previous {
-		r.outOfOrder++
+	sequence := &r.sequences[index]
+	sequence.mu.Lock()
+	previous := sequence.last[data.Topic]
+	violation := previous > 0 && data.Seq < previous
+	if data.Seq > previous {
+		sequence.last[data.Topic] = data.Seq
+	}
+	sequence.mu.Unlock()
+	if violation {
+		r.outOfOrder.Add(1)
+		r.orderingMu.Lock()
 		if len(r.ordering) < 100 {
 			r.ordering = append(r.ordering, OrderingViolation{
 				ReceiverMachine: r.machine,
@@ -119,9 +160,7 @@ func (r *deliveryRecorder) onData(account AccountRecord, data dataPacket) {
 				ClientMid:       mid,
 			})
 		}
-	}
-	if data.Seq > r.lastSeq[index][data.Topic] {
-		r.lastSeq[index][data.Topic] = data.Seq
+		r.orderingMu.Unlock()
 	}
 	if content.SentAt > 0 {
 		delay := time.Since(time.Unix(0, content.SentAt))
@@ -133,7 +172,7 @@ func (r *deliveryRecorder) onData(account AccountRecord, data dataPacket) {
 		if senderMachine == r.machine {
 			r.localLatency.Observe(delay)
 		} else if delay < 0 {
-			r.crossClockSkewSamples++
+			r.crossClockSkewSamples.Add(1)
 		} else {
 			r.crossLatency.Observe(delay)
 		}
@@ -150,26 +189,22 @@ func senderMachineFromClientMid(runID, mid string) string {
 }

 func (r *deliveryRecorder) metricCounts() (received, duplicates, outOfOrder, disconnects int64) {
-	r.mu.Lock()
-	duplicates = r.duplicates
-	outOfOrder = r.outOfOrder
-	r.mu.Unlock()
-	return r.received.Load(), duplicates, outOfOrder, r.disconnects.Load()
+	return r.received.Load(), r.duplicates.Load(), r.outOfOrder.Load(), r.disconnects.Load()
 }

 func (r *deliveryRecorder) beginHistory() { r.historyMode.Store(true) }

 func (r *deliveryRecorder) historyFound(mid string) bool {
-	r.mu.Lock()
-	defer r.mu.Unlock()
-	return r.history[mid]
+	shard := r.shard(mid)
+	shard.mu.RLock()
+	found := shard.history[mid]
+	shard.mu.RUnlock()
+	return found
 }

 func (r *deliveryRecorder) historyFoundAll(messageIDs []string) bool {
-	r.mu.Lock()
-	defer r.mu.Unlock()
 	for _, mid := range messageIDs {
-		if !r.history[mid] {
+		if !r.historyFound(mid) {
 			return false
 		}
 	}
@@ -182,28 +217,31 @@ func (r *deliveryRecorder) unackedObservedMessages(ledger []AcceptedMessage, sce
 		accepted[message.ClientMid] = true
 	}
 	runID, _ := r.runID.Load().(string)
-	r.mu.Lock()
-	defer r.mu.Unlock()
 	messages := make([]AcceptedMessage, 0)
-	for mid, obs := range r.observed {
-		if accepted[mid] || obs.seq <= 0 || obs.topic == "" {
-			continue
+	for i := range r.shards {
+		shard := &r.shards[i]
+		shard.mu.RLock()
+		for mid, obs := range shard.observed {
+			if accepted[mid] || obs.seq <= 0 || obs.topic == "" {
+				continue
+			}
+			kind := "p2p"
+			if strings.HasPrefix(obs.topic, "grp") {
+				kind = "group"
+			}
+			observedScenario := obs.scenario
+			if observedScenario == "" {
+				observedScenario = scenario
+			}
+			message := AcceptedMessage{RunID: runID, Scenario: observedScenario, Kind: kind,
+				ClientMid: mid, Topic: obs.topic, Seq: obs.seq, SenderMachine: obs.senderMachine,
+				SenderIndex: obs.senderIndex, PeerMachine: r.machine, PeerIndex: firstObservedIndex(obs.bits)}
+			if obs.sentAt > 0 {
+				message.SentAt = time.Unix(0, obs.sentAt).UTC()
+			}
+			messages = append(messages, message)
 		}
-		kind := "p2p"
-		if strings.HasPrefix(obs.topic, "grp") {
-			kind = "group"
-		}
-		observedScenario := obs.scenario
-		if observedScenario == "" {
-			observedScenario = scenario
-		}
-		message := AcceptedMessage{RunID: runID, Scenario: observedScenario, Kind: kind,
-			ClientMid: mid, Topic: obs.topic, Seq: obs.seq, SenderMachine: obs.senderMachine,
-			SenderIndex: obs.senderIndex, PeerMachine: r.machine, PeerIndex: firstObservedIndex(obs.bits)}
-		if obs.sentAt > 0 {
-			message.SentAt = time.Unix(0, obs.sentAt).UTC()
-		}
-		messages = append(messages, message)
+		shard.mu.RUnlock()
 	}
 	sort.Slice(messages, func(i, j int) bool { return messages[i].ClientMid < messages[j].ClientMid })
 	return messages
@@ -218,34 +256,88 @@ func firstObservedIndex(words []uint64) int {
 	return 0
 }

-func (r *deliveryRecorder) onDisconnect(_ AccountRecord, _ error) {
+func (r *deliveryRecorder) onDisconnect(account AccountRecord, err error) {
+	category := disconnectCategory(err)
+	r.disconnectMu.Lock()
 	r.disconnects.Add(1)
+	r.disconnectReasons[category]++
+	if len(r.disconnectSamples) < 100 {
+		message := ""
+		if err != nil {
+			message = err.Error()
+			if len(message) > 300 {
+				message = message[:300]
+			}
+		}
+		r.disconnectSamples = append(r.disconnectSamples, DisconnectSample{
+			ReceiverMachine: r.machine, ReceiverIndex: account.Index, Category: category,
+			Error: message, At: time.Now().UTC(),
+		})
+	}
+	r.disconnectMu.Unlock()
+}
+
+func disconnectCategory(err error) string {
+	if err == nil {
+		return "unknown"
+	}
+	if errors.Is(err, io.EOF) {
+		return "eof"
+	}
+	if errors.Is(err, net.ErrClosed) {
+		return "closed_connection"
+	}
+	var netErr net.Error
+	if errors.As(err, &netErr) && netErr.Timeout() {
+		return "timeout"
+	}
+	message := strings.ToLower(err.Error())
+	switch {
+	case strings.Contains(message, "close 1006") || strings.Contains(message, "abnormal closure"):
+		return "abnormal_close"
+	case strings.Contains(message, "close 1000") || strings.Contains(message, "normal closure"):
+		return "normal_close"
+	case strings.Contains(message, "connection reset"):
+		return "connection_reset"
+	case strings.Contains(message, "broken pipe"):
+		return "broken_pipe"
+	case strings.Contains(message, "timeout") || strings.Contains(message, "deadline exceeded"):
+		return "timeout"
+	case strings.Contains(message, "use of closed network connection"):
+		return "closed_connection"
+	default:
+		return "other"
+	}
 }

 func (r *deliveryRecorder) summarize(base AgentSummary, ledger []AcceptedMessage) AgentSummary {
 	base.DeliveryLatency = r.latency.Snapshot()
 	base.DeliveryLatencyLocal = r.localLatency.Snapshot()
 	base.DeliveryLatencyCross = r.crossLatency.Snapshot()
+	r.disconnectMu.Lock()
 	base.Disconnects = r.disconnects.Load()
+	base.DisconnectReasons = cloneCountMap(r.disconnectReasons)
+	base.DisconnectSamples = append([]DisconnectSample(nil), r.disconnectSamples...)
+	r.disconnectMu.Unlock()
 	runID, _ := r.runID.Load().(string)
 	accepted := make(map[string]AcceptedMessage, len(ledger))
 	const missingLimit = 100000

-	r.mu.Lock()
-	defer r.mu.Unlock()
-	base.Duplicates = r.duplicates
-	base.OutOfOrder = r.outOfOrder
-	base.CrossClockSkewSamples = r.crossClockSkewSamples
+	base.Duplicates = r.duplicates.Load()
+	base.OutOfOrder = r.outOfOrder.Load()
+	base.CrossClockSkewSamples = r.crossClockSkewSamples.Load()
+	r.orderingMu.Lock()
 	base.OrderingViolations = append([]OrderingViolation(nil), r.ordering...)
-	if r.malformed > 0 {
-		base.Errors = append(base.Errors, fmt.Sprintf("malformed run messages=%d", r.malformed))
+	r.orderingMu.Unlock()
+	if malformed := r.malformed.Load(); malformed > 0 {
+		base.Errors = append(base.Errors, fmt.Sprintf("malformed run messages=%d", malformed))
 	}
 	for _, message := range ledger {
 		if message.RunID != runID {
 			continue
 		}
 		accepted[message.ClientMid] = message
-		obs := r.observed[message.ClientMid]
+		obs := r.observationSnapshot(message.ClientMid)
 		switch message.Kind {
 		case "group":
 			base.ExpectedGroup += int64(r.localUsers)
@@ -273,10 +365,15 @@ func (r *deliveryRecorder) summarize(base AgentSummary, ledger []AcceptedMessage
 			}
 		}
 	}
-	for mid, obs := range r.observed {
-		if _, ok := accepted[mid]; !ok {
-			base.UnexpectedMessages += bitsetCount(obs.bits)
+	for i := range r.shards {
+		shard := &r.shards[i]
+		shard.mu.RLock()
+		for mid, obs := range shard.observed {
+			if _, ok := accepted[mid]; !ok {
+				base.UnexpectedMessages += bitsetCount(obs.bits)
+			}
 		}
+		shard.mu.RUnlock()
 	}
 	base.RealtimeLost = (base.ExpectedPeer - base.ReceivedPeer) + (base.ExpectedGroup - base.ReceivedGroup)
 	if len(base.Missing) == missingLimit {
@@ -291,14 +388,12 @@ func (r *deliveryRecorder) countRealtimeLostStored(ledger []AcceptedMessage, sto
 	for _, mid := range storedMessageIDs {
 		stored[mid] = true
 	}
-	r.mu.Lock()
-	defer r.mu.Unlock()
 	var count int64
 	for _, message := range ledger {
 		if message.RunID != runID || !stored[message.ClientMid] {
 			continue
 		}
-		obs := r.observed[message.ClientMid]
+		obs := r.observationSnapshot(message.ClientMid)
 		switch message.Kind {
 		case "group":
 			for index := 1; index <= r.localUsers; index++ {
@@ -315,6 +410,39 @@ func (r *deliveryRecorder) countRealtimeLostStored(ledger []AcceptedMessage, sto
 	return count
 }

+func (r *deliveryRecorder) shard(mid string) *observationShard {
+	var hash uint32 = 2166136261
+	for i := 0; i < len(mid); i++ {
+		hash ^= uint32(mid[i])
+		hash *= 16777619
+	}
+	return &r.shards[hash%uint32(len(r.shards))]
+}
+
+func (r *deliveryRecorder) observationSnapshot(mid string) *observation {
+	shard := r.shard(mid)
+	shard.mu.RLock()
+	obs := shard.observed[mid]
+	if obs != nil {
+		copyOfObservation := *obs
+		copyOfObservation.bits = append([]uint64(nil), obs.bits...)
+		obs = &copyOfObservation
+	}
+	shard.mu.RUnlock()
+	return obs
+}
+
+func cloneCountMap(source map[string]int64) map[string]int64 {
+	if len(source) == 0 {
+		return nil
+	}
+	clone := make(map[string]int64, len(source))
+	for key, value := range source {
+		clone[key] = value
+	}
+	return clone
+}
+
 func observationHas(obs *observation, receiverIndex int) bool {
 	if obs == nil || receiverIndex <= 0 {
 		return false