fix: harden high-fanout delivery under load

loadtest/im-pressure/recorder.go

@@ -17,6 +17,16 @@ import (
 const observationShardCount = 64
 
 type observation struct {
+	topic         string
+	seq           int64
+	senderMachine string
+	senderIndex   int
+	scenario      string
+	sentAt        int64
+	bits          []atomic.Uint64
+}
+
+type observationView struct {
 	topic         string
 	seq           int64
 	senderMachine string
@@ -121,23 +131,26 @@ func (r *deliveryRecorder) onData(account AccountRecord, data dataPacket) {
 		return
 	}
 	shard := r.shard(mid)
-	shard.mu.Lock()
+	shard.mu.RLock()
 	obs := shard.observed[mid]
+	shard.mu.RUnlock()
 	if obs == nil {
-		obs = &observation{topic: data.Topic, seq: data.Seq, senderMachine: content.SenderMachine,
-			senderIndex: content.SenderIndex, scenario: content.Scenario, sentAt: content.SentAt,
-			bits: make([]uint64, (r.localUsers+63)/64)}
-		shard.observed[mid] = obs
+		shard.mu.Lock()
+		obs = shard.observed[mid]
+		if obs == nil {
+			obs = &observation{topic: data.Topic, seq: data.Seq, senderMachine: content.SenderMachine,
+				senderIndex: content.SenderIndex, scenario: content.Scenario, sentAt: content.SentAt,
+				bits: make([]atomic.Uint64, (r.localUsers+63)/64)}
+			shard.observed[mid] = obs
+		}
+		shard.mu.Unlock()
 	}
 	word, bit := index/64, uint(index%64)
 	mask := uint64(1) << bit
-	if obs.bits[word]&mask != 0 {
-		shard.mu.Unlock()
+	if !markObservationBit(obs.bits, word, mask) {
 		r.duplicates.Add(1)
 		return
 	}
-	obs.bits[word] |= mask
-	shard.mu.Unlock()
 	r.received.Add(1)
 	sequence := &r.sequences[index]
 	sequence.mu.Lock()
@@ -235,7 +248,7 @@ func (r *deliveryRecorder) unackedObservedMessages(ledger []AcceptedMessage, sce
 			}
 			message := AcceptedMessage{RunID: runID, Scenario: observedScenario, Kind: kind,
 				ClientMid: mid, Topic: obs.topic, Seq: obs.seq, SenderMachine: obs.senderMachine,
-				SenderIndex: obs.senderIndex, PeerMachine: r.machine, PeerIndex: firstObservedIndex(obs.bits)}
+				SenderIndex: obs.senderIndex, PeerMachine: r.machine, PeerIndex: firstObservedAtomicIndex(obs.bits)}
 			if obs.sentAt > 0 {
 				message.SentAt = time.Unix(0, obs.sentAt).UTC()
 			}
@@ -256,6 +269,27 @@ func firstObservedIndex(words []uint64) int {
 	return 0
 }
 
+func firstObservedAtomicIndex(words []atomic.Uint64) int {
+	for wordIndex := range words {
+		if word := words[wordIndex].Load(); word != 0 {
+			return wordIndex*64 + bits.TrailingZeros64(word) + 1
+		}
+	}
+	return 0
+}
+
+func markObservationBit(words []atomic.Uint64, word int, mask uint64) bool {
+	for {
+		current := words[word].Load()
+		if current&mask != 0 {
+			return false
+		}
+		if words[word].CompareAndSwap(current, current|mask) {
+			return true
+		}
+	}
+}
+
 func (r *deliveryRecorder) onDisconnect(account AccountRecord, err error) {
 	category := disconnectCategory(err)
 	r.disconnectMu.Lock()
@@ -370,7 +404,7 @@ func (r *deliveryRecorder) summarize(base AgentSummary, ledger []AcceptedMessage
 		shard.mu.RLock()
 		for mid, obs := range shard.observed {
 			if _, ok := accepted[mid]; !ok {
-				base.UnexpectedMessages += bitsetCount(obs.bits)
+				base.UnexpectedMessages += atomicBitsetCount(obs.bits)
 			}
 		}
 		shard.mu.RUnlock()
@@ -419,17 +453,21 @@ func (r *deliveryRecorder) shard(mid string) *observationShard {
 	return &r.shards[hash%uint32(len(r.shards))]
 }
 
-func (r *deliveryRecorder) observationSnapshot(mid string) *observation {
+func (r *deliveryRecorder) observationSnapshot(mid string) *observationView {
 	shard := r.shard(mid)
 	shard.mu.RLock()
 	obs := shard.observed[mid]
-	if obs != nil {
-		copyOfObservation := *obs
-		copyOfObservation.bits = append([]uint64(nil), obs.bits...)
-		obs = &copyOfObservation
-	}
 	shard.mu.RUnlock()
-	return obs
+	if obs == nil {
+		return nil
+	}
+	view := &observationView{topic: obs.topic, seq: obs.seq, senderMachine: obs.senderMachine,
+		senderIndex: obs.senderIndex, scenario: obs.scenario, sentAt: obs.sentAt,
+		bits: make([]uint64, len(obs.bits))}
+	for i := range obs.bits {
+		view.bits[i] = obs.bits[i].Load()
+	}
+	return view
 }
 
 func cloneCountMap(source map[string]int64) map[string]int64 {
@@ -443,7 +481,7 @@ func cloneCountMap(source map[string]int64) map[string]int64 {
 	return clone
 }
 
-func observationHas(obs *observation, receiverIndex int) bool {
+func observationHas(obs *observationView, receiverIndex int) bool {
 	if obs == nil || receiverIndex <= 0 {
 		return false
 	}
@@ -452,6 +490,14 @@ func observationHas(obs *observation, receiverIndex int) bool {
 	return word < len(obs.bits) && obs.bits[word]&(uint64(1)<<bit) != 0
 }
 
+func atomicBitsetCount(words []atomic.Uint64) int64 {
+	var count int64
+	for i := range words {
+		count += int64(bits.OnesCount64(words[i].Load()))
+	}
+	return count
+}
+
 func bitsetCount(words []uint64) int64 {
 	var count int64
 	for _, word := range words {