test: harden group load verification

loadtest/im-pressure/recorder.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"math/bits"
+	"sort"
 	"strings"
 	"sync"
 	"sync/atomic"
@@ -11,9 +12,13 @@ import (
 )
 
 type observation struct {
-	topic string
-	seq   int64
-	bits  []uint64
+	topic         string
+	seq           int64
+	senderMachine string
+	senderIndex   int
+	scenario      string
+	sentAt        int64
+	bits          []uint64
 }
 
 type deliveryRecorder struct {
@@ -65,6 +70,8 @@ func (r *deliveryRecorder) onData(account AccountRecord, data dataPacket) {
 	var content struct {
 		RunID         string `json:"run_id"`
 		SenderMachine string `json:"sender_machine"`
+		SenderIndex   int    `json:"sender_index"`
+		Scenario      string `json:"scenario"`
 		SentAt        int64  `json:"sent_at_unix_nano"`
 	}
 	if json.Unmarshal(data.Content, &content) != nil || content.RunID != runID {
@@ -87,7 +94,9 @@ func (r *deliveryRecorder) onData(account AccountRecord, data dataPacket) {
 	defer r.mu.Unlock()
 	obs := r.observed[mid]
 	if obs == nil {
-		obs = &observation{topic: data.Topic, seq: data.Seq, bits: make([]uint64, (r.localUsers+63)/64)}
+		obs = &observation{topic: data.Topic, seq: data.Seq, senderMachine: content.SenderMachine,
+			senderIndex: content.SenderIndex, scenario: content.Scenario, sentAt: content.SentAt,
+			bits: make([]uint64, (r.localUsers+63)/64)}
 		r.observed[mid] = obs
 	}
 	word, bit := index/64, uint(index%64)
@@ -156,6 +165,48 @@ func (r *deliveryRecorder) historyFound(mid string) bool {
 	return r.history[mid]
 }
 
+func (r *deliveryRecorder) unackedObservedMessages(ledger []AcceptedMessage, scenario string) []AcceptedMessage {
+	accepted := make(map[string]bool, len(ledger))
+	for _, message := range ledger {
+		accepted[message.ClientMid] = true
+	}
+	runID, _ := r.runID.Load().(string)
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	messages := make([]AcceptedMessage, 0)
+	for mid, obs := range r.observed {
+		if accepted[mid] || obs.seq <= 0 || obs.topic == "" {
+			continue
+		}
+		kind := "p2p"
+		if strings.HasPrefix(obs.topic, "grp") {
+			kind = "group"
+		}
+		observedScenario := obs.scenario
+		if observedScenario == "" {
+			observedScenario = scenario
+		}
+		message := AcceptedMessage{RunID: runID, Scenario: observedScenario, Kind: kind,
+			ClientMid: mid, Topic: obs.topic, Seq: obs.seq, SenderMachine: obs.senderMachine,
+			SenderIndex: obs.senderIndex, PeerMachine: r.machine, PeerIndex: firstObservedIndex(obs.bits)}
+		if obs.sentAt > 0 {
+			message.SentAt = time.Unix(0, obs.sentAt).UTC()
+		}
+		messages = append(messages, message)
+	}
+	sort.Slice(messages, func(i, j int) bool { return messages[i].ClientMid < messages[j].ClientMid })
+	return messages
+}
+
+func firstObservedIndex(words []uint64) int {
+	for wordIndex, word := range words {
+		if word != 0 {
+			return wordIndex*64 + bits.TrailingZeros64(word) + 1
+		}
+	}
+	return 0
+}
+
 func (r *deliveryRecorder) onDisconnect(_ AccountRecord, _ error) {
 	r.disconnects.Add(1)
 }