pro/im_server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test: harden group load verification

Browse Source
This commit is contained in:
richjohnrambo
2026-09-28 00:58:24 +08:00
parent 622fc50418
commit cb2210aa3e
17 changed files with 491 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
loadtest/im-pressure/agent.go
+103 -15
View File
@@ -28,7 +28,6 @@ type agentRun struct {
recorder *deliveryRecorder
ack *DurationHistogram
attempted atomic.Int64
rejected atomic.Int64
ready atomic.Int64
publish403 atomic.Int64
publish502 atomic.Int64
@@ -207,11 +206,21 @@ func runAgent(ctx context.Context, cfg *Config, manifest *Manifest, machine, coo
if err = waitContext(ctx, drain); err != nil {
return err
}
observedLocal := run.recorder.unackedObservedMessages(ledger, start.Scenario)
if err = coord.submitObserved(ctx, machine, observedLocal); err != nil {
return err
}
observedLedger, err := coord.observed(ctx)
if err != nil {
return err
}
verificationLedger := mergeMessageLedgers(ledger, observedLedger)
base := AgentSummary{
RunID: start.RunID, Scenario: start.Scenario, Machine: machine, StartedAt: startedAt,
LoadEndedAt: loadEndedAt, FinishedAt: time.Now().UTC(), ExpectedConnections: cfg.UsersPerMachine,
ReadyConnections: run.clientCount(),
PublishAttempted: run.attempted.Load(), PublishAccepted: int64(len(local)), PublishRejected: run.rejected.Load(),
PublishAttempted: run.attempted.Load(), PublishAccepted: int64(len(local)),
PublishRejected: run.explicitRejected(), PublishOutcomeUnknown: run.outcomeUnknown(),
PublishErrors: run.publishErrorCounts(),
AckLatency: run.ack.Snapshot(), AcceptedLocal: local,
}
@@ -224,9 +233,12 @@ func runAgent(ctx context.Context, cfg *Config, manifest *Manifest, machine, coo
if err = waitContext(ctx, historySettle); err != nil {
return err
}
checked, found, persistenceMissing := run.reconcileHistory(ctx, ledger)
summary.HistoryChecked = int64(checked)
summary.HistoryFound = int64(found)
history := run.reconcileHistory(ctx, verificationLedger, ledger)
summary.HistoryChecked = int64(history.checked)
summary.HistoryFound = int64(history.found)
summary.HistoryUnackedChecked = int64(history.unackedChecked)
summary.HistoryUnackedFound = int64(history.unackedFound)
persistenceMissing := history.missing
summary.PersistenceLost = int64(len(persistenceMissing))
summary.RealtimeLostStored = run.recorder.countRealtimeLostStored(ledger)
if len(persistenceMissing) > 10000 {
@@ -256,8 +268,20 @@ type historyQuery struct {
limit int
}
func (r *agentRun) reconcileHistory(ctx context.Context, ledger []AcceptedMessage) (int, int, []string) {
type historyResult struct {
checked int
found int
unackedChecked int
unackedFound int
missing []string
}
func (r *agentRun) reconcileHistory(ctx context.Context, ledger, ackLedger []AcceptedMessage) historyResult {
expected := make(map[string]AcceptedMessage)
acked := make(map[string]bool, len(ackLedger))
for _, message := range ackLedger {
acked[message.ClientMid] = true
}
byTopic := make(map[string][]AcceptedMessage)
for _, message := range ledger {
if message.RunID != r.runID {
@@ -272,7 +296,7 @@ func (r *agentRun) reconcileHistory(ctx context.Context, ledger []AcceptedMessag
}
}
if len(expected) == 0 {
return 0, 0, nil
return historyResult{}
}
r.recorder.beginHistory()
queries := make([]historyQuery, 0, len(byTopic))
@@ -286,13 +310,14 @@ func (r *agentRun) reconcileHistory(ctx context.Context, ledger []AcceptedMessag
r.addError(fmt.Sprintf("history topic=%s has no verifier client", topicHash(topic)))
continue
}
for start := 0; start < len(messages); start += 200 {
end := start + 200
pageSize := r.historyPageSize()
for start := 0; start < len(messages); start += pageSize {
end := start + pageSize
if end > len(messages) {
end = len(messages)
}
queries = append(queries, historyQuery{client: client, topic: topic,
since: messages[start].Seq, before: messages[end-1].Seq + 1, limit: 256})
since: messages[start].Seq, before: messages[end-1].Seq + 1, limit: pageSize})
}
}
historyTimeout, _ := configDuration(r.cfg.HistoryTimeout)
@@ -315,7 +340,17 @@ func (r *agentRun) reconcileHistory(ctx context.Context, ledger []AcceptedMessag
missingIDs = append(missingIDs, message.ClientMid)
}
sort.Strings(missingIDs)
return len(expected), len(expected) - len(missing), missingIDs
result := historyResult{checked: len(expected), found: len(expected) - len(missing), missing: missingIDs}
for mid := range expected {
if acked[mid] {
continue
}
result.unackedChecked++
if r.recorder.historyFound(mid) {
result.unackedFound++
}
}
return result
}
func (r *agentRun) historyQueriesForMessages(messages []AcceptedMessage) []historyQuery {
@@ -333,13 +368,14 @@ func (r *agentRun) historyQueriesForMessages(messages []AcceptedMessage) []histo
if client == nil {
continue
}
for start := 0; start < len(topicMessages); start += 200 {
end := start + 200
pageSize := r.historyPageSize()
for start := 0; start < len(topicMessages); start += pageSize {
end := start + pageSize
if end > len(topicMessages) {
end = len(topicMessages)
}
queries = append(queries, historyQuery{client: client, topic: topic,
since: topicMessages[start].Seq, before: topicMessages[end-1].Seq + 1, limit: 256})
since: topicMessages[start].Seq, before: topicMessages[end-1].Seq + 1, limit: pageSize})
}
}
return queries
@@ -354,12 +390,23 @@ func (r *agentRun) issueHistoryQueries(ctx context.Context, queries []historyQue
return
}
jobs := make(chan historyQuery)
interval := 100 * time.Millisecond
if r.cfg != nil && r.cfg.HistoryInterval != "" {
interval = mustDuration(r.cfg.HistoryInterval)
}
limiter := time.NewTicker(interval)
defer limiter.Stop()
var wg sync.WaitGroup
for i := 0; i < workers; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for query := range jobs {
select {
case <-limiter.C:
case <-ctx.Done():
return
}
if err := query.client.GetData(ctx, query.topic, query.since, query.before, query.limit); err != nil && ctx.Err() == nil {
r.addError(fmt.Sprintf("history get topic=%s range=%d-%d: %v", topicHash(query.topic), query.since, query.before, err))
}
@@ -379,6 +426,13 @@ func (r *agentRun) issueHistoryQueries(ctx context.Context, queries []historyQue
wg.Wait()
}
func (r *agentRun) historyPageSize() int {
if r.cfg != nil && r.cfg.HistoryPageSize > 0 {
return r.cfg.HistoryPageSize
}
return 50
}
func missingHistory(recorder *deliveryRecorder, expected map[string]AcceptedMessage) []AcceptedMessage {
missing := make([]AcceptedMessage, 0)
for mid, message := range expected {
@@ -389,6 +443,33 @@ func missingHistory(recorder *deliveryRecorder, expected map[string]AcceptedMess
return missing
}
func mergeMessageLedgers(accepted, observed []AcceptedMessage) []AcceptedMessage {
byID := make(map[string]AcceptedMessage, len(accepted)+len(observed))
for _, message := range observed {
if message.ClientMid == "" {
continue
}
current, exists := byID[message.ClientMid]
currentIsPeer := current.PeerMachine != "" && current.PeerMachine != current.SenderMachine
candidateIsPeer := message.PeerMachine != "" && message.PeerMachine != message.SenderMachine
if !exists || (!currentIsPeer && candidateIsPeer) {
byID[message.ClientMid] = message
}
}
// ACK-derived records carry authoritative sender and timing fields.
for _, message := range accepted {
if message.ClientMid != "" {
byID[message.ClientMid] = message
}
}
merged := make([]AcceptedMessage, 0, len(byID))
for _, message := range byID {
merged = append(merged, message)
}
sort.Slice(merged, func(i, j int) bool { return merged[i].ClientMid < merged[j].ClientMid })
return merged
}
func (r *agentRun) addError(message string) {
r.mu.Lock()
defer r.mu.Unlock()
@@ -615,7 +696,6 @@ func (r *agentRun) publish(ctx context.Context, client *WSClient, topic, mid str
r.attempted.Add(1)
accepted, err := client.Publish(ctx, topic, mid, r.runID, r.scenario, messageIndex)
if err != nil {
r.rejected.Add(1)
r.classifyPublishError(err)
r.mu.Lock()
if len(r.errors) < 1000 {
@@ -654,6 +734,14 @@ func (r *agentRun) publishErrorCounts() map[string]int64 {
}
}
func (r *agentRun) explicitRejected() int64 {
return r.publish403.Load() + r.publish502.Load() + r.publish503.Load() + r.publishOther.Load()
}
func (r *agentRun) outcomeUnknown() int64 {
return r.publishTimeout.Load() + r.publishSocket.Load()
}
func (r *agentRun) addAccepted(message AcceptedMessage) {
r.mu.Lock()
r.accepted = append(r.accepted, message)