pro/im_server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: harden multi-pod delivery and load verification

Browse Source
This commit is contained in:
richjohnrambo
2026-09-27 03:37:32 +08:00
parent a2de100f2b
commit ce3250be64
33 changed files with 1340 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files
loadtest/im-pressure/agent.go
+150 -25
View File
@@ -3,12 +3,14 @@ package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"log"
"os"
"path/filepath"
"regexp"
"sort"
"strings"
"sync"
"sync/atomic"
"time"
@@ -17,16 +19,22 @@ import (
var runIDPattern = regexp.MustCompile(`^[a-zA-Z0-9_-]{1,24}$`)
type agentRun struct {
cfg *Config
machine string
runID string
scenario string
clients map[int]*WSClient
recorder *deliveryRecorder
ack *DurationHistogram
attempted atomic.Int64
rejected atomic.Int64
ready atomic.Int64
cfg *Config
machine string
runID string
scenario string
clients map[int]*WSClient
recorder *deliveryRecorder
ack *DurationHistogram
attempted atomic.Int64
rejected atomic.Int64
ready atomic.Int64
publish403 atomic.Int64
publish502 atomic.Int64
publish503 atomic.Int64
publishTimeout atomic.Int64
publishSocket atomic.Int64
publishOther atomic.Int64
mu sync.Mutex
accepted []AcceptedMessage
@@ -37,6 +45,58 @@ func validScenario(scenario string) bool {
return scenario == "p2p" || scenario == "group" || scenario == "mixed"
}
func preflightP2P(ctx context.Context, cfg *Config, manifest *Manifest, local []AccountRecord, apiKey string) error {
accounts := manifestAccountMap(manifest)
api := newAPIClient(cfg, apiKey)
jobs := make(chan AccountRecord)
errCh := make(chan error, len(local))
var wg sync.WaitGroup
workers := cfg.PreflightWorkers
if workers > len(local) {
workers = len(local)
}
for i := 0; i < workers; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for source := range jobs {
peerMachine := "a"
if source.Machine == "a" {
peerMachine = "b"
}
peer := accounts[accountKey(peerMachine, source.Index)]
if peer == nil {
errCh <- fmt.Errorf("p2p preflight %s-%04d: peer is missing", source.Machine, source.Index)
continue
}
conversation, err := api.ensureP2P(ctx, source, *peer)
if err == nil && (conversation.Relationship != "friend" || conversation.Topic != source.P2PTopic) {
err = fmt.Errorf("relationship=%s topic_match=%t", conversation.Relationship, conversation.Topic == source.P2PTopic)
}
if err != nil {
errCh <- fmt.Errorf("p2p preflight %s-%04d: %w", source.Machine, source.Index, err)
}
}
}()
}
for _, account := range local {
jobs <- account
}
close(jobs)
wg.Wait()
close(errCh)
failures := make([]string, 0, len(errCh))
for err := range errCh {
if len(failures) < 20 {
failures = append(failures, err.Error())
}
}
if len(failures) > 0 {
return fmt.Errorf("p2p permission preflight failed: %s", strings.Join(failures, "; "))
}
return nil
}
func runAgent(ctx context.Context, cfg *Config, manifest *Manifest, machine, coordinatorURL, output, metricsListen string) error {
if machine != "a" && machine != "b" {
return fmt.Errorf("machine must be a or b")
@@ -58,6 +118,11 @@ func runAgent(ctx context.Context, cfg *Config, manifest *Manifest, machine, coo
for i := range accounts {
accounts[i].Endpoint = cfg.Endpoints[machine][(accounts[i].Index-1)%len(cfg.Endpoints[machine])]
}
if !cfg.SkipP2PPreflight {
if err = preflightP2P(ctx, cfg, manifest, accounts, apiKey); err != nil {
return err
}
}
coord := newCoordinatorClient(coordinatorURL)
placeholderRunID := "pending"
run := &agentRun{cfg: cfg, machine: machine, clients: make(map[int]*WSClient), ack: NewDurationHistogram()}
@@ -110,7 +175,8 @@ func runAgent(ctx context.Context, cfg *Config, manifest *Manifest, machine, coo
RunID: start.RunID, Scenario: start.Scenario, Machine: machine, StartedAt: startedAt,
LoadEndedAt: loadEndedAt, FinishedAt: time.Now().UTC(), ReadyConnections: len(run.clients),
PublishAttempted: run.attempted.Load(), PublishAccepted: int64(len(local)), PublishRejected: run.rejected.Load(),
AckLatency: run.ack.Snapshot(), AcceptedLocal: local,
PublishErrors: run.publishErrorCounts(),
AckLatency: run.ack.Snapshot(), AcceptedLocal: local,
}
run.mu.Lock()
base.Errors = append(base.Errors, run.errors...)
@@ -187,31 +253,29 @@ func (r *agentRun) reconcileHistory(ctx context.Context, ledger []AcceptedMessag
since: messages[start].Seq, before: messages[end-1].Seq + 1, limit: 256})
}
}
for _, query := range queries {
if err := query.client.GetData(ctx, query.topic, query.since, query.before, query.limit); err != nil {
r.addError(fmt.Sprintf("history get topic=%s range=%d-%d: %v", topicHash(query.topic), query.since, query.before, err))
}
}
_ = waitContext(ctx, 2*time.Second)
deadline := time.Now().Add(120 * time.Second)
for {
historyTimeout, _ := configDuration(r.cfg.HistoryTimeout)
historyCtx, cancel := context.WithTimeout(ctx, historyTimeout)
defer cancel()
r.issueHistoryQueries(historyCtx, queries)
_ = waitContext(historyCtx, 2*time.Second)
for round := 0; round < 2; round++ {
missing := missingHistory(r.recorder, expected)
if len(missing) == 0 || time.Now().After(deadline) {
break
}
if waitContext(ctx, 5*time.Second) != nil {
if len(missing) == 0 || historyCtx.Err() != nil {
break
}
retryQueries := make([]historyQuery, 0, len(missing))
for _, message := range missing {
client := r.clients[message.PeerIndex]
if message.Kind == "group" {
client = r.clients[1]
}
if client != nil {
_ = client.GetData(ctx, message.Topic, message.Seq, message.Seq+1, 1)
retryQueries = append(retryQueries, historyQuery{client: client, topic: message.Topic,
since: message.Seq, before: message.Seq + 1, limit: 1})
}
}
_ = waitContext(ctx, 500*time.Millisecond)
r.issueHistoryQueries(historyCtx, retryQueries)
_ = waitContext(historyCtx, 2*time.Second)
}
missing := missingHistory(r.recorder, expected)
missingIDs := make([]string, 0, len(missing))
@@ -222,6 +286,40 @@ func (r *agentRun) reconcileHistory(ctx context.Context, ledger []AcceptedMessag
return len(expected), len(expected) - len(missing), missingIDs
}
func (r *agentRun) issueHistoryQueries(ctx context.Context, queries []historyQuery) {
workers := r.cfg.HistoryWorkers
if workers > len(queries) {
workers = len(queries)
}
if workers == 0 {
return
}
jobs := make(chan historyQuery)
var wg sync.WaitGroup
for i := 0; i < workers; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for query := range jobs {
if err := query.client.GetData(ctx, query.topic, query.since, query.before, query.limit); err != nil && ctx.Err() == nil {
r.addError(fmt.Sprintf("history get topic=%s range=%d-%d: %v", topicHash(query.topic), query.since, query.before, err))
}
}
}()
}
for _, query := range queries {
select {
case jobs <- query:
case <-ctx.Done():
close(jobs)
wg.Wait()
return
}
}
close(jobs)
wg.Wait()
}
func missingHistory(recorder *deliveryRecorder, expected map[string]AcceptedMessage) []AcceptedMessage {
missing := make([]AcceptedMessage, 0)
for mid, message := range expected {
@@ -413,6 +511,7 @@ func (r *agentRun) publish(ctx context.Context, client *WSClient, topic, mid str
accepted, err := client.Publish(ctx, topic, mid, r.runID, r.scenario, messageIndex)
if err != nil {
r.rejected.Add(1)
r.classifyPublishError(err)
r.mu.Lock()
if len(r.errors) < 1000 {
r.errors = append(r.errors, fmt.Sprintf("publish %s: %v", mid, err))
@@ -424,6 +523,32 @@ func (r *agentRun) publish(ctx context.Context, client *WSClient, topic, mid str
return accepted, nil
}
func (r *agentRun) classifyPublishError(err error) {
var ctrlErr *ctrlError
switch {
case errors.As(err, &ctrlErr) && ctrlErr.Code == 403:
r.publish403.Add(1)
case errors.As(err, &ctrlErr) && ctrlErr.Code == 502:
r.publish502.Add(1)
case errors.As(err, &ctrlErr) && ctrlErr.Code == 503:
r.publish503.Add(1)
case strings.Contains(err.Error(), "ctrl timeout") || errors.Is(err, context.DeadlineExceeded):
r.publishTimeout.Add(1)
case strings.Contains(err.Error(), "websocket") || strings.Contains(err.Error(), "broken pipe") ||
strings.Contains(err.Error(), "connection reset"):
r.publishSocket.Add(1)
default:
r.publishOther.Add(1)
}
}
func (r *agentRun) publishErrorCounts() map[string]int64 {
return map[string]int64{
"403": r.publish403.Load(), "502": r.publish502.Load(), "503": r.publish503.Load(),
"ctrl_timeout": r.publishTimeout.Load(), "socket": r.publishSocket.Load(), "other": r.publishOther.Load(),
}
}
func (r *agentRun) addAccepted(message AcceptedMessage) {
r.mu.Lock()
r.accepted = append(r.accepted, message)