pro/im_server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: harden high-fanout delivery under load

Browse Source
This commit is contained in:
richjohnrambo
2026-09-30 23:04:54 +08:00
parent 2b631ab176
commit 19584d0939
14 changed files with 491 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files
loadtest/im-pressure/agent.go
+25 -2
View File
@@ -26,6 +26,7 @@ type agentRun struct {
clients map[int]*WSClient
clientsMu sync.RWMutex
recorder *deliveryRecorder
callbackQueue *callbackQueueMetrics
ack *DurationHistogram
attempted atomic.Int64
ready atomic.Int64
@@ -125,7 +126,8 @@ func runAgent(ctx context.Context, cfg *Config, manifest *Manifest, machine, coo
}
coord := newCoordinatorClient(coordinatorURL)
placeholderRunID := "pending"
run := &agentRun{cfg: cfg, machine: machine, clients: make(map[int]*WSClient), ack: NewDurationHistogram()}
run := &agentRun{cfg: cfg, machine: machine, clients: make(map[int]*WSClient), ack: NewDurationHistogram(),
callbackQueue: &callbackQueueMetrics{}}
run.recorder = newDeliveryRecorder(placeholderRunID, machine, len(accounts))
stopMetrics, err := startAgentMetrics(metricsListen, run)
if err != nil {
@@ -206,6 +208,9 @@ func runAgent(ctx context.Context, cfg *Config, manifest *Manifest, machine, coo
if err = waitContext(ctx, drain); err != nil {
return err
}
if err = waitForCallbackQueue(ctx, run.callbackQueue, 30*time.Second); err != nil {
return err
}
observedLocal := run.recorder.unackedObservedMessages(ledger, start.Scenario)
if err = coord.submitObserved(ctx, machine, observedLocal); err != nil {
return err
@@ -223,6 +228,7 @@ func runAgent(ctx context.Context, cfg *Config, manifest *Manifest, machine, coo
PublishRejected: run.explicitRejected(), PublishOutcomeUnknown: run.outcomeUnknown(),
PublishErrors: run.publishErrorCounts(),
AckLatency: run.ack.Snapshot(), AcceptedLocal: local,
CallbackQueue: run.callbackQueue.snapshot(),
}
run.mu.Lock()
base.Errors = append(base.Errors, run.errors...)
@@ -270,6 +276,23 @@ func runAgent(ctx context.Context, cfg *Config, manifest *Manifest, machine, coo
return nil
}
func waitForCallbackQueue(ctx context.Context, queue *callbackQueueMetrics, timeout time.Duration) error {
deadline := time.NewTimer(timeout)
defer deadline.Stop()
ticker := time.NewTicker(10 * time.Millisecond)
defer ticker.Stop()
for !queue.isDrained() {
select {
case <-ctx.Done():
return ctx.Err()
case <-deadline.C:
return fmt.Errorf("data callback queue did not drain within %s: current=%d processing=%d",
timeout, queue.current.Load(), queue.processing.Load())
case <-ticker.C:
}
}
return nil
}
type historyQuery struct {
client *WSClient
topic string
@@ -667,7 +690,7 @@ func (r *agentRun) maintainConnections(ctx context.Context, accounts []AccountRe
func (r *agentRun) connectAccountWithRetry(ctx context.Context, account AccountRecord, groupTopic, apiKey, password string) (*WSClient, error) {
var lastErr error
for attempt := 1; attempt <= 6; attempt++ {
client, err := connectWSClient(ctx, r.cfg, account, apiKey, password, r.recorder.onData, r.recorder.onDisconnect)
client, err := connectWSClient(ctx, r.cfg, account, apiKey, password, r.recorder.onData, r.recorder.onDisconnect, r.callbackQueue)
if err == nil {
if err = client.Subscribe(ctx, account.P2PTopic); err == nil {
err = client.Subscribe(ctx, groupTopic)