pro/im_server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: trace delayed websocket acknowledgements

Browse Source
This commit is contained in:
richjohnrambo
2026-10-01 15:19:57 +08:00
parent 19584d0939
commit 73a594146f
19 changed files with 788 additions and 149 deletions
Download Patch File Download Diff File Expand all files Collapse all files
loadtest/im-pressure/agent.go
+132 -24
View File
@@ -19,27 +19,32 @@ import (
var runIDPattern = regexp.MustCompile(`^[a-zA-Z0-9_-]{1,24}$`)
type agentRun struct {
cfg *Config
machine string
runID string
scenario string
clients map[int]*WSClient
clientsMu sync.RWMutex
recorder *deliveryRecorder
callbackQueue *callbackQueueMetrics
ack *DurationHistogram
attempted atomic.Int64
ready atomic.Int64
publish403 atomic.Int64
publish502 atomic.Int64
publish503 atomic.Int64
publishTimeout atomic.Int64
publishSocket atomic.Int64
publishOther atomic.Int64
cfg *Config
machine string
runID string
scenario string
clients map[int]*WSClient
clientsMu sync.RWMutex
recorder *deliveryRecorder
callbackQueue *callbackQueueMetrics
ack *DurationHistogram
ctrlLatency *DurationHistogram
attempted atomic.Int64
ready atomic.Int64
publish403 atomic.Int64
publish502 atomic.Int64
publish503 atomic.Int64
publishTimeout atomic.Int64
publishSocket atomic.Int64
publishOther atomic.Int64
publishCtrlObserved atomic.Int64
publishCtrlOver20 atomic.Int64
publishLateCtrl atomic.Int64
mu sync.Mutex
accepted []AcceptedMessage
errors []string
mu sync.Mutex
accepted []AcceptedMessage
errors []string
ctrlSamples []CtrlObservationSample
}
func validScenario(scenario string) bool {
@@ -127,6 +132,7 @@ func runAgent(ctx context.Context, cfg *Config, manifest *Manifest, machine, coo
coord := newCoordinatorClient(coordinatorURL)
placeholderRunID := "pending"
run := &agentRun{cfg: cfg, machine: machine, clients: make(map[int]*WSClient), ack: NewDurationHistogram(),
ctrlLatency: NewDurationHistogram(),
callbackQueue: &callbackQueueMetrics{}}
run.recorder = newDeliveryRecorder(placeholderRunID, machine, len(accounts))
stopMetrics, err := startAgentMetrics(metricsListen, run)
@@ -211,6 +217,12 @@ func runAgent(ctx context.Context, cfg *Config, manifest *Manifest, machine, coo
if err = waitForCallbackQueue(ctx, run.callbackQueue, 30*time.Second); err != nil {
return err
}
quietPeriod, _ := configDuration(cfg.DeliveryQuiet)
quietTimeout, _ := configDuration(cfg.DeliveryTimeout)
log.Printf("agent=%s waiting for %s of delivery quiescence", machine, quietPeriod)
if err = waitForDeliveryQuiescence(ctx, run.recorder, run.callbackQueue, quietPeriod, quietTimeout); err != nil {
return err
}
observedLocal := run.recorder.unackedObservedMessages(ledger, start.Scenario)
if err = coord.submitObserved(ctx, machine, observedLocal); err != nil {
return err
@@ -233,7 +245,10 @@ func runAgent(ctx context.Context, cfg *Config, manifest *Manifest, machine, coo
run.mu.Lock()
base.Errors = append(base.Errors, run.errors...)
run.mu.Unlock()
summary := run.recorder.summarize(base, ledger)
// Include messages observed over WebSocket without a timely ctrl ACK. They
// are confirmed server-side publishes and must participate in realtime
// delivery accounting instead of being reported as unexpected traffic.
summary := run.recorder.summarize(base, verificationLedger)
historySettle, _ := configDuration(cfg.HistorySettle)
log.Printf("agent=%s waiting %s before history verification", machine, historySettle)
if err = waitContext(ctx, historySettle); err != nil {
@@ -256,7 +271,7 @@ func runAgent(ctx context.Context, cfg *Config, manifest *Manifest, machine, coo
summary.HistoryUnackedFound = int64(history.unackedFound)
persistenceMissing := history.missing
summary.PersistenceLost = int64(len(persistenceMissing))
summary.RealtimeLostStored = run.recorder.countRealtimeLostStored(ledger, storedMessageIDs)
summary.RealtimeLostStored = run.recorder.countRealtimeLostStored(verificationLedger, storedMessageIDs)
if len(persistenceMissing) > 10000 {
summary.PersistenceMissing = append([]string(nil), persistenceMissing[:10000]...)
summary.Errors = append(summary.Errors, "persistence missing details truncated at 10000")
@@ -264,6 +279,7 @@ func runAgent(ctx context.Context, cfg *Config, manifest *Manifest, machine, coo
summary.PersistenceMissing = persistenceMissing
}
summary.FinishedAt = time.Now().UTC()
run.applyCtrlObservations(&summary)
if err = writeAgentSummary(output, summary); err != nil {
return err
}
@@ -293,6 +309,51 @@ func waitForCallbackQueue(ctx context.Context, queue *callbackQueueMetrics, time
}
return nil
}
func waitForDeliveryQuiescence(ctx context.Context, recorder *deliveryRecorder, queue *callbackQueueMetrics,
quietPeriod, timeout time.Duration) error {
deadline := time.NewTimer(timeout)
defer deadline.Stop()
ticker := time.NewTicker(100 * time.Millisecond)
defer ticker.Stop()
lastReceived := recorder.received.Load()
stableSince := time.Now()
for {
select {
case <-ctx.Done():
return ctx.Err()
case <-deadline.C:
return fmt.Errorf("deliveries did not become quiescent within %s: received=%d queued=%d processing=%d",
timeout, recorder.received.Load(), queue.current.Load(), queue.processing.Load())
case now := <-ticker.C:
received := recorder.received.Load()
if received != lastReceived || !queue.isDrained() {
lastReceived = received
stableSince = now
continue
}
if now.Sub(stableSince) >= quietPeriod {
return nil
}
}
}
}
func assignAgentEndpoints(cfg *Config, machine string, accounts []AccountRecord) {
endpoints := cfg.Endpoints[machine]
dialIPs := cfg.DialIPs[machine]
machineOffset := 0
if machine == "b" {
machineOffset = 1
}
for i := range accounts {
accounts[i].Endpoint = endpoints[(accounts[i].Index-1)%len(endpoints)]
if len(dialIPs) > 0 {
slot := (accounts[i].Index-1)*2 + machineOffset
accounts[i].DialIP = strings.TrimSpace(dialIPs[slot%len(dialIPs)])
}
}
}
type historyQuery struct {
client *WSClient
topic string
@@ -690,7 +751,8 @@ func (r *agentRun) maintainConnections(ctx context.Context, accounts []AccountRe
func (r *agentRun) connectAccountWithRetry(ctx context.Context, account AccountRecord, groupTopic, apiKey, password string) (*WSClient, error) {
var lastErr error
for attempt := 1; attempt <= 6; attempt++ {
client, err := connectWSClient(ctx, r.cfg, account, apiKey, password, r.recorder.onData, r.recorder.onDisconnect, r.callbackQueue)
client, err := connectWSClient(ctx, r.cfg, account, apiKey, password, r.recorder.onData, r.recorder.onDisconnect,
r.observeCtrl, r.callbackQueue)
if err == nil {
if err = client.Subscribe(ctx, account.P2PTopic); err == nil {
err = client.Subscribe(ctx, groupTopic)
@@ -898,7 +960,53 @@ func (r *agentRun) explicitRejected() int64 {
}
func (r *agentRun) outcomeUnknown() int64 {
return r.publishTimeout.Load() + r.publishSocket.Load()
return r.unresolvedCtrl() + r.publishSocket.Load()
}
func (r *agentRun) unresolvedCtrl() int64 {
unresolved := r.publishTimeout.Load() - r.publishLateCtrl.Load()
if unresolved < 0 {
return 0
}
return unresolved
}
func (r *agentRun) observeCtrl(account AccountRecord, observation ctrlObservation) {
if observation.Kind != "pub" {
return
}
r.publishCtrlObserved.Add(1)
r.ctrlLatency.Observe(observation.Latency)
if observation.Latency > 20*time.Second {
r.publishCtrlOver20.Add(1)
}
if observation.Late {
r.publishLateCtrl.Add(1)
}
if observation.Late || observation.Latency > 20*time.Second {
r.mu.Lock()
if len(r.ctrlSamples) < 1000 {
r.ctrlSamples = append(r.ctrlSamples, CtrlObservationSample{
Machine: account.Machine, Index: account.Index, ID: observation.ID, Code: observation.Code,
SentAt: observation.SentAt, ReceivedAt: observation.ReceivedAt,
LatencyMS: float64(observation.Latency) / float64(time.Millisecond), Late: observation.Late,
})
}
r.mu.Unlock()
}
}
func (r *agentRun) applyCtrlObservations(summary *AgentSummary) {
summary.CtrlTrackingEnabled = true
summary.PublishCtrlObserved = r.publishCtrlObserved.Load()
summary.PublishCtrlOver20 = r.publishCtrlOver20.Load()
summary.PublishLateCtrl = r.publishLateCtrl.Load()
summary.PublishCtrlUnresolved = r.unresolvedCtrl()
summary.PublishOutcomeUnknown = r.outcomeUnknown()
summary.PublishCtrlLatency = r.ctrlLatency.Snapshot()
r.mu.Lock()
summary.CtrlObservationSamples = append([]CtrlObservationSample(nil), r.ctrlSamples...)
r.mu.Unlock()
}
func (r *agentRun) addAccepted(message AcceptedMessage) {