pro/im_server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: trace delayed websocket acknowledgements

Browse Source
This commit is contained in:
richjohnrambo
2026-10-01 15:19:57 +08:00
parent 19584d0939
commit 73a594146f
19 changed files with 788 additions and 149 deletions
Download Patch File Download Diff File Expand all files Collapse all files
loadtest/im-pressure/model.go
+83 -43
View File
@@ -6,6 +6,7 @@ import (
"encoding/json"
"errors"
"fmt"
"net"
"os"
"sort"
"strings"
@@ -21,6 +22,7 @@ type Config struct {
PasswordEnv string `json:"password_env"`
UsersPerMachine int `json:"users_per_machine"`
Endpoints map[string][]string `json:"endpoints"`
DialIPs map[string][]string `json:"dial_ips,omitempty"`
InsecureTLS bool `json:"insecure_tls"`
PrepareWorkers int `json:"prepare_workers"`
ConnectRamp string `json:"connect_ramp"`
@@ -30,6 +32,8 @@ type Config struct {
P2PInterval string `json:"p2p_interval"`
DrainP2P string `json:"drain_p2p"`
DrainGroup string `json:"drain_group"`
DeliveryQuiet string `json:"delivery_quiet_period"`
DeliveryTimeout string `json:"delivery_quiet_timeout"`
PhonePrefix string `json:"registration_phone_prefix"`
VerifyCodeEnv string `json:"verification_code_env"`
HistoryWorkers int `json:"history_workers"`
@@ -85,6 +89,18 @@ func loadConfig(path string) (*Config, error) {
if cfg.DrainGroup == "" {
cfg.DrainGroup = "60s"
}
if cfg.DeliveryQuiet == "" {
cfg.DeliveryQuiet = "10s"
}
if _, err = configDuration(cfg.DeliveryQuiet); err != nil {
return nil, fmt.Errorf("delivery_quiet_period: %w", err)
}
if cfg.DeliveryTimeout == "" {
cfg.DeliveryTimeout = "3m"
}
if _, err = configDuration(cfg.DeliveryTimeout); err != nil {
return nil, fmt.Errorf("delivery_quiet_timeout: %w", err)
}
if cfg.HistoryWorkers <= 0 {
cfg.HistoryWorkers = 12
}
@@ -140,6 +156,11 @@ func loadConfig(path string) (*Config, error) {
if len(cfg.Endpoints[machine]) == 0 {
return nil, fmt.Errorf("at least one endpoint is required for machine %s", machine)
}
for _, ip := range cfg.DialIPs[machine] {
if net.ParseIP(strings.TrimSpace(ip)) == nil {
return nil, fmt.Errorf("invalid dial IP %q for machine %s", ip, machine)
}
}
}
if cfg.PhonePrefix != "" {
if len(cfg.PhonePrefix) != 7 || strings.Trim(cfg.PhonePrefix, "0123456789") != "" {
@@ -214,6 +235,7 @@ type AccountRecord struct {
TokenExpires time.Time `json:"token_expires,omitempty"`
P2PTopic string `json:"p2p_topic,omitempty"`
Endpoint string `json:"endpoint,omitempty"`
DialIP string `json:"-"`
}
func newManifest(cfg *Config) *Manifest {
@@ -385,49 +407,67 @@ type AcceptedMessage struct {
}
type AgentSummary struct {
RunID string `json:"run_id"`
Scenario string `json:"scenario"`
Machine string `json:"machine"`
StartedAt time.Time `json:"started_at"`
LoadEndedAt time.Time `json:"load_ended_at"`
FinishedAt time.Time `json:"finished_at"`
ExpectedConnections int `json:"expected_connections"`
ReadyConnections int `json:"ready_connections"`
PublishAttempted int64 `json:"publish_attempted"`
PublishAccepted int64 `json:"publish_accepted"`
PublishRejected int64 `json:"publish_rejected"`
PublishOutcomeUnknown int64 `json:"publish_outcome_unknown"`
PublishErrors map[string]int64 `json:"publish_errors,omitempty"`
ExpectedPeer int64 `json:"expected_peer"`
ReceivedPeer int64 `json:"received_peer"`
ExpectedEcho int64 `json:"expected_echo"`
ReceivedEcho int64 `json:"received_echo"`
ExpectedGroup int64 `json:"expected_group"`
ReceivedGroup int64 `json:"received_group"`
RealtimeLost int64 `json:"realtime_lost"`
RealtimeLostStored int64 `json:"realtime_lost_but_persisted"`
Duplicates int64 `json:"duplicates"`
OutOfOrder int64 `json:"out_of_order"`
OrderingViolations []OrderingViolation `json:"ordering_violations,omitempty"`
Disconnects int64 `json:"disconnects"`
DisconnectReasons map[string]int64 `json:"disconnect_reasons,omitempty"`
DisconnectSamples []DisconnectSample `json:"disconnect_samples,omitempty"`
UnexpectedMessages int64 `json:"unexpected_messages"`
HistoryChecked int64 `json:"history_checked"`
HistoryFound int64 `json:"history_found"`
HistoryUnackedChecked int64 `json:"history_unacked_observed_checked"`
HistoryUnackedFound int64 `json:"history_unacked_observed_found"`
PersistenceLost int64 `json:"persistence_lost"`
AckLatency HistogramSnapshot `json:"ack_latency"`
DeliveryLatency HistogramSnapshot `json:"delivery_latency"`
DeliveryLatencyLocal HistogramSnapshot `json:"delivery_latency_local"`
DeliveryLatencyCross HistogramSnapshot `json:"delivery_latency_cross_machine"`
CrossClockSkewSamples int64 `json:"cross_machine_negative_latency_samples"`
CallbackQueue CallbackQueueSnapshot `json:"callback_queue"`
Errors []string `json:"errors,omitempty"`
Missing []MissingDelivery `json:"missing,omitempty"`
PersistenceMissing []string `json:"persistence_missing,omitempty"`
AcceptedLocal []AcceptedMessage `json:"accepted_local,omitempty"`
RunID string `json:"run_id"`
Scenario string `json:"scenario"`
Machine string `json:"machine"`
StartedAt time.Time `json:"started_at"`
LoadEndedAt time.Time `json:"load_ended_at"`
FinishedAt time.Time `json:"finished_at"`
ExpectedConnections int `json:"expected_connections"`
ReadyConnections int `json:"ready_connections"`
PublishAttempted int64 `json:"publish_attempted"`
PublishAccepted int64 `json:"publish_accepted"`
PublishRejected int64 `json:"publish_rejected"`
PublishOutcomeUnknown int64 `json:"publish_outcome_unknown"`
PublishErrors map[string]int64 `json:"publish_errors,omitempty"`
CtrlTrackingEnabled bool `json:"ctrl_tracking_enabled"`
PublishCtrlObserved int64 `json:"publish_ctrl_observed"`
PublishCtrlOver20 int64 `json:"publish_ctrl_over_20s"`
PublishLateCtrl int64 `json:"publish_ctrl_late"`
PublishCtrlUnresolved int64 `json:"publish_ctrl_unresolved"`
PublishCtrlLatency HistogramSnapshot `json:"publish_ctrl_latency"`
CtrlObservationSamples []CtrlObservationSample `json:"ctrl_observation_samples,omitempty"`
ExpectedPeer int64 `json:"expected_peer"`
ReceivedPeer int64 `json:"received_peer"`
ExpectedEcho int64 `json:"expected_echo"`
ReceivedEcho int64 `json:"received_echo"`
ExpectedGroup int64 `json:"expected_group"`
ReceivedGroup int64 `json:"received_group"`
RealtimeLost int64 `json:"realtime_lost"`
RealtimeLostStored int64 `json:"realtime_lost_but_persisted"`
Duplicates int64 `json:"duplicates"`
OutOfOrder int64 `json:"out_of_order"`
OrderingViolations []OrderingViolation `json:"ordering_violations,omitempty"`
Disconnects int64 `json:"disconnects"`
DisconnectReasons map[string]int64 `json:"disconnect_reasons,omitempty"`
DisconnectSamples []DisconnectSample `json:"disconnect_samples,omitempty"`
UnexpectedMessages int64 `json:"unexpected_messages"`
HistoryChecked int64 `json:"history_checked"`
HistoryFound int64 `json:"history_found"`
HistoryUnackedChecked int64 `json:"history_unacked_observed_checked"`
HistoryUnackedFound int64 `json:"history_unacked_observed_found"`
PersistenceLost int64 `json:"persistence_lost"`
AckLatency HistogramSnapshot `json:"ack_latency"`
DeliveryLatency HistogramSnapshot `json:"delivery_latency"`
DeliveryLatencyLocal HistogramSnapshot `json:"delivery_latency_local"`
DeliveryLatencyCross HistogramSnapshot `json:"delivery_latency_cross_machine"`
CrossClockSkewSamples int64 `json:"cross_machine_negative_latency_samples"`
CallbackQueue CallbackQueueSnapshot `json:"callback_queue"`
Errors []string `json:"errors,omitempty"`
Missing []MissingDelivery `json:"missing,omitempty"`
PersistenceMissing []string `json:"persistence_missing,omitempty"`
AcceptedLocal []AcceptedMessage `json:"accepted_local,omitempty"`
}
type CtrlObservationSample struct {
Machine string `json:"machine"`
Index int `json:"index"`
ID string `json:"id"`
Code int `json:"code"`
SentAt time.Time `json:"sent_at"`
ReceivedAt time.Time `json:"received_at"`
LatencyMS float64 `json:"latency_ms"`
Late bool `json:"late"`
}
type CallbackQueueSnapshot struct {