pro/im_server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test(load): add idle and tunable group scenarios

Browse Source
This commit is contained in:
richjohnrambo
2026-10-05 17:36:18 +08:00
parent 965f94f154
commit d3054f62b5
8 changed files with 385 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
loadtest/im-pressure/README.md
+7 -2
View File
@@ -197,8 +197,13 @@ mkdir -p "loadtest/im-pressure/results/${RUN_ID}"
两个 agent 都建立 1000 条连接后，协调器下发统一的 UTC `start_at`。场景可替换为：
- `p2p`：1000 对用户两侧同时发送，默认每人 10 条、间隔 1 秒。
- `group`：`users_per_machine=1000` 时每个 agent 有 500 个发送者，每人 2 条；全局每 100ms 一条，共 2000 条。其他容量按每个 agent 用户数的 50% 选发送者。
- `mixed`：`users_per_machine=1000` 时使用 300 对私聊用户和每个 agent 250 个群发送者；其他容量分别按 30% 和 25% 同比计算。
- `group`：默认每个 agent 有 `group_senders_per_machine` 个发送者，每人 `group_rounds` 条；两机合并后按 `group_publish_interval` 控制全局发送间隔。双机默认 500+500 发送者、100ms 一条，共 2000 条；单机 60% 默认 300+300 发送者。
- `mixed`：默认使用 `mixed_p2p_users_per_machine` 对私聊用户和 `mixed_group_senders_per_machine` 个群发送者。双机默认每机 300 个私聊用户、250 个群发送者；单机 60% 默认每机 180 个私聊用户、150 个群发送者。
- `idle`：只建立并维持 WS 连接，不发送业务消息，持续 `idle_duration`。该场景用于先确认 2000 长连接在入口、Pod、DNS、心跳和自动重连下是否稳定。
群聊容量阶梯建议只改 `group_publish_interval`，保持发送者和轮数不变：`125ms` 约等于 8 msg/s，`100ms` 约等于 10 msg/s，`83ms` 约等于 12 msg/s，`67ms` 约等于 15 msg/s，`56ms` 约等于 18 msg/s。先跑 `idle`，再从低速 `group` 逐级上探；某一级出现连接波动、ACK 长尾或 DB pool wait 明显上升时，重复该级确认后再进入下一档。
自动重连由 `reconnect_workers`、`reconnect_min_wait`、`reconnect_max_wait` 控制。断线账号会按指数退避并附带确定性抖动后重连，避免两台压测机在入口抖动时同时打出登录洪峰。
## 机器 A：回收并合并报告
loadtest/im-pressure/agent.go
+79 -11
View File
@@ -49,7 +49,7 @@ type agentRun struct {
}
func validScenario(scenario string) bool {
return scenario == "p2p" || scenario == "group" || scenario == "mixed"
return scenario == "p2p" || scenario == "group" || scenario == "mixed" || scenario == "idle"
}
func preflightP2P(ctx context.Context, cfg *Config, manifest *Manifest, local []AccountRecord, apiKey string) error {
@@ -680,10 +680,25 @@ func (r *agentRun) connect(ctx context.Context, accounts []AccountRecord, groupT
}
func (r *agentRun) maintainConnections(ctx context.Context, accounts []AccountRecord, groupTopic, apiKey, password string) {
type reconnectState struct {
attempt int
nextAt time.Time
}
byIndex := make(map[int]AccountRecord, len(accounts))
for _, account := range accounts {
byIndex[account.Index] = account
}
states := make(map[int]reconnectState, len(accounts))
var stateMu sync.Mutex
minWait := mustDuration(r.cfg.ReconnectMinWait)
maxWait := mustDuration(r.cfg.ReconnectMaxWait)
if maxWait < minWait {
maxWait = minWait
}
reconnectWorkers := r.cfg.ReconnectWorkers
if reconnectWorkers <= 0 {
reconnectWorkers = 8
}
ticker := time.NewTicker(time.Second)
defer ticker.Stop()
for {
@@ -692,7 +707,7 @@ func (r *agentRun) maintainConnections(ctx context.Context, accounts []AccountRe
}
jobs := make(chan AccountRecord)
var wg sync.WaitGroup
workers := 16
workers := reconnectWorkers
if workers > len(accounts) {
workers = len(accounts)
}
@@ -714,8 +729,17 @@ func (r *agentRun) maintainConnections(ctx context.Context, accounts []AccountRe
if ctx.Err() == nil {
log.Printf("account=%s reconnect failed: %v", account.Username, err)
}
stateMu.Lock()
state := states[index]
state.attempt++
state.nextAt = time.Now().Add(reconnectBackoff(minWait, maxWait, state.attempt, index))
states[index] = state
stateMu.Unlock()
continue
}
stateMu.Lock()
delete(states, index)
stateMu.Unlock()
r.setClient(index, client)
log.Printf("account=%s connection restored", account.Username)
}
@@ -732,6 +756,12 @@ func (r *agentRun) maintainConnections(ctx context.Context, accounts []AccountRe
if current != nil && current.alive() {
continue
}
stateMu.Lock()
nextAt := states[index].nextAt
stateMu.Unlock()
if !nextAt.IsZero() && time.Now().Before(nextAt) {
continue
}
select {
case jobs <- account:
case <-ctx.Done():
@@ -750,6 +780,25 @@ func (r *agentRun) maintainConnections(ctx context.Context, accounts []AccountRe
}
}
func reconnectBackoff(minWait, maxWait time.Duration, attempt, index int) time.Duration {
if attempt < 1 {
attempt = 1
}
delay := minWait
for i := 1; i < attempt && delay < maxWait; i++ {
delay *= 2
}
if delay > maxWait {
delay = maxWait
}
jitterBase := delay / 4
if jitterBase <= 0 {
return delay
}
jitter := time.Duration((index*97+attempt*53)%1000) * jitterBase / 1000
return delay + jitter
}
func (r *agentRun) connectAccountWithRetry(ctx context.Context, account AccountRecord, groupTopic, apiKey, password string) (*WSClient, error) {
var lastErr error
for attempt := 1; attempt <= 6; attempt++ {
@@ -787,12 +836,25 @@ func (r *agentRun) execute(ctx context.Context, start time.Time, groupTopic stri
case "p2p":
return r.scheduleP2P(ctx, start, r.cfg.UsersPerMachine, r.cfg.P2PMessages, mustDuration(r.cfg.P2PInterval))
case "group":
return r.scheduleGroup(ctx, start, groupTopic, 1, scaledUsers(r.cfg.UsersPerMachine, 1, 2), 2)
senders := r.cfg.GroupSenders
if senders <= 0 {
senders = scaledUsers(r.cfg.UsersPerMachine, 1, 2)
}
if senders > r.cfg.UsersPerMachine {
return fmt.Errorf("group_senders_per_machine=%d exceeds users_per_machine=%d", senders, r.cfg.UsersPerMachine)
}
return r.scheduleGroup(ctx, start, groupTopic, 1, senders, r.cfg.GroupRounds, mustDuration(r.cfg.GroupInterval))
case "mixed":
p2pUsers := scaledUsers(r.cfg.UsersPerMachine, 3, 10)
groupSenders := scaledUsers(r.cfg.UsersPerMachine, 1, 4)
p2pUsers := r.cfg.MixedP2PUsers
if p2pUsers <= 0 {
p2pUsers = scaledUsers(r.cfg.UsersPerMachine, 3, 10)
}
groupSenders := r.cfg.MixedGroupSenders
if groupSenders <= 0 {
groupSenders = scaledUsers(r.cfg.UsersPerMachine, 1, 4)
}
if p2pUsers+groupSenders > r.cfg.UsersPerMachine {
groupSenders = r.cfg.UsersPerMachine - p2pUsers
return fmt.Errorf("mixed_p2p_users_per_machine + mixed_group_senders_per_machine exceeds users_per_machine")
}
if groupSenders <= 0 {
return fmt.Errorf("users_per_machine=%d is too small for mixed scenario", r.cfg.UsersPerMachine)
@@ -803,7 +865,7 @@ func (r *agentRun) execute(ctx context.Context, start time.Time, groupTopic stri
go func() { defer wg.Done(); errCh <- r.scheduleP2P(ctx, start, p2pUsers, 5, 20*time.Second) }()
go func() {
defer wg.Done()
errCh <- r.scheduleGroup(ctx, start, groupTopic, p2pUsers+1, p2pUsers+groupSenders, 2)
errCh <- r.scheduleGroup(ctx, start, groupTopic, p2pUsers+1, p2pUsers+groupSenders, r.cfg.GroupRounds, mustDuration(r.cfg.GroupInterval))
}()
wg.Wait()
close(errCh)
@@ -813,6 +875,8 @@ func (r *agentRun) execute(ctx context.Context, start time.Time, groupTopic stri
}
}
return nil
case "idle":
return waitContext(ctx, mustDuration(r.cfg.IdleDuration))
default:
return fmt.Errorf("unknown scenario %s", r.scenario)
}
@@ -867,15 +931,19 @@ func (r *agentRun) scheduleP2P(ctx context.Context, start time.Time, users, mess
return ctx.Err()
}
func (r *agentRun) scheduleGroup(ctx context.Context, start time.Time, topic string, first, last, rounds int) error {
func (r *agentRun) scheduleGroup(ctx context.Context, start time.Time, topic string, first, last, rounds int, globalInterval time.Duration) error {
if first < 1 || last > r.clientCount() || first > last {
return fmt.Errorf("invalid group publisher range %d-%d", first, last)
}
count := last - first + 1
roundDuration := time.Duration(count) * 200 * time.Millisecond
if globalInterval <= 0 {
return fmt.Errorf("group publish interval must be positive")
}
perMachineInterval := 2 * globalInterval
roundDuration := time.Duration(count) * perMachineInterval
offset := time.Duration(0)
if r.machine == "b" {
offset = 100 * time.Millisecond
offset = globalInterval
}
var wg sync.WaitGroup
for index := first; index <= last; index++ {
@@ -884,7 +952,7 @@ func (r *agentRun) scheduleGroup(ctx context.Context, start time.Time, topic str
go func(index, position int) {
defer wg.Done()
for round := 1; round <= rounds; round++ {
target := start.Add(time.Duration(round-1)*roundDuration + time.Duration(position)*200*time.Millisecond + offset)
target := start.Add(time.Duration(round-1)*roundDuration + time.Duration(position)*perMachineInterval + offset)
if waitUntil(ctx, target) != nil {
return
}
loadtest/im-pressure/agent_history_test.go
+27
View File
@@ -76,6 +76,33 @@ func TestScenarioUsersScaleWithConfiguredCapacity(t *testing.T) {
}
}
func TestReconnectBackoffCapsAndJitters(t *testing.T) {
minWait := 2 * time.Second
maxWait := 30 * time.Second
first := reconnectBackoff(minWait, maxWait, 1, 10)
if first < minWait || first >= minWait+minWait/4 {
t.Fatalf("first backoff=%s outside expected jitter range", first)
}
capped := reconnectBackoff(minWait, maxWait, 10, 10)
if capped < maxWait || capped >= maxWait+maxWait/4 {
t.Fatalf("capped backoff=%s outside expected jitter range", capped)
}
if reconnectBackoff(minWait, maxWait, 3, 10) == reconnectBackoff(minWait, maxWait, 3, 11) {
t.Fatal("different account indexes should receive different jitter")
}
}
func TestValidScenarioIncludesIdle(t *testing.T) {
for _, scenario := range []string{"p2p", "group", "mixed", "idle"} {
if !validScenario(scenario) {
t.Fatalf("scenario %s should be valid", scenario)
}
}
if validScenario("unknown") {
t.Fatal("unknown scenario should be rejected")
}
}
func TestNewHistoryQueryIncludesInterleavedSequenceRange(t *testing.T) {
messages := make([]AcceptedMessage, 10)
for i := range messages {
loadtest/im-pressure/config.example.json
+9
View File
@@ -13,6 +13,9 @@
"insecure_tls": false,
"prepare_workers": 20,
"connect_ramp": "20s",
"reconnect_workers": 8,
"reconnect_min_wait": "2s",
"reconnect_max_wait": "30s",
"command_timeout": "20s",
"ws_ping_interval": "20s",
"preflight_workers": 32,
@@ -26,6 +29,12 @@
"close_ramp": "10s",
"p2p_messages_per_user": 10,
"p2p_interval": "1s",
"group_senders_per_machine": 500,
"group_rounds": 2,
"group_publish_interval": "100ms",
"mixed_p2p_users_per_machine": 300,
"mixed_group_senders_per_machine": 250,
"idle_duration": "15m",
"drain_p2p": "30s",
"drain_group": "60s",
"delivery_quiet_period": "10s",
loadtest/im-pressure/config.single-machine.example.json
+9
View File
@@ -17,6 +17,9 @@
"insecure_tls": false,
"prepare_workers": 20,
"connect_ramp": "30s",
"reconnect_workers": 8,
"reconnect_min_wait": "2s",
"reconnect_max_wait": "30s",
"command_timeout": "90s",
"ws_ping_interval": "20s",
"preflight_workers": 32,
@@ -30,6 +33,12 @@
"close_ramp": "60s",
"p2p_messages_per_user": 10,
"p2p_interval": "1s",
"group_senders_per_machine": 300,
"group_rounds": 2,
"group_publish_interval": "100ms",
"mixed_p2p_users_per_machine": 180,
"mixed_group_senders_per_machine": 150,
"idle_duration": "15m",
"drain_p2p": "30s",
"drain_group": "60s",
"delivery_quiet_period": "10s",
loadtest/im-pressure/main.go
+1 -1
View File
@@ -101,7 +101,7 @@ func coordinatorCommand(ctx context.Context, args []string) error {
fs := flag.NewFlagSet("coordinator", flag.ContinueOnError)
listen := fs.String("listen", ":8090", "coordinator listen address")
runID := fs.String("run-id", "", "unique run ID, max 24 characters")
scenario := fs.String("scenario", "", "p2p, group, or mixed")
scenario := fs.String("scenario", "", "p2p, group, mixed, or idle")
output := fs.String("output", "loadtest/im-pressure/results/coordinator.json", "coordinator result path")
expected := fs.Int("expected-agents", 2, "number of agents")
delay := fs.Duration("start-delay", 30*time.Second, "delay after all agents are ready")
loadtest/im-pressure/model.go
+85 -29
View File
@@ -16,35 +16,44 @@ import (
const manifestVersion = 1
type Config struct {
Tenant string `json:"tenant"`
HTTPBase string `json:"http_base"`
APIKeyEnv string `json:"api_key_env"`
PasswordEnv string `json:"password_env"`
UsersPerMachine int `json:"users_per_machine"`
Endpoints map[string][]string `json:"endpoints"`
DialIPs map[string][]string `json:"dial_ips,omitempty"`
InsecureTLS bool `json:"insecure_tls"`
PrepareWorkers int `json:"prepare_workers"`
ConnectRamp string `json:"connect_ramp"`
CommandTimeout string `json:"command_timeout"`
WSPingInterval string `json:"ws_ping_interval"`
P2PMessages int `json:"p2p_messages_per_user"`
P2PInterval string `json:"p2p_interval"`
DrainP2P string `json:"drain_p2p"`
DrainGroup string `json:"drain_group"`
DeliveryQuiet string `json:"delivery_quiet_period"`
DeliveryTimeout string `json:"delivery_quiet_timeout"`
PhonePrefix string `json:"registration_phone_prefix"`
VerifyCodeEnv string `json:"verification_code_env"`
HistoryWorkers int `json:"history_workers"`
HistoryPageSize int `json:"history_page_size"`
HistoryInterval string `json:"history_query_interval"`
HistoryPageWait string `json:"history_page_wait"`
HistorySettle string `json:"history_settle"`
HistoryTimeout string `json:"history_timeout"`
CloseRamp string `json:"close_ramp"`
PreflightWorkers int `json:"preflight_workers"`
SkipP2PPreflight bool `json:"skip_p2p_preflight"`
Tenant string `json:"tenant"`
HTTPBase string `json:"http_base"`
APIKeyEnv string `json:"api_key_env"`
PasswordEnv string `json:"password_env"`
UsersPerMachine int `json:"users_per_machine"`
Endpoints map[string][]string `json:"endpoints"`
DialIPs map[string][]string `json:"dial_ips,omitempty"`
InsecureTLS bool `json:"insecure_tls"`
PrepareWorkers int `json:"prepare_workers"`
ConnectRamp string `json:"connect_ramp"`
ReconnectWorkers int `json:"reconnect_workers"`
ReconnectMinWait string `json:"reconnect_min_wait"`
ReconnectMaxWait string `json:"reconnect_max_wait"`
CommandTimeout string `json:"command_timeout"`
WSPingInterval string `json:"ws_ping_interval"`
P2PMessages int `json:"p2p_messages_per_user"`
P2PInterval string `json:"p2p_interval"`
GroupSenders int `json:"group_senders_per_machine"`
GroupRounds int `json:"group_rounds"`
GroupInterval string `json:"group_publish_interval"`
MixedP2PUsers int `json:"mixed_p2p_users_per_machine"`
MixedGroupSenders int `json:"mixed_group_senders_per_machine"`
IdleDuration string `json:"idle_duration"`
DrainP2P string `json:"drain_p2p"`
DrainGroup string `json:"drain_group"`
DeliveryQuiet string `json:"delivery_quiet_period"`
DeliveryTimeout string `json:"delivery_quiet_timeout"`
PhonePrefix string `json:"registration_phone_prefix"`
VerifyCodeEnv string `json:"verification_code_env"`
HistoryWorkers int `json:"history_workers"`
HistoryPageSize int `json:"history_page_size"`
HistoryInterval string `json:"history_query_interval"`
HistoryPageWait string `json:"history_page_wait"`
HistorySettle string `json:"history_settle"`
HistoryTimeout string `json:"history_timeout"`
CloseRamp string `json:"close_ramp"`
PreflightWorkers int `json:"preflight_workers"`
SkipP2PPreflight bool `json:"skip_p2p_preflight"`
}
func loadConfig(path string) (*Config, error) {
@@ -68,6 +77,29 @@ func loadConfig(path string) (*Config, error) {
if cfg.ConnectRamp == "" {
cfg.ConnectRamp = "20s"
}
if cfg.ReconnectWorkers <= 0 {
cfg.ReconnectWorkers = 8
}
if cfg.ReconnectWorkers > 128 {
return nil, errors.New("reconnect_workers must not exceed 128")
}
if cfg.ReconnectMinWait == "" {
cfg.ReconnectMinWait = "2s"
}
if _, err = configDuration(cfg.ReconnectMinWait); err != nil {
return nil, fmt.Errorf("reconnect_min_wait: %w", err)
}
if cfg.ReconnectMaxWait == "" {
cfg.ReconnectMaxWait = "30s"
}
minReconnect, _ := configDuration(cfg.ReconnectMinWait)
maxReconnect, err := configDuration(cfg.ReconnectMaxWait)
if err != nil {
return nil, fmt.Errorf("reconnect_max_wait: %w", err)
}
if maxReconnect < minReconnect {
return nil, errors.New("reconnect_max_wait must be greater than or equal to reconnect_min_wait")
}
if cfg.CommandTimeout == "" {
cfg.CommandTimeout = "20s"
}
@@ -83,6 +115,30 @@ func loadConfig(path string) (*Config, error) {
if cfg.P2PInterval == "" {
cfg.P2PInterval = "1s"
}
if cfg.GroupRounds <= 0 {
cfg.GroupRounds = 2
}
if cfg.GroupInterval == "" {
cfg.GroupInterval = "100ms"
}
if _, err = configDuration(cfg.GroupInterval); err != nil {
return nil, fmt.Errorf("group_publish_interval: %w", err)
}
if cfg.GroupSenders < 0 {
return nil, errors.New("group_senders_per_machine must not be negative")
}
if cfg.MixedP2PUsers < 0 {
return nil, errors.New("mixed_p2p_users_per_machine must not be negative")
}
if cfg.MixedGroupSenders < 0 {
return nil, errors.New("mixed_group_senders_per_machine must not be negative")
}
if cfg.IdleDuration == "" {
cfg.IdleDuration = "15m"
}
if _, err = configDuration(cfg.IdleDuration); err != nil {
return nil, fmt.Errorf("idle_duration: %w", err)
}
if cfg.DrainP2P == "" {
cfg.DrainP2P = "30s"
}