pro/im_server
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: harden single-machine load handling

Browse Source
This commit is contained in:
richjohnrambo
2026-09-29 17:52:13 +08:00
parent ab73b69870
commit 4b00b5ae37
24 changed files with 1030 additions and 144 deletions
Download Patch File Download Diff File Expand all files Collapse all files
loadtest/im-pressure/.gitignore
+1
View File
@@ -1,5 +1,6 @@
/im-pressure
/config.json
/config.single-machine.json
/config.local*.json
/.env.test
/manifest*.json
loadtest/im-pressure/README.md
+6 -3
View File
@@ -2,6 +2,9 @@
该工具按配置指定的企业租户运行：两台压测机各承载 1000 个 WebSocket 用户，共 2000 个用户、1000 对好友和一个 2000 人群。当前测试环境使用 `allrich`，本轮实时参数见 [MACHINE-B.md](./MACHINE-B.md) 和 [allrich 双机压测执行记录](../../docs/test/IM-allrich双机压测执行记录-20260927.md)。
只使用一台物理压测机时，请使用 [单机 60% 容量压测方案](../../docs/test/IM-单机60%容量压力测试方案.md) 和 [config.single-machine.example.json](./config.single-machine.example.json)。单机依然运行 A/B 两个逻辑 agent，用于保留好友对、跨逻辑分片投递和合并报告的完整契约。
首轮实测数据与瓶颈分析见 [单机 60% 容量压测结果](../../docs/test/IM-单机60%容量压力测试结果-20260929.md)。
机器 B 上的 AI 或操作人请直接阅读 [MACHINE-B.md](./MACHINE-B.md)。该文件不重复讲方案，只给出机器 B 从核验到启动、观察和回传结果的完整命令。
## 两台机器的固定分工
@@ -194,8 +197,8 @@ mkdir -p "loadtest/im-pressure/results/${RUN_ID}"
两个 agent 都建立 1000 条连接后，协调器下发统一的 UTC `start_at`。场景可替换为：
- `p2p`：1000 对用户两侧同时发送，默认每人 10 条、间隔 1 秒。
- `group`：每台机器 500 个发送者，每人 2 条；全局每 100ms 一条，共 2000 条。
- `mixed`：300 对私聊用户每人 5 条，同时每台 250 个群发送者各发 2 条。
- `group`：`users_per_machine=1000` 时每个 agent 有 500 个发送者，每人 2 条；全局每 100ms 一条，共 2000 条。其他容量按每个 agent 用户数的 50% 选发送者。
- `mixed`：`users_per_machine=1000` 时使用 300 对私聊用户和每个 agent 250 个群发送者；其他容量分别按 30% 和 25% 同比计算。
## 机器 A：回收并合并报告
@@ -215,7 +218,7 @@ scp "${MACHINE_B_SSH}:${MACHINE_B_REPO}/loadtest/im-pressure/results/${RUN_ID}/a
-output "loadtest/im-pressure/results/${RUN_ID}/report.json"
```
`report.json` 给出 ACK、实时投递、重复、乱序、断连和历史持久化核验结果。当前正确性门禁要求上述错误计数全部为 0。
`report.json` 给出 ACK、实时投递、重复、乱序、断连和历史持久化核验结果。断连会额外输出 `disconnect_reasons` 分类计数和最多 100 条 `disconnect_samples` 原始错误样本。当前正确性门禁要求上述错误计数全部为 0。
每个 agent 在 `:9105/metrics` 暴露连接、发送、接收、重复、乱序、断连以及 Go/process 指标；端口冲突时可通过 `-metrics-listen` 修改，传空字符串可关闭。
loadtest/im-pressure/agent.go
+54 -8
View File
@@ -458,8 +458,18 @@ func newHistoryQuery(client *WSClient, topic string, messages []AcceptedMessage,
for _, message := range messages {
expected = append(expected, message.ClientMid)
}
return historyQuery{client: client, topic: topic, since: messages[0].Seq,
before: messages[len(messages)-1].Seq + 1, limit: limit, expected: expected}
since := messages[0].Seq
before := messages[len(messages)-1].Seq + 1
// P2P topics interleave both senders. The requested range must leave room
// for non-target messages between the expected sequence numbers.
if span := before - since; span > int64(limit) {
limit = int(span)
}
if limit > 200 {
limit = 200
}
return historyQuery{client: client, topic: topic, since: since,
before: before, limit: limit, expected: expected}
}
func (r *agentRun) waitHistoryPage(ctx context.Context, expected []string, timeout time.Duration) {
@@ -659,13 +669,24 @@ func (r *agentRun) execute(ctx context.Context, start time.Time, groupTopic stri
case "p2p":
return r.scheduleP2P(ctx, start, r.cfg.UsersPerMachine, r.cfg.P2PMessages, mustDuration(r.cfg.P2PInterval))
case "group":
return r.scheduleGroup(ctx, start, groupTopic, 1, 500, 2)
return r.scheduleGroup(ctx, start, groupTopic, 1, scaledUsers(r.cfg.UsersPerMachine, 1, 2), 2)
case "mixed":
p2pUsers := scaledUsers(r.cfg.UsersPerMachine, 3, 10)
groupSenders := scaledUsers(r.cfg.UsersPerMachine, 1, 4)
if p2pUsers+groupSenders > r.cfg.UsersPerMachine {
groupSenders = r.cfg.UsersPerMachine - p2pUsers
}
if groupSenders <= 0 {
return fmt.Errorf("users_per_machine=%d is too small for mixed scenario", r.cfg.UsersPerMachine)
}
var wg sync.WaitGroup
errCh := make(chan error, 2)
wg.Add(2)
go func() { defer wg.Done(); errCh <- r.scheduleP2P(ctx, start, 300, 5, 20*time.Second) }()
go func() { defer wg.Done(); errCh <- r.scheduleGroup(ctx, start, groupTopic, 301, 550, 2) }()
go func() { defer wg.Done(); errCh <- r.scheduleP2P(ctx, start, p2pUsers, 5, 20*time.Second) }()
go func() {
defer wg.Done()
errCh <- r.scheduleGroup(ctx, start, groupTopic, p2pUsers+1, p2pUsers+groupSenders, 2)
}()
wg.Wait()
close(errCh)
for err := range errCh {
@@ -679,6 +700,17 @@ func (r *agentRun) execute(ctx context.Context, start time.Time, groupTopic stri
}
}
func scaledUsers(total, numerator, denominator int) int {
count := total * numerator / denominator
if count < 1 {
return 1
}
if count > total {
return total
}
return count
}
func (r *agentRun) scheduleP2P(ctx context.Context, start time.Time, users, messages int, interval time.Duration) error {
var wg sync.WaitGroup
for index := 1; index <= users; index++ {
@@ -858,9 +890,10 @@ func (r *agentRun) drainDuration() (time.Duration, error) {
func (r *agentRun) close() {
clients := r.clientsSnapshot()
var interval time.Duration
if r.cfg != nil && r.cfg.CloseRamp != "" && len(clients) > 1 {
interval = mustDuration(r.cfg.CloseRamp) / time.Duration(len(clients)-1)
sort.Slice(clients, func(i, j int) bool { return clients[i].account.Index < clients[j].account.Index })
interval, initialDelay := closeSchedule(r.cfg, r.machine, len(clients))
if initialDelay > 0 {
time.Sleep(initialDelay)
}
for index, client := range clients {
client.Close()
@@ -870,6 +903,19 @@ func (r *agentRun) close() {
}
}
func closeSchedule(cfg *Config, machine string, clients int) (interval, initialDelay time.Duration) {
if cfg == nil || cfg.CloseRamp == "" || clients <= 1 {
return 0, 0
}
// The two logical agents share one global ramp. Interleaving their slots avoids
// doubling the disconnect rate when both agents begin cleanup at the same time.
interval = 2 * mustDuration(cfg.CloseRamp) / time.Duration(clients-1)
if machine == "b" {
initialDelay = interval / 2
}
return interval, initialDelay
}
func waitUntil(ctx context.Context, target time.Time) error {
delay := time.Until(target)
if delay <= 0 {
loadtest/im-pressure/agent_history_test.go
+48
View File
@@ -2,7 +2,9 @@ package main
import (
"errors"
"fmt"
"testing"
"time"
)
func TestHistoryQueriesForMessagesGroupsRangesByTopic(t *testing.T) {
@@ -51,6 +53,40 @@ func TestClassifyPublishErrorTreatsClosedConnectionAsUnknownSocketError(t *testi
}
}
func TestScenarioUsersScaleWithConfiguredCapacity(t *testing.T) {
tests := []struct {
users int
group int
mixedP2P int
mixedGroup int
}{
{users: 1000, group: 500, mixedP2P: 300, mixedGroup: 250},
{users: 600, group: 300, mixedP2P: 180, mixedGroup: 150},
}
for _, tc := range tests {
if got := scaledUsers(tc.users, 1, 2); got != tc.group {
t.Fatalf("users=%d group=%d want=%d", tc.users, got, tc.group)
}
if got := scaledUsers(tc.users, 3, 10); got != tc.mixedP2P {
t.Fatalf("users=%d mixed p2p=%d want=%d", tc.users, got, tc.mixedP2P)
}
if got := scaledUsers(tc.users, 1, 4); got != tc.mixedGroup {
t.Fatalf("users=%d mixed group=%d want=%d", tc.users, got, tc.mixedGroup)
}
}
}
func TestNewHistoryQueryIncludesInterleavedSequenceRange(t *testing.T) {
messages := make([]AcceptedMessage, 10)
for i := range messages {
messages[i] = AcceptedMessage{ClientMid: fmt.Sprintf("mid-%d", i), Seq: int64(100 + i*2)}
}
query := newHistoryQuery(nil, "p2p-one", messages, 10)
if query.since != 100 || query.before != 119 || query.limit != 19 {
t.Fatalf("history query range=%d-%d limit=%d want=100-119/19", query.since, query.before, query.limit)
}
}
func TestMergeMessageLedgersPrefersPeerObservationAndAck(t *testing.T) {
senderEcho := AcceptedMessage{ClientMid: "mid-1", SenderMachine: "a", PeerMachine: "a", Seq: 10}
peerDelivery := AcceptedMessage{ClientMid: "mid-1", SenderMachine: "a", PeerMachine: "b", PeerIndex: 7, Seq: 10}
@@ -65,3 +101,15 @@ func TestMergeMessageLedgersPrefersPeerObservationAndAck(t *testing.T) {
t.Fatalf("ACK ledger was not authoritative: %+v", merged)
}
}
func TestCloseScheduleInterleavesLogicalAgents(t *testing.T) {
cfg := &Config{CloseRamp: "60s"}
intervalA, delayA := closeSchedule(cfg, "a", 601)
intervalB, delayB := closeSchedule(cfg, "b", 601)
if intervalA != 200*time.Millisecond || intervalB != intervalA {
t.Fatalf("intervals: a=%s b=%s, want 200ms", intervalA, intervalB)
}
if delayA != 0 || delayB != 100*time.Millisecond {
t.Fatalf("delays: a=%s b=%s, want 0/100ms", delayA, delayB)
}
}
loadtest/im-pressure/config.single-machine.example.json
+31
View File
@@ -0,0 +1,31 @@
{
"tenant": "006",
"http_base": "https://im-api.example.com",
"api_key_env": "IM_API_KEY",
"password_env": "IM_LOAD_PASSWORD",
"verification_code_env": "IM_VERIFICATION_CODE",
"registration_phone_prefix": "",
"users_per_machine": 600,
"endpoints": {
"a": ["wss://im-api.example.com/v0/channels"],
"b": ["wss://im-api.example.com/v0/channels"]
},
"insecure_tls": false,
"prepare_workers": 20,
"connect_ramp": "30s",
"command_timeout": "20s",
"ws_ping_interval": "20s",
"preflight_workers": 32,
"skip_p2p_preflight": false,
"history_workers": 12,
"history_page_size": 10,
"history_query_interval": "500ms",
"history_page_wait": "3s",
"history_settle": "5m",
"history_timeout": "3m",
"close_ramp": "60s",
"p2p_messages_per_user": 10,
"p2p_interval": "1s",
"drain_p2p": "30s",
"drain_group": "60s"
}
loadtest/im-pressure/dbverify.go
+14 -11
View File
@@ -14,17 +14,18 @@ import (
)
type dbVerifyResult struct {
RunID string `json:"run_id"`
TenantCode string `json:"tenant_code"`
TenantID int64 `json:"tenant_id"`
Expected int `json:"expected"`
FoundExact int `json:"found_exact"`
Missing int `json:"missing"`
Mismatch int `json:"mismatch"`
DuplicateRows int `json:"duplicate_rows"`
Unexpected int `json:"unexpected_rows"`
Tables []string `json:"tables"`
MissingMIDs []string `json:"missing_client_mids,omitempty"`
RunID string `json:"run_id"`
TenantCode string `json:"tenant_code"`
TenantID int64 `json:"tenant_id"`
Expected int `json:"expected"`
FoundExact int `json:"found_exact"`
Missing int `json:"missing"`
Mismatch int `json:"mismatch"`
DuplicateRows int `json:"duplicate_rows"`
Unexpected int `json:"unexpected_rows"`
Tables []string `json:"tables"`
MissingMIDs []string `json:"missing_client_mids,omitempty"`
UnexpectedMIDs []string `json:"unexpected_client_mids,omitempty"`
}
type persistedIdentity struct {
@@ -145,9 +146,11 @@ func runDBVerify(ctx context.Context, coordinatorPath, output, tenantCode, dsnEn
for mid := range actual {
if _, ok := expected[mid]; !ok {
result.Unexpected++
result.UnexpectedMIDs = append(result.UnexpectedMIDs, mid)
}
}
sort.Strings(result.MissingMIDs)
sort.Strings(result.UnexpectedMIDs)
resultRaw, _ := json.MarshalIndent(result, "", " ")
if output == "" || output == "-" {
if _, err = fmt.Fprintln(os.Stdout, string(resultRaw)); err != nil {
loadtest/im-pressure/histogram.go
+18 -20
View File
@@ -2,7 +2,7 @@ package main
import (
"sort"
"sync"
"sync/atomic"
"time"
)
@@ -13,11 +13,10 @@ var latencyBounds = []time.Duration{
}
type DurationHistogram struct {
mu sync.Mutex
count int64
sum time.Duration
max time.Duration
bucket []int64
count atomic.Int64
sum atomic.Int64
max atomic.Int64
bucket []atomic.Int64
}
type HistogramSnapshot struct {
@@ -28,37 +27,36 @@ type HistogramSnapshot struct {
}
func NewDurationHistogram() *DurationHistogram {
return &DurationHistogram{bucket: make([]int64, len(latencyBounds)+1)}
return &DurationHistogram{bucket: make([]atomic.Int64, len(latencyBounds)+1)}
}
func (h *DurationHistogram) Observe(value time.Duration) {
if value < 0 {
return
}
h.mu.Lock()
defer h.mu.Unlock()
h.count++
h.sum += value
if value > h.max {
h.max = value
nanos := int64(value)
h.count.Add(1)
h.sum.Add(nanos)
for previous := h.max.Load(); nanos > previous; previous = h.max.Load() {
if h.max.CompareAndSwap(previous, nanos) {
break
}
}
idx := sort.Search(len(latencyBounds), func(i int) bool { return value <= latencyBounds[i] })
h.bucket[idx]++
h.bucket[idx].Add(1)
}
func (h *DurationHistogram) Snapshot() HistogramSnapshot {
h.mu.Lock()
defer h.mu.Unlock()
buckets := make(map[string]int64, len(h.bucket))
for i, count := range h.bucket {
for i := range h.bucket {
label := "+Inf"
if i < len(latencyBounds) {
label = latencyBounds[i].String()
}
buckets[label] = count
buckets[label] = h.bucket[i].Load()
}
return HistogramSnapshot{
Count: h.count, SumMS: float64(h.sum) / float64(time.Millisecond),
MaxMS: float64(h.max) / float64(time.Millisecond), Buckets: buckets,
Count: h.count.Load(), SumMS: float64(h.sum.Load()) / float64(time.Millisecond),
MaxMS: float64(h.max.Load()) / float64(time.Millisecond), Buckets: buckets,
}
}
loadtest/im-pressure/httpapi.go
+8 -3
View File
@@ -269,6 +269,11 @@ type groupMembersResponse struct {
}
func (a *apiClient) countGroupMembers(ctx context.Context, account AccountRecord, topic string) (int, error) {
members, err := a.groupMemberIDs(ctx, account, topic)
return len(members), err
}
func (a *apiClient) groupMemberIDs(ctx context.Context, account AccountRecord, topic string) (map[string]bool, error) {
cursor := ""
seen := make(map[string]bool)
for {
@@ -278,7 +283,7 @@ func (a *apiClient) countGroupMembers(ctx context.Context, account AccountRecord
}
var page groupMembersResponse
if err := a.do(ctx, http.MethodGet, path, account.Token, "", nil, &page); err != nil {
return 0, err
return nil, err
}
for _, item := range page.Items {
seen[item.UserID] = true
@@ -287,11 +292,11 @@ func (a *apiClient) countGroupMembers(ctx context.Context, account AccountRecord
break
}
if page.NextCursor == cursor {
return 0, fmt.Errorf("group member cursor did not advance: %s", cursor)
return nil, fmt.Errorf("group member cursor did not advance: %s", cursor)
}
cursor = page.NextCursor
}
return len(seen), nil
return seen, nil
}
func queryInt(value string, fallback int) int {
loadtest/im-pressure/merge.go
+12 -1
View File
@@ -38,6 +38,8 @@ type CombinedReport struct {
OutOfOrder int64 `json:"out_of_order"`
OrderingViolations []OrderingViolation `json:"ordering_violations,omitempty"`
Disconnects int64 `json:"disconnects"`
DisconnectReasons map[string]int64 `json:"disconnect_reasons,omitempty"`
DisconnectSamples []DisconnectSample `json:"disconnect_samples,omitempty"`
UnexpectedMessages int64 `json:"unexpected_messages"`
HistoryChecked int64 `json:"history_checked"`
HistoryFound int64 `json:"history_found"`
@@ -62,7 +64,7 @@ func runMerge(aPath, bPath, output string) error {
return fmt.Errorf("agent summaries do not belong to the same run")
}
report := CombinedReport{RunID: a.RunID, Scenario: a.Scenario, GeneratedAt: time.Now().UTC(), Agents: []AgentSummary{a, b}, Passed: true,
PublishErrors: make(map[string]int64)}
PublishErrors: make(map[string]int64), DisconnectReasons: make(map[string]int64)}
for _, summary := range report.Agents {
expectedConnections := summary.ExpectedConnections
if expectedConnections == 0 {
@@ -92,6 +94,15 @@ func runMerge(aPath, bPath, output string) error {
report.OutOfOrder += summary.OutOfOrder
report.OrderingViolations = append(report.OrderingViolations, summary.OrderingViolations...)
report.Disconnects += summary.Disconnects
for category, count := range summary.DisconnectReasons {
report.DisconnectReasons[category] += count
}
if remaining := 100 - len(report.DisconnectSamples); remaining > 0 {
if remaining > len(summary.DisconnectSamples) {
remaining = len(summary.DisconnectSamples)
}
report.DisconnectSamples = append(report.DisconnectSamples, summary.DisconnectSamples[:remaining]...)
}
report.UnexpectedMessages += summary.UnexpectedMessages
report.HistoryChecked += summary.HistoryChecked
report.HistoryFound += summary.HistoryFound
loadtest/im-pressure/merge_test.go
+9 -1
View File
@@ -14,11 +14,14 @@ func TestRunMergeComputesArrivalAndPersistenceGate(t *testing.T) {
ExpectedConnections: 1000,
PublishAttempted: 10, PublishAccepted: 10, ExpectedPeer: 10, ReceivedPeer: 9, UnexpectedMessages: 42,
PublishErrors: map[string]int64{"ctrl_timeout": 3, "socket": 2, "503": 1},
RealtimeLost: 1, HistoryChecked: 10, HistoryFound: 10}
RealtimeLost: 1, HistoryChecked: 10, HistoryFound: 10, Disconnects: 1,
DisconnectReasons: map[string]int64{"timeout": 1},
DisconnectSamples: []DisconnectSample{{ReceiverMachine: "a", ReceiverIndex: 3, Category: "timeout"}}}
b := AgentSummary{RunID: "run1", Scenario: "p2p", Machine: "b", ReadyConnections: 1000,
ExpectedConnections: 1000,
PublishAttempted: 10, PublishAccepted: 10, ExpectedPeer: 10, ReceivedPeer: 10,
HistoryChecked: 10, HistoryFound: 9, PersistenceLost: 1, OutOfOrder: 1,
Disconnects: 2, DisconnectReasons: map[string]int64{"timeout": 1, "connection_reset": 1},
OrderingViolations: []OrderingViolation{{ReceiverMachine: "b", ReceiverIndex: 7, PreviousSeq: 12, CurrentSeq: 11}}}
aPath, bPath, reportPath := filepath.Join(dir, "a.json"), filepath.Join(dir, "b.json"), filepath.Join(dir, "report.json")
writeSummaryForTest(t, aPath, a)
@@ -47,6 +50,11 @@ func TestRunMergeComputesArrivalAndPersistenceGate(t *testing.T) {
if len(report.OrderingViolations) != 1 || report.OrderingViolations[0].ReceiverIndex != 7 {
t.Fatalf("ordering violations were not merged: %+v", report.OrderingViolations)
}
if report.Disconnects != 3 || report.DisconnectReasons["timeout"] != 2 ||
report.DisconnectReasons["connection_reset"] != 1 || len(report.DisconnectSamples) != 1 {
t.Fatalf("disconnect diagnostics were not merged: count=%d reasons=%v samples=%+v",
report.Disconnects, report.DisconnectReasons, report.DisconnectSamples)
}
}
func TestMergedHistogramP95(t *testing.T) {
loadtest/im-pressure/model.go
+10
View File
@@ -410,6 +410,8 @@ type AgentSummary struct {
OutOfOrder int64 `json:"out_of_order"`
OrderingViolations []OrderingViolation `json:"ordering_violations,omitempty"`
Disconnects int64 `json:"disconnects"`
DisconnectReasons map[string]int64 `json:"disconnect_reasons,omitempty"`
DisconnectSamples []DisconnectSample `json:"disconnect_samples,omitempty"`
UnexpectedMessages int64 `json:"unexpected_messages"`
HistoryChecked int64 `json:"history_checked"`
HistoryFound int64 `json:"history_found"`
@@ -427,6 +429,14 @@ type AgentSummary struct {
AcceptedLocal []AcceptedMessage `json:"accepted_local,omitempty"`
}
type DisconnectSample struct {
ReceiverMachine string `json:"receiver_machine"`
ReceiverIndex int `json:"receiver_index"`
Category string `json:"category"`
Error string `json:"error"`
At time.Time `json:"at"`
}
type OrderingViolation struct {
ReceiverMachine string `json:"receiver_machine"`
ReceiverIndex int `json:"receiver_index"`
loadtest/im-pressure/model_test.go
+13
View File
@@ -24,6 +24,19 @@ func TestLoadConfigAppliesBoundedHistoryDefaults(t *testing.T) {
}
}
func TestSingleMachineExampleUsesSixtyPercentCapacity(t *testing.T) {
cfg, err := loadConfig("config.single-machine.example.json")
if err != nil {
t.Fatal(err)
}
if cfg.UsersPerMachine != 600 || cfg.CloseRamp != "60s" {
t.Fatalf("unexpected single-machine capacity: users=%d close_ramp=%s", cfg.UsersPerMachine, cfg.CloseRamp)
}
if len(cfg.Endpoints["a"]) != 1 || len(cfg.Endpoints["b"]) != 1 {
t.Fatalf("single-machine endpoints must define both logical agents: %+v", cfg.Endpoints)
}
}
func TestNewManifestDistributesUsersAndEndpoints(t *testing.T) {
cfg := &Config{Tenant: "006", UsersPerMachine: 3, Endpoints: map[string][]string{
"a": {"ws://a1", "ws://a2"}, "b": {"ws://b1"},
loadtest/im-pressure/prepare.go
+62 -16
View File
@@ -49,7 +49,7 @@ func runPrepare(ctx context.Context, cfg *Config, manifestPath string) error {
if err = saveManifest(manifestPath, manifest); err != nil {
return err
}
if err = prepareGroup(ctx, cfg, manifest, api); err != nil {
if err = prepareGroup(ctx, cfg, manifest, api, manifestPath); err != nil {
return err
}
if err = saveManifest(manifestPath, manifest); err != nil {
@@ -184,7 +184,7 @@ func prepareFriendships(ctx context.Context, cfg *Config, manifest *Manifest, ap
return err
}
func prepareGroup(ctx context.Context, cfg *Config, manifest *Manifest, api *apiClient) error {
func prepareGroup(ctx context.Context, cfg *Config, manifest *Manifest, api *apiClient, manifestPath string) error {
if err := parallelAccounts(ctx, manifest.Accounts, cfg.PrepareWorkers, func(account AccountRecord) error {
return retryPrepareOperation(ctx, "privacy-"+account.Username, func() error {
return api.disableGroupInviteVerification(ctx, account)
@@ -200,6 +200,9 @@ func prepareGroup(ctx context.Context, cfg *Config, manifest *Manifest, api *api
return err
}
manifest.GroupTopic = group.Topic
if err = saveManifest(manifestPath, manifest); err != nil {
return fmt.Errorf("checkpoint created group: %w", err)
}
log.Printf("created load-test group topic_hash=%s", topicHash(group.Topic))
}
memberIDs := make([]string, 0, len(manifest.Accounts)-1)
@@ -214,20 +217,9 @@ func prepareGroup(ctx context.Context, cfg *Config, manifest *Manifest, api *api
if end > len(memberIDs) {
end = len(memberIDs)
}
var response *groupInvitationResponse
err := retryPrepareOperation(ctx, fmt.Sprintf("group-invite-%03d", batch), func() error {
var inviteErr error
response, inviteErr = api.inviteGroup(ctx, owner, manifest.GroupTopic, batch, memberIDs[offset:end])
return inviteErr
})
if err != nil {
if err := ensureGroupMembers(ctx, api, owner, manifest.GroupTopic, batch, memberIDs[offset:end]); err != nil {
return fmt.Errorf("invite group batch %d: %w", batch, err)
}
for _, item := range response.Items {
if item.Status != "joined" && item.Status != "already_member" {
return fmt.Errorf("invite user %s status=%s reason=%s", item.UserID, item.Status, item.Reason)
}
}
}
count, err := api.countGroupMembers(ctx, owner, manifest.GroupTopic)
if err != nil {
@@ -240,9 +232,60 @@ func prepareGroup(ctx context.Context, cfg *Config, manifest *Manifest, api *api
return nil
}
func retryPrepareOperation(ctx context.Context, label string, operation func() error) error {
func ensureGroupMembers(ctx context.Context, api *apiClient, owner AccountRecord, topic string, batch int, desired []string) error {
var lastErr error
for attempt := 1; attempt <= 6; attempt++ {
for attempt := 1; attempt <= 12; attempt++ {
members, err := api.groupMemberIDs(ctx, owner, topic)
if err != nil {
lastErr = fmt.Errorf("list current members: %w", err)
} else {
missing := make([]string, 0, len(desired))
for _, userID := range desired {
if !members[userID] {
missing = append(missing, userID)
}
}
if len(missing) == 0 {
return nil
}
response, inviteErr := api.inviteGroup(ctx, owner, topic, batch, missing)
if inviteErr != nil {
lastErr = inviteErr
} else {
lastErr = nil
for _, item := range response.Items {
if item.Status != "joined" && item.Status != "already_member" {
lastErr = fmt.Errorf("invite user %s status=%s reason=%s", item.UserID, item.Status, item.Reason)
break
}
}
if lastErr == nil {
continue
}
}
}
if ctx.Err() != nil {
return ctx.Err()
}
if attempt == 12 {
break
}
delay := time.Duration(attempt*5) * time.Second
log.Printf("operation=group-invite-%03d attempt=%d failed; retry_in=%s: %v", batch, attempt, delay, lastErr)
if err := waitContext(ctx, delay); err != nil {
return err
}
}
return fmt.Errorf("missing members not recovered after retries: %w", lastErr)
}
func retryPrepareOperation(ctx context.Context, label string, operation func() error) error {
return retryOperation(ctx, label, 6, operation)
}
func retryOperation(ctx context.Context, label string, attempts int, operation func() error) error {
var lastErr error
for attempt := 1; attempt <= attempts; attempt++ {
if err := operation(); err == nil {
return nil
} else {
@@ -251,6 +294,9 @@ func retryPrepareOperation(ctx context.Context, label string, operation func() e
if ctx.Err() != nil {
return ctx.Err()
}
if attempt == attempts {
break
}
delay := time.Duration(attempt*5) * time.Second
log.Printf("operation=%s attempt=%d failed; retry_in=%s: %v", label, attempt, delay, lastErr)
if err := waitContext(ctx, delay); err != nil {
loadtest/im-pressure/recorder.go
+205 -77
View File
@@ -2,8 +2,11 @@ package main
import (
"encoding/json"
"errors"
"fmt"
"io"
"math/bits"
"net"
"sort"
"strings"
"sync"
@@ -11,6 +14,8 @@ import (
"time"
)
const observationShardCount = 64
type observation struct {
topic string
seq int64
@@ -21,6 +26,17 @@ type observation struct {
bits []uint64
}
type observationShard struct {
mu sync.RWMutex
observed map[string]*observation
history map[string]bool
}
type sequenceState struct {
mu sync.Mutex
last map[string]int64
}
type deliveryRecorder struct {
machine string
localUsers int
@@ -32,26 +48,35 @@ type deliveryRecorder struct {
runID atomic.Value
historyMode atomic.Bool
mu sync.Mutex
observed map[string]*observation
lastSeq []map[string]int64
duplicates int64
outOfOrder int64
ordering []OrderingViolation
malformed int64
crossClockSkewSamples int64
history map[string]bool
shards []observationShard
sequences []sequenceState
duplicates atomic.Int64
outOfOrder atomic.Int64
malformed atomic.Int64
crossClockSkewSamples atomic.Int64
orderingMu sync.Mutex
ordering []OrderingViolation
disconnectMu sync.Mutex
disconnectReasons map[string]int64
disconnectSamples []DisconnectSample
}
func newDeliveryRecorder(runID, machine string, localUsers int) *deliveryRecorder {
lastSeq := make([]map[string]int64, localUsers)
for i := range lastSeq {
lastSeq[i] = make(map[string]int64)
shards := make([]observationShard, observationShardCount)
for i := range shards {
shards[i].observed = make(map[string]*observation)
shards[i].history = make(map[string]bool)
}
sequences := make([]sequenceState, localUsers)
for i := range sequences {
sequences[i].last = make(map[string]int64)
}
recorder := &deliveryRecorder{
machine: machine, localUsers: localUsers, latency: NewDurationHistogram(),
localLatency: NewDurationHistogram(), crossLatency: NewDurationHistogram(),
observed: make(map[string]*observation), lastSeq: lastSeq, history: make(map[string]bool),
shards: shards, sequences: sequences, disconnectReasons: make(map[string]int64),
}
recorder.runID.Store(runID)
return recorder
@@ -59,7 +84,13 @@ func newDeliveryRecorder(runID, machine string, localUsers int) *deliveryRecorde
func (r *deliveryRecorder) setRunID(runID string) { r.runID.Store(runID) }
func (r *deliveryRecorder) resetDisconnects() { r.disconnects.Store(0) }
func (r *deliveryRecorder) resetDisconnects() {
r.disconnectMu.Lock()
r.disconnects.Store(0)
r.disconnectReasons = make(map[string]int64)
r.disconnectSamples = nil
r.disconnectMu.Unlock()
}
func (r *deliveryRecorder) onData(account AccountRecord, data dataPacket) {
runID, _ := r.runID.Load().(string)
@@ -75,40 +106,50 @@ func (r *deliveryRecorder) onData(account AccountRecord, data dataPacket) {
SentAt int64 `json:"sent_at_unix_nano"`
}
if json.Unmarshal(data.Content, &content) != nil || content.RunID != runID {
r.mu.Lock()
r.malformed++
r.mu.Unlock()
r.malformed.Add(1)
return
}
if r.historyMode.Load() {
r.mu.Lock()
r.history[mid] = true
r.mu.Unlock()
shard := r.shard(mid)
shard.mu.Lock()
shard.history[mid] = true
shard.mu.Unlock()
return
}
index := account.Index - 1
if index < 0 || index >= r.localUsers {
return
}
r.mu.Lock()
defer r.mu.Unlock()
obs := r.observed[mid]
shard := r.shard(mid)
shard.mu.Lock()
obs := shard.observed[mid]
if obs == nil {
obs = &observation{topic: data.Topic, seq: data.Seq, senderMachine: content.SenderMachine,
senderIndex: content.SenderIndex, scenario: content.Scenario, sentAt: content.SentAt,
bits: make([]uint64, (r.localUsers+63)/64)}
r.observed[mid] = obs
shard.observed[mid] = obs
}
word, bit := index/64, uint(index%64)
mask := uint64(1) << bit
if obs.bits[word]&mask != 0 {
r.duplicates++
shard.mu.Unlock()
r.duplicates.Add(1)
return
}
obs.bits[word] |= mask
shard.mu.Unlock()
r.received.Add(1)
if previous := r.lastSeq[index][data.Topic]; previous > 0 && data.Seq < previous {
r.outOfOrder++
sequence := &r.sequences[index]
sequence.mu.Lock()
previous := sequence.last[data.Topic]
violation := previous > 0 && data.Seq < previous
if data.Seq > previous {
sequence.last[data.Topic] = data.Seq
}
sequence.mu.Unlock()
if violation {
r.outOfOrder.Add(1)
r.orderingMu.Lock()
if len(r.ordering) < 100 {
r.ordering = append(r.ordering, OrderingViolation{
ReceiverMachine: r.machine,
@@ -119,9 +160,7 @@ func (r *deliveryRecorder) onData(account AccountRecord, data dataPacket) {
ClientMid: mid,
})
}
}
if data.Seq > r.lastSeq[index][data.Topic] {
r.lastSeq[index][data.Topic] = data.Seq
r.orderingMu.Unlock()
}
if content.SentAt > 0 {
delay := time.Since(time.Unix(0, content.SentAt))
@@ -133,7 +172,7 @@ func (r *deliveryRecorder) onData(account AccountRecord, data dataPacket) {
if senderMachine == r.machine {
r.localLatency.Observe(delay)
} else if delay < 0 {
r.crossClockSkewSamples++
r.crossClockSkewSamples.Add(1)
} else {
r.crossLatency.Observe(delay)
}
@@ -150,26 +189,22 @@ func senderMachineFromClientMid(runID, mid string) string {
}
func (r *deliveryRecorder) metricCounts() (received, duplicates, outOfOrder, disconnects int64) {
r.mu.Lock()
duplicates = r.duplicates
outOfOrder = r.outOfOrder
r.mu.Unlock()
return r.received.Load(), duplicates, outOfOrder, r.disconnects.Load()
return r.received.Load(), r.duplicates.Load(), r.outOfOrder.Load(), r.disconnects.Load()
}
func (r *deliveryRecorder) beginHistory() { r.historyMode.Store(true) }
func (r *deliveryRecorder) historyFound(mid string) bool {
r.mu.Lock()
defer r.mu.Unlock()
return r.history[mid]
shard := r.shard(mid)
shard.mu.RLock()
found := shard.history[mid]
shard.mu.RUnlock()
return found
}
func (r *deliveryRecorder) historyFoundAll(messageIDs []string) bool {
r.mu.Lock()
defer r.mu.Unlock()
for _, mid := range messageIDs {
if !r.history[mid] {
if !r.historyFound(mid) {
return false
}
}
@@ -182,28 +217,31 @@ func (r *deliveryRecorder) unackedObservedMessages(ledger []AcceptedMessage, sce
accepted[message.ClientMid] = true
}
runID, _ := r.runID.Load().(string)
r.mu.Lock()
defer r.mu.Unlock()
messages := make([]AcceptedMessage, 0)
for mid, obs := range r.observed {
if accepted[mid] || obs.seq <= 0 || obs.topic == "" {
continue
for i := range r.shards {
shard := &r.shards[i]
shard.mu.RLock()
for mid, obs := range shard.observed {
if accepted[mid] || obs.seq <= 0 || obs.topic == "" {
continue
}
kind := "p2p"
if strings.HasPrefix(obs.topic, "grp") {
kind = "group"
}
observedScenario := obs.scenario
if observedScenario == "" {
observedScenario = scenario
}
message := AcceptedMessage{RunID: runID, Scenario: observedScenario, Kind: kind,
ClientMid: mid, Topic: obs.topic, Seq: obs.seq, SenderMachine: obs.senderMachine,
SenderIndex: obs.senderIndex, PeerMachine: r.machine, PeerIndex: firstObservedIndex(obs.bits)}
if obs.sentAt > 0 {
message.SentAt = time.Unix(0, obs.sentAt).UTC()
}
messages = append(messages, message)
}
kind := "p2p"
if strings.HasPrefix(obs.topic, "grp") {
kind = "group"
}
observedScenario := obs.scenario
if observedScenario == "" {
observedScenario = scenario
}
message := AcceptedMessage{RunID: runID, Scenario: observedScenario, Kind: kind,
ClientMid: mid, Topic: obs.topic, Seq: obs.seq, SenderMachine: obs.senderMachine,
SenderIndex: obs.senderIndex, PeerMachine: r.machine, PeerIndex: firstObservedIndex(obs.bits)}
if obs.sentAt > 0 {
message.SentAt = time.Unix(0, obs.sentAt).UTC()
}
messages = append(messages, message)
shard.mu.RUnlock()
}
sort.Slice(messages, func(i, j int) bool { return messages[i].ClientMid < messages[j].ClientMid })
return messages
@@ -218,34 +256,88 @@ func firstObservedIndex(words []uint64) int {
return 0
}
func (r *deliveryRecorder) onDisconnect(_ AccountRecord, _ error) {
func (r *deliveryRecorder) onDisconnect(account AccountRecord, err error) {
category := disconnectCategory(err)
r.disconnectMu.Lock()
r.disconnects.Add(1)
r.disconnectReasons[category]++
if len(r.disconnectSamples) < 100 {
message := ""
if err != nil {
message = err.Error()
if len(message) > 300 {
message = message[:300]
}
}
r.disconnectSamples = append(r.disconnectSamples, DisconnectSample{
ReceiverMachine: r.machine, ReceiverIndex: account.Index, Category: category,
Error: message, At: time.Now().UTC(),
})
}
r.disconnectMu.Unlock()
}
func disconnectCategory(err error) string {
if err == nil {
return "unknown"
}
if errors.Is(err, io.EOF) {
return "eof"
}
if errors.Is(err, net.ErrClosed) {
return "closed_connection"
}
var netErr net.Error
if errors.As(err, &netErr) && netErr.Timeout() {
return "timeout"
}
message := strings.ToLower(err.Error())
switch {
case strings.Contains(message, "close 1006") || strings.Contains(message, "abnormal closure"):
return "abnormal_close"
case strings.Contains(message, "close 1000") || strings.Contains(message, "normal closure"):
return "normal_close"
case strings.Contains(message, "connection reset"):
return "connection_reset"
case strings.Contains(message, "broken pipe"):
return "broken_pipe"
case strings.Contains(message, "timeout") || strings.Contains(message, "deadline exceeded"):
return "timeout"
case strings.Contains(message, "use of closed network connection"):
return "closed_connection"
default:
return "other"
}
}
func (r *deliveryRecorder) summarize(base AgentSummary, ledger []AcceptedMessage) AgentSummary {
base.DeliveryLatency = r.latency.Snapshot()
base.DeliveryLatencyLocal = r.localLatency.Snapshot()
base.DeliveryLatencyCross = r.crossLatency.Snapshot()
r.disconnectMu.Lock()
base.Disconnects = r.disconnects.Load()
base.DisconnectReasons = cloneCountMap(r.disconnectReasons)
base.DisconnectSamples = append([]DisconnectSample(nil), r.disconnectSamples...)
r.disconnectMu.Unlock()
runID, _ := r.runID.Load().(string)
accepted := make(map[string]AcceptedMessage, len(ledger))
const missingLimit = 100000
r.mu.Lock()
defer r.mu.Unlock()
base.Duplicates = r.duplicates
base.OutOfOrder = r.outOfOrder
base.CrossClockSkewSamples = r.crossClockSkewSamples
base.Duplicates = r.duplicates.Load()
base.OutOfOrder = r.outOfOrder.Load()
base.CrossClockSkewSamples = r.crossClockSkewSamples.Load()
r.orderingMu.Lock()
base.OrderingViolations = append([]OrderingViolation(nil), r.ordering...)
if r.malformed > 0 {
base.Errors = append(base.Errors, fmt.Sprintf("malformed run messages=%d", r.malformed))
r.orderingMu.Unlock()
if malformed := r.malformed.Load(); malformed > 0 {
base.Errors = append(base.Errors, fmt.Sprintf("malformed run messages=%d", malformed))
}
for _, message := range ledger {
if message.RunID != runID {
continue
}
accepted[message.ClientMid] = message
obs := r.observed[message.ClientMid]
obs := r.observationSnapshot(message.ClientMid)
switch message.Kind {
case "group":
base.ExpectedGroup += int64(r.localUsers)
@@ -273,10 +365,15 @@ func (r *deliveryRecorder) summarize(base AgentSummary, ledger []AcceptedMessage
}
}
}
for mid, obs := range r.observed {
if _, ok := accepted[mid]; !ok {
base.UnexpectedMessages += bitsetCount(obs.bits)
for i := range r.shards {
shard := &r.shards[i]
shard.mu.RLock()
for mid, obs := range shard.observed {
if _, ok := accepted[mid]; !ok {
base.UnexpectedMessages += bitsetCount(obs.bits)
}
}
shard.mu.RUnlock()
}
base.RealtimeLost = (base.ExpectedPeer - base.ReceivedPeer) + (base.ExpectedGroup - base.ReceivedGroup)
if len(base.Missing) == missingLimit {
@@ -291,14 +388,12 @@ func (r *deliveryRecorder) countRealtimeLostStored(ledger []AcceptedMessage, sto
for _, mid := range storedMessageIDs {
stored[mid] = true
}
r.mu.Lock()
defer r.mu.Unlock()
var count int64
for _, message := range ledger {
if message.RunID != runID || !stored[message.ClientMid] {
continue
}
obs := r.observed[message.ClientMid]
obs := r.observationSnapshot(message.ClientMid)
switch message.Kind {
case "group":
for index := 1; index <= r.localUsers; index++ {
@@ -315,6 +410,39 @@ func (r *deliveryRecorder) countRealtimeLostStored(ledger []AcceptedMessage, sto
return count
}
func (r *deliveryRecorder) shard(mid string) *observationShard {
var hash uint32 = 2166136261
for i := 0; i < len(mid); i++ {
hash ^= uint32(mid[i])
hash *= 16777619
}
return &r.shards[hash%uint32(len(r.shards))]
}
func (r *deliveryRecorder) observationSnapshot(mid string) *observation {
shard := r.shard(mid)
shard.mu.RLock()
obs := shard.observed[mid]
if obs != nil {
copyOfObservation := *obs
copyOfObservation.bits = append([]uint64(nil), obs.bits...)
obs = &copyOfObservation
}
shard.mu.RUnlock()
return obs
}
func cloneCountMap(source map[string]int64) map[string]int64 {
if len(source) == 0 {
return nil
}
clone := make(map[string]int64, len(source))
for key, value := range source {
clone[key] = value
}
return clone
}
func observationHas(obs *observation, receiverIndex int) bool {
if obs == nil || receiverIndex <= 0 {
return false
loadtest/im-pressure/recorder_test.go
+62
View File
@@ -2,6 +2,9 @@ package main
import (
"encoding/json"
"errors"
"fmt"
"sync"
"testing"
"time"
)
@@ -111,6 +114,65 @@ func TestDeliveryRecorderSeparatesLocalAndCrossMachineLatency(t *testing.T) {
}
}
func TestDeliveryRecorderConcurrentGroupFanout(t *testing.T) {
const users = 32
const messages = 64
recorder := newDeliveryRecorder("run1", "a", users)
ledger := make([]AcceptedMessage, messages)
for messageIndex := range ledger {
ledger[messageIndex] = AcceptedMessage{
RunID: "run1", Kind: "group", Topic: "grp-one", Seq: 1,
ClientMid: fmt.Sprintf("run1-g-b-%04d-01", messageIndex+1), SenderMachine: "b",
}
}
var writers sync.WaitGroup
for receiver := 1; receiver <= users; receiver++ {
for _, message := range ledger {
writers.Add(1)
go func(receiver int, message AcceptedMessage) {
defer writers.Done()
recorder.onData(AccountRecord{Index: receiver}, testData(message, time.Now().Add(-time.Millisecond)))
}(receiver, message)
}
}
writers.Wait()
summary := recorder.summarize(AgentSummary{}, ledger)
want := int64(users * messages)
if summary.ExpectedGroup != want || summary.ReceivedGroup != want || summary.RealtimeLost != 0 {
t.Fatalf("group expected/received/lost = %d/%d/%d, want %d/%d/0",
summary.ExpectedGroup, summary.ReceivedGroup, summary.RealtimeLost, want, want)
}
if summary.Duplicates != 0 || summary.DeliveryLatency.Count != want {
t.Fatalf("duplicates=%d latency samples=%d want=0/%d", summary.Duplicates, summary.DeliveryLatency.Count, want)
}
}
func TestDeliveryRecorderDisconnectDiagnostics(t *testing.T) {
recorder := newDeliveryRecorder("run1", "a", 1)
recorder.onDisconnect(AccountRecord{Index: 1}, errors.New("websocket: close 1006 (abnormal closure): unexpected EOF"))
recorder.onDisconnect(AccountRecord{Index: 1}, errors.New("read tcp: connection reset by peer"))
summary := recorder.summarize(AgentSummary{}, nil)
if summary.Disconnects != 2 || summary.DisconnectReasons["abnormal_close"] != 1 ||
summary.DisconnectReasons["connection_reset"] != 1 || len(summary.DisconnectSamples) != 2 {
t.Fatalf("unexpected disconnect diagnostics: count=%d reasons=%v samples=%+v",
summary.Disconnects, summary.DisconnectReasons, summary.DisconnectSamples)
}
recorder.resetDisconnects()
summary = recorder.summarize(AgentSummary{}, nil)
if summary.Disconnects != 0 || len(summary.DisconnectReasons) != 0 || len(summary.DisconnectSamples) != 0 {
t.Fatalf("disconnect diagnostics were not reset: %+v", summary)
}
}
func TestBitsetCountCountsEachBitOnce(t *testing.T) {
if got := bitsetCount([]uint64{0b1011, 0b100000}); got != 4 {
t.Fatalf("bitset count=%d want=4", got)
}
}
func testData(message AcceptedMessage, sentAt time.Time) dataPacket {
content, _ := json.Marshal(map[string]any{"run_id": message.RunID, "scenario": message.Scenario,
"sender_machine": message.SenderMachine, "sender_index": message.SenderIndex,