pro/im_server
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

perf: expose and reduce message publish latency

Browse Source
This commit is contained in:
richjohnrambo
2026-09-27 21:08:35 +08:00
parent 8e778726e0
commit bde02e8eac
15 changed files with 274 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files
loadtest/im-pressure/README.md
+3 -1
View File
@@ -40,7 +40,7 @@
- 如果由机器 A 集中采集 agent 指标，机器 A 必须能访问 `http://机器B内网IP:9105/metrics`。
- 两台机器都必须能访问配置中的 HTTP API 和各自的 WS 地址。
- `8090` 是无鉴权的压测协调端口，只允许压测内网访问，不要暴露到公网。
- 两台机器必须使用 NTP/chrony 同步时间；端到端延迟依赖两边时钟一致。
- 两台机器必须使用 NTP/chrony 同步时间。报告会分别输出同机时钟延迟和跨机器单向延迟；跨机器出现负延迟样本时，后者仅用于诊断，不参与门禁。
## 统计口径
@@ -51,6 +51,8 @@
- 每轮建立 WS 后、正式计时前，以 32 个 worker 对本机 1000 个账号做好友关系、P2P Topic 和发布权限预检；任一异常会终止本轮。
- 历史回查默认每机使用 12 个 worker、总超时 3 分钟，并在超时内持续重试缺失区间。历史读取并发能力应使用独立场景测试，不与落库吞吐测试混在一起。
- publish 错误完整分类为 `403`、`502`、`503`、`ctrl_timeout`、`socket_error` 和 `other`，不受文本错误样本上限影响。
- `delivery_local_p95_ms` 使用发送方本机收到 echo 的同机时钟样本，是投递延迟门禁口径；兼容字段 `delivery_p95_ms` 与它保持一致。
- `delivery_cross_machine_p95_ms` 是跨机器单向观测值，只有两机时钟同步达标时才有意义；`cross_machine_negative_latency_samples > 0` 明确表示存在时钟偏差。
- `manifest.json` 含登录 token，文件权限为 `0600`，不得提交到 Git 或放入结果包。
## 两台机器初始化
loadtest/im-pressure/merge.go
+41 -30
View File
@@ -10,35 +10,38 @@ import (
)
type CombinedReport struct {
RunID string `json:"run_id"`
Scenario string `json:"scenario"`
GeneratedAt time.Time `json:"generated_at"`
Agents []AgentSummary `json:"agents"`
ExpectedConnections int `json:"expected_connections"`
ReadyConnections int `json:"ready_connections"`
PublishAttempted int64 `json:"publish_attempted"`
PublishAccepted int64 `json:"publish_accepted"`
PublishRejected int64 `json:"publish_rejected"`
PublishErrors map[string]int64 `json:"publish_errors,omitempty"`
PublishSuccessRate float64 `json:"publish_success_rate"`
AckP95MS float64 `json:"ack_p95_ms"`
DeliveryP95MS float64 `json:"delivery_p95_ms"`
ExpectedDeliveries int64 `json:"expected_deliveries"`
ReceivedDeliveries int64 `json:"received_deliveries"`
RealtimeLost int64 `json:"realtime_lost"`
RealtimeLostStored int64 `json:"realtime_lost_but_persisted"`
ArrivalRate float64 `json:"arrival_rate"`
LossRate float64 `json:"loss_rate"`
Duplicates int64 `json:"duplicates"`
OutOfOrder int64 `json:"out_of_order"`
OrderingViolations []OrderingViolation `json:"ordering_violations,omitempty"`
Disconnects int64 `json:"disconnects"`
HistoryChecked int64 `json:"history_checked"`
HistoryFound int64 `json:"history_found"`
PersistenceLost int64 `json:"persistence_lost"`
Missing []MissingDelivery `json:"missing,omitempty"`
Passed bool `json:"passed"`
FailureReasons []string `json:"failure_reasons,omitempty"`
RunID string `json:"run_id"`
Scenario string `json:"scenario"`
GeneratedAt time.Time `json:"generated_at"`
Agents []AgentSummary `json:"agents"`
ExpectedConnections int `json:"expected_connections"`
ReadyConnections int `json:"ready_connections"`
PublishAttempted int64 `json:"publish_attempted"`
PublishAccepted int64 `json:"publish_accepted"`
PublishRejected int64 `json:"publish_rejected"`
PublishErrors map[string]int64 `json:"publish_errors,omitempty"`
PublishSuccessRate float64 `json:"publish_success_rate"`
AckP95MS float64 `json:"ack_p95_ms"`
DeliveryP95MS float64 `json:"delivery_p95_ms"`
DeliveryLocalP95MS float64 `json:"delivery_local_p95_ms"`
DeliveryCrossP95MS float64 `json:"delivery_cross_machine_p95_ms"`
CrossClockSkewSamples int64 `json:"cross_machine_negative_latency_samples"`
ExpectedDeliveries int64 `json:"expected_deliveries"`
ReceivedDeliveries int64 `json:"received_deliveries"`
RealtimeLost int64 `json:"realtime_lost"`
RealtimeLostStored int64 `json:"realtime_lost_but_persisted"`
ArrivalRate float64 `json:"arrival_rate"`
LossRate float64 `json:"loss_rate"`
Duplicates int64 `json:"duplicates"`
OutOfOrder int64 `json:"out_of_order"`
OrderingViolations []OrderingViolation `json:"ordering_violations,omitempty"`
Disconnects int64 `json:"disconnects"`
HistoryChecked int64 `json:"history_checked"`
HistoryFound int64 `json:"history_found"`
PersistenceLost int64 `json:"persistence_lost"`
Missing []MissingDelivery `json:"missing,omitempty"`
Passed bool `json:"passed"`
FailureReasons []string `json:"failure_reasons,omitempty"`
}
func runMerge(aPath, bPath, output string) error {
@@ -79,6 +82,7 @@ func runMerge(aPath, bPath, output string) error {
report.HistoryChecked += summary.HistoryChecked
report.HistoryFound += summary.HistoryFound
report.PersistenceLost += summary.PersistenceLost
report.CrossClockSkewSamples += summary.CrossClockSkewSamples
report.Missing = append(report.Missing, summary.Missing...)
if len(summary.Errors) > 0 {
report.FailureReasons = append(report.FailureReasons, fmt.Sprintf("agent %s errors=%d", summary.Machine, len(summary.Errors)))
@@ -88,7 +92,14 @@ func runMerge(aPath, bPath, output string) error {
report.PublishSuccessRate = float64(report.PublishAccepted) / float64(report.PublishAttempted)
}
report.AckP95MS = mergedAckP95(report.Agents)
report.DeliveryP95MS = mergedHistogramP95(report.Agents, func(summary AgentSummary) HistogramSnapshot { return summary.DeliveryLatency })
report.DeliveryLocalP95MS = mergedHistogramP95(report.Agents, func(summary AgentSummary) HistogramSnapshot { return summary.DeliveryLatencyLocal })
report.DeliveryCrossP95MS = mergedHistogramP95(report.Agents, func(summary AgentSummary) HistogramSnapshot { return summary.DeliveryLatencyCross })
// delivery_p95_ms remains the compatibility field, but now uses only same-clock
// samples. Cross-machine wall clocks cannot produce a valid one-way latency.
report.DeliveryP95MS = report.DeliveryLocalP95MS
if report.DeliveryP95MS == 0 {
report.DeliveryP95MS = mergedHistogramP95(report.Agents, func(summary AgentSummary) HistogramSnapshot { return summary.DeliveryLatency })
}
if report.ExpectedDeliveries > 0 {
report.ArrivalRate = float64(report.ReceivedDeliveries) / float64(report.ExpectedDeliveries)
report.LossRate = float64(report.RealtimeLost) / float64(report.ExpectedDeliveries)
loadtest/im-pressure/merge_test.go
+31
View File
@@ -80,6 +80,37 @@ func TestMergedAckP95FallsBackWhenRawSamplesAreIncomplete(t *testing.T) {
}
}
func TestRunMergeUsesSameClockDeliveryLatencyForGate(t *testing.T) {
dir := t.TempDir()
local := HistogramSnapshot{Count: 10, MaxMS: 200, Buckets: map[string]int64{"200ms": 10}}
cross := HistogramSnapshot{Count: 10, MaxMS: 5000, Buckets: map[string]int64{"5s": 10}}
a := AgentSummary{RunID: "run1", Scenario: "p2p", Machine: "a", ExpectedConnections: 1,
ReadyConnections: 1, PublishAttempted: 1, PublishAccepted: 1, ExpectedPeer: 1, ReceivedPeer: 1,
DeliveryLatencyLocal: local, DeliveryLatencyCross: cross, CrossClockSkewSamples: 3}
b := a
b.Machine = "b"
aPath, bPath, reportPath := filepath.Join(dir, "a.json"), filepath.Join(dir, "b.json"), filepath.Join(dir, "report.json")
writeSummaryForTest(t, aPath, a)
writeSummaryForTest(t, bPath, b)
if err := runMerge(aPath, bPath, reportPath); err != nil {
t.Fatal(err)
}
var report CombinedReport
raw, err := os.ReadFile(reportPath)
if err != nil {
t.Fatal(err)
}
if err = json.Unmarshal(raw, &report); err != nil {
t.Fatal(err)
}
if report.DeliveryP95MS != 200 || report.DeliveryLocalP95MS != 200 || report.DeliveryCrossP95MS != 5000 {
t.Fatalf("unexpected delivery percentiles: %+v", report)
}
if report.CrossClockSkewSamples != 6 || !report.Passed {
t.Fatalf("cross-machine clock skew must not fail the same-clock latency gate: %+v", report)
}
}
func writeSummaryForTest(t *testing.T, path string, summary AgentSummary) {
t.Helper()
raw, err := json.Marshal(summary)
loadtest/im-pressure/model.go
+37 -34
View File
@@ -350,40 +350,43 @@ type AcceptedMessage struct {
}
type AgentSummary struct {
RunID string `json:"run_id"`
Scenario string `json:"scenario"`
Machine string `json:"machine"`
StartedAt time.Time `json:"started_at"`
LoadEndedAt time.Time `json:"load_ended_at"`
FinishedAt time.Time `json:"finished_at"`
ExpectedConnections int `json:"expected_connections"`
ReadyConnections int `json:"ready_connections"`
PublishAttempted int64 `json:"publish_attempted"`
PublishAccepted int64 `json:"publish_accepted"`
PublishRejected int64 `json:"publish_rejected"`
PublishErrors map[string]int64 `json:"publish_errors,omitempty"`
ExpectedPeer int64 `json:"expected_peer"`
ReceivedPeer int64 `json:"received_peer"`
ExpectedEcho int64 `json:"expected_echo"`
ReceivedEcho int64 `json:"received_echo"`
ExpectedGroup int64 `json:"expected_group"`
ReceivedGroup int64 `json:"received_group"`
RealtimeLost int64 `json:"realtime_lost"`
RealtimeLostStored int64 `json:"realtime_lost_but_persisted"`
Duplicates int64 `json:"duplicates"`
OutOfOrder int64 `json:"out_of_order"`
OrderingViolations []OrderingViolation `json:"ordering_violations,omitempty"`
Disconnects int64 `json:"disconnects"`
UnexpectedMessages int64 `json:"unexpected_messages"`
HistoryChecked int64 `json:"history_checked"`
HistoryFound int64 `json:"history_found"`
PersistenceLost int64 `json:"persistence_lost"`
AckLatency HistogramSnapshot `json:"ack_latency"`
DeliveryLatency HistogramSnapshot `json:"delivery_latency"`
Errors []string `json:"errors,omitempty"`
Missing []MissingDelivery `json:"missing,omitempty"`
PersistenceMissing []string `json:"persistence_missing,omitempty"`
AcceptedLocal []AcceptedMessage `json:"accepted_local,omitempty"`
RunID string `json:"run_id"`
Scenario string `json:"scenario"`
Machine string `json:"machine"`
StartedAt time.Time `json:"started_at"`
LoadEndedAt time.Time `json:"load_ended_at"`
FinishedAt time.Time `json:"finished_at"`
ExpectedConnections int `json:"expected_connections"`
ReadyConnections int `json:"ready_connections"`
PublishAttempted int64 `json:"publish_attempted"`
PublishAccepted int64 `json:"publish_accepted"`
PublishRejected int64 `json:"publish_rejected"`
PublishErrors map[string]int64 `json:"publish_errors,omitempty"`
ExpectedPeer int64 `json:"expected_peer"`
ReceivedPeer int64 `json:"received_peer"`
ExpectedEcho int64 `json:"expected_echo"`
ReceivedEcho int64 `json:"received_echo"`
ExpectedGroup int64 `json:"expected_group"`
ReceivedGroup int64 `json:"received_group"`
RealtimeLost int64 `json:"realtime_lost"`
RealtimeLostStored int64 `json:"realtime_lost_but_persisted"`
Duplicates int64 `json:"duplicates"`
OutOfOrder int64 `json:"out_of_order"`
OrderingViolations []OrderingViolation `json:"ordering_violations,omitempty"`
Disconnects int64 `json:"disconnects"`
UnexpectedMessages int64 `json:"unexpected_messages"`
HistoryChecked int64 `json:"history_checked"`
HistoryFound int64 `json:"history_found"`
PersistenceLost int64 `json:"persistence_lost"`
AckLatency HistogramSnapshot `json:"ack_latency"`
DeliveryLatency HistogramSnapshot `json:"delivery_latency"`
DeliveryLatencyLocal HistogramSnapshot `json:"delivery_latency_local"`
DeliveryLatencyCross HistogramSnapshot `json:"delivery_latency_cross_machine"`
CrossClockSkewSamples int64 `json:"cross_machine_negative_latency_samples"`
Errors []string `json:"errors,omitempty"`
Missing []MissingDelivery `json:"missing,omitempty"`
PersistenceMissing []string `json:"persistence_missing,omitempty"`
AcceptedLocal []AcceptedMessage `json:"accepted_local,omitempty"`
}
type OrderingViolation struct {
loadtest/im-pressure/recorder.go
+47 -18
View File
@@ -17,22 +17,25 @@ type observation struct {
}
type deliveryRecorder struct {
machine string
localUsers int
latency *DurationHistogram
disconnects atomic.Int64
received atomic.Int64
runID atomic.Value
historyMode atomic.Bool
machine string
localUsers int
latency *DurationHistogram
localLatency *DurationHistogram
crossLatency *DurationHistogram
disconnects atomic.Int64
received atomic.Int64
runID atomic.Value
historyMode atomic.Bool
mu sync.Mutex
observed map[string]*observation
lastSeq []map[string]int64
duplicates int64
outOfOrder int64
ordering []OrderingViolation
malformed int64
history map[string]bool
mu sync.Mutex
observed map[string]*observation
lastSeq []map[string]int64
duplicates int64
outOfOrder int64
ordering []OrderingViolation
malformed int64
crossClockSkewSamples int64
history map[string]bool
}
func newDeliveryRecorder(runID, machine string, localUsers int) *deliveryRecorder {
@@ -42,6 +45,7 @@ func newDeliveryRecorder(runID, machine string, localUsers int) *deliveryRecorde
}
recorder := &deliveryRecorder{
machine: machine, localUsers: localUsers, latency: NewDurationHistogram(),
localLatency: NewDurationHistogram(), crossLatency: NewDurationHistogram(),
observed: make(map[string]*observation), lastSeq: lastSeq, history: make(map[string]bool),
}
recorder.runID.Store(runID)
@@ -57,8 +61,9 @@ func (r *deliveryRecorder) onData(account AccountRecord, data dataPacket) {
return
}
var content struct {
RunID string `json:"run_id"`
SentAt int64 `json:"sent_at_unix_nano"`
RunID string `json:"run_id"`
SenderMachine string `json:"sender_machine"`
SentAt int64 `json:"sent_at_unix_nano"`
}
if json.Unmarshal(data.Content, &content) != nil || content.RunID != runID {
r.mu.Lock()
@@ -108,10 +113,31 @@ func (r *deliveryRecorder) onData(account AccountRecord, data dataPacket) {
r.lastSeq[index][data.Topic] = data.Seq
}
if content.SentAt > 0 {
r.latency.Observe(time.Since(time.Unix(0, content.SentAt)))
delay := time.Since(time.Unix(0, content.SentAt))
r.latency.Observe(delay)
senderMachine := content.SenderMachine
if senderMachine == "" {
senderMachine = senderMachineFromClientMid(runID, mid)
}
if senderMachine == r.machine {
r.localLatency.Observe(delay)
} else if delay < 0 {
r.crossClockSkewSamples++
} else {
r.crossLatency.Observe(delay)
}
}
}
func senderMachineFromClientMid(runID, mid string) string {
prefix := strings.TrimPrefix(mid, runID+"-")
parts := strings.SplitN(prefix, "-", 3)
if len(parts) < 2 {
return ""
}
return parts[1]
}
func (r *deliveryRecorder) metricCounts() (received, duplicates, outOfOrder, disconnects int64) {
r.mu.Lock()
duplicates = r.duplicates
@@ -134,6 +160,8 @@ func (r *deliveryRecorder) onDisconnect(_ AccountRecord, _ error) {
func (r *deliveryRecorder) summarize(base AgentSummary, ledger []AcceptedMessage) AgentSummary {
base.DeliveryLatency = r.latency.Snapshot()
base.DeliveryLatencyLocal = r.localLatency.Snapshot()
base.DeliveryLatencyCross = r.crossLatency.Snapshot()
base.Disconnects = r.disconnects.Load()
runID, _ := r.runID.Load().(string)
accepted := make(map[string]AcceptedMessage, len(ledger))
@@ -143,6 +171,7 @@ func (r *deliveryRecorder) summarize(base AgentSummary, ledger []AcceptedMessage
defer r.mu.Unlock()
base.Duplicates = r.duplicates
base.OutOfOrder = r.outOfOrder
base.CrossClockSkewSamples = r.crossClockSkewSamples
base.OrderingViolations = append([]OrderingViolation(nil), r.ordering...)
if r.malformed > 0 {
base.Errors = append(base.Errors, fmt.Sprintf("malformed run messages=%d", r.malformed))
loadtest/im-pressure/recorder_test.go
+17
View File
@@ -66,6 +66,23 @@ func TestDeliveryRecorderCapturesOrderingViolationDetails(t *testing.T) {
}
}
func TestDeliveryRecorderSeparatesLocalAndCrossMachineLatency(t *testing.T) {
recorder := newDeliveryRecorder("run1", "a", 1)
local := AcceptedMessage{RunID: "run1", ClientMid: "run1-p-a-0001-01", Topic: "p2p-one", Seq: 1}
cross := AcceptedMessage{RunID: "run1", ClientMid: "run1-p-b-0001-01", Topic: "p2p-one", Seq: 2}
recorder.onData(AccountRecord{Index: 1}, testData(local, time.Now().Add(-10*time.Millisecond)))
recorder.onData(AccountRecord{Index: 1}, testData(cross, time.Now().Add(time.Second)))
summary := recorder.summarize(AgentSummary{}, nil)
if summary.DeliveryLatencyLocal.Count != 1 || summary.DeliveryLatencyCross.Count != 0 {
t.Fatalf("unexpected split latency counts: local=%d cross=%d",
summary.DeliveryLatencyLocal.Count, summary.DeliveryLatencyCross.Count)
}
if summary.CrossClockSkewSamples != 1 {
t.Fatalf("cross clock skew samples=%d", summary.CrossClockSkewSamples)
}
}
func testData(message AcceptedMessage, sentAt time.Time) dataPacket {
content, _ := json.Marshal(map[string]any{"run_id": message.RunID, "sent_at_unix_nano": sentAt.UnixNano()})
return dataPacket{Topic: message.Topic, Seq: message.Seq,
loadtest/im-pressure/wsclient.go
+2 -1
View File
@@ -245,7 +245,8 @@ func (c *WSClient) Publish(ctx context.Context, topic, clientMid, runID, scenari
"id": id, "topic": topic, "noecho": false,
"head": map[string]any{"mime": "application/json", "x-im-schema": 1, "x-im-type": "text", "x-im-client-mid": clientMid},
"content": map[string]any{"text": "loadtest", "run_id": runID, "scenario": scenario,
"sender_index": c.account.Index, "message_index": messageIndex, "sent_at_unix_nano": sentAt.UnixNano()},
"sender_machine": c.account.Machine, "sender_index": c.account.Index,
"message_index": messageIndex, "sent_at_unix_nano": sentAt.UnixNano()},
}})
if err != nil {
return AcceptedMessage{}, err