pro/im_server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: harden multi-pod delivery and load verification

Browse Source
This commit is contained in:
richjohnrambo
2026-09-27 03:37:32 +08:00
parent a2de100f2b
commit ce3250be64
33 changed files with 1340 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files
loadtest/im-pressure/merge.go
+57 -1
View File
@@ -18,6 +18,10 @@ type CombinedReport struct {
PublishAttempted int64 `json:"publish_attempted"`
PublishAccepted int64 `json:"publish_accepted"`
PublishRejected int64 `json:"publish_rejected"`
PublishErrors map[string]int64 `json:"publish_errors,omitempty"`
PublishSuccessRate float64 `json:"publish_success_rate"`
AckP95MS float64 `json:"ack_p95_ms"`
DeliveryP95MS float64 `json:"delivery_p95_ms"`
ExpectedDeliveries int64 `json:"expected_deliveries"`
ReceivedDeliveries int64 `json:"received_deliveries"`
RealtimeLost int64 `json:"realtime_lost"`
@@ -47,12 +51,16 @@ func runMerge(aPath, bPath, output string) error {
if a.RunID != b.RunID || a.Scenario != b.Scenario {
return fmt.Errorf("agent summaries do not belong to the same run")
}
report := CombinedReport{RunID: a.RunID, Scenario: a.Scenario, GeneratedAt: time.Now().UTC(), Agents: []AgentSummary{a, b}, Passed: true}
report := CombinedReport{RunID: a.RunID, Scenario: a.Scenario, GeneratedAt: time.Now().UTC(), Agents: []AgentSummary{a, b}, Passed: true,
PublishErrors: make(map[string]int64)}
for _, summary := range report.Agents {
report.ReadyConnections += summary.ReadyConnections
report.PublishAttempted += summary.PublishAttempted
report.PublishAccepted += summary.PublishAccepted
report.PublishRejected += summary.PublishRejected
for category, count := range summary.PublishErrors {
report.PublishErrors[category] += count
}
report.ExpectedDeliveries += summary.ExpectedPeer + summary.ExpectedGroup
report.ReceivedDeliveries += summary.ReceivedPeer + summary.ReceivedGroup
report.RealtimeLost += summary.RealtimeLost
@@ -68,6 +76,11 @@ func runMerge(aPath, bPath, output string) error {
report.FailureReasons = append(report.FailureReasons, fmt.Sprintf("agent %s errors=%d", summary.Machine, len(summary.Errors)))
}
}
if report.PublishAttempted > 0 {
report.PublishSuccessRate = float64(report.PublishAccepted) / float64(report.PublishAttempted)
}
report.AckP95MS = mergedHistogramP95(report.Agents, func(summary AgentSummary) HistogramSnapshot { return summary.AckLatency })
report.DeliveryP95MS = mergedHistogramP95(report.Agents, func(summary AgentSummary) HistogramSnapshot { return summary.DeliveryLatency })
if report.ExpectedDeliveries > 0 {
report.ArrivalRate = float64(report.ReceivedDeliveries) / float64(report.ExpectedDeliveries)
report.LossRate = float64(report.RealtimeLost) / float64(report.ExpectedDeliveries)
@@ -75,6 +88,17 @@ func runMerge(aPath, bPath, output string) error {
if report.ReadyConnections != 2000 {
report.FailureReasons = append(report.FailureReasons, fmt.Sprintf("ready connections=%d expected=2000", report.ReadyConnections))
}
if report.PublishAttempted > 0 && report.PublishSuccessRate < 0.9999 {
report.FailureReasons = append(report.FailureReasons,
fmt.Sprintf("publish success rate=%.6f expected>=0.999900", report.PublishSuccessRate))
}
if report.AckP95MS > 500 {
report.FailureReasons = append(report.FailureReasons, fmt.Sprintf("ack p95=%.0fms expected<=500ms", report.AckP95MS))
}
if report.DeliveryP95MS > 1000 {
report.FailureReasons = append(report.FailureReasons,
fmt.Sprintf("delivery p95=%.0fms expected<=1000ms", report.DeliveryP95MS))
}
if report.PublishRejected > 0 || report.RealtimeLost > 0 || report.PersistenceLost > 0 || report.Duplicates > 0 || report.OutOfOrder > 0 || report.Disconnects > 0 {
report.FailureReasons = append(report.FailureReasons, "message correctness gate failed")
}
@@ -88,6 +112,38 @@ func runMerge(aPath, bPath, output string) error {
return writeJSONFile(output, report)
}
func mergedHistogramP95(agents []AgentSummary, selectHistogram func(AgentSummary) HistogramSnapshot) float64 {
var count int64
buckets := make([]int64, len(latencyBounds)+1)
maxMS := float64(0)
for _, agent := range agents {
histogram := selectHistogram(agent)
count += histogram.Count
if histogram.MaxMS > maxMS {
maxMS = histogram.MaxMS
}
for i, bound := range latencyBounds {
buckets[i] += histogram.Buckets[bound.String()]
}
buckets[len(latencyBounds)] += histogram.Buckets["+Inf"]
}
if count == 0 {
return 0
}
target := (count*95 + 99) / 100
var cumulative int64
for i, bucket := range buckets {
cumulative += bucket
if cumulative >= target {
if i == len(latencyBounds) {
return maxMS
}
return float64(latencyBounds[i]) / float64(time.Millisecond)
}
}
return maxMS
}
func readAgentSummary(path string) (AgentSummary, error) {
var summary AgentSummary
raw, err := os.ReadFile(path)