pro/im_server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: trace delayed websocket acknowledgements

Browse Source
This commit is contained in:
richjohnrambo
2026-10-01 15:19:57 +08:00
parent 19584d0939
commit 73a594146f
19 changed files with 788 additions and 149 deletions
Download Patch File Download Diff File Expand all files Collapse all files
loadtest/im-pressure/merge.go
+59 -47
View File
@@ -10,45 +10,51 @@ import (
)
type CombinedReport struct {
RunID string `json:"run_id"`
Scenario string `json:"scenario"`
GeneratedAt time.Time `json:"generated_at"`
Agents []AgentSummary `json:"agents"`
ExpectedConnections int `json:"expected_connections"`
ReadyConnections int `json:"ready_connections"`
PublishAttempted int64 `json:"publish_attempted"`
PublishAccepted int64 `json:"publish_accepted"`
PublishRejected int64 `json:"publish_rejected"`
PublishExplicitReject int64 `json:"publish_explicitly_rejected"`
PublishOutcomeUnknown int64 `json:"publish_outcome_unknown"`
PublishErrors map[string]int64 `json:"publish_errors,omitempty"`
PublishSuccessRate float64 `json:"publish_success_rate"`
AckP95MS float64 `json:"ack_p95_ms"`
DeliveryP95MS float64 `json:"delivery_p95_ms"`
DeliveryLocalP95MS float64 `json:"delivery_local_p95_ms"`
DeliveryCrossP95MS float64 `json:"delivery_cross_machine_p95_ms"`
CrossClockSkewSamples int64 `json:"cross_machine_negative_latency_samples"`
ExpectedDeliveries int64 `json:"expected_deliveries"`
ReceivedDeliveries int64 `json:"received_deliveries"`
RealtimeLost int64 `json:"realtime_lost"`
RealtimeLostStored int64 `json:"realtime_lost_but_persisted"`
ArrivalRate float64 `json:"arrival_rate"`
LossRate float64 `json:"loss_rate"`
Duplicates int64 `json:"duplicates"`
OutOfOrder int64 `json:"out_of_order"`
OrderingViolations []OrderingViolation `json:"ordering_violations,omitempty"`
Disconnects int64 `json:"disconnects"`
DisconnectReasons map[string]int64 `json:"disconnect_reasons,omitempty"`
DisconnectSamples []DisconnectSample `json:"disconnect_samples,omitempty"`
UnexpectedMessages int64 `json:"unexpected_messages"`
HistoryChecked int64 `json:"history_checked"`
HistoryFound int64 `json:"history_found"`
HistoryUnackedChecked int64 `json:"history_unacked_observed_checked"`
HistoryUnackedFound int64 `json:"history_unacked_observed_found"`
PersistenceLost int64 `json:"persistence_lost"`
Missing []MissingDelivery `json:"missing,omitempty"`
Passed bool `json:"passed"`
FailureReasons []string `json:"failure_reasons,omitempty"`
RunID string `json:"run_id"`
Scenario string `json:"scenario"`
GeneratedAt time.Time `json:"generated_at"`
Agents []AgentSummary `json:"agents"`
ExpectedConnections int `json:"expected_connections"`
ReadyConnections int `json:"ready_connections"`
PublishAttempted int64 `json:"publish_attempted"`
PublishAccepted int64 `json:"publish_accepted"`
PublishRejected int64 `json:"publish_rejected"`
PublishExplicitReject int64 `json:"publish_explicitly_rejected"`
PublishOutcomeUnknown int64 `json:"publish_outcome_unknown"`
PublishErrors map[string]int64 `json:"publish_errors,omitempty"`
PublishCtrlObserved int64 `json:"publish_ctrl_observed"`
PublishCtrlOver20 int64 `json:"publish_ctrl_over_20s"`
PublishLateCtrl int64 `json:"publish_ctrl_late"`
PublishCtrlUnresolved int64 `json:"publish_ctrl_unresolved"`
PublishCtrlP95MS float64 `json:"publish_ctrl_p95_ms"`
CtrlObservationSamples []CtrlObservationSample `json:"ctrl_observation_samples,omitempty"`
PublishSuccessRate float64 `json:"publish_success_rate"`
AckP95MS float64 `json:"ack_p95_ms"`
DeliveryP95MS float64 `json:"delivery_p95_ms"`
DeliveryLocalP95MS float64 `json:"delivery_local_p95_ms"`
DeliveryCrossP95MS float64 `json:"delivery_cross_machine_p95_ms"`
CrossClockSkewSamples int64 `json:"cross_machine_negative_latency_samples"`
ExpectedDeliveries int64 `json:"expected_deliveries"`
ReceivedDeliveries int64 `json:"received_deliveries"`
RealtimeLost int64 `json:"realtime_lost"`
RealtimeLostStored int64 `json:"realtime_lost_but_persisted"`
ArrivalRate float64 `json:"arrival_rate"`
LossRate float64 `json:"loss_rate"`
Duplicates int64 `json:"duplicates"`
OutOfOrder int64 `json:"out_of_order"`
OrderingViolations []OrderingViolation `json:"ordering_violations,omitempty"`
Disconnects int64 `json:"disconnects"`
DisconnectReasons map[string]int64 `json:"disconnect_reasons,omitempty"`
DisconnectSamples []DisconnectSample `json:"disconnect_samples,omitempty"`
UnexpectedMessages int64 `json:"unexpected_messages"`
HistoryChecked int64 `json:"history_checked"`
HistoryFound int64 `json:"history_found"`
HistoryUnackedChecked int64 `json:"history_unacked_observed_checked"`
HistoryUnackedFound int64 `json:"history_unacked_observed_found"`
PersistenceLost int64 `json:"persistence_lost"`
Missing []MissingDelivery `json:"missing,omitempty"`
Passed bool `json:"passed"`
FailureReasons []string `json:"failure_reasons,omitempty"`
}
func runMerge(aPath, bPath, output string) error {
@@ -78,12 +84,24 @@ func runMerge(aPath, bPath, output string) error {
report.PublishErrors[category] += count
}
unknown := summary.PublishOutcomeUnknown
if unknown == 0 {
if summary.CtrlTrackingEnabled {
unknown = summary.PublishCtrlUnresolved + summary.PublishErrors["socket"]
} else if unknown == 0 {
unknown = summary.PublishErrors["ctrl_timeout"] + summary.PublishErrors["socket"]
}
explicit := summary.PublishErrors["403"] + summary.PublishErrors["502"] +
summary.PublishErrors["503"] + summary.PublishErrors["other"]
report.PublishOutcomeUnknown += unknown
report.PublishCtrlObserved += summary.PublishCtrlObserved
report.PublishCtrlOver20 += summary.PublishCtrlOver20
report.PublishLateCtrl += summary.PublishLateCtrl
report.PublishCtrlUnresolved += summary.PublishCtrlUnresolved
if remaining := 1000 - len(report.CtrlObservationSamples); remaining > 0 {
if remaining > len(summary.CtrlObservationSamples) {
remaining = len(summary.CtrlObservationSamples)
}
report.CtrlObservationSamples = append(report.CtrlObservationSamples, summary.CtrlObservationSamples[:remaining]...)
}
report.PublishExplicitReject += explicit
report.PublishRejected += explicit
report.ExpectedDeliveries += summary.ExpectedPeer + summary.ExpectedGroup
@@ -119,6 +137,7 @@ func runMerge(aPath, bPath, output string) error {
report.PublishSuccessRate = float64(report.PublishAccepted) / float64(report.PublishAttempted)
}
report.AckP95MS = mergedAckP95(report.Agents)
report.PublishCtrlP95MS = mergedHistogramP95(report.Agents, func(summary AgentSummary) HistogramSnapshot { return summary.PublishCtrlLatency })
report.DeliveryLocalP95MS = mergedHistogramP95(report.Agents, func(summary AgentSummary) HistogramSnapshot { return summary.DeliveryLatencyLocal })
report.DeliveryCrossP95MS = mergedHistogramP95(report.Agents, func(summary AgentSummary) HistogramSnapshot { return summary.DeliveryLatencyCross })
// delivery_p95_ms remains the compatibility field, but now uses only same-clock
@@ -147,13 +166,6 @@ func runMerge(aPath, bPath, output string) error {
report.FailureReasons = append(report.FailureReasons,
fmt.Sprintf("deliveries observed without ACK ledger=%d", report.UnexpectedMessages))
}
if report.AckP95MS > 500 {
report.FailureReasons = append(report.FailureReasons, fmt.Sprintf("ack p95=%.3fms expected<=500ms", report.AckP95MS))
}
if report.DeliveryP95MS > 1000 {
report.FailureReasons = append(report.FailureReasons,
fmt.Sprintf("delivery p95=%.0fms expected<=1000ms", report.DeliveryP95MS))
}
if report.PublishRejected > 0 || report.RealtimeLost > 0 || report.PersistenceLost > 0 || report.Duplicates > 0 || report.OutOfOrder > 0 || report.Disconnects > 0 || report.UnexpectedMessages > 0 {
report.FailureReasons = append(report.FailureReasons, "message correctness gate failed")
}