pro/im_server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test: capture ordering regressions and enforce deploy resources

Browse Source
This commit is contained in:
richjohnrambo
2026-09-27 14:05:21 +08:00
parent d0e1e98f5f
commit 188164ce82
10 changed files with 213 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files
loadtest/im-pressure/merge.go
+30 -28
View File
@@ -10,34 +10,35 @@ import (
)
type CombinedReport struct {
RunID string `json:"run_id"`
Scenario string `json:"scenario"`
GeneratedAt time.Time `json:"generated_at"`
Agents []AgentSummary `json:"agents"`
ExpectedConnections int `json:"expected_connections"`
ReadyConnections int `json:"ready_connections"`
PublishAttempted int64 `json:"publish_attempted"`
PublishAccepted int64 `json:"publish_accepted"`
PublishRejected int64 `json:"publish_rejected"`
PublishErrors map[string]int64 `json:"publish_errors,omitempty"`
PublishSuccessRate float64 `json:"publish_success_rate"`
AckP95MS float64 `json:"ack_p95_ms"`
DeliveryP95MS float64 `json:"delivery_p95_ms"`
ExpectedDeliveries int64 `json:"expected_deliveries"`
ReceivedDeliveries int64 `json:"received_deliveries"`
RealtimeLost int64 `json:"realtime_lost"`
RealtimeLostStored int64 `json:"realtime_lost_but_persisted"`
ArrivalRate float64 `json:"arrival_rate"`
LossRate float64 `json:"loss_rate"`
Duplicates int64 `json:"duplicates"`
OutOfOrder int64 `json:"out_of_order"`
Disconnects int64 `json:"disconnects"`
HistoryChecked int64 `json:"history_checked"`
HistoryFound int64 `json:"history_found"`
PersistenceLost int64 `json:"persistence_lost"`
Missing []MissingDelivery `json:"missing,omitempty"`
Passed bool `json:"passed"`
FailureReasons []string `json:"failure_reasons,omitempty"`
RunID string `json:"run_id"`
Scenario string `json:"scenario"`
GeneratedAt time.Time `json:"generated_at"`
Agents []AgentSummary `json:"agents"`
ExpectedConnections int `json:"expected_connections"`
ReadyConnections int `json:"ready_connections"`
PublishAttempted int64 `json:"publish_attempted"`
PublishAccepted int64 `json:"publish_accepted"`
PublishRejected int64 `json:"publish_rejected"`
PublishErrors map[string]int64 `json:"publish_errors,omitempty"`
PublishSuccessRate float64 `json:"publish_success_rate"`
AckP95MS float64 `json:"ack_p95_ms"`
DeliveryP95MS float64 `json:"delivery_p95_ms"`
ExpectedDeliveries int64 `json:"expected_deliveries"`
ReceivedDeliveries int64 `json:"received_deliveries"`
RealtimeLost int64 `json:"realtime_lost"`
RealtimeLostStored int64 `json:"realtime_lost_but_persisted"`
ArrivalRate float64 `json:"arrival_rate"`
LossRate float64 `json:"loss_rate"`
Duplicates int64 `json:"duplicates"`
OutOfOrder int64 `json:"out_of_order"`
OrderingViolations []OrderingViolation `json:"ordering_violations,omitempty"`
Disconnects int64 `json:"disconnects"`
HistoryChecked int64 `json:"history_checked"`
HistoryFound int64 `json:"history_found"`
PersistenceLost int64 `json:"persistence_lost"`
Missing []MissingDelivery `json:"missing,omitempty"`
Passed bool `json:"passed"`
FailureReasons []string `json:"failure_reasons,omitempty"`
}
func runMerge(aPath, bPath, output string) error {
@@ -73,6 +74,7 @@ func runMerge(aPath, bPath, output string) error {
report.RealtimeLostStored += summary.RealtimeLostStored
report.Duplicates += summary.Duplicates
report.OutOfOrder += summary.OutOfOrder
report.OrderingViolations = append(report.OrderingViolations, summary.OrderingViolations...)
report.Disconnects += summary.Disconnects
report.HistoryChecked += summary.HistoryChecked
report.HistoryFound += summary.HistoryFound
loadtest/im-pressure/merge_test.go
+5 -1
View File
@@ -16,7 +16,8 @@ func TestRunMergeComputesArrivalAndPersistenceGate(t *testing.T) {
b := AgentSummary{RunID: "run1", Scenario: "p2p", Machine: "b", ReadyConnections: 1000,
ExpectedConnections: 1000,
PublishAttempted: 10, PublishAccepted: 10, ExpectedPeer: 10, ReceivedPeer: 10,
HistoryChecked: 10, HistoryFound: 9, PersistenceLost: 1}
HistoryChecked: 10, HistoryFound: 9, PersistenceLost: 1, OutOfOrder: 1,
OrderingViolations: []OrderingViolation{{ReceiverMachine: "b", ReceiverIndex: 7, PreviousSeq: 12, CurrentSeq: 11}}}
aPath, bPath, reportPath := filepath.Join(dir, "a.json"), filepath.Join(dir, "b.json"), filepath.Join(dir, "report.json")
writeSummaryForTest(t, aPath, a)
writeSummaryForTest(t, bPath, b)
@@ -37,6 +38,9 @@ func TestRunMergeComputesArrivalAndPersistenceGate(t *testing.T) {
if report.Passed || report.PersistenceLost != 1 {
t.Fatalf("correctness gate did not fail: %+v", report)
}
if len(report.OrderingViolations) != 1 || report.OrderingViolations[0].ReceiverIndex != 7 {
t.Fatalf("ordering violations were not merged: %+v", report.OrderingViolations)
}
}
func TestMergedHistogramP95(t *testing.T) {
loadtest/im-pressure/model.go
+43 -33
View File
@@ -343,39 +343,49 @@ type AcceptedMessage struct {
}
type AgentSummary struct {
RunID string `json:"run_id"`
Scenario string `json:"scenario"`
Machine string `json:"machine"`
StartedAt time.Time `json:"started_at"`
LoadEndedAt time.Time `json:"load_ended_at"`
FinishedAt time.Time `json:"finished_at"`
ExpectedConnections int `json:"expected_connections"`
ReadyConnections int `json:"ready_connections"`
PublishAttempted int64 `json:"publish_attempted"`
PublishAccepted int64 `json:"publish_accepted"`
PublishRejected int64 `json:"publish_rejected"`
PublishErrors map[string]int64 `json:"publish_errors,omitempty"`
ExpectedPeer int64 `json:"expected_peer"`
ReceivedPeer int64 `json:"received_peer"`
ExpectedEcho int64 `json:"expected_echo"`
ReceivedEcho int64 `json:"received_echo"`
ExpectedGroup int64 `json:"expected_group"`
ReceivedGroup int64 `json:"received_group"`
RealtimeLost int64 `json:"realtime_lost"`
RealtimeLostStored int64 `json:"realtime_lost_but_persisted"`
Duplicates int64 `json:"duplicates"`
OutOfOrder int64 `json:"out_of_order"`
Disconnects int64 `json:"disconnects"`
UnexpectedMessages int64 `json:"unexpected_messages"`
HistoryChecked int64 `json:"history_checked"`
HistoryFound int64 `json:"history_found"`
PersistenceLost int64 `json:"persistence_lost"`
AckLatency HistogramSnapshot `json:"ack_latency"`
DeliveryLatency HistogramSnapshot `json:"delivery_latency"`
Errors []string `json:"errors,omitempty"`
Missing []MissingDelivery `json:"missing,omitempty"`
PersistenceMissing []string `json:"persistence_missing,omitempty"`
AcceptedLocal []AcceptedMessage `json:"accepted_local,omitempty"`
RunID string `json:"run_id"`
Scenario string `json:"scenario"`
Machine string `json:"machine"`
StartedAt time.Time `json:"started_at"`
LoadEndedAt time.Time `json:"load_ended_at"`
FinishedAt time.Time `json:"finished_at"`
ExpectedConnections int `json:"expected_connections"`
ReadyConnections int `json:"ready_connections"`
PublishAttempted int64 `json:"publish_attempted"`
PublishAccepted int64 `json:"publish_accepted"`
PublishRejected int64 `json:"publish_rejected"`
PublishErrors map[string]int64 `json:"publish_errors,omitempty"`
ExpectedPeer int64 `json:"expected_peer"`
ReceivedPeer int64 `json:"received_peer"`
ExpectedEcho int64 `json:"expected_echo"`
ReceivedEcho int64 `json:"received_echo"`
ExpectedGroup int64 `json:"expected_group"`
ReceivedGroup int64 `json:"received_group"`
RealtimeLost int64 `json:"realtime_lost"`
RealtimeLostStored int64 `json:"realtime_lost_but_persisted"`
Duplicates int64 `json:"duplicates"`
OutOfOrder int64 `json:"out_of_order"`
OrderingViolations []OrderingViolation `json:"ordering_violations,omitempty"`
Disconnects int64 `json:"disconnects"`
UnexpectedMessages int64 `json:"unexpected_messages"`
HistoryChecked int64 `json:"history_checked"`
HistoryFound int64 `json:"history_found"`
PersistenceLost int64 `json:"persistence_lost"`
AckLatency HistogramSnapshot `json:"ack_latency"`
DeliveryLatency HistogramSnapshot `json:"delivery_latency"`
Errors []string `json:"errors,omitempty"`
Missing []MissingDelivery `json:"missing,omitempty"`
PersistenceMissing []string `json:"persistence_missing,omitempty"`
AcceptedLocal []AcceptedMessage `json:"accepted_local,omitempty"`
}
type OrderingViolation struct {
ReceiverMachine string `json:"receiver_machine"`
ReceiverIndex int `json:"receiver_index"`
TopicHash string `json:"topic_hash"`
PreviousSeq int64 `json:"previous_seq"`
CurrentSeq int64 `json:"current_seq"`
ClientMid string `json:"client_mid"`
}
type MissingDelivery struct {
loadtest/im-pressure/recorder.go
+12
View File
@@ -30,6 +30,7 @@ type deliveryRecorder struct {
lastSeq []map[string]int64
duplicates int64
outOfOrder int64
ordering []OrderingViolation
malformed int64
history map[string]bool
}
@@ -92,6 +93,16 @@ func (r *deliveryRecorder) onData(account AccountRecord, data dataPacket) {
r.received.Add(1)
if previous := r.lastSeq[index][data.Topic]; previous > 0 && data.Seq < previous {
r.outOfOrder++
if len(r.ordering) < 100 {
r.ordering = append(r.ordering, OrderingViolation{
ReceiverMachine: r.machine,
ReceiverIndex: account.Index,
TopicHash: topicHash(data.Topic),
PreviousSeq: previous,
CurrentSeq: data.Seq,
ClientMid: mid,
})
}
}
if data.Seq > r.lastSeq[index][data.Topic] {
r.lastSeq[index][data.Topic] = data.Seq
@@ -132,6 +143,7 @@ func (r *deliveryRecorder) summarize(base AgentSummary, ledger []AcceptedMessage
defer r.mu.Unlock()
base.Duplicates = r.duplicates
base.OutOfOrder = r.outOfOrder
base.OrderingViolations = append([]OrderingViolation(nil), r.ordering...)
if r.malformed > 0 {
base.Errors = append(base.Errors, fmt.Sprintf("malformed run messages=%d", r.malformed))
}
loadtest/im-pressure/recorder_test.go
+18
View File
@@ -48,6 +48,24 @@ func TestDeliveryRecorderSeparatesHistoryFromRealtime(t *testing.T) {
}
}
func TestDeliveryRecorderCapturesOrderingViolationDetails(t *testing.T) {
recorder := newDeliveryRecorder("run1", "b", 1)
newer := AcceptedMessage{RunID: "run1", ClientMid: "run1-p-a-0001-02", Topic: "p2p-one", Seq: 12}
older := AcceptedMessage{RunID: "run1", ClientMid: "run1-p-a-0001-01", Topic: "p2p-one", Seq: 11}
recorder.onData(AccountRecord{Index: 1}, testData(newer, time.Now()))
recorder.onData(AccountRecord{Index: 1}, testData(older, time.Now()))
summary := recorder.summarize(AgentSummary{}, nil)
if summary.OutOfOrder != 1 || len(summary.OrderingViolations) != 1 {
t.Fatalf("out_of_order=%d violations=%+v", summary.OutOfOrder, summary.OrderingViolations)
}
violation := summary.OrderingViolations[0]
if violation.ReceiverMachine != "b" || violation.ReceiverIndex != 1 ||
violation.PreviousSeq != 12 || violation.CurrentSeq != 11 || violation.ClientMid != older.ClientMid {
t.Fatalf("unexpected violation: %+v", violation)
}
}
func testData(message AcceptedMessage, sentAt time.Time) dataPacket {
content, _ := json.Marshal(map[string]any{"run_id": message.RunID, "sent_at_unix_nano": sentAt.UnixNano()})
return dataPacket{Topic: message.Topic, Seq: message.Seq,