fix: harden single-machine load handling

This commit is contained in:
richjohnrambo
2026-09-29 17:52:13 +08:00
parent ab73b69870
commit 4b00b5ae37
24 changed files with 1030 additions and 144 deletions
+62
View File
@@ -2,6 +2,9 @@ package main
import (
"encoding/json"
"errors"
"fmt"
"sync"
"testing"
"time"
)
@@ -111,6 +114,65 @@ func TestDeliveryRecorderSeparatesLocalAndCrossMachineLatency(t *testing.T) {
}
}
func TestDeliveryRecorderConcurrentGroupFanout(t *testing.T) {
const users = 32
const messages = 64
recorder := newDeliveryRecorder("run1", "a", users)
ledger := make([]AcceptedMessage, messages)
for messageIndex := range ledger {
ledger[messageIndex] = AcceptedMessage{
RunID: "run1", Kind: "group", Topic: "grp-one", Seq: 1,
ClientMid: fmt.Sprintf("run1-g-b-%04d-01", messageIndex+1), SenderMachine: "b",
}
}
var writers sync.WaitGroup
for receiver := 1; receiver <= users; receiver++ {
for _, message := range ledger {
writers.Add(1)
go func(receiver int, message AcceptedMessage) {
defer writers.Done()
recorder.onData(AccountRecord{Index: receiver}, testData(message, time.Now().Add(-time.Millisecond)))
}(receiver, message)
}
}
writers.Wait()
summary := recorder.summarize(AgentSummary{}, ledger)
want := int64(users * messages)
if summary.ExpectedGroup != want || summary.ReceivedGroup != want || summary.RealtimeLost != 0 {
t.Fatalf("group expected/received/lost = %d/%d/%d, want %d/%d/0",
summary.ExpectedGroup, summary.ReceivedGroup, summary.RealtimeLost, want, want)
}
if summary.Duplicates != 0 || summary.DeliveryLatency.Count != want {
t.Fatalf("duplicates=%d latency samples=%d want=0/%d", summary.Duplicates, summary.DeliveryLatency.Count, want)
}
}
func TestDeliveryRecorderDisconnectDiagnostics(t *testing.T) {
recorder := newDeliveryRecorder("run1", "a", 1)
recorder.onDisconnect(AccountRecord{Index: 1}, errors.New("websocket: close 1006 (abnormal closure): unexpected EOF"))
recorder.onDisconnect(AccountRecord{Index: 1}, errors.New("read tcp: connection reset by peer"))
summary := recorder.summarize(AgentSummary{}, nil)
if summary.Disconnects != 2 || summary.DisconnectReasons["abnormal_close"] != 1 ||
summary.DisconnectReasons["connection_reset"] != 1 || len(summary.DisconnectSamples) != 2 {
t.Fatalf("unexpected disconnect diagnostics: count=%d reasons=%v samples=%+v",
summary.Disconnects, summary.DisconnectReasons, summary.DisconnectSamples)
}
recorder.resetDisconnects()
summary = recorder.summarize(AgentSummary{}, nil)
if summary.Disconnects != 0 || len(summary.DisconnectReasons) != 0 || len(summary.DisconnectSamples) != 0 {
t.Fatalf("disconnect diagnostics were not reset: %+v", summary)
}
}
func TestBitsetCountCountsEachBitOnce(t *testing.T) {
if got := bitsetCount([]uint64{0b1011, 0b100000}); got != 4 {
t.Fatalf("bitset count=%d want=4", got)
}
}
func testData(message AcceptedMessage, sentAt time.Time) dataPacket {
content, _ := json.Marshal(map[string]any{"run_id": message.RunID, "scenario": message.Scenario,
"sender_machine": message.SenderMachine, "sender_index": message.SenderIndex,