198 lines
8.1 KiB
Go
198 lines
8.1 KiB
Go
package main
|
|
|
|
import (
|
|
"encoding/json"
|
|
"os"
|
|
"path/filepath"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func TestRunMergeComputesArrivalAndPersistenceGate(t *testing.T) {
|
|
dir := t.TempDir()
|
|
a := AgentSummary{RunID: "run1", Scenario: "p2p", Machine: "a", ReadyConnections: 1000,
|
|
ExpectedConnections: 1000,
|
|
PublishAttempted: 10, PublishAccepted: 10, ExpectedPeer: 10, ReceivedPeer: 9, UnexpectedMessages: 42,
|
|
CtrlTrackingEnabled: true, PublishCtrlObserved: 9, PublishCtrlOver20: 2, PublishLateCtrl: 2,
|
|
PublishCtrlUnresolved: 1,
|
|
PublishErrors: map[string]int64{"ctrl_timeout": 3, "socket": 2, "503": 1},
|
|
RealtimeLost: 1, HistoryChecked: 10, HistoryFound: 10, Disconnects: 1,
|
|
DisconnectReasons: map[string]int64{"timeout": 1},
|
|
DisconnectSamples: []DisconnectSample{{ReceiverMachine: "a", ReceiverIndex: 3, Category: "timeout"}}}
|
|
b := AgentSummary{RunID: "run1", Scenario: "p2p", Machine: "b", ReadyConnections: 1000,
|
|
ExpectedConnections: 1000,
|
|
PublishAttempted: 10, PublishAccepted: 10, ExpectedPeer: 10, ReceivedPeer: 10,
|
|
HistoryChecked: 10, HistoryFound: 9, PersistenceLost: 1, OutOfOrder: 1,
|
|
CtrlTrackingEnabled: true, PublishCtrlObserved: 10,
|
|
Disconnects: 2, DisconnectReasons: map[string]int64{"timeout": 1, "connection_reset": 1},
|
|
OrderingViolations: []OrderingViolation{{ReceiverMachine: "b", ReceiverIndex: 7, PreviousSeq: 12, CurrentSeq: 11}}}
|
|
aPath, bPath, reportPath := filepath.Join(dir, "a.json"), filepath.Join(dir, "b.json"), filepath.Join(dir, "report.json")
|
|
writeSummaryForTest(t, aPath, a)
|
|
writeSummaryForTest(t, bPath, b)
|
|
if err := runMerge(aPath, bPath, reportPath); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var report CombinedReport
|
|
raw, err := os.ReadFile(reportPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err = json.Unmarshal(raw, &report); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if report.ExpectedDeliveries != 20 || report.ReceivedDeliveries != 19 || report.ArrivalRate != 0.95 || report.LossRate != 0.05 {
|
|
t.Fatalf("unexpected delivery totals: %+v", report)
|
|
}
|
|
if report.Passed || report.PersistenceLost != 1 {
|
|
t.Fatalf("correctness gate did not fail: %+v", report)
|
|
}
|
|
if report.UnexpectedMessages != 42 || report.PublishOutcomeUnknown != 3 ||
|
|
report.PublishExplicitReject != 1 || report.PublishRejected != 1 {
|
|
t.Fatalf("ambiguous publish outcomes were not merged: %+v", report)
|
|
}
|
|
if report.PublishCtrlObserved != 19 || report.PublishCtrlOver20 != 2 || report.PublishLateCtrl != 2 ||
|
|
report.PublishCtrlUnresolved != 1 {
|
|
t.Fatalf("ctrl observations were not merged: %+v", report)
|
|
}
|
|
if len(report.OrderingViolations) != 1 || report.OrderingViolations[0].ReceiverIndex != 7 {
|
|
t.Fatalf("ordering violations were not merged: %+v", report.OrderingViolations)
|
|
}
|
|
if report.Disconnects != 3 || report.DisconnectReasons["timeout"] != 2 ||
|
|
report.DisconnectReasons["connection_reset"] != 1 || len(report.DisconnectSamples) != 1 {
|
|
t.Fatalf("disconnect diagnostics were not merged: count=%d reasons=%v samples=%+v",
|
|
report.Disconnects, report.DisconnectReasons, report.DisconnectSamples)
|
|
}
|
|
}
|
|
|
|
func TestMergedHistogramP95(t *testing.T) {
|
|
agents := []AgentSummary{
|
|
{AckLatency: HistogramSnapshot{Count: 10, MaxMS: 900, Buckets: map[string]int64{"200ms": 9, "1s": 1}}},
|
|
{AckLatency: HistogramSnapshot{Count: 10, MaxMS: 300, Buckets: map[string]int64{"200ms": 10}}},
|
|
}
|
|
if got := mergedHistogramP95(agents, func(summary AgentSummary) HistogramSnapshot { return summary.AckLatency }); got != 200 {
|
|
t.Fatalf("p95=%v", got)
|
|
}
|
|
}
|
|
|
|
func TestMergedHistogramP95DoesNotUseMaximumForLongTail(t *testing.T) {
|
|
histogram := NewDurationHistogram()
|
|
for i := 0; i < 94; i++ {
|
|
histogram.Observe(5 * time.Second)
|
|
}
|
|
for i := 0; i < 5; i++ {
|
|
histogram.Observe(25 * time.Second)
|
|
}
|
|
histogram.Observe(70 * time.Second)
|
|
agents := []AgentSummary{{PublishCtrlLatency: histogram.Snapshot()}}
|
|
if got := mergedHistogramP95(agents, func(summary AgentSummary) HistogramSnapshot {
|
|
return summary.PublishCtrlLatency
|
|
}); got != 30000 {
|
|
t.Fatalf("p95=%v want=30000", got)
|
|
}
|
|
}
|
|
|
|
func TestMergedAckP95UsesExactAcceptedTimestamps(t *testing.T) {
|
|
base := time.Date(2026, 9, 27, 8, 0, 0, 0, time.UTC)
|
|
agents := []AgentSummary{{
|
|
AckLatency: HistogramSnapshot{Count: 4, MaxMS: 900, Buckets: map[string]int64{"1s": 4}},
|
|
AcceptedLocal: []AcceptedMessage{
|
|
{SentAt: base, AckAt: base.Add(100 * time.Millisecond)},
|
|
{SentAt: base, AckAt: base.Add(200 * time.Millisecond)},
|
|
{SentAt: base, AckAt: base.Add(300 * time.Millisecond)},
|
|
{SentAt: base, AckAt: base.Add(400 * time.Millisecond)},
|
|
},
|
|
}}
|
|
if got := mergedAckP95(agents); got != 400 {
|
|
t.Fatalf("exact p95=%v want=400", got)
|
|
}
|
|
}
|
|
|
|
func TestMergedAckP95FallsBackWhenRawSamplesAreIncomplete(t *testing.T) {
|
|
agents := []AgentSummary{{
|
|
AckLatency: HistogramSnapshot{Count: 2, MaxMS: 900, Buckets: map[string]int64{"1s": 2}},
|
|
AcceptedLocal: []AcceptedMessage{{SentAt: time.Now(), AckAt: time.Now().Add(100 * time.Millisecond)}},
|
|
}}
|
|
if got := mergedAckP95(agents); got != 1000 {
|
|
t.Fatalf("fallback p95=%v want=1000", got)
|
|
}
|
|
}
|
|
|
|
func TestRunMergeReportsSameClockDeliveryLatency(t *testing.T) {
|
|
dir := t.TempDir()
|
|
local := HistogramSnapshot{Count: 10, MaxMS: 200, Buckets: map[string]int64{"200ms": 10}}
|
|
cross := HistogramSnapshot{Count: 10, MaxMS: 5000, Buckets: map[string]int64{"5s": 10}}
|
|
a := AgentSummary{RunID: "run1", Scenario: "p2p", Machine: "a", ExpectedConnections: 1,
|
|
ReadyConnections: 1, PublishAttempted: 1, PublishAccepted: 1, ExpectedPeer: 1, ReceivedPeer: 1,
|
|
DeliveryLatencyLocal: local, DeliveryLatencyCross: cross, CrossClockSkewSamples: 3}
|
|
b := a
|
|
b.Machine = "b"
|
|
aPath, bPath, reportPath := filepath.Join(dir, "a.json"), filepath.Join(dir, "b.json"), filepath.Join(dir, "report.json")
|
|
writeSummaryForTest(t, aPath, a)
|
|
writeSummaryForTest(t, bPath, b)
|
|
if err := runMerge(aPath, bPath, reportPath); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var report CombinedReport
|
|
raw, err := os.ReadFile(reportPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err = json.Unmarshal(raw, &report); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if report.DeliveryP95MS != 200 || report.DeliveryLocalP95MS != 200 || report.DeliveryCrossP95MS != 5000 {
|
|
t.Fatalf("unexpected delivery percentiles: %+v", report)
|
|
}
|
|
if report.CrossClockSkewSamples != 6 || !report.Passed {
|
|
t.Fatalf("cross-machine clock skew must not fail correctness acceptance: %+v", report)
|
|
}
|
|
}
|
|
|
|
func TestRunMergeTreatsLateSuccessAsFinalAcceptance(t *testing.T) {
|
|
dir := t.TempDir()
|
|
late := AgentSummary{
|
|
RunID: "run1", Scenario: "group", Machine: "a", ExpectedConnections: 1, ReadyConnections: 1,
|
|
PublishAttempted: 1, PublishAccepted: 0, CtrlTrackingEnabled: true,
|
|
PublishCtrlObserved: 1, PublishLateCtrl: 1, PublishCtrlUnresolved: 0,
|
|
PublishErrors: map[string]int64{"ctrl_timeout": 1},
|
|
CtrlObservationSamples: []CtrlObservationSample{{Code: 202, Late: true}},
|
|
Errors: []string{"publish message-1: ctrl timeout id=pub-a-1"},
|
|
}
|
|
timely := AgentSummary{
|
|
RunID: "run1", Scenario: "group", Machine: "b", ExpectedConnections: 1, ReadyConnections: 1,
|
|
PublishAttempted: 1, PublishAccepted: 1, CtrlTrackingEnabled: true,
|
|
PublishCtrlObserved: 1, PublishErrors: map[string]int64{},
|
|
}
|
|
aPath, bPath, reportPath := filepath.Join(dir, "a.json"), filepath.Join(dir, "b.json"), filepath.Join(dir, "report.json")
|
|
writeSummaryForTest(t, aPath, late)
|
|
writeSummaryForTest(t, bPath, timely)
|
|
if err := runMerge(aPath, bPath, reportPath); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var report CombinedReport
|
|
raw, err := os.ReadFile(reportPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err = json.Unmarshal(raw, &report); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if report.PublishAccepted != 1 || report.PublishLateAccepted != 1 || report.PublishFinalAccepted != 2 || report.PublishSuccessRate != 1 {
|
|
t.Fatalf("late acceptance was not included in final publish outcome: %+v", report)
|
|
}
|
|
if !report.Passed || len(report.FailureReasons) != 0 {
|
|
t.Fatalf("resolved late ACK must not fail the report: %+v", report.FailureReasons)
|
|
}
|
|
}
|
|
|
|
func writeSummaryForTest(t *testing.T, path string, summary AgentSummary) {
|
|
t.Helper()
|
|
raw, err := json.Marshal(summary)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err = os.WriteFile(path, raw, 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|