fix: preserve cross-pod message order

loadtest/im-pressure/agent.go

@@ -40,6 +40,7 @@ type agentRun struct {
 	publishCtrlObserved atomic.Int64
 	publishCtrlOver20   atomic.Int64
 	publishLateCtrl     atomic.Int64
+	publishLateAccepted atomic.Int64
 
 	mu          sync.Mutex
 	accepted    []AcceptedMessage
@@ -982,6 +983,9 @@ func (r *agentRun) observeCtrl(account AccountRecord, observation ctrlObservatio
 	}
 	if observation.Late {
 		r.publishLateCtrl.Add(1)
+		if observation.Code >= 200 && observation.Code < 300 {
+			r.publishLateAccepted.Add(1)
+		}
 	}
 	if observation.Late || observation.Latency > 20*time.Second {
 		r.mu.Lock()
@@ -1001,6 +1005,7 @@ func (r *agentRun) applyCtrlObservations(summary *AgentSummary) {
 	summary.PublishCtrlObserved = r.publishCtrlObserved.Load()
 	summary.PublishCtrlOver20 = r.publishCtrlOver20.Load()
 	summary.PublishLateCtrl = r.publishLateCtrl.Load()
+	summary.PublishLateAccepted = r.publishLateAccepted.Load()
 	summary.PublishCtrlUnresolved = r.unresolvedCtrl()
 	summary.PublishOutcomeUnknown = r.outcomeUnknown()
 	summary.PublishCtrlLatency = r.ctrlLatency.Snapshot()

loadtest/im-pressure/merge.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
+	"strings"
 	"time"
 )
 
@@ -18,6 +19,8 @@ type CombinedReport struct {
 	ReadyConnections       int                     `json:"ready_connections"`
 	PublishAttempted       int64                   `json:"publish_attempted"`
 	PublishAccepted        int64                   `json:"publish_accepted"`
+	PublishLateAccepted    int64                   `json:"publish_late_accepted"`
+	PublishFinalAccepted   int64                   `json:"publish_final_accepted"`
 	PublishRejected        int64                   `json:"publish_rejected"`
 	PublishExplicitReject  int64                   `json:"publish_explicitly_rejected"`
 	PublishOutcomeUnknown  int64                   `json:"publish_outcome_unknown"`
@@ -80,6 +83,17 @@ func runMerge(aPath, bPath, output string) error {
 		report.ReadyConnections += summary.ReadyConnections
 		report.PublishAttempted += summary.PublishAttempted
 		report.PublishAccepted += summary.PublishAccepted
+		lateAccepted := summary.PublishLateAccepted
+		// Backfill summaries generated before publish_ctrl_late_accepted was added.
+		if summary.CtrlTrackingEnabled && lateAccepted == 0 {
+			for _, sample := range summary.CtrlObservationSamples {
+				if sample.Late && sample.Code >= 200 && sample.Code < 300 {
+					lateAccepted++
+				}
+			}
+		}
+		report.PublishLateAccepted += lateAccepted
+		report.PublishFinalAccepted += summary.PublishAccepted + lateAccepted
 		for category, count := range summary.PublishErrors {
 			report.PublishErrors[category] += count
 		}
@@ -129,12 +143,20 @@ func runMerge(aPath, bPath, output string) error {
 		report.PersistenceLost += summary.PersistenceLost
 		report.CrossClockSkewSamples += summary.CrossClockSkewSamples
 		report.Missing = append(report.Missing, summary.Missing...)
-		if len(summary.Errors) > 0 {
-			report.FailureReasons = append(report.FailureReasons, fmt.Sprintf("agent %s errors=%d", summary.Machine, len(summary.Errors)))
+		actionableErrors := len(summary.Errors)
+		lateTimeouts := int(lateAccepted)
+		for _, message := range summary.Errors {
+			if lateTimeouts > 0 && strings.Contains(message, "ctrl timeout") {
+				actionableErrors--
+				lateTimeouts--
+			}
+		}
+		if actionableErrors > 0 {
+			report.FailureReasons = append(report.FailureReasons, fmt.Sprintf("agent %s unresolved errors=%d", summary.Machine, actionableErrors))
 		}
 	}
 	if report.PublishAttempted > 0 {
-		report.PublishSuccessRate = float64(report.PublishAccepted) / float64(report.PublishAttempted)
+		report.PublishSuccessRate = float64(report.PublishFinalAccepted) / float64(report.PublishAttempted)
 	}
 	report.AckP95MS = mergedAckP95(report.Agents)
 	report.PublishCtrlP95MS = mergedHistogramP95(report.Agents, func(summary AgentSummary) HistogramSnapshot { return summary.PublishCtrlLatency })

loadtest/im-pressure/merge_test.go

@@ -148,6 +148,43 @@ func TestRunMergeReportsSameClockDeliveryLatency(t *testing.T) {
 	}
 }
 
+func TestRunMergeTreatsLateSuccessAsFinalAcceptance(t *testing.T) {
+	dir := t.TempDir()
+	late := AgentSummary{
+		RunID: "run1", Scenario: "group", Machine: "a", ExpectedConnections: 1, ReadyConnections: 1,
+		PublishAttempted: 1, PublishAccepted: 0, CtrlTrackingEnabled: true,
+		PublishCtrlObserved: 1, PublishLateCtrl: 1, PublishCtrlUnresolved: 0,
+		PublishErrors:          map[string]int64{"ctrl_timeout": 1},
+		CtrlObservationSamples: []CtrlObservationSample{{Code: 202, Late: true}},
+		Errors:                 []string{"publish message-1: ctrl timeout id=pub-a-1"},
+	}
+	timely := AgentSummary{
+		RunID: "run1", Scenario: "group", Machine: "b", ExpectedConnections: 1, ReadyConnections: 1,
+		PublishAttempted: 1, PublishAccepted: 1, CtrlTrackingEnabled: true,
+		PublishCtrlObserved: 1, PublishErrors: map[string]int64{},
+	}
+	aPath, bPath, reportPath := filepath.Join(dir, "a.json"), filepath.Join(dir, "b.json"), filepath.Join(dir, "report.json")
+	writeSummaryForTest(t, aPath, late)
+	writeSummaryForTest(t, bPath, timely)
+	if err := runMerge(aPath, bPath, reportPath); err != nil {
+		t.Fatal(err)
+	}
+	var report CombinedReport
+	raw, err := os.ReadFile(reportPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err = json.Unmarshal(raw, &report); err != nil {
+		t.Fatal(err)
+	}
+	if report.PublishAccepted != 1 || report.PublishLateAccepted != 1 || report.PublishFinalAccepted != 2 || report.PublishSuccessRate != 1 {
+		t.Fatalf("late acceptance was not included in final publish outcome: %+v", report)
+	}
+	if !report.Passed || len(report.FailureReasons) != 0 {
+		t.Fatalf("resolved late ACK must not fail the report: %+v", report.FailureReasons)
+	}
+}
+
 func writeSummaryForTest(t *testing.T, path string, summary AgentSummary) {
 	t.Helper()
 	raw, err := json.Marshal(summary)

loadtest/im-pressure/model.go

@@ -424,6 +424,7 @@ type AgentSummary struct {
 	PublishCtrlObserved    int64                   `json:"publish_ctrl_observed"`
 	PublishCtrlOver20      int64                   `json:"publish_ctrl_over_20s"`
 	PublishLateCtrl        int64                   `json:"publish_ctrl_late"`
+	PublishLateAccepted    int64                   `json:"publish_ctrl_late_accepted"`
 	PublishCtrlUnresolved  int64                   `json:"publish_ctrl_unresolved"`
 	PublishCtrlLatency     HistogramSnapshot       `json:"publish_ctrl_latency"`
 	CtrlObservationSamples []CtrlObservationSample `json:"ctrl_observation_samples,omitempty"`