perf: expose and reduce message publish latency

loadtest/im-pressure/recorder.go

@@ -17,22 +17,25 @@ type observation struct {
 }
 
 type deliveryRecorder struct {
-	machine     string
-	localUsers  int
-	latency     *DurationHistogram
-	disconnects atomic.Int64
-	received    atomic.Int64
-	runID       atomic.Value
-	historyMode atomic.Bool
+	machine      string
+	localUsers   int
+	latency      *DurationHistogram
+	localLatency *DurationHistogram
+	crossLatency *DurationHistogram
+	disconnects  atomic.Int64
+	received     atomic.Int64
+	runID        atomic.Value
+	historyMode  atomic.Bool
 
-	mu         sync.Mutex
-	observed   map[string]*observation
-	lastSeq    []map[string]int64
-	duplicates int64
-	outOfOrder int64
-	ordering   []OrderingViolation
-	malformed  int64
-	history    map[string]bool
+	mu                    sync.Mutex
+	observed              map[string]*observation
+	lastSeq               []map[string]int64
+	duplicates            int64
+	outOfOrder            int64
+	ordering              []OrderingViolation
+	malformed             int64
+	crossClockSkewSamples int64
+	history               map[string]bool
 }
 
 func newDeliveryRecorder(runID, machine string, localUsers int) *deliveryRecorder {
@@ -42,6 +45,7 @@ func newDeliveryRecorder(runID, machine string, localUsers int) *deliveryRecorde
 	}
 	recorder := &deliveryRecorder{
 		machine: machine, localUsers: localUsers, latency: NewDurationHistogram(),
+		localLatency: NewDurationHistogram(), crossLatency: NewDurationHistogram(),
 		observed: make(map[string]*observation), lastSeq: lastSeq, history: make(map[string]bool),
 	}
 	recorder.runID.Store(runID)
@@ -57,8 +61,9 @@ func (r *deliveryRecorder) onData(account AccountRecord, data dataPacket) {
 		return
 	}
 	var content struct {
-		RunID  string `json:"run_id"`
-		SentAt int64  `json:"sent_at_unix_nano"`
+		RunID         string `json:"run_id"`
+		SenderMachine string `json:"sender_machine"`
+		SentAt        int64  `json:"sent_at_unix_nano"`
 	}
 	if json.Unmarshal(data.Content, &content) != nil || content.RunID != runID {
 		r.mu.Lock()
@@ -108,10 +113,31 @@ func (r *deliveryRecorder) onData(account AccountRecord, data dataPacket) {
 		r.lastSeq[index][data.Topic] = data.Seq
 	}
 	if content.SentAt > 0 {
-		r.latency.Observe(time.Since(time.Unix(0, content.SentAt)))
+		delay := time.Since(time.Unix(0, content.SentAt))
+		r.latency.Observe(delay)
+		senderMachine := content.SenderMachine
+		if senderMachine == "" {
+			senderMachine = senderMachineFromClientMid(runID, mid)
+		}
+		if senderMachine == r.machine {
+			r.localLatency.Observe(delay)
+		} else if delay < 0 {
+			r.crossClockSkewSamples++
+		} else {
+			r.crossLatency.Observe(delay)
+		}
 	}
 }
 
+func senderMachineFromClientMid(runID, mid string) string {
+	prefix := strings.TrimPrefix(mid, runID+"-")
+	parts := strings.SplitN(prefix, "-", 3)
+	if len(parts) < 2 {
+		return ""
+	}
+	return parts[1]
+}
+
 func (r *deliveryRecorder) metricCounts() (received, duplicates, outOfOrder, disconnects int64) {
 	r.mu.Lock()
 	duplicates = r.duplicates
@@ -134,6 +160,8 @@ func (r *deliveryRecorder) onDisconnect(_ AccountRecord, _ error) {
 
 func (r *deliveryRecorder) summarize(base AgentSummary, ledger []AcceptedMessage) AgentSummary {
 	base.DeliveryLatency = r.latency.Snapshot()
+	base.DeliveryLatencyLocal = r.localLatency.Snapshot()
+	base.DeliveryLatencyCross = r.crossLatency.Snapshot()
 	base.Disconnects = r.disconnects.Load()
 	runID, _ := r.runID.Load().(string)
 	accepted := make(map[string]AcceptedMessage, len(ledger))
@@ -143,6 +171,7 @@ func (r *deliveryRecorder) summarize(base AgentSummary, ledger []AcceptedMessage
 	defer r.mu.Unlock()
 	base.Duplicates = r.duplicates
 	base.OutOfOrder = r.outOfOrder
+	base.CrossClockSkewSamples = r.crossClockSkewSamples
 	base.OrderingViolations = append([]OrderingViolation(nil), r.ordering...)
 	if r.malformed > 0 {
 		base.Errors = append(base.Errors, fmt.Sprintf("malformed run messages=%d", r.malformed))