fix: harden high-fanout delivery under load

loadtest/im-pressure/callback_queue.go

@@ -0,0 +1,70 @@
+package main
+
+import (
+	"sync/atomic"
+	"time"
+)
+
+type callbackQueueMetrics struct {
+	current    atomic.Int64
+	processing atomic.Int64
+	maximum    atomic.Int64
+	enqueued   atomic.Int64
+	completed  atomic.Int64
+	waitNanos  atomic.Int64
+	maxWait    atomic.Int64
+}
+
+func (m *callbackQueueMetrics) enqueue() {
+	if m == nil {
+		return
+	}
+	current := m.current.Add(1)
+	m.enqueued.Add(1)
+	updateAtomicMax(&m.maximum, current)
+}
+
+func (m *callbackQueueMetrics) startCallback(queuedAt time.Time) {
+	if m == nil {
+		return
+	}
+	wait := time.Since(queuedAt).Nanoseconds()
+	if wait < 0 {
+		wait = 0
+	}
+	m.current.Add(-1)
+	m.processing.Add(1)
+	m.waitNanos.Add(wait)
+	updateAtomicMax(&m.maxWait, wait)
+}
+
+func (m *callbackQueueMetrics) finishCallback() {
+	if m == nil {
+		return
+	}
+	m.processing.Add(-1)
+	m.completed.Add(1)
+}
+
+func (m *callbackQueueMetrics) snapshot() CallbackQueueSnapshot {
+	if m == nil {
+		return CallbackQueueSnapshot{}
+	}
+	return CallbackQueueSnapshot{
+		Current: m.current.Load(), Processing: m.processing.Load(), Maximum: m.maximum.Load(), Enqueued: m.enqueued.Load(),
+		Completed: m.completed.Load(), WaitSecondsTotal: float64(m.waitNanos.Load()) / float64(time.Second),
+		WaitSecondsMax: float64(m.maxWait.Load()) / float64(time.Second),
+	}
+}
+
+func (m *callbackQueueMetrics) isDrained() bool {
+	return m == nil || (m.current.Load() == 0 && m.processing.Load() == 0)
+}
+
+func updateAtomicMax(target *atomic.Int64, value int64) {
+	for current := target.Load(); value > current; current = target.Load() {
+		if target.CompareAndSwap(current, value) {
+			return
+		}
+	}
+}