fix: harden high-fanout delivery under load

This commit is contained in:
richjohnrambo
2026-09-30 23:04:54 +08:00
parent 2b631ab176
commit 19584d0939
14 changed files with 491 additions and 90 deletions
+21
View File
@@ -63,6 +63,27 @@ func startAgentMetrics(address string, run *agentRun) (func(), error) {
_, _, _, disconnects := run.recorder.metricCounts()
return float64(disconnects)
}),
prometheus.NewGaugeFunc(prometheus.GaugeOpts{Name: "im_load_agent_callback_queue_current", Help: "Data callbacks currently waiting to start.", ConstLabels: labels}, func() float64 {
return float64(run.callbackQueue.current.Load())
}),
prometheus.NewGaugeFunc(prometheus.GaugeOpts{Name: "im_load_agent_callback_queue_max", Help: "Peak data callbacks waiting to start.", ConstLabels: labels}, func() float64 {
return float64(run.callbackQueue.maximum.Load())
}),
prometheus.NewGaugeFunc(prometheus.GaugeOpts{Name: "im_load_agent_callback_processing", Help: "Data callbacks currently processing.", ConstLabels: labels}, func() float64 {
return float64(run.callbackQueue.processing.Load())
}),
prometheus.NewCounterFunc(prometheus.CounterOpts{Name: "im_load_agent_callback_queue_enqueued_total", Help: "Data callbacks enqueued.", ConstLabels: labels}, func() float64 {
return float64(run.callbackQueue.enqueued.Load())
}),
prometheus.NewCounterFunc(prometheus.CounterOpts{Name: "im_load_agent_callback_queue_completed_total", Help: "Data callbacks which completed processing.", ConstLabels: labels}, func() float64 {
return float64(run.callbackQueue.completed.Load())
}),
prometheus.NewCounterFunc(prometheus.CounterOpts{Name: "im_load_agent_callback_queue_wait_seconds_total", Help: "Cumulative time spent waiting to start data callbacks.", ConstLabels: labels}, func() float64 {
return float64(run.callbackQueue.waitNanos.Load()) / float64(time.Second)
}),
prometheus.NewGaugeFunc(prometheus.GaugeOpts{Name: "im_load_agent_callback_queue_wait_seconds_max", Help: "Maximum time spent waiting to start a data callback.", ConstLabels: labels}, func() float64 {
return float64(run.callbackQueue.maxWait.Load()) / float64(time.Second)
}),
)
mux := http.NewServeMux()
mux.Handle("/metrics", promhttp.HandlerFor(registry, promhttp.HandlerOpts{}))