feat: expose websocket delivery pressure metrics

server/hdl_websock.go

@@ -11,12 +11,17 @@ package main
 
 import (
 	"encoding/json"
+	"errors"
+	"io"
+	"net"
 	"net/http"
 	"sync/atomic"
+	"syscall"
 	"time"
 
 	"github.com/gorilla/websocket"
 	"github.com/tinode/chat/server/logs"
+	immetrics "github.com/tinode/chat/server/metrics"
 	"github.com/tinode/chat/server/store/types"
 )
 
@@ -56,9 +61,10 @@ func (sess *Session) readLoop() {
 		// Read a ClientComMessage
 		_, raw, err := sess.ws.ReadMessage()
 		if err != nil {
-			if websocket.IsUnexpectedCloseError(err, websocket.CloseGoingAway, websocket.CloseAbnormalClosure,
-				websocket.CloseNormalClosure) {
-				logs.Err.Println("ws: readLoop", sess.sid, err)
+			reason := classifyWSError(err)
+			immetrics.IncWSDisconnect(reason)
+			if websocket.IsUnexpectedCloseError(err, websocket.CloseGoingAway, websocket.CloseNormalClosure) {
+				logs.Err.Printf("ws: readLoop sid=%s uid=%s reason=%s err=%v", sess.sid, sess.uid.UserId(), reason, err)
 			}
 			return
 		}
@@ -68,17 +74,15 @@ func (sess *Session) readLoop() {
 }
 
 func (sess *Session) sendMessage(msg any) bool {
-	if len(sess.send) > sendQueueLimit {
-		logs.Err.Println("ws: outbound queue limit exceeded", sess.sid)
+	if depth := len(sess.send); depth > sendQueueLimit {
+		immetrics.ObserveWSWrite("text", "rejected", "queue_limit", time.Now(), 0)
+		logs.Err.Printf("ws: outbound queue limit exceeded sid=%s uid=%s depth=%d capacity=%d",
+			sess.sid, sess.uid.UserId(), depth, cap(sess.send))
 		return false
 	}
 
 	statsInc("OutgoingMessagesWebsockTotal", 1)
-	if err := wsWrite(sess.ws, websocket.TextMessage, msg); err != nil {
-		if websocket.IsUnexpectedCloseError(err, websocket.CloseGoingAway, websocket.CloseAbnormalClosure,
-			websocket.CloseNormalClosure) {
-			logs.Err.Println("ws: writeLoop", sess.sid, err)
-		}
+	if err := sess.writeWSFrame(websocket.TextMessage, msg); err != nil {
 		return false
 	}
 	return true
@@ -100,6 +104,7 @@ func (sess *Session) writeLoop() {
 				// Channel closed.
 				return
 			}
+			sess.observeOutboundDequeue(false)
 			switch v := msg.(type) {
 			case []*ServerComMessage: // batch of unserialized messages
 				for _, msg := range v {
@@ -128,7 +133,7 @@ func (sess *Session) writeLoop() {
 		case msg := <-sess.stop:
 			// Shutdown requested, don't care if the message is delivered
 			if msg != nil {
-				wsWrite(sess.ws, websocket.TextMessage, msg)
+				sess.writeWSFrame(websocket.TextMessage, msg)
 			}
 			return
 
@@ -136,17 +141,70 @@ func (sess *Session) writeLoop() {
 			sess.delSub(topic)
 
 		case <-ticker.C:
-			if err := wsWrite(sess.ws, websocket.PingMessage, nil); err != nil {
-				if websocket.IsUnexpectedCloseError(err, websocket.CloseGoingAway, websocket.CloseAbnormalClosure,
-					websocket.CloseNormalClosure) {
-					logs.Err.Println("ws: writeLoop ping", sess.sid, err)
-				}
+			if err := sess.writeWSFrame(websocket.PingMessage, nil); err != nil {
 				return
 			}
 		}
 	}
 }
 
+func (sess *Session) writeWSFrame(mt int, msg any) error {
+	frame := "text"
+	if mt == websocket.PingMessage {
+		frame = "ping"
+	}
+	payloadBytes := 0
+	if bits, ok := msg.([]byte); ok {
+		payloadBytes = len(bits)
+	}
+	started := time.Now()
+	err := wsWrite(sess.ws, mt, msg)
+	if err == nil {
+		immetrics.ObserveWSWrite(frame, "ok", "none", started, payloadBytes)
+		return nil
+	}
+	reason := classifyWSError(err)
+	immetrics.ObserveWSWrite(frame, "error", reason, started, payloadBytes)
+	logs.Err.Printf("ws: writeLoop sid=%s uid=%s frame=%s queue=%d capacity=%d reason=%s err=%v",
+		sess.sid, sess.uid.UserId(), frame, len(sess.send), cap(sess.send), reason, err)
+	return err
+}
+
+func classifyWSError(err error) string {
+	if err == nil {
+		return "none"
+	}
+	var closeErr *websocket.CloseError
+	if errors.As(err, &closeErr) {
+		switch closeErr.Code {
+		case websocket.CloseNormalClosure:
+			return "close_normal"
+		case websocket.CloseGoingAway:
+			return "close_going_away"
+		case websocket.CloseAbnormalClosure:
+			return "close_abnormal"
+		default:
+			return "close_other"
+		}
+	}
+	var netErr net.Error
+	if errors.As(err, &netErr) && netErr.Timeout() {
+		return "timeout"
+	}
+	switch {
+	case errors.Is(err, syscall.ECONNRESET):
+		return "connection_reset"
+	case errors.Is(err, syscall.EPIPE):
+		return "broken_pipe"
+	case errors.Is(err, net.ErrClosed):
+		return "closed"
+	case errors.Is(err, io.EOF):
+		return "eof"
+	default:
+		return "other"
+	}
+}
+
 // Writes a message with the given message type (mt) and payload.
 func wsWrite(ws *websocket.Conn, mt int, msg any) error {
 	var bits []byte