fix: preserve cross-pod message order

server/cluster.go

@@ -489,14 +489,16 @@ func clusterReqShard(msg *ClusterReq, count int) int {
 	return int(h.Sum32() % uint32(count))
 }
 
-// masterToProxyAsync forwards response from topic master to topic proxy
-// in a fire-and-forget manner.
-func (n *ClusterNode) masterToProxyAsync(msg *ClusterResp) error {
-	var unused bool
-	if c := n.callAsync("Cluster.TopicProxy", msg, &unused, nil); c.Error != nil {
-		return c.Error
+// masterToProxy forwards responses synchronously. A multiplexing session drains
+// one topic in order, so waiting for each RPC prevents the RPC server from
+// concurrently enqueueing later sequence IDs before earlier ones.
+func (n *ClusterNode) masterToProxy(msg *ClusterResp) error {
+	var rejected bool
+	err := n.call("Cluster.TopicProxy", msg, &rejected)
+	if err == nil && rejected {
+		return errClusterOverloaded
 	}
-	return nil
+	return err
 }
 
 // route routes server message within the cluster.
@@ -1437,7 +1439,7 @@ func (sess *Session) clusterWriteLoop(forTopic string) {
 			srvMsg.RcptTo = forTopic
 			response.RcptTo = forTopic
 
-			if err := sess.clnode.masterToProxyAsync(response); err != nil {
+			if err := sess.clnode.masterToProxy(response); err != nil {
 				logs.Warn.Printf("cluster: response to proxy failed \"%s\": %s", sess.sid, err.Error())
 				return
 			}
@@ -1454,6 +1456,12 @@ func (sess *Session) clusterWriteLoop(forTopic string) {
 		case <-sess.detach:
 			return
 		default:
+			sess.clusterWriteScheduled.Store(false)
+			// An enqueue can race with releasing ownership. Either the enqueuer
+			// schedules the next drain or this check does; the CAS prevents both.
+			if len(sess.send) > 0 {
+				sess.scheduleClusterWriteLoop()
+			}
 			terminate = false
 			return
 		}