fix: harden message seq allocation against stale redis replicas

- use sentinel failover client for group message runtime when sentinel_master_name/sentinel_addrs are configured, so seq counter reads never hit a stale replica after failover - re-seed missing redis seq counter from DB topic seqid instead of trusting the caller in-memory baseline - on unique-key conflict during async persist, re-allocate seq from DB latest, bump redis counter and fix the idempotency record so resends with the same client mid return the final seq - add uk(topic, seqid) unique keys for im_messages in tenant DDL
2026-09-15 12:41:38 +08:00
parent 4f6a996922
commit 5dd0a42002
4 changed files with 205 additions and 32 deletions
@@ -32,6 +32,14 @@ type redisConfig struct {
 	CacheTTL    int    `json:"cache_ttl_seconds"`
 	ShortTTL    int    `json:"short_state_ttl_seconds"`
 	GroupFanout bool   `json:"group_fanout_enabled"`
+
+	// Sentinel-aware connection. When set, the runtime uses a failover client
+	// instead of the single Addr. Required whenever Redis runs behind Sentinel:
+	// a plain client pinned to one address will hit replicas after failover,
+	// where writes fail with READONLY and reads return stale values (a stale
+	// seq counter can hand out already-persisted seq values).
+	SentinelMasterName string   `json:"sentinel_master_name"`
+	SentinelAddrs      []string `json:"sentinel_addrs"`
 }

 type groupMessageUniqueRecord struct {
@@ -77,6 +85,8 @@ type groupMessageRuntime interface {
 	Close() error
 	Claim(tenantID types.TenantID, topic string, from types.Uid, clientMsgID string) (*groupMessageUniqueRecord, bool, error)
 	NextSeq(tenantID types.TenantID, topic string, baseline int64) (int64, error)
+	PeekSeq(tenantID types.TenantID, topic string) (int64, error)
+	BumpSeqFloor(tenantID types.TenantID, topic string, seq int64) error
 	MarkPending(tenantID types.TenantID, topic string, from types.Uid, clientMsgID string, seq int64) error
 	MarkDone(tenantID types.TenantID, topic string, from types.Uid, clientMsgID string, seq int64, msgID string) error
 	MarkFailed(tenantID types.TenantID, topic string, from types.Uid, clientMsgID string) error
@@ -126,10 +136,50 @@ func ReserveMessageSeq(tenantID types.TenantID, topic string, baseline int64) (i
 	if groupMsgRuntime == nil || !tenantID.IsValid() || topic == "" {
 		return 0, false, nil
 	}
-	seq, err := groupMsgRuntime.NextSeq(tenantID, topic, baseline)
+	seq, err := nextMessageSeqWithFloor(tenantID, topic, baseline)
 	return seq, true, err
 }

+// TopicLastSeqID returns the seq last persisted for the topic in the DB, or 0
+// when the topic is unknown. It is used as a floor when re-seeding the Redis
+// seq counter.
+func TopicLastSeqID(tenantID types.TenantID, topic string) int64 {
+	if Topics == nil || !tenantID.IsValid() || topic == "" {
+		return 0
+	}
+	stopic, err := Topics.Get(tenantID, topic)
+	if err != nil || stopic == nil {
+		return 0
+	}
+	return stopic.SeqId
+}
+
+// nextMessageSeqWithFloor allocates the next seq for the topic. When the Redis
+// counter key is missing (Redis restart, eviction or flush) it re-seeds from
+// the DB topic seq instead of trusting the caller's in-memory baseline, which
+// may be stale and would otherwise hand out already-persisted seq values.
+func nextMessageSeqWithFloor(tenantID types.TenantID, topic string, baseline int64) (int64, error) {
+	if baseline < 0 {
+		baseline = 0
+	}
+	if current, err := groupMsgRuntime.PeekSeq(tenantID, topic); err == nil && current <= 0 {
+		if floor := TopicLastSeqID(tenantID, topic); floor > baseline {
+			baseline = floor
+		}
+	}
+	return groupMsgRuntime.NextSeq(tenantID, topic, baseline)
+}
+
+// BumpMessageSeqFloor pushes the Redis seq counter forward to seq (never
+// backward). Used after the async persister re-allocates a seq following a
+// uniqueness conflict so subsequent allocations do not collide again.
+func BumpMessageSeqFloor(tenantID types.TenantID, topic string, seq int64) error {
+	if groupMsgRuntime == nil || !tenantID.IsValid() || topic == "" || seq <= 0 {
+		return nil
+	}
+	return groupMsgRuntime.BumpSeqFloor(tenantID, topic, seq)
+}
+
 func MarkMessageUniqueDone(tenantID types.TenantID, topic string, from types.Uid, clientMsgID string, seq int64, msgID string) error {
 	if groupMsgRuntime == nil || !tenantID.IsValid() || topic == "" || from.IsZero() || clientMsgID == "" {
 		return nil
@@ -407,15 +457,31 @@ func initGroupMessageRuntime(raw json.RawMessage) error {
 	dialTimeout := timeoutOrDefault(cfg.DialMS, 2*time.Second)
 	readTimeout := timeoutOrDefault(cfg.ReadMS, 2*time.Second)
 	writeTimeout := timeoutOrDefault(cfg.WriteMS, 2*time.Second)
-	client := redis.NewClient(&redis.Options{
-		Addr:         cfg.Addr,
-		Username:     cfg.Username,
-		Password:     cfg.Password,
-		DB:           cfg.DB,
-		DialTimeout:  dialTimeout,
-		ReadTimeout:  readTimeout,
-		WriteTimeout: writeTimeout,
-	})
+
+	var client redis.UniversalClient
+	if cfg.SentinelMasterName != "" && len(cfg.SentinelAddrs) > 0 {
+		client = redis.NewFailoverClient(&redis.FailoverOptions{
+			MasterName:       cfg.SentinelMasterName,
+			SentinelAddrs:    cfg.SentinelAddrs,
+			SentinelPassword: cfg.Password,
+			Username:         cfg.Username,
+			Password:         cfg.Password,
+			DB:               cfg.DB,
+			DialTimeout:      dialTimeout,
+			ReadTimeout:      readTimeout,
+			WriteTimeout:     writeTimeout,
+		})
+	} else {
+		client = redis.NewClient(&redis.Options{
+			Addr:         cfg.Addr,
+			Username:     cfg.Username,
+			Password:     cfg.Password,
+			DB:           cfg.DB,
+			DialTimeout:  dialTimeout,
+			ReadTimeout:  readTimeout,
+			WriteTimeout: writeTimeout,
+		})
+	}
 	ctx, cancel := context.WithTimeout(context.Background(), dialTimeout)
 	defer cancel()
 	if err := client.Ping(ctx).Err(); err != nil {
@@ -441,7 +507,7 @@ func timeoutOrDefault(ms int, fallback time.Duration) time.Duration {
 }

 type redisGroupMessageRuntime struct {
-	client *redis.Client
+	client redis.UniversalClient
 	prefix string
 	cfg    redisConfig
 }
@@ -509,6 +575,35 @@ return redis.call("INCR", KEYS[1])
 	return seq, nil
 }

+func (r *redisGroupMessageRuntime) PeekSeq(tenantID types.TenantID, topic string) (int64, error) {
+	value, err := r.client.Get(context.Background(), r.seqKey(tenantID, topic)).Result()
+	if err == redis.Nil {
+		return 0, nil
+	}
+	if err != nil {
+		return 0, err
+	}
+	seq, err := strconv.ParseInt(value, 10, 64)
+	if err != nil {
+		return 0, err
+	}
+	return seq, nil
+}
+
+func (r *redisGroupMessageRuntime) BumpSeqFloor(tenantID types.TenantID, topic string, seq int64) error {
+	if seq <= 0 {
+		return nil
+	}
+	_, err := r.client.Eval(context.Background(), `
+local current = redis.call("GET", KEYS[1])
+if not current or tonumber(current) < tonumber(ARGV[1]) then
+  redis.call("SET", KEYS[1], ARGV[1])
+end
+return redis.call("GET", KEYS[1])
+`, []string{r.seqKey(tenantID, topic)}, seq).Int64()
+	return err
+}
+
 func (r *redisGroupMessageRuntime) MarkDone(tenantID types.TenantID, topic string, from types.Uid, clientMsgID string, seq int64, msgID string) error {
 	if clientMsgID == "" {
 		return nil