pro/im_server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: shard group messages by month

Browse Source
This commit is contained in:
richjohnrambo
2026-09-15 23:53:51 +08:00
parent b7c2536527
commit 139681fadb
25 changed files with 1718 additions and 247 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/store/group_message_runtime.go
+171 -14
View File
@@ -54,6 +54,14 @@ type MessageUniqueClaim struct {
Seq int64
}
// GroupMessageShardRoute describes the Seq range stored in one monthly table.
type GroupMessageShardRoute struct {
Month string `db:"month"`
MinSeq int64 `db:"min_seq"`
MaxSeq int64 `db:"max_seq"`
Count int64 `db:"count"`
}
type groupFanoutEnvelope struct {
OriginNodeID string `json:"origin_node_id"`
Message json.RawMessage `json:"message"`
@@ -84,10 +92,13 @@ type LoginSession struct {
type groupMessageRuntime interface {
Close() error
Claim(tenantID types.TenantID, topic string, from types.Uid, clientMsgID string) (*groupMessageUniqueRecord, bool, error)
NextSeq(tenantID types.TenantID, topic string, baseline int64) (int64, error)
NextSeq(tenantID types.TenantID, topic string, baseline int64, candidateMonth string) (int64, string, error)
PeekSeq(tenantID types.TenantID, topic string) (int64, error)
BumpSeqFloor(tenantID types.TenantID, topic string, seq int64) error
MarkPending(tenantID types.TenantID, topic string, from types.Uid, clientMsgID string, seq int64) error
PutShardRoute(tenantID types.TenantID, topic string, route GroupMessageShardRoute) error
GetShardRoutes(tenantID types.TenantID, topic string) ([]GroupMessageShardRoute, bool, error)
DeleteShardRoutes(tenantID types.TenantID, topic string) error
MarkDone(tenantID types.TenantID, topic string, from types.Uid, clientMsgID string, seq int64, msgID string) error
MarkFailed(tenantID types.TenantID, topic string, from types.Uid, clientMsgID string) error
CacheHotMessage(tenantID types.TenantID, msg *types.Message) error
@@ -132,12 +143,33 @@ func ClaimMessageUnique(tenantID types.TenantID, topic string, from types.Uid, c
return &MessageUniqueClaim{Status: rec.Status, Seq: rec.Seq}, claimed, true, err
}
func ReserveMessageSeq(tenantID types.TenantID, topic string, baseline int64) (int64, bool, error) {
func ReserveMessageSeq(tenantID types.TenantID, topic string, baseline int64, candidateMonth string) (int64, string, bool, error) {
if groupMsgRuntime == nil || !tenantID.IsValid() || topic == "" {
return 0, false, nil
return 0, candidateMonth, false, nil
}
seq, err := nextMessageSeqWithFloor(tenantID, topic, baseline)
return seq, true, err
seq, month, err := nextMessageSeqWithFloor(tenantID, topic, baseline, candidateMonth)
return seq, month, true, err
}
func PutGroupMessageShardRoute(tenantID types.TenantID, topic string, route GroupMessageShardRoute) error {
if groupMsgRuntime == nil || !tenantID.IsValid() || topic == "" || route.Month == "" {
return nil
}
return groupMsgRuntime.PutShardRoute(tenantID, topic, route)
}
func GetGroupMessageShardRoutes(tenantID types.TenantID, topic string) ([]GroupMessageShardRoute, bool, error) {
if groupMsgRuntime == nil || !tenantID.IsValid() || topic == "" {
return nil, false, nil
}
return groupMsgRuntime.GetShardRoutes(tenantID, topic)
}
func DeleteGroupMessageShardRoutes(tenantID types.TenantID, topic string) error {
if groupMsgRuntime == nil || !tenantID.IsValid() || topic == "" {
return nil
}
return groupMsgRuntime.DeleteShardRoutes(tenantID, topic)
}
// TopicLastSeqID returns the seq last persisted for the topic in the DB, or 0
@@ -158,7 +190,7 @@ func TopicLastSeqID(tenantID types.TenantID, topic string) int64 {
// counter key is missing (Redis restart, eviction or flush) it re-seeds from
// the DB topic seq instead of trusting the caller's in-memory baseline, which
// may be stale and would otherwise hand out already-persisted seq values.
func nextMessageSeqWithFloor(tenantID types.TenantID, topic string, baseline int64) (int64, error) {
func nextMessageSeqWithFloor(tenantID types.TenantID, topic string, baseline int64, candidateMonth string) (int64, string, error) {
if baseline < 0 {
baseline = 0
}
@@ -167,7 +199,7 @@ func nextMessageSeqWithFloor(tenantID types.TenantID, topic string, baseline int
baseline = floor
}
}
return groupMsgRuntime.NextSeq(tenantID, topic, baseline)
return groupMsgRuntime.NextSeq(tenantID, topic, baseline, candidateMonth)
}
// BumpMessageSeqFloor pushes the Redis seq counter forward to seq (never
@@ -555,24 +587,129 @@ func (r *redisGroupMessageRuntime) Claim(tenantID types.TenantID, topic string,
return &current, false, nil
}
func (r *redisGroupMessageRuntime) NextSeq(tenantID types.TenantID, topic string, baseline int64) (int64, error) {
func (r *redisGroupMessageRuntime) NextSeq(tenantID types.TenantID, topic string, baseline int64, candidateMonth string) (int64, string, error) {
if baseline < 0 {
baseline = 0
}
redisNow, err := r.client.Time(context.Background()).Result()
if err != nil {
return 0, "", err
}
candidateMonth = redisNow.UTC().Format("200601")
if !validShardMonth(candidateMonth) {
return 0, "", types.ErrMalformed
}
seqKey := r.seqKey(tenantID, topic)
seq, err := r.client.Eval(context.Background(), `
result, err := r.client.Eval(context.Background(), `
local current = redis.call("GET", KEYS[1])
if not current then
redis.call("SET", KEYS[1], ARGV[1])
elseif tonumber(current) < tonumber(ARGV[1]) then
redis.call("SET", KEYS[1], ARGV[1])
end
return redis.call("INCR", KEYS[1])
`, []string{seqKey}, baseline).Int64()
local active_month = redis.call("GET", KEYS[2])
if not active_month or active_month < ARGV[2] then
redis.call("SET", KEYS[2], ARGV[2])
active_month = ARGV[2]
end
local seq = redis.call("INCR", KEYS[1])
return {seq, active_month}
`, []string{seqKey, r.activeMonthKey(tenantID)}, baseline, candidateMonth).Slice()
if err != nil {
return 0, err
return 0, "", err
}
return seq, nil
if len(result) != 2 {
return 0, "", fmt.Errorf("redis group message sequence returned %d values", len(result))
}
seq, err := strconv.ParseInt(fmt.Sprint(result[0]), 10, 64)
if err != nil {
return 0, "", err
}
month := fmt.Sprint(result[1])
if !validShardMonth(month) {
return 0, "", fmt.Errorf("redis returned invalid active month %q", month)
}
return seq, month, nil
}
func (r *redisGroupMessageRuntime) PutShardRoute(tenantID types.TenantID, topic string, route GroupMessageShardRoute) error {
if !validShardMonth(route.Month) || route.MinSeq <= 0 || route.MaxSeq < route.MinSeq {
return types.ErrMalformed
}
_, err := r.client.Eval(context.Background(), `
local min_seq = tonumber(ARGV[2])
local max_seq = tonumber(ARGV[3])
local count = tonumber(ARGV[4])
local meta = redis.call("HGET", KEYS[2], ARGV[1])
if meta then
local old_min, old_max, old_count = string.match(meta, "^(%d+):(%d+):(%d+)$")
if old_min then
min_seq = math.min(min_seq, tonumber(old_min))
max_seq = math.max(max_seq, tonumber(old_max))
if count == 1 then
count = tonumber(old_count) + 1
else
count = math.max(count, tonumber(old_count))
end
end
end
redis.call("ZADD", KEYS[1], min_seq, ARGV[1])
redis.call("HSET", KEYS[2], ARGV[1], min_seq .. ":" .. max_seq .. ":" .. count)
return 1
`, []string{r.shardRouteKey(tenantID, topic), r.shardRouteMetaKey(tenantID, topic)},
route.Month, route.MinSeq, route.MaxSeq, route.Count).Result()
return err
}
func (r *redisGroupMessageRuntime) GetShardRoutes(tenantID types.TenantID, topic string) ([]GroupMessageShardRoute, bool, error) {
ctx := context.Background()
months, err := r.client.ZRevRange(ctx, r.shardRouteKey(tenantID, topic), 0, -1).Result()
if err != nil {
return nil, false, err
}
if len(months) == 0 {
return nil, false, nil
}
values, err := r.client.HMGet(ctx, r.shardRouteMetaKey(tenantID, topic), months...).Result()
if err != nil {
return nil, false, err
}
routes := make([]GroupMessageShardRoute, 0, len(months))
for i, month := range months {
if values[i] == nil {
continue
}
parts := strings.Split(fmt.Sprint(values[i]), ":")
if len(parts) != 3 {
continue
}
minSeq, minErr := strconv.ParseInt(parts[0], 10, 64)
maxSeq, maxErr := strconv.ParseInt(parts[1], 10, 64)
count, countErr := strconv.ParseInt(parts[2], 10, 64)
if minErr == nil && maxErr == nil && countErr == nil && validShardMonth(month) {
routes = append(routes, GroupMessageShardRoute{Month: month, MinSeq: minSeq, MaxSeq: maxSeq, Count: count})
}
}
return routes, len(routes) > 0, nil
}
func (r *redisGroupMessageRuntime) DeleteShardRoutes(tenantID types.TenantID, topic string) error {
return r.client.Del(context.Background(), r.shardRouteKey(tenantID, topic), r.shardRouteMetaKey(tenantID, topic)).Err()
}
func validShardMonth(month string) bool {
if len(month) != 6 {
return false
}
for i := range month {
if month[i] < '0' || month[i] > '9' {
return false
}
}
parsed, err := time.Parse("200601", month)
return err == nil && parsed.Format("200601") == month
}
func (r *redisGroupMessageRuntime) PeekSeq(tenantID types.TenantID, topic string) (int64, error) {
@@ -1182,7 +1319,19 @@ func expandRanges(ranges []types.Range, limit int) []int64 {
}
func (r *redisGroupMessageRuntime) seqKey(tenantID types.TenantID, topic string) string {
return r.key(tenantID, "topic", topic, "last_seq")
return r.shardKey(tenantID, "topic", topic, "last_seq")
}
func (r *redisGroupMessageRuntime) activeMonthKey(tenantID types.TenantID) string {
return r.shardKey(tenantID, "groupmsg", "active_month")
}
func (r *redisGroupMessageRuntime) shardRouteKey(tenantID types.TenantID, topic string) string {
return r.shardKey(tenantID, "groupmsg", "route", topic)
}
func (r *redisGroupMessageRuntime) shardRouteMetaKey(tenantID types.TenantID, topic string) string {
return r.shardKey(tenantID, "groupmsg", "route_meta", topic)
}
func (r *redisGroupMessageRuntime) uniqueKey(tenantID types.TenantID, topic string, from types.Uid, clientMsgID string) string {
@@ -1276,3 +1425,11 @@ func (r *redisGroupMessageRuntime) key(tenantID types.TenantID, parts ...string)
all = append(all, parts...)
return strings.Join(all, ":")
}
func (r *redisGroupMessageRuntime) shardKey(tenantID types.TenantID, parts ...string) string {
prefix := strings.TrimSuffix(r.prefix, ":")
slot := "{" + strconv.FormatInt(int64(tenantID), 10) + "}"
all := []string{prefix, slot}
all = append(all, parts...)
return strings.Join(all, ":")
}