pro/im_server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'dev' of https://git.118im.com/im_team/im_server into dev

Browse Source
This commit is contained in:
Austin
2026-09-16 10:23:33 +08:00
parent 4280b535c2 68b196215f
commit 846615db69
37 changed files with 3311 additions and 273 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/db/adapter.go
+3
View File
@@ -332,6 +332,7 @@ type TenantBusinessAdapter interface {
TenantTopicUpdateOnMessage(t.TenantID, string, *t.Message) error
TenantTopicUpdateSubCnt(t.TenantID, string) error
TenantTopicUpdate(t.TenantID, string, map[string]any) error
TenantTopicBumpMutationVersion(t.TenantID, string, time.Time) (int64, error)
TenantTopicOwnerChange(t.TenantID, string, t.Uid) error
TenantSubscriptionGet(t.TenantID, string, t.Uid, bool) (*t.Subscription, error)
@@ -351,6 +352,7 @@ type TenantBusinessAdapter interface {
TenantMessageDeleteList(t.TenantID, string, *t.DelMessage) error
TenantMessageGetDeleted(t.TenantID, string, t.Uid, *t.QueryOpt) ([]t.DelMessage, error)
TenantMessageSearch(t.TenantID, string, t.Uid, string, string, string, int) ([]t.Message, string, error)
TenantMessageMutations(t.TenantID, string, t.Uid, int64, int) ([]t.MessageMutation, string, bool, int64, error)
TenantGroupMessageSave(t.TenantID, *t.Message) error
TenantGroupMessageGetByClientID(t.TenantID, string, t.Uid, string) (*t.Message, error)
TenantGroupMessageGetOne(t.TenantID, string, int64) (*t.Message, error)
@@ -387,6 +389,7 @@ type TenantBusinessAdapter interface {
TenantGroupAnnouncementPut(t.TenantID, *t.GroupAnnouncement, int) (*t.GroupAnnouncement, error)
TenantGroupAnnouncementRead(t.TenantID, string, string, t.Uid, time.Time) error
TenantGroupMuteList(t.TenantID, string, string, int) ([]t.GroupMute, string, error)
TenantGroupMuteGet(t.TenantID, string, t.Uid) (*t.GroupMute, error)
TenantGroupMuteUpsert(t.TenantID, *t.GroupMute) (*t.GroupMute, error)
TenantGroupMuteDelete(t.TenantID, string, t.Uid, t.Uid, time.Time) error
TenantGroupFileList(t.TenantID, string, string, string, string, int) ([]t.GroupFile, string, error)
server/db/mysql/adapter.go
+38 -43
View File
@@ -9,7 +9,6 @@ import (
"database/sql"
"encoding/json"
"errors"
"fmt"
"hash/fnv"
"sort"
"strconv"
@@ -49,6 +48,7 @@ type adapter struct {
tenantRegistryDB string
tenantAdapters map[t.TenantID]*adapter
tenantMu sync.Mutex
groupShardWindow string
}
const (
@@ -375,7 +375,7 @@ func (a *adapter) CheckDbVersion() error {
}
// Version returns adapter version.
func (adapter) Version() int {
func (*adapter) Version() int {
return adpVersion
}
@@ -1819,26 +1819,19 @@ func (a *adapter) UserCreate(user *t.User) error {
user.Phone = canonicalPhone
phone = canonicalPhone
}
releaseAccountNoLock, err := lockUserAccountNoSequence(ctx, tx)
accountNo, hasAccountNo, err := nextUserAccountNo(ctx, tx)
if err != nil {
return err
}
defer releaseAccountNoLock()
accountNo, err := nextUserAccountNo(ctx, tx)
if err != nil {
return err
insert := "INSERT INTO im_users(id,username,createdat,updatedat,state,access,phone,public,trusted) VALUES(?,?,?,?,?,?,?,?,?)"
args := []any{decoded_uid, user.Username, user.CreatedAt, user.UpdatedAt, user.State, user.Access, phone,
common.ToJSON(user.Public), common.ToJSON(user.Trusted)}
if hasAccountNo {
insert = "INSERT INTO im_users(id,account_no,username,createdat,updatedat,state,access,phone,public,trusted) VALUES(?,?,?,?,?,?,?,?,?,?)"
args = []any{decoded_uid, accountNo, user.Username, user.CreatedAt, user.UpdatedAt, user.State, user.Access, phone,
common.ToJSON(user.Public), common.ToJSON(user.Trusted)}
}
if _, err = tx.Exec("INSERT INTO im_users(id,account_no,username,createdat,updatedat,state,access,phone,public,trusted) VALUES(?,?,?,?,?,?,?,?,?,?)",
decoded_uid,
accountNo,
user.Username,
user.CreatedAt,
user.UpdatedAt,
user.State,
user.Access,
phone,
common.ToJSON(user.Public),
common.ToJSON(user.Trusted)); err != nil {
if _, err = tx.Exec(insert, args...); err != nil {
if isDupe(err) {
return t.ErrDuplicate
}
@@ -1859,35 +1852,36 @@ func (a *adapter) UserCreate(user *t.User) error {
return tx.Commit()
}
func lockUserAccountNoSequence(ctx context.Context, tx *sqlx.Tx) (func(), error) {
var locked sql.NullInt64
if err := tx.QueryRowContext(ctx, "SELECT GET_LOCK(CONCAT(DATABASE(), ':im_users_account_no'), 10)").Scan(&locked); err != nil {
if isPXCStrictModeNamedLockError(err) {
return func() {}, nil
}
return nil, err
// nextUserAccountNo allocates a tenant-local, human-facing account number.
// The sequence row serializes concurrent registrations across server nodes.
func nextUserAccountNo(ctx context.Context, tx *sqlx.Tx) (uint32, bool, error) {
var columns int
if err := tx.GetContext(ctx, &columns, `SELECT COUNT(*) FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME='im_users' AND COLUMN_NAME='account_no'`); err != nil {
return 0, false, err
}
if !locked.Valid || locked.Int64 != 1 {
return nil, fmt.Errorf("lock user account number sequence timeout")
if columns == 0 {
return 0, false, nil
}
return func() {
_, _ = tx.ExecContext(ctx, "SELECT RELEASE_LOCK(CONCAT(DATABASE(), ':im_users_account_no'))")
}, nil
}
func nextUserAccountNo(ctx context.Context, tx *sqlx.Tx) (int64, error) {
var current sql.NullInt64
err := tx.QueryRowContext(ctx, "SELECT account_no FROM im_users ORDER BY account_no DESC LIMIT 1 FOR UPDATE").Scan(&current)
if err == sql.ErrNoRows {
return 1000, nil
}
result, err := tx.ExecContext(ctx, `UPDATE im_user_account_sequences
SET next_value=LAST_INSERT_ID(next_value+1) WHERE id=1`)
if err != nil {
return 0, err
return 0, true, err
}
if !current.Valid || current.Int64 < 999 {
return 1000, nil
if affected, affectedErr := result.RowsAffected(); affectedErr != nil {
return 0, true, affectedErr
} else if affected != 1 {
return 0, true, errors.New("mysql: user account sequence is not initialized")
}
return current.Int64 + 1, nil
var next uint64
if err = tx.GetContext(ctx, &next, "SELECT LAST_INSERT_ID()"); err != nil {
return 0, true, err
}
if next > uint64(^uint32(0)) {
return 0, true, errors.New("mysql: user account number exhausted")
}
return uint32(next), true, nil
}
// TenantAuthAddRecord adds the user's Basic authentication record.
@@ -2747,7 +2741,7 @@ func (a *adapter) TopicGet(topic string) (*t.Topic, error) {
// Fetch topic by name
var tt = new(t.Topic)
if err := a.db.GetContext(ctx, tt,
"SELECT createdat,updatedat,state,stateat,touchedat,name AS id,usebt,access,owner,seqid,delid,subcnt,public,trusted,tags,aux "+
"SELECT createdat,updatedat,state,stateat,touchedat,name AS id,usebt,access,owner,seqid,delid,subcnt,IFNULL(mutation_version,0) AS mutation_version,public,trusted,tags,aux "+
"FROM im_topics WHERE name=?", topic); err != nil {
if err == sql.ErrNoRows {
// Nothing found - clear the error
@@ -2901,7 +2895,7 @@ func (a *adapter) TopicsForUser(uid t.Uid, keepDeleted bool, opts *t.QueryOpt) (
// Fetch grp topics and join to subscriptions.
if len(topq) > 0 {
q = "SELECT updatedat,state,touchedat,name AS id,usebt,access,seqid,delid,subcnt,public,trusted " +
q = "SELECT updatedat,state,touchedat,name AS id,usebt,access,seqid,delid,subcnt,IFNULL(mutation_version,0) AS mutation_version,public,trusted " +
"FROM im_topics WHERE name IN (?)"
q, args, _ = sqlx.In(q, topq)
@@ -2943,6 +2937,7 @@ func (a *adapter) TopicsForUser(uid t.Uid, keepDeleted bool, opts *t.QueryOpt) (
sub.SetState(top.State)
sub.SetTouchedAt(top.TouchedAt)
sub.SetSeqId(top.SeqId)
sub.Private = ensurePrivateMutationVersion(sub.Private, top.MutationVersion)
if t.GetTopicCat(sub.Topic) == t.TopicCatGrp {
sub.SetSubCnt(top.SubCnt)
sub.SetPublic(common.FromJSON(top.Public))
server/db/mysql/group_message_shards.go
+208
View File
@@ -0,0 +1,208 @@
//go:build mysql
// +build mysql
package mysql
import (
"context"
"fmt"
"time"
"github.com/jmoiron/sqlx"
"github.com/tinode/chat/server/store"
t "github.com/tinode/chat/server/store/types"
)
const legacyGroupMessageTable = "im_group_messages"
func ensureGroupMessageShardInfrastructure(exec sqlExecutor) error {
statements := []string{
`CREATE TABLE IF NOT EXISTS im_group_message_shards (
shard_month CHAR(6) NOT NULL, table_name VARCHAR(64) NOT NULL,
shard_start DATETIME(3) NOT NULL, shard_end DATETIME(3) NOT NULL,
status VARCHAR(16) NOT NULL DEFAULT 'prepared', row_count BIGINT NOT NULL DEFAULT 0,
min_created_at DATETIME(3), max_created_at DATETIME(3),
created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
updated_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3),
PRIMARY KEY(shard_month), UNIQUE INDEX uk_im_group_message_shards_table(table_name),
INDEX idx_im_group_message_shards_status(status, shard_month)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci`,
`CREATE TABLE IF NOT EXISTS im_group_message_topic_shards (
topic VARCHAR(64) NOT NULL, shard_month CHAR(6) NOT NULL,
min_seq BIGINT NOT NULL, max_seq BIGINT NOT NULL, message_count BIGINT NOT NULL DEFAULT 0,
min_created_at DATETIME(3) NOT NULL, max_created_at DATETIME(3) NOT NULL,
created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
updated_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3),
PRIMARY KEY(topic, shard_month), INDEX idx_im_group_message_topic_shards_seq(topic, max_seq),
INDEX idx_im_group_message_topic_shards_month(shard_month, topic)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci`,
`CREATE TABLE IF NOT EXISTS im_group_message_dedup (
topic VARCHAR(64) NOT NULL, from_user BIGINT NOT NULL, client_msg_id VARCHAR(64) NOT NULL,
seqid BIGINT NOT NULL, shard_month CHAR(6) NOT NULL, message_id BIGINT,
status VARCHAR(16) NOT NULL DEFAULT 'pending', expires_at DATETIME(3) NOT NULL,
created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
updated_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3),
PRIMARY KEY(topic, from_user, client_msg_id), INDEX idx_im_group_message_dedup_expires(expires_at),
INDEX idx_im_group_message_dedup_topic_seq(topic, seqid)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci`,
}
for _, statement := range statements {
if _, err := exec.Exec(statement); err != nil {
return err
}
}
return ensureGroupMessageShardWindow(exec, time.Now().UTC())
}
func ensureGroupMessageShardWindow(exec sqlExecutor, now time.Time) error {
first := time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, time.UTC)
for _, offset := range []int{-1, 0, 1, 2} {
start := first.AddDate(0, offset, 0)
status := "prepared"
if offset == -1 {
status = "sealing"
} else if offset == 0 {
status = "writable"
}
if err := ensureGroupMessageShard(exec, start, status); err != nil {
return err
}
}
_, err := exec.Exec("UPDATE im_group_message_shards SET status='sealed' WHERE status='sealing' AND shard_end<?", first.AddDate(0, 0, -7))
return err
}
func (a *adapter) ensureCurrentGroupMessageShardWindow() error {
month := time.Now().UTC().Format("200601")
a.tenantMu.Lock()
defer a.tenantMu.Unlock()
if a.groupShardWindow == month {
return nil
}
if err := ensureGroupMessageShardWindow(a.db, time.Now().UTC()); err != nil {
return err
}
a.groupShardWindow = month
return nil
}
func ensureGroupMessageShard(exec sqlExecutor, start time.Time, status string) error {
month := start.UTC().Format("200601")
table, ok := groupMessageShardTable(month)
if !ok {
return fmt.Errorf("mysql: invalid group message shard month %q", month)
}
ddl := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s (
message_id BIGINT NOT NULL, topic VARCHAR(64) NOT NULL, group_id BIGINT NOT NULL,
seqid BIGINT NOT NULL, from_user BIGINT NOT NULL, client_msg_id VARCHAR(64),
msg_type VARCHAR(32) NOT NULL DEFAULT 'text', status VARCHAR(32) NOT NULL DEFAULT 'normal',
content JSON, head JSON, mentioned_user_ids JSON, mention_all TINYINT(1) NOT NULL DEFAULT 0,
reply_seqid BIGINT, root_seqid BIGINT, file_count INT NOT NULL DEFAULT 0,
content_size INT NOT NULL DEFAULT 0, has_risk TINYINT(1) NOT NULL DEFAULT 0,
audit_status VARCHAR(32) NOT NULL DEFAULT 'pass', version INT NOT NULL DEFAULT 1,
recalled_at DATETIME(3), recalled_by BIGINT, deleted_at DATETIME(3), expires_at DATETIME(3),
created_at DATETIME(3) NOT NULL, updated_at DATETIME(3) NOT NULL,
PRIMARY KEY(message_id), UNIQUE INDEX uk_%s_group_seq(group_id, seqid),
UNIQUE INDEX uk_%s_topic_seq(topic, seqid),
UNIQUE INDEX uk_%s_sender_client(topic, from_user, client_msg_id),
INDEX idx_%s_group_status_seq(group_id, status, seqid),
INDEX idx_%s_topic_created(topic, created_at), INDEX idx_%s_topic_from_seq(topic, from_user, seqid),
INDEX idx_%s_audit_created(audit_status, created_at), INDEX idx_%s_expires(expires_at)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci`,
table, month, month, month, month, month, month, month, month)
if _, err := exec.Exec(ddl); err != nil {
return err
}
end := start.AddDate(0, 1, 0)
_, err := exec.Exec(`INSERT INTO im_group_message_shards
(shard_month,table_name,shard_start,shard_end,status)
VALUES(?,?,?,?,?) ON DUPLICATE KEY UPDATE table_name=VALUES(table_name),
shard_start=VALUES(shard_start),shard_end=VALUES(shard_end),
status=IF(status IN ('sealed','archived'),status,VALUES(status))`,
month, table, start, end, status)
return err
}
func groupMessageShardTable(month string) (string, bool) {
if len(month) != 6 {
return "", false
}
for i := range month {
if month[i] < '0' || month[i] > '9' {
return "", false
}
}
parsed, err := time.Parse("200601", month)
if err != nil || parsed.Format("200601") != month {
return "", false
}
return "im_group_messages_" + month, true
}
func groupMessageTablesForTopic(q sqlx.QueryerContext, ctx context.Context, tenantID t.TenantID, topic string) ([]string, error) {
routes, cached, err := store.GetGroupMessageShardRoutes(tenantID, topic)
redisAvailable := err == nil
if err != nil {
cached = false
}
if !cached {
routes = nil
if err = sqlx.SelectContext(ctx, q, &routes,
`SELECT shard_month AS month,min_seq,max_seq,message_count AS count
FROM im_group_message_topic_shards WHERE topic=? ORDER BY shard_month DESC`, topic); err != nil {
return nil, err
}
if redisAvailable {
for _, route := range routes {
_ = store.PutGroupMessageShardRoute(tenantID, topic, route)
}
}
}
tables := make([]string, 0, len(routes)+1)
for _, route := range routes {
if table, ok := groupMessageShardTable(route.Month); ok {
tables = append(tables, table)
}
}
return append(tables, legacyGroupMessageTable), nil
}
func groupMessageTablesForSeq(q sqlx.QueryerContext, ctx context.Context, tenantID t.TenantID, topic string, seq int64) ([]string, error) {
routes, cached, err := store.GetGroupMessageShardRoutes(tenantID, topic)
redisAvailable := err == nil
if err != nil {
cached = false
}
if !cached {
routes = nil
if err = sqlx.SelectContext(ctx, q, &routes,
`SELECT shard_month AS month,min_seq,max_seq,message_count AS count
FROM im_group_message_topic_shards WHERE topic=? AND min_seq<=? AND max_seq>=?
ORDER BY shard_month DESC`, topic, seq, seq); err != nil {
return nil, err
}
if redisAvailable {
for _, route := range routes {
_ = store.PutGroupMessageShardRoute(tenantID, topic, route)
}
}
}
tables := make([]string, 0, 2)
for _, route := range routes {
if route.MinSeq <= seq && route.MaxSeq >= seq {
if table, ok := groupMessageShardTable(route.Month); ok {
tables = append(tables, table)
}
}
}
return append(tables, legacyGroupMessageTable), nil
}
func groupMessageMonthFromTable(table string) string {
const prefix = "im_group_messages_"
if len(table) == len(prefix)+6 && table[:len(prefix)] == prefix {
return table[len(prefix):]
}
return ""
}
server/db/mysql/group_schema.go
+1 -1
View File
@@ -167,5 +167,5 @@ func ensureIMGroupBusinessTables(exec sqlExecutor) error {
return err
}
}
return nil
return ensureGroupMessageShardInfrastructure(exec)
}
server/db/mysql/group_schema_test.go
+18 -2
View File
@@ -22,8 +22,8 @@ func TestEnsureIMGroupBusinessTablesUsesSelectedTenantDatabase(t *testing.T) {
if err := ensureIMGroupBusinessTables(exec); err != nil {
t.Fatal(err)
}
if len(exec.statements) != 13 {
t.Fatalf("executed %d group schema statements, want 13", len(exec.statements))
if len(exec.statements) != 25 {
t.Fatalf("executed %d group schema statements, want 25", len(exec.statements))
}
for _, statement := range exec.statements {
if strings.Contains(strings.ToLower(statement), "tinode.") || strings.Contains(strings.ToLower(statement), "create database") {
@@ -46,4 +46,20 @@ func TestEnsureIMGroupBusinessTablesUsesSelectedTenantDatabase(t *testing.T) {
if !strings.Contains(joined, "json_set(public, '$.fn'") {
t.Fatal("group schema is missing historical public.fn backfill")
}
for _, expected := range []string{"im_group_message_shards", "im_group_message_topic_shards", "im_group_message_dedup", "primary key(message_id)"} {
if !strings.Contains(joined, expected) {
t.Fatalf("group shard schema is missing %q", expected)
}
}
}
func TestGroupMessageShardTableRejectsUnsafeMonth(t *testing.T) {
if table, ok := groupMessageShardTable("202609"); !ok || table != "im_group_messages_202609" {
t.Fatalf("unexpected valid shard: table=%q ok=%v", table, ok)
}
for _, month := range []string{"", "20269", "202613", "2026_09", "202609;DROP TABLE im_users"} {
if table, ok := groupMessageShardTable(month); ok || table != "" {
t.Fatalf("unsafe shard month accepted: month=%q table=%q", month, table)
}
}
}
server/db/mysql/tenant_adapter.go
+485 -120
View File
@@ -172,6 +172,15 @@ func ensureP2PPrivatePeerRemark(private any, peerID, remark string) any {
return data
}
func ensurePrivateMutationVersion(private any, version int64) any {
data, _ := private.(map[string]any)
if data == nil {
data = map[string]any{}
}
data["mutation_version"] = version
return data
}
func clearProfilePublicFields(public map[string]any) {
for _, key := range []string{"display_name", "nickname", "fn", "name", "avatar_file_id", "photo", "avatar", "signature", "gender", "region", "public_account"} {
delete(public, key)
@@ -261,6 +270,9 @@ func finalizeTenantMessage(id t.TenantID, msg t.Message, recalledBy sql.NullInt6
func normalizeGroupMessage(id t.TenantID, msg t.Message, recalledBy sql.NullInt64) t.Message {
msg.TenantID = id
if rawID, err := strconv.ParseInt(msg.Id, 10, 64); err == nil && rawID > 0 {
msg.SetUid(store.EncodeUid(rawID))
}
msg.From = common.EncodeUidString(msg.From).String()
if recalledBy.Valid {
msg.RecalledBy = common.EncodeUidString(strconv.FormatInt(recalledBy.Int64, 10)).String()
@@ -309,6 +321,17 @@ func messageHeadBool(head t.KVMap, key string) bool {
}
}
func (a *adapter) tenantTopicBumpMutationVersionTx(ctx context.Context, tx *sqlx.Tx, id t.TenantID, topic string, at time.Time) error {
if a.tenantScopedDB {
_, err := tx.ExecContext(ctx, "UPDATE im_topics SET mutation_version=mutation_version+1,updatedat=?,touchedat=? WHERE name=?",
at, at, topic)
return err
}
_, err := tx.ExecContext(ctx, "UPDATE im_topics SET mutation_version=mutation_version+1,updatedat=?,touchedat=? WHERE tenant_id=? AND name=?",
at, at, id, topic)
return err
}
func stampTenantFile(id t.TenantID, file *t.FileDef) *t.FileDef {
if file != nil {
file.TenantID = id
@@ -888,7 +911,7 @@ func (a *adapter) TenantTopicGet(id t.TenantID, topic string) (*t.Topic, error)
}
var top t.Topic
err = a.db.GetContext(ctx, &top,
"SELECT tenant_id,createdat,updatedat,state,stateat,touchedat,name AS id,usebt,access,owner,seqid,delid,subcnt,public,trusted,tags,aux "+
"SELECT tenant_id,createdat,updatedat,state,stateat,touchedat,name AS id,usebt,access,owner,seqid,delid,subcnt,IFNULL(mutation_version,0) AS mutation_version,public,trusted,tags,aux "+
"FROM im_topics WHERE tenant_id=? AND name=?", id, topic)
if err == sql.ErrNoRows {
return nil, nil
@@ -1013,7 +1036,7 @@ func (a *adapter) TenantTopicsForUser(id t.TenantID, uid t.Uid, keepDeleted bool
return nil, err
}
if len(topq) > 0 {
q = "SELECT tenant_id,updatedat,state,touchedat,name AS id,usebt,access,seqid,delid,subcnt,public,trusted FROM im_topics WHERE tenant_id=? AND name IN (?)"
q = "SELECT tenant_id,updatedat,state,touchedat,name AS id,usebt,access,seqid,delid,subcnt,IFNULL(mutation_version,0) AS mutation_version,public,trusted FROM im_topics WHERE tenant_id=? AND name IN (?)"
q, args, _ = sqlx.In(q, id, topq)
if !keepDeleted {
q += " AND state!=?"
@@ -1042,6 +1065,7 @@ func (a *adapter) TenantTopicsForUser(id t.TenantID, uid t.Uid, keepDeleted bool
sub.SetState(top.State)
sub.SetTouchedAt(top.TouchedAt)
sub.SetSeqId(top.SeqId)
sub.Private = ensurePrivateMutationVersion(sub.Private, top.MutationVersion)
if t.GetTopicCat(sub.Topic) == t.TopicCatGrp {
sub.SetSubCnt(top.SubCnt)
sub.SetPublic(common.FromJSON(top.Public))
@@ -1369,44 +1393,70 @@ func tenantMessageDeleteList(tx *sqlx.Tx, tenantID t.TenantID, topic string, toD
return nil
}
func groupMessageDeleteList(tx *sqlx.Tx, topic string, toDel *t.DelMessage) error {
func groupMessageDeleteList(tx *sqlx.Tx, tenantID t.TenantID, topic string, toDel *t.DelMessage) error {
var err error
ctx := context.Background()
tables, err := groupMessageTablesForTopic(tx, ctx, tenantID, topic)
if err != nil {
return err
}
if toDel == nil {
if _, err = tx.Exec("DELETE FROM im_dellog WHERE topic=?", topic); err != nil {
return err
}
_, err = tx.Exec("DELETE FROM im_group_messages WHERE topic=?", topic)
for _, table := range tables {
if _, err = tx.Exec("DELETE FROM "+table+" WHERE topic=?", topic); err != nil {
return err
}
}
if _, err = tx.Exec("DELETE FROM im_group_message_topic_shards WHERE topic=?", topic); err != nil {
return err
}
_, err = tx.Exec("DELETE FROM im_group_message_dedup WHERE topic=?", topic)
return err
}
delRanges := toDel.SeqIdRanges
if toDel.DeletedFor == "" {
where := "m.topic=?"
args := []any{topic}
if len(delRanges) > 0 {
rSQL, rArgs := common.RangesToSql(delRanges)
where += " AND m.seqid " + rSQL
args = append(args, rArgs...)
}
where += " AND m.deleted_at IS NULL"
if newerThan := toDel.GetNewerThan(); newerThan != nil {
where += " AND m.created_at>?"
args = append(args, newerThan)
}
var seqIDs []int64
if err = tx.Select(&seqIDs, "SELECT seqid FROM im_group_messages AS m WHERE "+where, args...); err != nil {
return err
seqIDs := make([]int64, 0)
for _, table := range tables {
where := "m.topic=?"
args := []any{topic}
if len(delRanges) > 0 {
rSQL, rArgs := common.RangesToSql(delRanges)
where += " AND m.seqid " + rSQL
args = append(args, rArgs...)
}
where += " AND m.deleted_at IS NULL"
if newerThan := toDel.GetNewerThan(); newerThan != nil {
where += " AND m.created_at>?"
args = append(args, newerThan)
}
var tableSeqIDs []int64
if err = tx.Select(&tableSeqIDs, "SELECT seqid FROM "+table+" AS m WHERE "+where, args...); err != nil {
return err
}
seqIDs = append(seqIDs, tableSeqIDs...)
}
if len(seqIDs) == 0 {
return nil
}
sort.Slice(seqIDs, func(i, j int) bool { return seqIDs[i] < seqIDs[j] })
delRanges = t.SliceToRanges(seqIDs)
uniqueSeqIDs := seqIDs[:0]
for _, seqID := range seqIDs {
if len(uniqueSeqIDs) == 0 || uniqueSeqIDs[len(uniqueSeqIDs)-1] != seqID {
uniqueSeqIDs = append(uniqueSeqIDs, seqID)
}
}
delRanges = t.SliceToRanges(uniqueSeqIDs)
rSQL, rArgs := common.RangesToSql(delRanges)
where = "m.topic=? AND m.seqid " + rSQL
args = append([]any{topic}, rArgs...)
if _, err = tx.Exec("UPDATE im_group_messages AS m SET m.updated_at=?,m.deleted_at=?,m.status='deleted',m.from_user=0,m.head=NULL,m.content=NULL,m.content_size=0 WHERE "+
where, append([]any{t.TimeNow(), t.TimeNow()}, args...)...); err != nil {
return err
where := "m.topic=? AND m.seqid " + rSQL
args := append([]any{topic}, rArgs...)
now := t.TimeNow()
for _, table := range tables {
if _, err = tx.Exec("UPDATE "+table+" AS m SET m.updated_at=?,m.deleted_at=?,m.status='deleted',m.from_user=0,m.head=NULL,m.content=NULL,m.content_size=0 WHERE "+
where, append([]any{now, now}, args...)...); err != nil {
return err
}
}
}
stmt, err := tx.Prepare("INSERT INTO im_dellog(topic,deletedfor,delid,low,hi) VALUES(?,?,?,?,?)")
@@ -1460,7 +1510,7 @@ func (a *adapter) TenantTopicDelete(id t.TenantID, topic string, isChan, hard bo
return err
}
if t.GetTopicCat(topic) == t.TopicCatGrp {
err = groupMessageDeleteList(tx, topic, nil)
err = groupMessageDeleteList(tx, id, topic, nil)
} else {
err = tenantMessageDeleteList(tx, id, topic, nil)
}
@@ -1575,6 +1625,49 @@ func (a *adapter) TenantTopicUpdate(id t.TenantID, topic string, update map[stri
return tx.Commit()
}
func (a *adapter) TenantTopicBumpMutationVersion(id t.TenantID, topic string, at time.Time) (int64, error) {
tenantAdp, err := a.routedTenantAdapter(id)
if err != nil || topic == "" {
return 0, t.ErrMalformed
}
if tenantAdp != a {
return tenantAdp.TenantTopicBumpMutationVersion(id, topic, at)
}
if at.IsZero() {
at = time.Now().UTC().Round(time.Millisecond)
}
ctx, cancel := a.getContextForTx()
if cancel != nil {
defer cancel()
}
tx, err := a.db.BeginTxx(ctx, nil)
if err != nil {
return 0, err
}
defer func() {
if err != nil {
tx.Rollback()
}
}()
if err = a.tenantTopicBumpMutationVersionTx(ctx, tx, id, topic, at); err != nil {
return 0, err
}
var version int64
query := "SELECT mutation_version FROM im_topics WHERE name=?"
args := []any{topic}
if !a.tenantScopedDB {
query = "SELECT mutation_version FROM im_topics WHERE tenant_id=? AND name=?"
args = []any{id, topic}
}
if err = tx.GetContext(ctx, &version, query, args...); err != nil {
return 0, err
}
if err = tx.Commit(); err != nil {
return 0, err
}
return version, nil
}
func (a *adapter) TenantTopicOwnerChange(id t.TenantID, topic string, owner t.Uid) error {
tenantAdp, err := a.routedTenantAdapter(id)
if err != nil || topic == "" {
@@ -1989,6 +2082,9 @@ func (a *adapter) TenantGroupMessageSave(id t.TenantID, msg *t.Message) error {
if tenantAdp != a {
return tenantAdp.TenantGroupMessageSave(id, msg)
}
if err = a.ensureCurrentGroupMessageShardWindow(); err != nil {
return err
}
ctx, cancel := a.getContext()
if cancel != nil {
defer cancel()
@@ -1997,24 +2093,77 @@ func (a *adapter) TenantGroupMessageSave(id t.TenantID, msg *t.Message) error {
if err != nil {
return err
}
if msg.ShardMonth == "" {
msg.ShardMonth = msg.CreatedAt.UTC().Format("200601")
}
table, ok := groupMessageShardTable(msg.ShardMonth)
if !ok || msg.Uid().IsZero() {
return t.ErrMalformed
}
if msg.ShardMonth == time.Now().UTC().Format("200601") {
if _, err = a.db.ExecContext(ctx, "UPDATE im_group_message_shards SET status='writable' WHERE shard_month=? AND status='prepared'", msg.ShardMonth); err != nil {
return err
}
}
var shardStatus string
if err = a.db.QueryRowContext(ctx, "SELECT status FROM im_group_message_shards WHERE shard_month=? AND table_name=?", msg.ShardMonth, table).Scan(&shardStatus); err != nil {
if err == sql.ErrNoRows {
return fmt.Errorf("mysql: group message shard %s is not registered", msg.ShardMonth)
}
return err
}
if shardStatus != "writable" && shardStatus != "sealing" {
return fmt.Errorf("mysql: group message shard %s is %s", msg.ShardMonth, shardStatus)
}
clientMsgID := sql.NullString{String: msg.ClientMsgID, Valid: msg.ClientMsgID != ""}
contentJSON := common.ToJSON(msg.Content)
res, err := a.db.ExecContext(ctx,
`INSERT INTO im_group_messages
(topic,group_id,seqid,from_user,client_msg_id,msg_type,status,content,head,mentioned_user_ids,mention_all,
tx, err := a.db.BeginTxx(ctx, nil)
if err != nil {
return err
}
defer tx.Rollback()
messageID := store.DecodeUid(msg.Uid())
if clientMsgID.Valid {
if _, err = tx.ExecContext(ctx, `INSERT INTO im_group_message_dedup
(topic,from_user,client_msg_id,seqid,shard_month,message_id,status,expires_at)
VALUES(?,?,?,?,?,?,'done',?)`, msg.Topic, decodeUID(t.ParseUid(msg.From)), clientMsgID,
msg.SeqId, msg.ShardMonth, messageID, time.Now().UTC().Add(30*24*time.Hour)); err != nil {
if isDupe(err) {
return t.ErrDuplicate
}
return err
}
}
query := fmt.Sprintf(`INSERT INTO %s
(message_id,topic,group_id,seqid,from_user,client_msg_id,msg_type,status,content,head,mentioned_user_ids,mention_all,
file_count,content_size,version,created_at,updated_at)
VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,
msg.Topic, groupID, msg.SeqId, decodeUID(t.ParseUid(msg.From)), clientMsgID, messageHeadType(msg.Head), "normal",
VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`, table)
_, err = tx.ExecContext(ctx, query,
messageID, msg.Topic, groupID, msg.SeqId, decodeUID(t.ParseUid(msg.From)), clientMsgID, messageHeadType(msg.Head), "normal",
contentJSON, msg.Head, messageHeadJSON(msg.Head, "mentioned_user_ids"), messageHeadBool(msg.Head, "mention_all"),
0, len(contentJSON), messageVersion(msg.Version), msg.CreatedAt, msg.UpdatedAt)
if err == nil {
lastID, _ := res.LastInsertId()
msg.SetUid(t.Uid(lastID))
}
if isDupe(err) {
return t.ErrDuplicate
}
return err
if err != nil {
return err
}
if _, err = tx.ExecContext(ctx, `INSERT INTO im_group_message_topic_shards
(topic,shard_month,min_seq,max_seq,message_count,min_created_at,max_created_at)
VALUES(?,?,?,?,1,?,?) ON DUPLICATE KEY UPDATE
min_seq=LEAST(min_seq,VALUES(min_seq)),max_seq=GREATEST(max_seq,VALUES(max_seq)),
message_count=message_count+1,min_created_at=LEAST(min_created_at,VALUES(min_created_at)),
max_created_at=GREATEST(max_created_at,VALUES(max_created_at))`,
msg.Topic, msg.ShardMonth, msg.SeqId, msg.SeqId, msg.CreatedAt, msg.CreatedAt); err != nil {
return err
}
if _, err = tx.ExecContext(ctx, `UPDATE im_group_message_shards SET row_count=row_count+1,
min_created_at=IF(min_created_at IS NULL,?,LEAST(min_created_at,?)),
max_created_at=IF(max_created_at IS NULL,?,GREATEST(max_created_at,?)) WHERE shard_month=?`,
msg.CreatedAt, msg.CreatedAt, msg.CreatedAt, msg.CreatedAt, msg.ShardMonth); err != nil {
return err
}
return tx.Commit()
}
func (a *adapter) TenantMessageGetByClientID(id t.TenantID, from t.Uid, clientMsgID string) (*t.Message, error) {
@@ -2067,22 +2216,44 @@ func (a *adapter) TenantGroupMessageGetByClientID(id t.TenantID, topic string, f
if cancel != nil {
defer cancel()
}
row := struct {
t.Message
RecalledBy sql.NullInt64 `db:"recalled_by"`
}{}
err = a.db.QueryRowxContext(ctx,
"SELECT id,created_at AS createdat,updated_at AS updatedat,deleted_at AS deletedat,0 AS delid,seqid,topic,from_user AS `from`,IFNULL(client_msg_id,'') AS client_msg_id,version,recalled_at,recalled_by,head,content "+
"FROM im_group_messages WHERE topic=? AND from_user=? AND client_msg_id=?",
topic, decodeUID(from), clientMsgID).StructScan(&row)
if err == sql.ErrNoRows {
return nil, nil
var seq int64
if err = a.db.QueryRowContext(ctx, `SELECT seqid FROM im_group_message_dedup
WHERE topic=? AND from_user=? AND client_msg_id=?`, topic, decodeUID(from), clientMsgID).Scan(&seq); err == nil {
msg, getErr := tenantGroupMessageGetOne(a.db, ctx, id, topic, seq, "")
if getErr == nil {
return msg, nil
}
if getErr != sql.ErrNoRows {
return nil, getErr
}
} else if err != sql.ErrNoRows {
return nil, err
}
tables, err := groupMessageTablesForTopic(a.db, ctx, id, topic)
if err != nil {
return nil, err
}
msg := normalizeGroupMessage(id, row.Message, row.RecalledBy)
return &msg, nil
for _, table := range tables {
month := groupMessageMonthFromTable(table)
idColumn := "id"
if month != "" {
idColumn = "message_id"
}
row := struct {
t.Message
RecalledBy sql.NullInt64 `db:"recalled_by"`
}{}
query := fmt.Sprintf("SELECT %s AS id,created_at AS createdat,updated_at AS updatedat,deleted_at AS deletedat,0 AS delid,seqid,topic,from_user AS `from`,IFNULL(client_msg_id,'') AS client_msg_id,version,recalled_at,recalled_by,head,content,'%s' AS shard_month FROM %s WHERE topic=? AND from_user=? AND client_msg_id=?", idColumn, month, table)
err = a.db.QueryRowxContext(ctx, query, topic, decodeUID(from), clientMsgID).StructScan(&row)
if err == nil {
msg := normalizeGroupMessage(id, row.Message, row.RecalledBy)
return &msg, nil
}
if err != sql.ErrNoRows {
return nil, err
}
}
return nil, nil
}
func (a *adapter) TenantMessageGetOne(id t.TenantID, topic string, seq int64) (*t.Message, error) {
@@ -2173,6 +2344,9 @@ func (a *adapter) TenantMessageRecall(id t.TenantID, topic string, seq int64, ac
if err != nil {
return nil, err
}
if err = a.tenantTopicBumpMutationVersionTx(ctx, tx, id, topic, recalledAt); err != nil {
return nil, err
}
if err = tx.Commit(); err != nil {
return nil, err
}
@@ -2203,7 +2377,7 @@ func (a *adapter) TenantGroupMessageRecall(id t.TenantID, topic string, seq int6
tx.Rollback()
}
}()
msg, err := tenantGroupMessageGetOne(tx, ctx, id, topic, seq, " FOR UPDATE")
msg, messageTable, err := tenantGroupMessageGetOneWithTable(tx, ctx, id, topic, seq, " FOR UPDATE")
if err != nil {
if err == sql.ErrNoRows {
return nil, t.ErrNotFound
@@ -2213,12 +2387,15 @@ func (a *adapter) TenantGroupMessageRecall(id t.TenantID, topic string, seq int6
if msg.Version != expectedVersion || msg.RecalledAt != nil || msg.DeletedAt != nil {
return nil, t.ErrDuplicate
}
_, err = tx.ExecContext(ctx,
"UPDATE im_group_messages SET updated_at=?,version=version+1,status='recalled',recalled_at=?,recalled_by=?,head=NULL,content=NULL WHERE topic=? AND seqid=? AND version=?",
_, err = tx.ExecContext(ctx, fmt.Sprintf(
"UPDATE %s SET updated_at=?,version=version+1,status='recalled',recalled_at=?,recalled_by=?,head=NULL,content=NULL WHERE topic=? AND seqid=? AND version=?", messageTable),
recalledAt, recalledAt, decodeUID(actor), topic, seq, expectedVersion)
if err != nil {
return nil, err
}
if err = a.tenantTopicBumpMutationVersionTx(ctx, tx, id, topic, recalledAt); err != nil {
return nil, err
}
if err = tx.Commit(); err != nil {
return nil, err
}
@@ -2272,6 +2449,9 @@ func (a *adapter) TenantMessageEdit(id t.TenantID, topic string, seq int64, acto
if err != nil {
return nil, err
}
if err = a.tenantTopicBumpMutationVersionTx(ctx, tx, id, topic, editedAt); err != nil {
return nil, err
}
if err = tx.Commit(); err != nil {
return nil, err
}
@@ -2302,7 +2482,7 @@ func (a *adapter) TenantGroupMessageEdit(id t.TenantID, topic string, seq int64,
tx.Rollback()
}
}()
msg, err := tenantGroupMessageGetOne(tx, ctx, id, topic, seq, " FOR UPDATE")
msg, messageTable, err := tenantGroupMessageGetOneWithTable(tx, ctx, id, topic, seq, " FOR UPDATE")
if err != nil {
if err == sql.ErrNoRows {
return nil, t.ErrNotFound
@@ -2318,12 +2498,15 @@ func (a *adapter) TenantGroupMessageEdit(id t.TenantID, topic string, seq int64,
return nil, err
}
contentJSON := common.ToJSON(content)
_, err = tx.ExecContext(ctx,
"UPDATE im_group_messages SET updated_at=?,version=version+1,msg_type=?,head=?,content=?,content_size=? WHERE topic=? AND seqid=? AND version=?",
_, err = tx.ExecContext(ctx, fmt.Sprintf(
"UPDATE %s SET updated_at=?,version=version+1,msg_type=?,head=?,content=?,content_size=? WHERE topic=? AND seqid=? AND version=?", messageTable),
editedAt, messageHeadType(head), head, contentJSON, len(contentJSON), topic, seq, expectedVersion)
if err != nil {
return nil, err
}
if err = a.tenantTopicBumpMutationVersionTx(ctx, tx, id, topic, editedAt); err != nil {
return nil, err
}
if err = tx.Commit(); err != nil {
return nil, err
}
@@ -2407,58 +2590,84 @@ func (a *adapter) TenantGroupMessageGetAll(id t.TenantID, topic string, uid t.Ui
return tenantAdp.TenantGroupMessageGetAll(id, topic, uid, opts)
}
limit := a.maxMessageResults
args := []any{store.DecodeUid(uid), topic}
seq := ""
seqClause := ""
seqArgs := make([]any, 0)
if opts != nil {
seq = "AND m.seqid "
seqClause = "AND m.seqid "
if len(opts.IdRanges) > 0 {
constr, addArgs := common.RangesToSql(opts.IdRanges)
seq += constr
args = append(args, addArgs...)
seqClause += constr
seqArgs = append(seqArgs, addArgs...)
} else {
seq += "BETWEEN ? AND ?"
seqClause += "BETWEEN ? AND ?"
if opts.Since > 0 {
args = append(args, opts.Since)
seqArgs = append(seqArgs, opts.Since)
} else {
args = append(args, 0)
seqArgs = append(seqArgs, 0)
}
if opts.Before > 1 {
args = append(args, opts.Before-1)
seqArgs = append(seqArgs, opts.Before-1)
} else {
args = append(args, 1<<31-1)
seqArgs = append(seqArgs, int64(^uint64(0)>>1))
}
}
if opts.Limit > 0 && opts.Limit < limit {
limit = opts.Limit
}
}
args = append(args, limit)
ctx, cancel := a.getContext()
if cancel != nil {
defer cancel()
}
rows, err := a.db.QueryxContext(ctx,
"SELECT m.id,m.created_at AS createdat,m.updated_at AS updatedat,m.deleted_at AS deletedat,0 AS delid,m.seqid,m.topic,m.from_user AS `from`,IFNULL(m.client_msg_id,'') AS client_msg_id,m.version,m.recalled_at,m.recalled_by,m.head,m.content "+
"FROM im_group_messages AS m LEFT JOIN im_dellog AS d "+
"ON d.topic COLLATE utf8mb4_unicode_ci=m.topic AND m.seqid BETWEEN d.low AND d.hi-1 AND d.deletedfor=? "+
"WHERE m.status!='deleted' AND m.topic=? "+seq+" AND d.deletedfor IS NULL ORDER BY m.seqid DESC LIMIT ?",
args...)
tables, err := groupMessageTablesForTopic(a.db, ctx, id, topic)
if err != nil {
return nil, err
}
defer rows.Close()
msgs := make([]t.Message, 0, limit)
for rows.Next() {
row := struct {
t.Message
RecalledBy sql.NullInt64 `db:"recalled_by"`
}{}
if err = rows.StructScan(&row); err != nil {
seen := make(map[int64]struct{}, limit)
for _, table := range tables {
if len(msgs) >= limit {
break
}
month := groupMessageMonthFromTable(table)
idColumn := "id"
if month != "" {
idColumn = "message_id"
}
args := []any{store.DecodeUid(uid), topic}
args = append(args, seqArgs...)
args = append(args, limit-len(msgs))
query := fmt.Sprintf("SELECT m.%s AS id,m.created_at AS createdat,m.updated_at AS updatedat,m.deleted_at AS deletedat,0 AS delid,m.seqid,m.topic,m.from_user AS `from`,IFNULL(m.client_msg_id,'') AS client_msg_id,m.version,m.recalled_at,m.recalled_by,m.head,m.content,'%s' AS shard_month FROM %s AS m LEFT JOIN im_dellog AS d ON d.topic COLLATE utf8mb4_unicode_ci=m.topic AND m.seqid BETWEEN d.low AND d.hi-1 AND d.deletedfor=? WHERE m.status!='deleted' AND m.topic=? %s AND d.deletedfor IS NULL ORDER BY m.seqid DESC LIMIT ?", idColumn, month, table, seqClause)
rows, queryErr := a.db.QueryxContext(ctx, query, args...)
if queryErr != nil {
return nil, queryErr
}
for rows.Next() {
row := struct {
t.Message
RecalledBy sql.NullInt64 `db:"recalled_by"`
}{}
if err = rows.StructScan(&row); err != nil {
rows.Close()
return nil, err
}
if _, exists := seen[row.SeqId]; exists {
continue
}
seen[row.SeqId] = struct{}{}
msgs = append(msgs, normalizeGroupMessage(id, row.Message, row.RecalledBy))
}
if err = rows.Err(); err != nil {
rows.Close()
return nil, err
}
msgs = append(msgs, normalizeGroupMessage(id, row.Message, row.RecalledBy))
rows.Close()
}
return msgs, rows.Err()
sort.Slice(msgs, func(i, j int) bool { return msgs[i].SeqId > msgs[j].SeqId })
if len(msgs) > limit {
msgs = msgs[:limit]
}
return msgs, nil
}
func (a *adapter) TenantMessageSearch(id t.TenantID, topic string, uid t.Uid, queryText, messageType, cursor string, limit int) ([]t.Message, string, error) {
@@ -2543,7 +2752,7 @@ func (a *adapter) tenantGroupMessageSearch(id t.TenantID, topic string, uid t.Ui
where := []string{"m.status!='deleted'", "m.topic=?"}
args := []any{topic}
if cursor != "" {
parsed, err := strconv.Atoi(cursor)
parsed, err := strconv.ParseInt(cursor, 10, 64)
if err != nil || parsed <= 0 {
return nil, "", t.ErrMalformed
}
@@ -2564,43 +2773,182 @@ func (a *adapter) tenantGroupMessageSearch(id t.TenantID, topic string, uid t.Ui
if cancel != nil {
defer cancel()
}
queryArgs := append([]any{store.DecodeUid(uid)}, args...)
queryArgs = append(queryArgs, limit+1)
rows, err := a.db.QueryxContext(ctx,
"SELECT m.id,m.created_at AS createdat,m.updated_at AS updatedat,m.deleted_at AS deletedat,0 AS delid,m.seqid,m.topic,m.from_user AS `from`,IFNULL(m.client_msg_id,'') AS client_msg_id,m.version,m.recalled_at,m.recalled_by,m.head,m.content "+
"FROM im_group_messages AS m LEFT JOIN im_dellog AS d "+
"ON d.topic COLLATE utf8mb4_unicode_ci=m.topic AND m.seqid BETWEEN d.low AND d.hi-1 AND d.deletedfor=? "+
"WHERE "+strings.Join(where, " AND ")+" AND d.deletedfor IS NULL ORDER BY m.seqid DESC LIMIT ?",
queryArgs...)
tables, err := groupMessageTablesForTopic(a.db, ctx, id, topic)
if err != nil {
return nil, "", err
}
defer rows.Close()
msgs := make([]t.Message, 0, limit)
nextCursor := ""
lastSeq := int64(0)
for rows.Next() {
row := struct {
t.Message
RecalledBy sql.NullInt64 `db:"recalled_by"`
}{}
if err = rows.StructScan(&row); err != nil {
msgs := make([]t.Message, 0, limit+1)
seen := make(map[int64]struct{}, limit+1)
for _, table := range tables {
if len(msgs) > limit {
break
}
month := groupMessageMonthFromTable(table)
idColumn := "id"
if month != "" {
idColumn = "message_id"
}
queryArgs := append([]any{store.DecodeUid(uid)}, args...)
queryArgs = append(queryArgs, limit+1-len(msgs))
query := fmt.Sprintf("SELECT m.%s AS id,m.created_at AS createdat,m.updated_at AS updatedat,m.deleted_at AS deletedat,0 AS delid,m.seqid,m.topic,m.from_user AS `from`,IFNULL(m.client_msg_id,'') AS client_msg_id,m.version,m.recalled_at,m.recalled_by,m.head,m.content,'%s' AS shard_month FROM %s AS m LEFT JOIN im_dellog AS d ON d.topic COLLATE utf8mb4_unicode_ci=m.topic AND m.seqid BETWEEN d.low AND d.hi-1 AND d.deletedfor=? WHERE %s AND d.deletedfor IS NULL ORDER BY m.seqid DESC LIMIT ?", idColumn, month, table, strings.Join(where, " AND "))
rows, queryErr := a.db.QueryxContext(ctx, query, queryArgs...)
if queryErr != nil {
return nil, "", queryErr
}
for rows.Next() {
row := struct {
t.Message
RecalledBy sql.NullInt64 `db:"recalled_by"`
}{}
if err = rows.StructScan(&row); err != nil {
rows.Close()
return nil, "", err
}
if _, exists := seen[row.SeqId]; exists {
continue
}
seen[row.SeqId] = struct{}{}
msgs = append(msgs, normalizeGroupMessage(id, row.Message, row.RecalledBy))
}
if err = rows.Err(); err != nil {
rows.Close()
return nil, "", err
}
msg := normalizeGroupMessage(id, row.Message, row.RecalledBy)
if len(msgs) == limit {
nextCursor = strconv.FormatInt(lastSeq, 10)
continue
}
msgs = append(msgs, msg)
lastSeq = msg.SeqId
rows.Close()
}
if err = rows.Err(); err != nil {
return nil, "", err
sort.Slice(msgs, func(i, j int) bool { return msgs[i].SeqId > msgs[j].SeqId })
nextCursor := ""
if len(msgs) > limit {
msgs = msgs[:limit]
lastSeq := msgs[len(msgs)-1].SeqId
nextCursor = strconv.FormatInt(lastSeq, 10)
}
return msgs, nextCursor, nil
}
func (a *adapter) TenantMessageMutations(id t.TenantID, topic string, uid t.Uid, sinceVersion int64, limit int) ([]t.MessageMutation, string, bool, int64, error) {
if !a.tenantScopedDB {
tenantAdp, err := a.routedTenantAdapter(id)
if err != nil {
return nil, "", false, 0, err
}
if tenantAdp != a {
return tenantAdp.TenantMessageMutations(id, topic, uid, sinceVersion, limit)
}
}
if err := validTenant(id); err != nil || topic == "" || uid.IsZero() || sinceVersion < 0 {
return nil, "", false, 0, t.ErrMalformed
}
if limit <= 0 || limit > 100 {
limit = 50
}
ctx, cancel := a.getContext()
if cancel != nil {
defer cancel()
}
var currentVersion int64
if a.tenantScopedDB {
if err := a.db.GetContext(ctx, &currentVersion, "SELECT IFNULL(mutation_version,0) FROM im_topics WHERE name=?", topic); err != nil {
if err == sql.ErrNoRows {
return nil, "", false, 0, t.ErrNotFound
}
return nil, "", false, 0, err
}
} else {
if err := a.db.GetContext(ctx, &currentVersion, "SELECT IFNULL(mutation_version,0) FROM im_topics WHERE tenant_id=? AND name=?", id, topic); err != nil {
if err == sql.ErrNoRows {
return nil, "", false, 0, t.ErrNotFound
}
return nil, "", false, 0, err
}
}
if sinceVersion >= currentVersion {
return []t.MessageMutation{}, strconv.FormatInt(currentVersion, 10), false, currentVersion, nil
}
if t.GetTopicCat(topic) == t.TopicCatGrp {
return a.tenantGroupMessageMutations(ctx, id, topic, uid, limit, currentVersion)
}
return a.tenantP2PMessageMutations(ctx, id, topic, uid, limit, currentVersion)
}
func (a *adapter) tenantP2PMessageMutations(ctx context.Context, id t.TenantID, topic string, uid t.Uid, limit int, currentVersion int64) ([]t.MessageMutation, string, bool, int64, error) {
rows, err := a.db.QueryxContext(ctx,
"SELECT m.seqid,m.version,m.updatedat AS mutated_at,m.recalled_at,m.recalled_by,rev.actor "+
"FROM im_messages AS m "+
"LEFT JOIN im_dellog AS d ON d.topic=m.topic AND m.seqid BETWEEN d.low AND d.hi-1 AND d.deletedfor=? "+
"LEFT JOIN im_message_revisions AS rev ON rev.topic=m.topic AND rev.seqid=m.seqid AND rev.version=m.version-1 "+
"WHERE m.topic=? AND d.deletedfor IS NULL AND (m.version>1 OR m.recalled_at IS NOT NULL OR m.deletedat IS NOT NULL) "+
"ORDER BY m.updatedat DESC,m.seqid DESC LIMIT ?",
decodeUID(uid), topic, limit+1)
if err != nil {
return nil, "", false, 0, err
}
defer rows.Close()
return scanTenantMessageMutations(rows, limit, currentVersion)
}
func (a *adapter) tenantGroupMessageMutations(ctx context.Context, id t.TenantID, topic string, uid t.Uid, limit int, currentVersion int64) ([]t.MessageMutation, string, bool, int64, error) {
rows, err := a.db.QueryxContext(ctx,
"SELECT m.seqid,m.version,m.updated_at AS mutated_at,m.recalled_at,m.recalled_by,rev.actor "+
"FROM im_group_messages AS m "+
"LEFT JOIN im_dellog AS d ON d.topic COLLATE utf8mb4_unicode_ci=m.topic AND m.seqid BETWEEN d.low AND d.hi-1 AND d.deletedfor=? "+
"LEFT JOIN im_message_revisions AS rev ON rev.topic=m.topic AND rev.seqid=m.seqid AND rev.version=m.version-1 "+
"WHERE m.topic=? AND d.deletedfor IS NULL AND (m.version>1 OR m.recalled_at IS NOT NULL OR m.deleted_at IS NOT NULL OR m.status='recalled') "+
"ORDER BY m.updated_at DESC,m.seqid DESC LIMIT ?",
decodeUID(uid), topic, limit+1)
if err != nil {
return nil, "", false, 0, err
}
defer rows.Close()
return scanTenantMessageMutations(rows, limit, currentVersion)
}
func scanTenantMessageMutations(rows *sqlx.Rows, limit int, currentVersion int64) ([]t.MessageMutation, string, bool, int64, error) {
items := make([]t.MessageMutation, 0, limit)
hasMore := false
for rows.Next() {
row := struct {
SeqId int64 `db:"seqid"`
Version int `db:"version"`
MutatedAt time.Time `db:"mutated_at"`
RecalledAt sql.NullTime `db:"recalled_at"`
RecalledBy sql.NullInt64 `db:"recalled_by"`
Actor sql.NullInt64 `db:"actor"`
}{}
if err := rows.StructScan(&row); err != nil {
return nil, "", false, 0, err
}
if len(items) == limit {
hasMore = true
continue
}
event := "message.edited"
operator := ""
if row.Actor.Valid {
operator = store.EncodeUid(row.Actor.Int64).String()
}
if row.RecalledAt.Valid {
event = "message.recalled"
if row.RecalledBy.Valid {
operator = store.EncodeUid(row.RecalledBy.Int64).String()
}
row.MutatedAt = row.RecalledAt.Time
}
items = append(items, t.MessageMutation{
SeqId: row.SeqId,
Version: row.Version,
Event: event,
Operator: operator,
MutatedAt: row.MutatedAt,
MutationVersion: currentVersion,
})
}
if err := rows.Err(); err != nil {
return nil, "", false, 0, err
}
return items, strconv.FormatInt(currentVersion, 10), hasMore, currentVersion, nil
}
func (a *adapter) TenantCallCreate(id t.TenantID, record *t.CallRecord) error {
if !a.tenantScopedDB {
tenantAdp, err := a.routedTenantAdapter(id)
@@ -6902,19 +7250,36 @@ func (a *adapter) groupIDByTopic(ctx context.Context, topic string) (int64, erro
}
func tenantGroupMessageGetOne(q sqlx.QueryerContext, ctx context.Context, tenantID t.TenantID, topic string, seq int64, suffix string) (*t.Message, error) {
row := struct {
t.Message
RecalledBy sql.NullInt64 `db:"recalled_by"`
}{}
err := sqlx.GetContext(ctx, q, &row,
"SELECT id,created_at AS createdat,updated_at AS updatedat,deleted_at AS deletedat,0 AS delid,seqid,topic,from_user AS `from`,IFNULL(client_msg_id,'') AS client_msg_id,version,recalled_at,recalled_by,head,content "+
"FROM im_group_messages WHERE topic=? AND seqid=?"+suffix,
topic, seq)
msg, _, err := tenantGroupMessageGetOneWithTable(q, ctx, tenantID, topic, seq, suffix)
return msg, err
}
func tenantGroupMessageGetOneWithTable(q sqlx.QueryerContext, ctx context.Context, tenantID t.TenantID, topic string, seq int64, suffix string) (*t.Message, string, error) {
tables, err := groupMessageTablesForSeq(q, ctx, tenantID, topic, seq)
if err != nil {
return nil, err
return nil, "", err
}
msg := normalizeGroupMessage(tenantID, row.Message, row.RecalledBy)
return &msg, nil
for _, table := range tables {
month := groupMessageMonthFromTable(table)
idColumn := "id"
if month != "" {
idColumn = "message_id"
}
row := struct {
t.Message
RecalledBy sql.NullInt64 `db:"recalled_by"`
}{}
query := fmt.Sprintf("SELECT %s AS id,created_at AS createdat,updated_at AS updatedat,deleted_at AS deletedat,0 AS delid,seqid,topic,from_user AS `from`,IFNULL(client_msg_id,'') AS client_msg_id,version,recalled_at,recalled_by,head,content,'%s' AS shard_month FROM %s WHERE topic=? AND seqid=?%s", idColumn, month, table, suffix)
err = sqlx.GetContext(ctx, q, &row, query, topic, seq)
if err == nil {
msg := normalizeGroupMessage(tenantID, row.Message, row.RecalledBy)
return &msg, table, nil
}
if err != sql.ErrNoRows {
return nil, "", err
}
}
return nil, "", sql.ErrNoRows
}
func messageVersion(version int) int {
@@ -7395,7 +7760,7 @@ func (a *adapter) TenantGroupMessageDeleteList(id t.TenantID, topic string, del
tx.Rollback()
}
}()
if err = groupMessageDeleteList(tx, topic, del); err != nil {
if err = groupMessageDeleteList(tx, id, topic, del); err != nil {
return err
}
return tx.Commit()
server/db/mysql/tenant_groups.go
+18
View File
@@ -930,6 +930,24 @@ func (a *adapter) TenantGroupMuteList(id t.TenantID, topic, cursor string, limit
return items, next, rows.Err()
}
func (a *adapter) TenantGroupMuteGet(id t.TenantID, topic string, uid t.Uid) (*t.GroupMute, error) {
if tenantAdp, err := a.routeGroupTenant(id); err != nil {
return nil, err
} else if tenantAdp != a {
return tenantAdp.TenantGroupMuteGet(id, topic, uid)
}
if topic == "" || uid.IsZero() {
return nil, t.ErrMalformed
}
now := time.Now().UTC()
ctx, cancel := a.getContext()
if cancel != nil {
defer cancel()
}
return a.groupMuteByQuery(ctx, "SELECT "+groupMuteSelectColumns+" FROM im_group_mutes WHERE topic=? AND user_id=? AND status=? AND (expires_at IS NULL OR expires_at>?)",
topic, decodeUID(uid), t.GroupMuteStatusActive, now)
}
func (a *adapter) TenantGroupMuteUpsert(id t.TenantID, mute *t.GroupMute) (*t.GroupMute, error) {
if tenantAdp, err := a.routeGroupTenant(id); err != nil {
return nil, err
server/db/mysql/tenant_security_audit.go
+18 -9
View File
@@ -98,7 +98,7 @@ func (a *adapter) TenantMessageAdminApply(id t.TenantID, action *t.MessageAdminA
return tx.Commit()
}
sender, beforeStatus, err := lockAdminMessage(ctx, tx, action.ConversationType, action.Topic, action.SeqID)
sender, beforeStatus, messageTable, err := lockAdminMessage(ctx, tx, id, action.ConversationType, action.Topic, action.SeqID)
if err != nil {
return err
}
@@ -108,7 +108,7 @@ func (a *adapter) TenantMessageAdminApply(id t.TenantID, action *t.MessageAdminA
afterStatus = "deleted"
}
if action.ConversationType == "group" {
_, err = tx.ExecContext(ctx, "UPDATE im_group_messages SET status=?,deleted_at=?,updated_at=? WHERE topic=? AND seqid=?", afterStatus, action.ProcessedAt, action.ProcessedAt, action.Topic, action.SeqID)
_, err = tx.ExecContext(ctx, "UPDATE "+messageTable+" SET status=?,deleted_at=?,updated_at=? WHERE topic=? AND seqid=?", afterStatus, action.ProcessedAt, action.ProcessedAt, action.Topic, action.SeqID)
} else {
_, err = tx.ExecContext(ctx, "UPDATE im_messages SET deletedat=?,updatedat=? WHERE topic=? AND seqid=?", action.ProcessedAt, action.ProcessedAt, action.Topic, action.SeqID)
}
@@ -127,17 +127,26 @@ func (a *adapter) TenantMessageAdminApply(id t.TenantID, action *t.MessageAdminA
return tx.Commit()
}
func lockAdminMessage(ctx context.Context, tx *sqlx.Tx, conversationType, topic string, seqID int64) (int64, string, error) {
func lockAdminMessage(ctx context.Context, tx *sqlx.Tx, tenantID t.TenantID, conversationType, topic string, seqID int64) (int64, string, string, error) {
if conversationType == "group" {
tables, err := groupMessageTablesForSeq(tx, ctx, tenantID, topic, seqID)
if err != nil {
return 0, "", "", err
}
var row struct {
Sender int64 `db:"from_user"`
Status string `db:"status"`
}
err := tx.GetContext(ctx, &row, "SELECT from_user,status FROM im_group_messages WHERE topic=? AND seqid=? FOR UPDATE", topic, seqID)
if errors.Is(err, sql.ErrNoRows) {
return 0, "", t.ErrNotFound
for _, table := range tables {
err = tx.GetContext(ctx, &row, "SELECT from_user,status FROM "+table+" WHERE topic=? AND seqid=? FOR UPDATE", topic, seqID)
if err == nil {
return row.Sender, row.Status, table, nil
}
if !errors.Is(err, sql.ErrNoRows) {
return 0, "", "", err
}
}
return row.Sender, row.Status, err
return 0, "", "", t.ErrNotFound
}
var row struct {
Sender int64 `db:"from"`
@@ -146,7 +155,7 @@ func lockAdminMessage(ctx context.Context, tx *sqlx.Tx, conversationType, topic
}
err := tx.GetContext(ctx, &row, "SELECT `from`,deletedat,recalled_at FROM im_messages WHERE topic=? AND seqid=? FOR UPDATE", topic, seqID)
if errors.Is(err, sql.ErrNoRows) {
return 0, "", t.ErrNotFound
return 0, "", "", t.ErrNotFound
}
status := "normal"
if row.RecalledAt.Valid {
@@ -155,7 +164,7 @@ func lockAdminMessage(ctx context.Context, tx *sqlx.Tx, conversationType, topic
if row.DeletedAt.Valid {
status = "deleted"
}
return row.Sender, status, err
return row.Sender, status, "im_messages", err
}
func nullableJSON(raw []byte) any {
server/db/mysql/user_identity_schema.go
+20
View File
@@ -150,6 +150,9 @@ func ensureUserIdentitySchema(db *sqlx.DB) error {
if _, err = conn.ExecContext(ctx, "ALTER TABLE im_users MODIFY COLUMN username VARCHAR(32) NOT NULL"); err != nil {
return err
}
if err = ensureUserAccountSequence(ctx, conn); err != nil {
return err
}
if err = migrateBasicAuthSchema(ctx, conn, usersScoped); err != nil {
return err
}
@@ -171,6 +174,23 @@ func ensureUserIdentitySchema(db *sqlx.DB) error {
return nil
}
func ensureUserAccountSequence(ctx context.Context, conn *sqlx.Conn) error {
exists, err := identityColumnExists(ctx, conn, "im_users", "account_no")
if err != nil || !exists {
return err
}
if _, err = conn.ExecContext(ctx, `CREATE TABLE IF NOT EXISTS im_user_account_sequences(
id TINYINT UNSIGNED NOT NULL,next_value INT UNSIGNED NOT NULL,
updated_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3),
PRIMARY KEY(id)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci`); err != nil {
return err
}
_, err = conn.ExecContext(ctx, `INSERT INTO im_user_account_sequences(id,next_value)
SELECT 1,GREATEST(COALESCE(MAX(account_no),999),999) FROM im_users
ON DUPLICATE KEY UPDATE next_value=GREATEST(next_value,VALUES(next_value))`)
return err
}
func backfillUsernames(ctx context.Context, conn *sqlx.Conn, scoped bool) error {
prefix := "0 AS tenant_id,u.id AS user_id,"
join := "a.userid=u.id"
server/im_http.go
+171 -12
View File
@@ -4074,6 +4074,8 @@ func serveIMTopicMessageAction(wrt http.ResponseWriter, req *http.Request) {
serveIMTopicDraft(wrt, req)
case strings.HasSuffix(req.URL.Path, "/calls"):
serveIMTopicCalls(wrt, req)
case strings.HasSuffix(req.URL.Path, "/message-mutations"):
serveIMMessageMutations(wrt, req)
case strings.HasSuffix(req.URL.Path, "/messages/search"):
serveIMMessageSearch(wrt, req)
case strings.Contains(req.URL.Path, "/messages/") && strings.HasSuffix(req.URL.Path, "/context"):
@@ -4198,6 +4200,40 @@ func serveIMMessageSearch(wrt http.ResponseWriter, req *http.Request) {
writeIMSuccess(wrt, req, map[string]any{"topic": topic, "items": items, "next_cursor": nextCursor})
}
func serveIMMessageMutations(wrt http.ResponseWriter, req *http.Request) {
if req.Method != http.MethodGet {
writeIMError(wrt, req, http.StatusMethodNotAllowed, 30002, imReasonMethodNotAllowed, "method not allowed", nil)
return
}
principal, topic, ok := authenticateTopicReader(wrt, req, topicFromMessageMutationsPath(req.URL.Path))
if !ok {
return
}
sinceVersion, err := parseIMMutationSince(req.URL.Query().Get("since_version"), req.URL.Query().Get("since"))
if err != nil {
writeIMError(wrt, req, http.StatusBadRequest, 30000, imReasonMalformedRequest, "malformed request", map[string]any{"field": "since_version"})
return
}
limit := parseIMLimit(req.URL.Query().Get("limit"), 50, 100)
mutations, nextCursor, hasMore, mutationVersion, err := store.Messages.Mutations(principal.TenantID, topic, principal.Uid, sinceVersion, limit)
if err != nil {
writeIMRelationError(wrt, req, err)
return
}
items := make([]map[string]any, 0, len(mutations))
for _, mutation := range mutations {
items = append(items, messageMutationEventPayload(mutation))
}
writeIMSuccess(wrt, req, map[string]any{
"topic": topic,
"items": items,
"next_cursor": nextCursor,
"has_more": hasMore,
"mutation_version": mutationVersion,
"expired": false,
})
}
func serveIMMessageContext(wrt http.ResponseWriter, req *http.Request) {
if req.Method != http.MethodGet {
writeIMError(wrt, req, http.StatusMethodNotAllowed, 30002, imReasonMethodNotAllowed, "method not allowed", nil)
@@ -5583,6 +5619,11 @@ func serveIMGroupInvitations(wrt http.ResponseWriter, req *http.Request, topic s
}
if joined {
item["status"] = types.GroupInvitationStatusJoined
if imReliableHTTPEventsEnabled(req) {
publishIMGroupMemberJoinedSystemNotice(principal.TenantID, topic, principal.Uid, uid,
"有成员通过邀请加入了群聊。",
map[string]any{"source": "direct_invitation"})
}
} else {
item["status"] = "already_member"
}
@@ -5827,6 +5868,11 @@ func serveIMGroupJoinRequests(wrt http.ResponseWriter, req *http.Request, topic
}
publishIMGroupRealtimeEvent(principal.TenantID, topic, principal.Uid, "group.member.joined",
map[string]any{"user_id": principal.Uid.UserId(), "joined": joined, "source": "open_group"})
if joined && imReliableHTTPEventsEnabled(req) {
publishIMGroupMemberJoinedSystemNotice(principal.TenantID, topic, principal.Uid, principal.Uid,
"有成员加入了群聊。",
map[string]any{"source": "open_group"})
}
writeIMSuccess(wrt, req, map[string]any{"topic": topic, "status": "joined", "joined": joined, "source": "open_group"})
return
}
@@ -6347,19 +6393,34 @@ func serveIMGroupMutes(wrt http.ResponseWriter, req *http.Request, topic, tail s
if !ok {
return
}
if !(selfSub.ModeGiven & selfSub.ModeWant).IsAdmin() {
writeIMError(wrt, req, http.StatusForbidden, 30005, imReasonPermissionDenied, "permission denied", nil)
return
}
isAdmin := (selfSub.ModeGiven & selfSub.ModeWant).IsAdmin()
switch {
case tail == "mutes" && req.Method == http.MethodGet:
items, nextCursor, err := store.GroupBusiness.ListMutes(principal.TenantID, topic, strings.TrimSpace(req.URL.Query().Get("cursor")), parseIMLimit(req.URL.Query().Get("limit"), 50, 100))
if err != nil && !errors.Is(err, types.ErrNotFound) {
writeIMRelationError(wrt, req, err)
return
var items []types.GroupMute
nextCursor := ""
if isAdmin {
var err error
items, nextCursor, err = store.GroupBusiness.ListMutes(principal.TenantID, topic, strings.TrimSpace(req.URL.Query().Get("cursor")), parseIMLimit(req.URL.Query().Get("limit"), 50, 100))
if err != nil && !errors.Is(err, types.ErrNotFound) {
writeIMRelationError(wrt, req, err)
return
}
} else {
item, err := store.GroupBusiness.GetMute(principal.TenantID, topic, principal.Uid)
if err != nil && !errors.Is(err, types.ErrNotFound) {
writeIMRelationError(wrt, req, err)
return
}
if item != nil {
items = append(items, *item)
}
}
writeIMSuccess(wrt, req, map[string]any{"topic": topic, "items": groupMutePayloads(items), "next_cursor": nextCursor})
case (tail == "mutes" || tail == "mutes/batch") && req.Method == http.MethodPost:
if !isAdmin {
writeIMError(wrt, req, http.StatusForbidden, 30005, imReasonPermissionDenied, "permission denied", nil)
return
}
var body imGroupMuteBatchRequest
if !decodeIMJSONBody(wrt, req, &body) {
return
@@ -6400,6 +6461,10 @@ func serveIMGroupMutes(wrt http.ResponseWriter, req *http.Request, topic, tail s
map[string]any{"topic": topic, "items": groupMutePayloads(mutes)})
writeIMSuccess(wrt, req, map[string]any{"topic": topic, "items": groupMutePayloads(mutes)})
case strings.HasPrefix(tail, "mutes/") && req.Method == http.MethodDelete:
if !isAdmin {
writeIMError(wrt, req, http.StatusForbidden, 30005, imReasonPermissionDenied, "permission denied", nil)
return
}
uid := types.ParseUserId(strings.TrimSpace(strings.TrimPrefix(tail, "mutes/")))
if uid.IsZero() {
writeIMError(wrt, req, http.StatusBadRequest, 30000, imReasonMalformedRequest, "malformed request", map[string]any{"field": "user_id"})
@@ -8712,6 +8777,17 @@ func messageSearchPayload(msg types.Message, query string) map[string]any {
}
}
func messageMutationEventPayload(mutation types.MessageMutation) map[string]any {
return map[string]any{
"seq": mutation.SeqId,
"version": mutation.Version,
"event": mutation.Event,
"operator": mutation.Operator,
"mutated_at": mutation.MutatedAt.UTC().Format(time.RFC3339Nano),
"mutation_version": mutation.MutationVersion,
}
}
func messageContextPayload(msg types.Message) map[string]any {
return map[string]any{
"seq": msg.SeqId,
@@ -11583,7 +11659,9 @@ func broadcastIMConversationSystemMessage(tenantID types.TenantID, msg *types.Me
logs.Warn.Printf("im system message: hub route queue full topic=%s seq=%d", msg.Topic, msg.SeqId)
}
deliverIMConversationSystemMessageToP2PSessions(tenantID, msg, out)
publishRealtimeTopicMessage(msg.Topic, out)
if !publishRealtimeP2PConversationSystemMessage(tenantID, msg.Topic, out) {
publishRealtimeTopicMessage(msg.Topic, out)
}
broadcastIMConversationSystemPresence(tenantID, msg, timestamp)
}
@@ -11607,6 +11685,28 @@ func publishIMGroupInvitationNotification(tenantID types.TenantID, inv *types.Gr
}
}
func publishIMGroupMemberJoinedSystemNotice(tenantID types.TenantID, topic string, inviter, target types.Uid, text string, extra map[string]any) {
if target.IsZero() {
return
}
actors := make([]imSystemMessageActor, 0, 2)
if !inviter.IsZero() && inviter != target {
actors = append(actors, imSystemMessageActor{
UserID: inviter.UserId(),
Click: map[string]any{"type": "user_profile", "target_id": inviter.UserId()},
})
}
actors = append(actors, imSystemMessageActor{
UserID: target.UserId(),
Click: map[string]any{"type": "user_profile", "target_id": target.UserId()},
})
publishIMConversationSystemNotice(tenantID, topic,
"system_action", "group_member_joined", text,
actors,
[]imSystemMessageAction{{Type: "group_profile", Label: "查看群聊", TargetID: topic}},
extra)
}
func deliverIMConversationSystemMessageToP2PSessions(tenantID types.TenantID, msg *types.Message, out *ServerComMessage) {
if globals.sessionStore == nil || msg == nil || out == nil || out.Data == nil {
return
@@ -11629,9 +11729,10 @@ func deliverIMConversationSystemMessageToP2PSessions(tenantID types.TenantID, ms
if session.getSub(msg.Topic) != nil {
return true
}
msgCopy := out.copy()
msgCopy.RcptTo = session.uid.UserId()
msgCopy.Data.Topic = msg.Topic
msgCopy, ok := p2pConversationSystemMessageForUser(out, msg.Topic, session.uid)
if !ok {
return true
}
if !session.queueOut(msgCopy) {
logs.Warn.Printf("im system message: session queue full topic=%s seq=%d user=%s", msg.Topic, msg.SeqId, session.uid.UserId())
}
@@ -11639,6 +11740,43 @@ func deliverIMConversationSystemMessageToP2PSessions(tenantID types.TenantID, ms
})
}
func publishRealtimeP2PConversationSystemMessage(tenantID types.TenantID, topic string, out *ServerComMessage) bool {
uid1, uid2, err := types.ParseP2P(topic)
if err != nil {
return false
}
for _, uid := range []types.Uid{uid1, uid2} {
msgCopy, ok := p2pConversationSystemMessageForUser(out, topic, uid)
if !ok {
continue
}
publishRealtimeUserMessage(uid, msgCopy)
}
return true
}
func p2pConversationSystemMessageForUser(out *ServerComMessage, topic string, uid types.Uid) (*ServerComMessage, bool) {
if out == nil || uid.IsZero() {
return nil, false
}
viewTopic, err := types.P2PNameForUser(uid, topic)
if err != nil || viewTopic == "" {
return nil, false
}
msgCopy := out.copy()
msgCopy.RcptTo = uid.UserId()
if msgCopy.Data != nil {
msgCopy.Data.Topic = viewTopic
}
if msgCopy.Info != nil {
msgCopy.Info.Topic = viewTopic
}
if msgCopy.Pres != nil {
msgCopy.Pres.Topic = viewTopic
}
return msgCopy, true
}
func broadcastIMConversationSystemPresence(tenantID types.TenantID, msg *types.Message, timestamp time.Time) {
if globals.hub == nil || msg == nil || msg.SeqId <= 0 || msg.Topic == "" {
return
@@ -11724,6 +11862,12 @@ func topicFromMessagesSearchPath(path string) string {
return topicFromPathWithSuffix(path, prefix, suffix)
}
func topicFromMessageMutationsPath(path string) string {
const prefix = "/v1/im/topics/"
const suffix = "/message-mutations"
return topicFromPathWithSuffix(path, prefix, suffix)
}
func topicFromMediaPath(path string) string {
const prefix = "/v1/im/topics/"
const suffix = "/media"
@@ -11751,6 +11895,21 @@ func topicFromPathWithSuffix(path, prefix, suffix string) string {
return strings.Trim(rest, "/")
}
func parseIMMutationSince(primary, fallback string) (int64, error) {
value := strings.TrimSpace(primary)
if value == "" {
value = strings.TrimSpace(fallback)
}
if value == "" {
return 0, nil
}
parsed, err := strconv.ParseInt(value, 10, 64)
if err != nil || parsed < 0 {
return 0, strconv.ErrSyntax
}
return parsed, nil
}
func messageContextPath(path string) (string, int64, bool) {
const prefix = "/v1/im/topics/"
idx := strings.Index(path, prefix)
server/im_http_test.go
+251 -1
View File
@@ -548,6 +548,15 @@ func (s *testGroupBusinessStore) ListMutes(_ types.TenantID, _ string, _ string,
return s.mutes, "", nil
}
func (s *testGroupBusinessStore) GetMute(_ types.TenantID, _ string, uid types.Uid) (*types.GroupMute, error) {
for i := range s.mutes {
if s.mutes[i].UserID == uid && s.mutes[i].Status == types.GroupMuteStatusActive {
return &s.mutes[i], nil
}
}
return nil, types.ErrNotFound
}
func (s *testGroupBusinessStore) UpsertMute(_ types.TenantID, mute *types.GroupMute) (*types.GroupMute, error) {
mute.Version = 1
s.mutes = append(s.mutes, *mute)
@@ -5370,6 +5379,74 @@ func TestServeIMGroupQRCodesAndResolve(t *testing.T) {
}
}
func TestServeIMGroupInvitationDirectJoinWritesSystemNotice(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
mockStore := expectIMBearer(t, ctrl, "token-1", 7, 42)
mockStore.EXPECT().IsOpen().Return(true).AnyTimes()
oldSubs := store.Subs
oldTopics := store.Topics
oldMessages := store.Messages
oldProfiles := store.Profiles
oldGroupBusiness := store.GroupBusiness
subs := mock_store.NewMockSubsPersistenceInterface(ctrl)
topics := mock_store.NewMockTopicsPersistenceInterface(ctrl)
messages := mock_store.NewMockMessagesPersistenceInterface(ctrl)
store.Subs = subs
store.Topics = topics
store.Messages = messages
store.Profiles = nil
store.GroupBusiness = &testGroupBusinessStore{}
t.Cleanup(func() {
store.Subs = oldSubs
store.Topics = oldTopics
store.Messages = oldMessages
store.Profiles = oldProfiles
store.GroupBusiness = oldGroupBusiness
})
subs.EXPECT().Get(types.TenantID(7), "grpAlpha", types.Uid(42), false).Return(&types.Subscription{
TenantID: 7, User: types.Uid(42).String(), Topic: "grpAlpha", ModeWant: types.ModeCFull, ModeGiven: types.ModeCFull,
}, nil)
subs.EXPECT().Get(types.TenantID(7), "grpAlpha", types.Uid(43), false).Return(nil, types.ErrNotFound)
topics.EXPECT().Get(types.TenantID(7), "grpAlpha").Return(&types.Topic{TenantID: 7, SeqId: 7}, nil)
messages.EXPECT().Save(types.TenantID(7), gomock.Any(), nil, false).DoAndReturn(
func(_ types.TenantID, msg *types.Message, _ []string, _ bool) (*types.MessageSaveResult, error) {
if msg.Topic != "grpAlpha" || msg.SeqId != 8 {
t.Fatalf("unexpected system notice message: %#v", msg)
}
if msg.Head["x-im-template"] != "group_member_joined" || msg.Head["x-im-type"] != "system_action" {
t.Fatalf("unexpected system notice head: %#v", msg.Head)
}
content := msg.Content.(map[string]any)
if content["text"] != "有成员通过邀请加入了群聊。" || content["template"] != "group_member_joined" {
t.Fatalf("unexpected system notice content: %#v", content)
}
extra := content["extra"].(map[string]any)
if extra["source"] != "direct_invitation" {
t.Fatalf("unexpected system notice extra: %#v", extra)
}
return &types.MessageSaveResult{SeqId: msg.SeqId}, nil
})
body := fmt.Sprintf(`{"user_ids":["%s"],"message":"欢迎"}`, types.Uid(43).UserId())
req := httptest.NewRequest(http.MethodPost, "/v1/im/groups/grpAlpha/invitations", strings.NewReader(body))
req.Header.Set("Authorization", "Bearer token-1")
req.Header.Set("Idempotency-Key", "invite-1")
req = req.WithContext(context.WithValue(req.Context(), imHTTPProtocolValidated, true))
rec := httptest.NewRecorder()
serveIMGroupResource(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("invite status: got %d body=%s", rec.Code, rec.Body.String())
}
data := decodeIMHTTPResponse(t, rec).Data.(map[string]any)
items := data["items"].([]any)
if len(items) != 1 || items[0].(map[string]any)["status"] != types.GroupInvitationStatusJoined {
t.Fatalf("unexpected invitation response: %#v", data)
}
}
func TestServeIMGroupAnnouncementGovernanceJoinRequest(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
@@ -5625,6 +5702,56 @@ func TestServeIMGroupAdminsMutesAndOwnerTransfer(t *testing.T) {
}
}
func TestServeIMGroupMutesMemberGetsOwnMuteOnly(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
oldStore := store.Store
oldSubs := store.Subs
oldGroupBusiness := store.GroupBusiness
mockStore := mock_store.NewMockPersistentStorageInterface(ctrl)
tokenAuth := mock_auth.NewMockAuthHandler(ctrl)
mockStore.EXPECT().GetLogicalAuthHandler("token").Return(tokenAuth).Times(2)
tokenAuth.EXPECT().IsInitialized().Return(true).Times(2)
tokenAuth.EXPECT().Authenticate(auth.AuthContext{}, []byte("token-1")).
Return(&auth.Rec{TenantID: 7, Uid: 43, AuthLevel: auth.LevelAuth}, nil, nil).Times(2)
store.Store = mockStore
subs := mock_store.NewMockSubsPersistenceInterface(ctrl)
store.Subs = subs
memberSub := &types.Subscription{TenantID: 7, User: types.Uid(43).String(), Topic: "grpAlpha", ModeWant: types.ModeCPublic, ModeGiven: types.ModeCPublic}
subs.EXPECT().Get(types.TenantID(7), "grpAlpha", types.Uid(43), false).Return(memberSub, nil).Times(2)
groupStore := &testGroupBusinessStore{mutes: []types.GroupMute{
{Topic: "grpAlpha", UserID: types.Uid(42), MutedBy: types.Uid(99), MutedAt: time.Now().UTC(), Status: types.GroupMuteStatusActive, Version: 1},
{Topic: "grpAlpha", UserID: types.Uid(43), MutedBy: types.Uid(99), MutedAt: time.Now().UTC(), Reason: "test", Status: types.GroupMuteStatusActive, Version: 2},
}}
store.GroupBusiness = groupStore
defer func() {
store.Store = oldStore
store.Subs = oldSubs
store.GroupBusiness = oldGroupBusiness
}()
getReq := httptest.NewRequest(http.MethodGet, "/v1/im/groups/grpAlpha/mutes", nil)
getReq.Header.Set("Authorization", "Bearer token-1")
getRec := httptest.NewRecorder()
serveIMGroupResource(getRec, getReq)
if getRec.Code != http.StatusOK {
t.Fatalf("member mute get status: got %d body=%s", getRec.Code, getRec.Body.String())
}
data := decodeIMHTTPResponse(t, getRec).Data.(map[string]any)
items := data["items"].([]any)
if len(items) != 1 || items[0].(map[string]any)["user_id"] != types.Uid(43).UserId() {
t.Fatalf("member should only see own mute item: %#v", items)
}
postReq := httptest.NewRequest(http.MethodPost, "/v1/im/groups/grpAlpha/mutes/batch", bytes.NewBufferString(`{"user_ids":["`+types.Uid(42).UserId()+`"]}`))
postReq.Header.Set("Authorization", "Bearer token-1")
postRec := httptest.NewRecorder()
serveIMGroupResource(postRec, postReq)
if postRec.Code != http.StatusForbidden {
t.Fatalf("member mute post status: got %d body=%s", postRec.Code, postRec.Body.String())
}
}
func TestServeIMGroupDissolveConsumesActionToken(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
@@ -7647,6 +7774,62 @@ func TestServeIMMessageRecall(t *testing.T) {
}
}
func TestServeIMMessageMutations(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
actor := types.Uid(42)
expectIMBearer(t, ctrl, "token-1", 7, actor)
oldSubs := store.Subs
oldMessages := store.Messages
subs := mock_store.NewMockSubsPersistenceInterface(ctrl)
messages := mock_store.NewMockMessagesPersistenceInterface(ctrl)
store.Subs = subs
store.Messages = messages
defer func() {
store.Subs = oldSubs
store.Messages = oldMessages
}()
subs.EXPECT().Get(types.TenantID(7), "grpRecall", actor, false).Return(&types.Subscription{
TenantID: 7,
User: actor.String(),
Topic: "grpRecall",
ModeWant: types.ModeCPublic,
ModeGiven: types.ModeCPublic,
}, nil)
mutatedAt := time.Date(2026, 9, 15, 10, 0, 0, 0, time.UTC)
messages.EXPECT().Mutations(types.TenantID(7), "grpRecall", actor, int64(1), 20).Return([]types.MessageMutation{{
SeqId: 9,
Version: 2,
Event: "message.recalled",
Operator: actor.UserId(),
MutatedAt: mutatedAt,
MutationVersion: 3,
}}, "3", false, int64(3), nil)
req := httptest.NewRequest(http.MethodGet, "/v1/im/topics/grpRecall/message-mutations?since_version=1&limit=20", nil)
req.Header.Set("Authorization", "Bearer token-1")
rec := httptest.NewRecorder()
serveIMTopicMessageAction(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("status: got %d, want %d; body=%s", rec.Code, http.StatusOK, rec.Body.String())
}
data := decodeIMHTTPResponse(t, rec).Data.(map[string]any)
if data["mutation_version"].(float64) != 3 || data["next_cursor"] != "3" {
t.Fatalf("unexpected mutation response: %#v", data)
}
items := data["items"].([]any)
if len(items) != 1 {
t.Fatalf("unexpected mutation items: %#v", items)
}
item := items[0].(map[string]any)
if item["seq"].(float64) != 9 || item["version"].(float64) != 2 || item["event"] != "message.recalled" {
t.Fatalf("unexpected mutation item: %#v", item)
}
}
func TestServeIMMessageBatchRecallPartialSuccess(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
@@ -8234,7 +8417,11 @@ func TestPublishIMConversationSystemNoticeDeliversToOnlineP2PSessions(t *testing
if !ok || msg.Data == nil {
t.Fatalf("unexpected outbound for %s: %#v", uid.UserId(), raw)
}
if msg.Data.Topic != topic || msg.Data.SeqId != 1 {
wantTopic, err := types.P2PNameForUser(uid, topic)
if err != nil {
t.Fatal(err)
}
if msg.Data.Topic != wantTopic || msg.Data.SeqId != 1 {
t.Fatalf("unexpected p2p data for %s: %#v", uid.UserId(), msg.Data)
}
if msg.Data.Head["x-im-template"] != "friend_added" || msg.Data.Head["x-im-type"] != "system_notice" {
@@ -8652,6 +8839,69 @@ func TestServeIMUserProfile(t *testing.T) {
}
}
func TestServeIMUserProfileBlockedByMeReturnsMinimalProfile(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
expectIMBearer(t, ctrl, "token-1", 7, 1)
oldUsers := store.Users
oldRelations := store.Relations
oldProfiles := store.Profiles
users := mock_store.NewMockUsersPersistenceInterface(ctrl)
peer := types.Uid(2)
users.EXPECT().Get(types.TenantID(7), peer).Return(&types.User{
ObjHeader: types.ObjHeader{Id: peer.String()},
TenantID: 7,
Public: map[string]any{
"display_name": "Alice",
"nickname": "Alice",
"avatar_file_id": "file-1",
"signature": "hello",
"gender": "female",
"region": "Shanghai",
"public_account": "alice001",
},
}, nil)
store.Users = users
store.Relations = &testRelationsStore{context: &types.RelationshipContext{
TenantID: 7,
UserID: 1,
PeerID: peer,
Topic: types.Uid(1).P2PName(peer),
Relationship: types.RelationshipBlockedByMe,
Remark: "老同事",
}}
store.Profiles = &testProfilesStore{}
defer func() {
store.Users = oldUsers
store.Relations = oldRelations
store.Profiles = oldProfiles
}()
req := httptest.NewRequest(http.MethodGet, "/v1/im/users/"+peer.UserId()+"/profile", nil)
req.Header.Set("Authorization", "Bearer token-1")
rec := httptest.NewRecorder()
serveIMUserResource(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("status: got %d, want %d; body=%s", rec.Code, http.StatusOK, rec.Body.String())
}
data := decodeIMHTTPResponse(t, rec).Data.(map[string]any)
if data["relationship"] != types.RelationshipBlockedByMe || data["display_name"] != "老同事" ||
data["avatar_file_id"] != "file-1" || data["topic"] == "" {
t.Fatalf("unexpected blocked profile response: %#v", data)
}
capabilities := data["capabilities"].(map[string]any)
if capabilities["can_send"] != false || capabilities["can_call"] != false || capabilities["can_add_friend"] != false {
t.Fatalf("blocked profile should disable actions: %#v", data)
}
for _, field := range []string{"signature", "gender", "region", "public_account"} {
if _, ok := data[field]; ok {
t.Fatalf("blocked profile should omit private field %q: %#v", field, data)
}
}
}
func TestServeIMUserQRCodes(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
server/internal_admin_query_resources.go
+84 -16
View File
@@ -63,19 +63,20 @@ var internalTenantResourceSpecs = map[string]*internalQuerySpec{
"private-messages": privateMessagesInternalQuerySpec(),
"group-messages": groupMessagesInternalQuerySpec(),
"user-login-logs": {
Select: `SELECT id,CAST(user_id AS CHAR) AS user_id,username,account_type,account_mask,login_type,event_type,session_id,device_id,platform,app_version,ip,region,user_agent,risk_level,request_id,login_at,logout_at,metadata`,
Select: `SELECT id,user_id,username,account_type,account_mask,login_type,event_type,session_id,device_id,platform,app_version,ip,region,user_agent,risk_level,request_id,login_at,logout_at,metadata`,
From: "FROM im_user_login_logs", DefaultSort: "ORDER BY id DESC", DetailKey: "id",
Filters: []internalQueryFilter{{"user_id", "user_id", "=", "uid"}, {"account", "account_mask", "LIKE", "string"}, {"login_type", "login_type", "=", "string"}, {"event_type", "event_type", "=", "string"}, {"platform", "platform", "=", "string"}, {"ip", "ip", "LIKE", "string"}, {"login_at_start", "login_at", ">=", "time"}, {"login_at_end", "login_at", "<=", "time"}},
},
"login-failure-logs": {
Select: `SELECT id,CAST(user_id AS CHAR) AS user_id,account_type,account_mask,failure_type,reason,detail,ip,region,device_id,platform,app_version,user_agent,risk_level,risk_action,request_id,attempt_at,metadata`,
Select: `SELECT id,user_id,account_type,account_mask,failure_type,reason,detail,ip,region,device_id,platform,app_version,user_agent,risk_level,risk_action,request_id,attempt_at,metadata`,
From: "FROM im_login_failure_logs", DefaultSort: "ORDER BY id DESC", DetailKey: "id",
Filters: []internalQueryFilter{{"user_id", "user_id", "=", "uid"}, {"account", "account_mask", "LIKE", "string"}, {"failure_type", "failure_type", "=", "string"}, {"platform", "platform", "=", "string"}, {"ip", "ip", "LIKE", "string"}, {"attempt_at_start", "attempt_at", ">=", "time"}, {"attempt_at_end", "attempt_at", "<=", "time"}},
},
"audit-logs": {
Select: "SELECT id,request_id,actor_user_id,action,target_type,target_id,ip,user_agent,detail,created_at",
From: "FROM im_audit_logs", DefaultSort: "ORDER BY id DESC", DetailKey: "id",
Filters: []internalQueryFilter{{"actor_user_id", "actor_user_id", "=", "uid"}, {"action", "action", "=", "string"}, {"target_type", "target_type", "=", "string"}, {"target_id", "target_id", "=", "string"}, {"created_at_start", "created_at", ">=", "time"}, {"created_at_end", "created_at", "<=", "time"}},
Select: `SELECT id,request_id,actor_uid AS actor_user_id,action,target_type,target_id,ip,user_agent,
JSON_OBJECT('actor_role',actor_role,'before_snapshot',before_snapshot,'after_snapshot',after_snapshot,'reason',reason,'metadata',metadata) AS detail,created_at`,
From: "FROM im_audit_logs", DefaultSort: "ORDER BY id DESC", DetailKey: "id",
Filters: []internalQueryFilter{{"actor_user_id", "actor_uid", "=", "uid"}, {"action", "action", "=", "string"}, {"target_type", "target_type", "=", "string"}, {"target_id", "target_id", "=", "string"}, {"created_at_start", "created_at", ">=", "time"}, {"created_at_end", "created_at", "<=", "time"}},
},
"realname-verifications": {
Select: "SELECT id,user_id,submission_id,status,masked_name AS real_name,masked_id_number AS id_card_no_masked,review_reason AS reject_reason,verified_at AS reviewed_at,created_at,updated_at",
@@ -136,6 +137,14 @@ func serveInternalAdminResourceList(wrt http.ResponseWriter, req *http.Request,
writeIMError(wrt, req, http.StatusNotFound, 30004, imReasonMalformedRequest, "resource not found", nil)
return
}
if resource == "group-messages" {
var err error
spec, err = groupMessagesInternalQuerySpecForTenant(tenantID)
if err != nil {
writeInternalQueryError(wrt, req, err)
return
}
}
query, countQuery, args, page, perPage, err := buildInternalAdminListQuery(req, spec, scope)
if err != nil {
writeIMError(wrt, req, http.StatusBadRequest, 30000, imReasonMalformedRequest, "malformed request", nil)
@@ -256,7 +265,7 @@ func internalAdminQuerySpec(resource string) (*internalQuerySpec, bool) {
func usersInternalQuerySpec() *internalQuerySpec {
return &internalQuerySpec{
Select: `SELECT CAST(u.id AS CHAR) AS user_id,u.account_no AS display_no,u.username,u.phone,u.state,u.stateat AS state_at,u.access,u.lastseen AS last_seen,u.useragent AS user_agent,u.public,u.trusted,
Select: `SELECT u.id AS user_id,u.account_no AS display_no,u.username,u.phone,u.state,u.stateat AS state_at,u.access,u.lastseen AS last_seen,u.useragent AS user_agent,u.public,u.trusted,
p.nickname,p.public_account,p.gender,p.region,p.avatar_file_id,
COALESCE(
NULLIF(avatar.location,''),
@@ -283,7 +292,7 @@ func groupsInternalQuerySpec() *internalQuerySpec {
return &internalQuerySpec{
Select: `SELECT t.name AS topic,t.state,t.owner AS owner_id,t.seqid AS seq_id,t.delid AS del_id,t.subcnt AS sub_count,t.touchedat AS touched_at,t.createdat AS created_at,t.updatedat AS updated_at,t.public,t.trusted,t.access,t.tags,t.aux,
gs.name AS group_name,gs.avatar_file_id,gs.description,gs.group_status,gs.join_approval,gs.mute_all,gs.member_upload_files,gs.member_private_chat,gs.member_invite,gs.max_member_count,gs.max_admin_count,gs.member_count AS business_member_count,gs.version AS settings_version,gs.updated_at AS settings_updated_at`,
From: "FROM im_topics AS t LEFT JOIN im_group_settings AS gs ON gs.topic=t.name",
From: "FROM (SELECT * FROM im_topics WHERE name LIKE 'grp%') AS t LEFT JOIN im_group_settings AS gs ON gs.topic=t.name",
DefaultSort: "ORDER BY t.id DESC",
DetailKey: "t.name",
Filters: []internalQueryFilter{{"topic", "t.name", "LIKE", "string"}, {"owner_id", "t.owner", "=", "uid"}, {"state", "t.state", "=", "int"}, {"group_status", "gs.group_status", "=", "string"}, {"created_at_start", "t.createdat", ">=", "time"}, {"created_at_end", "t.createdat", "<=", "time"}, {"touched_at_start", "t.touchedat", ">=", "time"}, {"touched_at_end", "t.touchedat", "<=", "time"}},
@@ -296,9 +305,9 @@ func privateMessagesInternalQuerySpec() *internalQuerySpec {
contentExcerpt := `LEFT(COALESCE(NULLIF(JSON_UNQUOTE(JSON_EXTRACT(m.content,'$.txt')),''),CASE JSON_TYPE(m.content) WHEN 'STRING' THEN JSON_UNQUOTE(m.content) ELSE CONCAT('[',` + contentType + `,']') END,''),500)`
status := `CASE WHEN aa.action='delete' THEN 'deleted' WHEN aa.action='hide' THEN 'hidden' WHEN m.recalled_at IS NOT NULL THEN 'recalled' WHEN m.deletedat IS NOT NULL THEN 'deleted' ELSE 'normal' END`
return &internalQuerySpec{
Select: `SELECT m.topic,m.seqid AS seq_id,CAST(m.` + "`from`" + ` AS CHAR) AS from_user_id,
Select: `SELECT m.topic,m.seqid AS seq_id,m.` + "`from`" + ` AS from_user_id,
COALESCE(su.username,'') AS sender_username,COALESCE(sp.nickname,'') AS sender_nickname,
CAST(rs.userid AS CHAR) AS to_user_id,COALESCE(ru.username,'') AS receiver_username,COALESCE(rp.nickname,'') AS receiver_nickname,
rs.userid AS to_user_id,COALESCE(ru.username,'') AS receiver_username,COALESCE(rp.nickname,'') AS receiver_nickname,
` + contentType + ` AS content_type,` + contentExcerpt + ` AS content_excerpt,
` + status + ` AS status,m.createdat AS created_at,m.updatedat AS updated_at`,
From: `FROM (SELECT * FROM im_messages WHERE topic LIKE 'p2p%') AS m
@@ -318,7 +327,7 @@ func groupMessagesInternalQuerySpec() *internalQuerySpec {
contentType := `CASE m.msg_type WHEN 'call' THEN 'system' WHEN 'system_notice' THEN 'system' WHEN 'contact_card' THEN 'card' WHEN 'forward_bundle' THEN 'card' WHEN 'link' THEN 'text' ELSE m.msg_type END`
contentExcerpt := `LEFT(COALESCE(NULLIF(JSON_UNQUOTE(JSON_EXTRACT(m.content,'$.txt')),''),CASE JSON_TYPE(m.content) WHEN 'STRING' THEN JSON_UNQUOTE(m.content) ELSE CONCAT('[',` + contentType + `,']') END,''),500)`
return &internalQuerySpec{
Select: `SELECT m.topic,m.seqid AS seq_id,CAST(m.from_user AS CHAR) AS from_user_id,
Select: `SELECT m.topic,m.seqid AS seq_id,m.from_user AS from_user_id,
COALESCE(u.username,'') AS sender_username,COALESCE(p.nickname,'') AS sender_nickname,
COALESCE(gs.name,'') AS group_name,m.topic AS group_no,` + contentType + ` AS content_type,
` + contentExcerpt + ` AS content_excerpt,
@@ -346,14 +355,68 @@ func groupMessagesInternalQuerySpec() *internalQuerySpec {
}
}
func groupMessagesInternalQuerySpecForTenant(tenantID types.TenantID) (*internalQuerySpec, error) {
source, err := internalGroupMessageSource(tenantID)
if err != nil {
return nil, err
}
spec := groupMessagesInternalQuerySpec()
spec.From = strings.Replace(spec.From, "FROM im_group_messages AS m", "FROM "+source+" AS m", 1)
spec.DefaultSort = "ORDER BY m.created_at DESC,m.id DESC"
return spec, nil
}
func internalGroupMessageSource(tenantID types.TenantID) (string, error) {
result, err := store.InternalAdminQueries.Query(tenantID,
"SELECT table_name FROM im_group_message_shards WHERE status<>'archived' ORDER BY shard_month DESC",
"SELECT COUNT(*) FROM im_group_message_shards WHERE status<>'archived'", nil, 1, 200)
if err != nil {
return "", err
}
selects := make([]string, 0)
for _, item := range result.Items {
table := strings.TrimSpace(fmt.Sprint(item["table_name"]))
if !validInternalGroupMessageTable(table) {
continue
}
selects = append(selects, fmt.Sprintf(`SELECT message_id AS id,topic,seqid,from_user,msg_type,status,content,head,reply_seqid,mentioned_user_ids,created_at,updated_at FROM %s`, table))
}
selects = append(selects, `SELECT id,topic COLLATE utf8mb4_0900_ai_ci AS topic,seqid,from_user,
msg_type COLLATE utf8mb4_0900_ai_ci AS msg_type,status COLLATE utf8mb4_0900_ai_ci AS status,
content,head,reply_seqid,mentioned_user_ids,created_at,updated_at
FROM im_group_messages AS legacy WHERE NOT EXISTS (
SELECT 1 FROM im_group_message_topic_shards AS route
WHERE route.topic=legacy.topic COLLATE utf8mb4_0900_ai_ci AND legacy.seqid BETWEEN route.min_seq AND route.max_seq)`)
return "(" + strings.Join(selects, " UNION ALL ") + ")", nil
}
func validInternalGroupMessageTable(table string) bool {
const prefix = "im_group_messages_"
if len(table) != len(prefix)+6 || !strings.HasPrefix(table, prefix) {
return false
}
for _, ch := range table[len(prefix):] {
if ch < '0' || ch > '9' {
return false
}
}
return true
}
func serveInternalDashboardOverview(wrt http.ResponseWriter, req *http.Request, tenantID types.TenantID) {
row, err := store.InternalAdminQueries.QueryOne(tenantID, `SELECT
groupMessages, err := internalGroupMessageSource(tenantID)
if err != nil {
writeInternalQueryError(wrt, req, err)
return
}
query := fmt.Sprintf(`SELECT
(SELECT COUNT(*) FROM im_users WHERE state<>20) AS total_users,
(SELECT COUNT(*) FROM im_users WHERE state=0 AND lastseen>=UTC_DATE()) AS active_users,
(SELECT COUNT(*) FROM im_group_settings WHERE group_status='active') AS total_groups,
(SELECT COUNT(*) FROM im_topics WHERE state<>20 AND (name LIKE 'p2p%' OR name LIKE 'grp%')) AS total_conversations,
((SELECT COUNT(*) FROM im_messages WHERE topic LIKE 'p2p%')+(SELECT COUNT(*) FROM im_group_messages)) AS total_messages,
((SELECT COUNT(*) FROM im_messages WHERE topic LIKE 'p2p%' AND createdat>=UTC_DATE())+(SELECT COUNT(*) FROM im_group_messages WHERE created_at>=UTC_DATE())) AS today_messages`, nil)
(SELECT COUNT(*) FROM im_topics WHERE state<>20 AND (name LIKE 'p2p%%' OR name LIKE 'grp%%')) AS total_conversations,
((SELECT COUNT(*) FROM im_messages WHERE topic LIKE 'p2p%%')+(SELECT COUNT(*) FROM %s AS gm)) AS total_messages,
((SELECT COUNT(*) FROM im_messages WHERE topic LIKE 'p2p%%' AND createdat>=UTC_DATE())+(SELECT COUNT(*) FROM %s AS gm WHERE gm.created_at>=UTC_DATE())) AS today_messages`, groupMessages, groupMessages)
row, err := store.InternalAdminQueries.QueryOne(tenantID, query, nil)
if err != nil {
writeInternalQueryError(wrt, req, err)
return
@@ -366,14 +429,19 @@ func serveInternalDashboardTrends(wrt http.ResponseWriter, req *http.Request, te
if days > 31 {
days = 31
}
groupMessages, err := internalGroupMessageSource(tenantID)
if err != nil {
writeInternalQueryError(wrt, req, err)
return
}
query := fmt.Sprintf(`WITH RECURSIVE dates AS (
SELECT UTC_DATE()-INTERVAL %d DAY AS day
UNION ALL SELECT day+INTERVAL 1 DAY FROM dates WHERE day<UTC_DATE()
) SELECT DATE_FORMAT(d.day,'%%Y-%%m-%%d') AS date,
(SELECT COUNT(*) FROM im_users AS u WHERE u.createdat>=d.day AND u.createdat<d.day+INTERVAL 1 DAY) AS user_count,
((SELECT COUNT(*) FROM im_messages AS pm WHERE pm.topic LIKE 'p2p%%' AND pm.createdat>=d.day AND pm.createdat<d.day+INTERVAL 1 DAY)+
(SELECT COUNT(*) FROM im_group_messages AS gm WHERE gm.created_at>=d.day AND gm.created_at<d.day+INTERVAL 1 DAY)) AS message_count
FROM dates AS d ORDER BY d.day ASC`, days-1)
(SELECT COUNT(*) FROM %s AS gm WHERE gm.created_at>=d.day AND gm.created_at<d.day+INTERVAL 1 DAY)) AS message_count
FROM dates AS d ORDER BY d.day ASC`, days-1, groupMessages)
result, err := store.InternalAdminQueries.Query(tenantID, query, fmt.Sprintf("SELECT %d", days), nil, 1, days)
if err != nil {
writeInternalQueryError(wrt, req, err)
server/internal_api.go
+23 -25
View File
@@ -376,12 +376,10 @@ func authorizeInternalAPI(wrt http.ResponseWriter, req *http.Request, capability
writeIMError(wrt, req, http.StatusUnauthorized, 30043, imReasonAuthRequired, "invalid internal signature", nil)
return false
}
if req.Method != http.MethodGet {
requestClaim := internalRequestClaimValue(signingText)
if !claimInternalNonce(serviceName, nonce, requestClaim, time.Now().Add(window)) {
writeIMError(wrt, req, http.StatusUnauthorized, 30043, imReasonAuthRequired, "internal nonce replayed", nil)
return false
}
requestClaim := internalRequestClaimValue(signingText)
if !claimInternalNonce(serviceName, nonce, requestClaim, time.Now().Add(window)) {
writeIMError(wrt, req, http.StatusUnauthorized, 30043, imReasonAuthRequired, "internal nonce replayed", nil)
return false
}
return true
}
@@ -566,7 +564,11 @@ func serveInternalMessageQuery(wrt http.ResponseWriter, req *http.Request, tenan
where := "m.topic=? AND m.seqid=?"
if strings.HasPrefix(topic, "grp") {
resource = "group-messages"
spec = groupMessagesInternalQuerySpec()
spec, err = groupMessagesInternalQuerySpecForTenant(tenantID)
if err != nil {
writeInternalQueryError(wrt, req, err)
return
}
}
spec.Select += ",m.head,JSON_UNQUOTE(m.content) AS content"
row, err := store.InternalAdminQueries.QueryOne(tenantID, spec.Select+" "+spec.From+" WHERE "+where, []any{topic, seq})
@@ -579,26 +581,22 @@ func serveInternalMessageQuery(wrt http.ResponseWriter, req *http.Request, tenan
}
func serveInternalFileQuery(wrt http.ResponseWriter, req *http.Request, tenantID types.TenantID, fileID string) {
if fileID == "" || store.Files == nil {
if fileID == "" {
writeIMError(wrt, req, http.StatusBadRequest, 30000, imReasonMalformedRequest, "malformed request", map[string]any{"field": "file_id"})
return
}
file, err := store.Files.Get(tenantID, fileID)
fileDBID, ok := parseInternalAdminDBID(fileID)
if !ok {
writeIMError(wrt, req, http.StatusBadRequest, 30000, imReasonMalformedRequest, "malformed request", map[string]any{"field": "file_id"})
return
}
spec, _ := internalAdminQuerySpec("files")
row, err := store.InternalAdminQueries.QueryOne(tenantID, spec.Select+" "+spec.From+" WHERE id=?", []any{fileDBID})
if err != nil {
writeInternalQueryError(wrt, req, err)
return
}
writeIMSuccess(wrt, req, map[string]any{
"file_id": file.Id,
"status": file.Status,
"user_id": internalAdminUIDStringRaw(file.User),
"mime_type": file.MimeType,
"size": file.Size,
"location": file.Location,
"etag": file.ETag,
"created_at": file.CreatedAt,
"updated_at": file.UpdatedAt,
})
writeIMSuccess(wrt, req, normalizeInternalAdminRowForAPI(row))
}
func serveInternalKafkaDeadLettersQuery(wrt http.ResponseWriter, req *http.Request, tenantID types.TenantID) {
@@ -705,6 +703,7 @@ func internalKafkaDeadLetterPayload(dead *types.KafkaDeadLetter, includeEnvelope
}
func writeInternalQueryError(wrt http.ResponseWriter, req *http.Request, err error) {
logs.Warn.Printf("internal admin query failed: method=%s path=%s err=%v", req.Method, req.URL.RequestURI(), err)
switch {
case errors.Is(err, types.ErrUserNotFound), errors.Is(err, types.ErrTopicNotFound), errors.Is(err, types.ErrNotFound):
writeIMError(wrt, req, http.StatusNotFound, 30004, imReasonMalformedRequest, "resource not found", nil)
@@ -772,12 +771,12 @@ func verifyInternalSignature(secret, signingText, signature string) bool {
func claimInternalNonce(serviceName, nonce, requestClaim string, expiresAt time.Time) bool {
ttl := time.Until(expiresAt)
nonceKey := fmt.Sprintf("%s:%s", serviceName, nonce)
if claimed, sameValue, enabled, err := store.ClaimShortStateValue("internal_api_nonce", nonceKey, requestClaim, ttl); enabled {
if claimed, _, enabled, err := store.ClaimShortStateValue("internal_api_nonce", nonceKey, requestClaim, ttl); enabled {
if err != nil {
logs.Warn.Println("internal api: failed to claim nonce in redis", err)
return false
}
return claimed || sameValue
return claimed
}
now := time.Now()
internalAPI.nonces.Range(func(key, value any) bool {
@@ -786,9 +785,8 @@ func claimInternalNonce(serviceName, nonce, requestClaim string, expiresAt time.
}
return true
})
if existing, loaded := internalAPI.nonces.LoadOrStore(nonceKey, internalNonceClaim{value: requestClaim, expiresAt: expiresAt}); loaded {
claim, ok := existing.(internalNonceClaim)
return ok && claim.value == requestClaim
if _, loaded := internalAPI.nonces.LoadOrStore(nonceKey, internalNonceClaim{value: requestClaim, expiresAt: expiresAt}); loaded {
return false
}
return true
}
server/internal_api_test.go
+15
View File
@@ -5,6 +5,7 @@ import (
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/golang/mock/gomock"
"github.com/tinode/chat/server/store"
@@ -18,6 +19,20 @@ func (internalAdminQueryNotFound) Query(types.TenantID, string, string, []any, i
return nil, types.ErrNotFound
}
func TestClaimInternalNonceRejectsReplay(t *testing.T) {
oldConfig := internalAPI.config
internalAPI = internalAPIRuntime{config: oldConfig}
defer func() { internalAPI = internalAPIRuntime{config: oldConfig} }()
expiresAt := time.Now().Add(time.Minute)
if !claimInternalNonce("manager-test", "single-use-nonce", "request-claim", expiresAt) {
t.Fatal("first nonce claim was rejected")
}
if claimInternalNonce("manager-test", "single-use-nonce", "request-claim", expiresAt) {
t.Fatal("replayed nonce was accepted")
}
}
func (internalAdminQueryNotFound) QueryOne(types.TenantID, string, []any) (map[string]any, error) {
return nil, types.ErrNotFound
}
server/kafka_message_persist.go
+19 -3
View File
@@ -18,6 +18,7 @@ type kafkaMessagePersistPayload struct {
AttachmentURLs []string `json:"attachment_urls,omitempty"`
ReadBySender bool `json:"read_by_sender,omitempty"`
MessageKind string `json:"message_kind"`
ShardMonth string `json:"shard_month,omitempty"`
}
var publishKafkaMessagePersist = publishKafkaEnvelope
@@ -34,6 +35,11 @@ func saveMessageMaybeKafka(ctx context.Context, tenantID types.TenantID, msg *ty
if msg == nil || msg.TenantID != tenantID || msg.Topic == "" || msg.SeqId <= 0 {
return nil, types.ErrMalformed
}
msg.InitTimes()
if msg.Uid().IsZero() {
msg.SetUid(store.Store.GetUid())
}
isGroupMessage := types.GetTopicCat(msg.Topic) == types.TopicCatGrp
clientMsgID, err := store.NormalizeClientMsgID(msg.Head)
if err != nil {
return nil, err
@@ -56,15 +62,21 @@ func saveMessageMaybeKafka(ctx context.Context, tenantID types.TenantID, msg *ty
return nil, types.ErrDuplicate
}
}
if seq, runtimeReady, seqErr := store.ReserveMessageSeq(tenantID, msg.Topic, msg.SeqId-1); seqErr != nil {
candidateMonth := msg.CreatedAt.UTC().Format("200601")
if seq, shardMonth, runtimeReady, seqErr := store.ReserveMessageSeq(tenantID, msg.Topic, msg.SeqId-1, candidateMonth); seqErr != nil {
if clientMsgID != "" {
_ = store.MarkMessageUniqueFailed(tenantID, msg.Topic, fromUid, clientMsgID)
}
return nil, seqErr
} else if runtimeReady {
msg.SeqId = seq
if isGroupMessage {
msg.ShardMonth = shardMonth
}
}
if msg.ShardMonth == "" && isGroupMessage {
msg.ShardMonth = candidateMonth
}
msg.InitTimes()
if err = enqueueKafkaMessagePersist(ctx, tenantID, msg, attachmentURLs, readBySender); err != nil {
if clientMsgID != "" {
if markErr := store.MarkMessageUniquePending(tenantID, msg.Topic, fromUid, clientMsgID, msg.SeqId); markErr != nil {
@@ -79,7 +91,7 @@ func saveMessageMaybeKafka(ctx context.Context, tenantID types.TenantID, msg *ty
logs.Warn.Printf("kafka message persist: failed to mark message unique key done topic=%s seq=%d: %v", msg.Topic, msg.SeqId, err)
}
}
if types.GetTopicCat(msg.Topic) == types.TopicCatGrp {
if isGroupMessage {
if cacheErr := store.CachePendingHotMessage(tenantID, msg); cacheErr != nil {
logs.Warn.Printf("kafka message persist: failed to cache pending hot message after publish topic=%s seq=%d: %v", msg.Topic, msg.SeqId, cacheErr)
}
@@ -97,6 +109,7 @@ func enqueueKafkaMessagePersist(ctx context.Context, tenantID types.TenantID, ms
AttachmentURLs: attachmentURLs,
ReadBySender: readBySender,
MessageKind: kind,
ShardMonth: msg.ShardMonth,
})
if err != nil {
return err
@@ -147,6 +160,9 @@ func handleKafkaMessagePersistFromTopic(ctx context.Context, envelope kafkaEnvel
if payload.Message.TenantID == types.ZeroTenantID {
payload.Message.TenantID = tenantID
}
if payload.Message.ShardMonth == "" {
payload.Message.ShardMonth = payload.ShardMonth
}
if payload.Message.TenantID != tenantID {
return markKafkaDeadLetter(ctx, tenantID, envelope, consume, "TENANT_MISMATCH", "message tenant does not match envelope tenant")
}
server/media/oss/oss.go
+10 -1
View File
@@ -254,7 +254,7 @@ func (oh *ossHandler) Headers(tenantID types.TenantID, method string, url *url.U
if redirURL != "" {
return http.Header{
"Location": {redirURL},
"Location": {normalizeSignedRedirectURL(redirURL)},
"ETag": {`"` + fdef.ETag + `"`},
"Content-Type": {"application/json; charset=utf-8"},
"Cache-Control": {oh.conf.CacheControl},
@@ -263,6 +263,15 @@ func (oh *ossHandler) Headers(tenantID types.TenantID, method string, url *url.U
return nil, 0, nil
}
func normalizeSignedRedirectURL(rawURL string) string {
parsed, err := url.Parse(strings.TrimSpace(rawURL))
if err != nil || parsed.Scheme != "http" || parsed.Host == "" {
return rawURL
}
parsed.Scheme = "https"
return parsed.String()
}
// Upload processes request for a file upload.
func (oh *ossHandler) Upload(fdef *types.FileDef, file io.Reader) (string, int64, error) {
key, err := tenantObjectKey(fdef.TenantID, fdef.Uid().String32(), fdef.CreatedAt)
server/media/oss/oss_tenant_test.go
+11
View File
@@ -58,6 +58,17 @@ func TestIsOSSObjectNotFound(t *testing.T) {
}
}
func TestNormalizeSignedRedirectURL(t *testing.T) {
rawURL := "http://im-oss-t1.oss-cn-shenzhen.aliyuncs.com/tenants/1/avatar.jpg?Expires=1&OSSAccessKeyId=ak&Signature=sig"
want := "https://im-oss-t1.oss-cn-shenzhen.aliyuncs.com/tenants/1/avatar.jpg?Expires=1&OSSAccessKeyId=ak&Signature=sig"
if got := normalizeSignedRedirectURL(rawURL); got != want {
t.Fatalf("normalizeSignedRedirectURL()=%q, want %q", got, want)
}
if got := normalizeSignedRedirectURL("/v0/file/s/avatar.jpg"); got != "/v0/file/s/avatar.jpg" {
t.Fatalf("relative URL must be unchanged, got %q", got)
}
}
func TestLoadSysConfig(t *testing.T) {
original := store.ThirdPartyConfig
store.ThirdPartyConfig = fakeThirdPartyConfig{
server/store/group_message_runtime.go
+171 -14
View File
@@ -54,6 +54,14 @@ type MessageUniqueClaim struct {
Seq int64
}
// GroupMessageShardRoute describes the Seq range stored in one monthly table.
type GroupMessageShardRoute struct {
Month string `db:"month"`
MinSeq int64 `db:"min_seq"`
MaxSeq int64 `db:"max_seq"`
Count int64 `db:"count"`
}
type groupFanoutEnvelope struct {
OriginNodeID string `json:"origin_node_id"`
Message json.RawMessage `json:"message"`
@@ -84,10 +92,13 @@ type LoginSession struct {
type groupMessageRuntime interface {
Close() error
Claim(tenantID types.TenantID, topic string, from types.Uid, clientMsgID string) (*groupMessageUniqueRecord, bool, error)
NextSeq(tenantID types.TenantID, topic string, baseline int64) (int64, error)
NextSeq(tenantID types.TenantID, topic string, baseline int64, candidateMonth string) (int64, string, error)
PeekSeq(tenantID types.TenantID, topic string) (int64, error)
BumpSeqFloor(tenantID types.TenantID, topic string, seq int64) error
MarkPending(tenantID types.TenantID, topic string, from types.Uid, clientMsgID string, seq int64) error
PutShardRoute(tenantID types.TenantID, topic string, route GroupMessageShardRoute) error
GetShardRoutes(tenantID types.TenantID, topic string) ([]GroupMessageShardRoute, bool, error)
DeleteShardRoutes(tenantID types.TenantID, topic string) error
MarkDone(tenantID types.TenantID, topic string, from types.Uid, clientMsgID string, seq int64, msgID string) error
MarkFailed(tenantID types.TenantID, topic string, from types.Uid, clientMsgID string) error
CacheHotMessage(tenantID types.TenantID, msg *types.Message) error
@@ -132,12 +143,33 @@ func ClaimMessageUnique(tenantID types.TenantID, topic string, from types.Uid, c
return &MessageUniqueClaim{Status: rec.Status, Seq: rec.Seq}, claimed, true, err
}
func ReserveMessageSeq(tenantID types.TenantID, topic string, baseline int64) (int64, bool, error) {
func ReserveMessageSeq(tenantID types.TenantID, topic string, baseline int64, candidateMonth string) (int64, string, bool, error) {
if groupMsgRuntime == nil || !tenantID.IsValid() || topic == "" {
return 0, false, nil
return 0, candidateMonth, false, nil
}
seq, err := nextMessageSeqWithFloor(tenantID, topic, baseline)
return seq, true, err
seq, month, err := nextMessageSeqWithFloor(tenantID, topic, baseline, candidateMonth)
return seq, month, true, err
}
func PutGroupMessageShardRoute(tenantID types.TenantID, topic string, route GroupMessageShardRoute) error {
if groupMsgRuntime == nil || !tenantID.IsValid() || topic == "" || route.Month == "" {
return nil
}
return groupMsgRuntime.PutShardRoute(tenantID, topic, route)
}
func GetGroupMessageShardRoutes(tenantID types.TenantID, topic string) ([]GroupMessageShardRoute, bool, error) {
if groupMsgRuntime == nil || !tenantID.IsValid() || topic == "" {
return nil, false, nil
}
return groupMsgRuntime.GetShardRoutes(tenantID, topic)
}
func DeleteGroupMessageShardRoutes(tenantID types.TenantID, topic string) error {
if groupMsgRuntime == nil || !tenantID.IsValid() || topic == "" {
return nil
}
return groupMsgRuntime.DeleteShardRoutes(tenantID, topic)
}
// TopicLastSeqID returns the seq last persisted for the topic in the DB, or 0
@@ -158,7 +190,7 @@ func TopicLastSeqID(tenantID types.TenantID, topic string) int64 {
// counter key is missing (Redis restart, eviction or flush) it re-seeds from
// the DB topic seq instead of trusting the caller's in-memory baseline, which
// may be stale and would otherwise hand out already-persisted seq values.
func nextMessageSeqWithFloor(tenantID types.TenantID, topic string, baseline int64) (int64, error) {
func nextMessageSeqWithFloor(tenantID types.TenantID, topic string, baseline int64, candidateMonth string) (int64, string, error) {
if baseline < 0 {
baseline = 0
}
@@ -167,7 +199,7 @@ func nextMessageSeqWithFloor(tenantID types.TenantID, topic string, baseline int
baseline = floor
}
}
return groupMsgRuntime.NextSeq(tenantID, topic, baseline)
return groupMsgRuntime.NextSeq(tenantID, topic, baseline, candidateMonth)
}
// BumpMessageSeqFloor pushes the Redis seq counter forward to seq (never
@@ -555,24 +587,129 @@ func (r *redisGroupMessageRuntime) Claim(tenantID types.TenantID, topic string,
return &current, false, nil
}
func (r *redisGroupMessageRuntime) NextSeq(tenantID types.TenantID, topic string, baseline int64) (int64, error) {
func (r *redisGroupMessageRuntime) NextSeq(tenantID types.TenantID, topic string, baseline int64, candidateMonth string) (int64, string, error) {
if baseline < 0 {
baseline = 0
}
redisNow, err := r.client.Time(context.Background()).Result()
if err != nil {
return 0, "", err
}
candidateMonth = redisNow.UTC().Format("200601")
if !validShardMonth(candidateMonth) {
return 0, "", types.ErrMalformed
}
seqKey := r.seqKey(tenantID, topic)
seq, err := r.client.Eval(context.Background(), `
result, err := r.client.Eval(context.Background(), `
local current = redis.call("GET", KEYS[1])
if not current then
redis.call("SET", KEYS[1], ARGV[1])
elseif tonumber(current) < tonumber(ARGV[1]) then
redis.call("SET", KEYS[1], ARGV[1])
end
return redis.call("INCR", KEYS[1])
`, []string{seqKey}, baseline).Int64()
local active_month = redis.call("GET", KEYS[2])
if not active_month or active_month < ARGV[2] then
redis.call("SET", KEYS[2], ARGV[2])
active_month = ARGV[2]
end
local seq = redis.call("INCR", KEYS[1])
return {seq, active_month}
`, []string{seqKey, r.activeMonthKey(tenantID)}, baseline, candidateMonth).Slice()
if err != nil {
return 0, err
return 0, "", err
}
return seq, nil
if len(result) != 2 {
return 0, "", fmt.Errorf("redis group message sequence returned %d values", len(result))
}
seq, err := strconv.ParseInt(fmt.Sprint(result[0]), 10, 64)
if err != nil {
return 0, "", err
}
month := fmt.Sprint(result[1])
if !validShardMonth(month) {
return 0, "", fmt.Errorf("redis returned invalid active month %q", month)
}
return seq, month, nil
}
func (r *redisGroupMessageRuntime) PutShardRoute(tenantID types.TenantID, topic string, route GroupMessageShardRoute) error {
if !validShardMonth(route.Month) || route.MinSeq <= 0 || route.MaxSeq < route.MinSeq {
return types.ErrMalformed
}
_, err := r.client.Eval(context.Background(), `
local min_seq = tonumber(ARGV[2])
local max_seq = tonumber(ARGV[3])
local count = tonumber(ARGV[4])
local meta = redis.call("HGET", KEYS[2], ARGV[1])
if meta then
local old_min, old_max, old_count = string.match(meta, "^(%d+):(%d+):(%d+)$")
if old_min then
min_seq = math.min(min_seq, tonumber(old_min))
max_seq = math.max(max_seq, tonumber(old_max))
if count == 1 then
count = tonumber(old_count) + 1
else
count = math.max(count, tonumber(old_count))
end
end
end
redis.call("ZADD", KEYS[1], min_seq, ARGV[1])
redis.call("HSET", KEYS[2], ARGV[1], min_seq .. ":" .. max_seq .. ":" .. count)
return 1
`, []string{r.shardRouteKey(tenantID, topic), r.shardRouteMetaKey(tenantID, topic)},
route.Month, route.MinSeq, route.MaxSeq, route.Count).Result()
return err
}
func (r *redisGroupMessageRuntime) GetShardRoutes(tenantID types.TenantID, topic string) ([]GroupMessageShardRoute, bool, error) {
ctx := context.Background()
months, err := r.client.ZRevRange(ctx, r.shardRouteKey(tenantID, topic), 0, -1).Result()
if err != nil {
return nil, false, err
}
if len(months) == 0 {
return nil, false, nil
}
values, err := r.client.HMGet(ctx, r.shardRouteMetaKey(tenantID, topic), months...).Result()
if err != nil {
return nil, false, err
}
routes := make([]GroupMessageShardRoute, 0, len(months))
for i, month := range months {
if values[i] == nil {
continue
}
parts := strings.Split(fmt.Sprint(values[i]), ":")
if len(parts) != 3 {
continue
}
minSeq, minErr := strconv.ParseInt(parts[0], 10, 64)
maxSeq, maxErr := strconv.ParseInt(parts[1], 10, 64)
count, countErr := strconv.ParseInt(parts[2], 10, 64)
if minErr == nil && maxErr == nil && countErr == nil && validShardMonth(month) {
routes = append(routes, GroupMessageShardRoute{Month: month, MinSeq: minSeq, MaxSeq: maxSeq, Count: count})
}
}
return routes, len(routes) > 0, nil
}
func (r *redisGroupMessageRuntime) DeleteShardRoutes(tenantID types.TenantID, topic string) error {
return r.client.Del(context.Background(), r.shardRouteKey(tenantID, topic), r.shardRouteMetaKey(tenantID, topic)).Err()
}
func validShardMonth(month string) bool {
if len(month) != 6 {
return false
}
for i := range month {
if month[i] < '0' || month[i] > '9' {
return false
}
}
parsed, err := time.Parse("200601", month)
return err == nil && parsed.Format("200601") == month
}
func (r *redisGroupMessageRuntime) PeekSeq(tenantID types.TenantID, topic string) (int64, error) {
@@ -1182,7 +1319,19 @@ func expandRanges(ranges []types.Range, limit int) []int64 {
}
func (r *redisGroupMessageRuntime) seqKey(tenantID types.TenantID, topic string) string {
return r.key(tenantID, "topic", topic, "last_seq")
return r.shardKey(tenantID, "topic", topic, "last_seq")
}
func (r *redisGroupMessageRuntime) activeMonthKey(tenantID types.TenantID) string {
return r.shardKey(tenantID, "groupmsg", "active_month")
}
func (r *redisGroupMessageRuntime) shardRouteKey(tenantID types.TenantID, topic string) string {
return r.shardKey(tenantID, "groupmsg", "route", topic)
}
func (r *redisGroupMessageRuntime) shardRouteMetaKey(tenantID types.TenantID, topic string) string {
return r.shardKey(tenantID, "groupmsg", "route_meta", topic)
}
func (r *redisGroupMessageRuntime) uniqueKey(tenantID types.TenantID, topic string, from types.Uid, clientMsgID string) string {
@@ -1276,3 +1425,11 @@ func (r *redisGroupMessageRuntime) key(tenantID types.TenantID, parts ...string)
all = append(all, parts...)
return strings.Join(all, ":")
}
func (r *redisGroupMessageRuntime) shardKey(tenantID types.TenantID, parts ...string) string {
prefix := strings.TrimSuffix(r.prefix, ":")
slot := "{" + strconv.FormatInt(int64(tenantID), 10) + "}"
all := []string{prefix, slot}
all = append(all, parts...)
return strings.Join(all, ":")
}
server/store/group_message_runtime_test.go
+37 -11
View File
@@ -13,6 +13,7 @@ type testGroupMessageRuntime struct {
seqs map[string]int64
shortStates map[string]bool
deletedPrefixes []string
deletedObjects []string
}
func newTestGroupMessageRuntime() *testGroupMessageRuntime {
@@ -35,12 +36,24 @@ func (r *testGroupMessageRuntime) Claim(tenantID types.TenantID, topic string, f
return rec, true, nil
}
func (r *testGroupMessageRuntime) NextSeq(_ types.TenantID, topic string, baseline int64) (int64, error) {
func (r *testGroupMessageRuntime) NextSeq(_ types.TenantID, topic string, baseline int64, candidateMonth string) (int64, string, error) {
if r.seqs[topic] < baseline {
r.seqs[topic] = baseline
}
r.seqs[topic]++
return r.seqs[topic], nil
return r.seqs[topic], candidateMonth, nil
}
func (r *testGroupMessageRuntime) PutShardRoute(types.TenantID, string, GroupMessageShardRoute) error {
return nil
}
func (r *testGroupMessageRuntime) GetShardRoutes(types.TenantID, string) ([]GroupMessageShardRoute, bool, error) {
return nil, false, nil
}
func (r *testGroupMessageRuntime) DeleteShardRoutes(types.TenantID, string) error {
return nil
}
func (r *testGroupMessageRuntime) PeekSeq(_ types.TenantID, topic string) (int64, error) {
@@ -140,7 +153,8 @@ func (r *testGroupMessageRuntime) GetCachedObject(types.TenantID, string, string
return false, nil
}
func (r *testGroupMessageRuntime) DeleteCachedObject(types.TenantID, string, string) error {
func (r *testGroupMessageRuntime) DeleteCachedObject(_ types.TenantID, namespace, id string) error {
r.deletedObjects = append(r.deletedObjects, namespace+":"+id)
return nil
}
@@ -234,9 +248,9 @@ func TestMessageRuntimeSeqAndClientMidIdempotency(t *testing.T) {
if err != nil || !ready || !claimed || claim == nil || claim.Status != "pending" {
t.Fatalf("unexpected first claim: claim=%#v claimed=%v ready=%v err=%v", claim, claimed, ready, err)
}
seq, ready, err := ReserveMessageSeq(tenantID, topic, 184)
if err != nil || !ready || seq != 185 {
t.Fatalf("unexpected first seq: seq=%d ready=%v err=%v", seq, ready, err)
seq, month, ready, err := ReserveMessageSeq(tenantID, topic, 184, "202609")
if err != nil || !ready || seq != 185 || month != "202609" {
t.Fatalf("unexpected first seq: seq=%d month=%s ready=%v err=%v", seq, month, ready, err)
}
if err = MarkMessageUniqueDone(tenantID, topic, from, "mid-1", seq, ""); err != nil {
t.Fatalf("mark done: %v", err)
@@ -251,9 +265,9 @@ func TestMessageRuntimeSeqAndClientMidIdempotency(t *testing.T) {
if err != nil || !ready || !claimed || claim == nil {
t.Fatalf("unexpected second claim: claim=%#v claimed=%v ready=%v err=%v", claim, claimed, ready, err)
}
seq, ready, err = ReserveMessageSeq(tenantID, topic, 184)
if err != nil || !ready || seq != 186 {
t.Fatalf("unexpected second seq: seq=%d ready=%v err=%v", seq, ready, err)
seq, month, ready, err = ReserveMessageSeq(tenantID, topic, 184, "202609")
if err != nil || !ready || seq != 186 || month != "202609" {
t.Fatalf("unexpected second seq: seq=%d month=%s ready=%v err=%v", seq, month, ready, err)
}
}
@@ -266,7 +280,7 @@ func TestBumpMessageSeqFloorNeverMovesBackward(t *testing.T) {
tenantID := types.TenantID(7)
topic := types.Uid(1).P2PName(types.Uid(2))
seq, ready, err := ReserveMessageSeq(tenantID, topic, 0)
seq, _, ready, err := ReserveMessageSeq(tenantID, topic, 0, "")
if err != nil || !ready || seq != 1 {
t.Fatalf("unexpected first seq: seq=%d ready=%v err=%v", seq, ready, err)
}
@@ -283,7 +297,7 @@ func TestBumpMessageSeqFloorNeverMovesBackward(t *testing.T) {
t.Fatalf("floor must never move backward: current=%d err=%v", current, err)
}
// The next allocation must continue above the bumped floor.
if seq, _, err := ReserveMessageSeq(tenantID, topic, 0); err != nil || seq != 6 {
if seq, _, _, err := ReserveMessageSeq(tenantID, topic, 0, ""); err != nil || seq != 6 {
t.Fatalf("unexpected seq after bump: seq=%d err=%v", seq, err)
}
}
@@ -299,3 +313,15 @@ func TestInvalidateGroupMemberPagesUsesTopicKeyPrefix(t *testing.T) {
t.Fatalf("unexpected deleted prefixes: %#v", runtime.deletedPrefixes)
}
}
func TestInvalidateGroupSettingsUsesTopicCacheKey(t *testing.T) {
oldRuntime := groupMsgRuntime
runtime := newTestGroupMessageRuntime()
groupMsgRuntime = runtime
t.Cleanup(func() { groupMsgRuntime = oldRuntime })
invalidateGroupSettings(7, " grpAlpha ")
if len(runtime.deletedObjects) != 1 || runtime.deletedObjects[0] != "group_settings:grpAlpha" {
t.Fatalf("unexpected deleted objects: %#v", runtime.deletedObjects)
}
}
server/store/mock_store/mock_store.go
+33
View File
@@ -1008,6 +1008,21 @@ func (mr *MockTopicsPersistenceInterfaceMockRecorder) OwnerChange(tenantID, topi
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "OwnerChange", reflect.TypeOf((*MockTopicsPersistenceInterface)(nil).OwnerChange), tenantID, topic, newOwner)
}
// BumpMutationVersion mocks base method.
func (m *MockTopicsPersistenceInterface) BumpMutationVersion(tenantID types.TenantID, topic string, at time.Time) (int64, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "BumpMutationVersion", tenantID, topic, at)
ret0, _ := ret[0].(int64)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// BumpMutationVersion indicates an expected call of BumpMutationVersion.
func (mr *MockTopicsPersistenceInterfaceMockRecorder) BumpMutationVersion(tenantID, topic, at interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BumpMutationVersion", reflect.TypeOf((*MockTopicsPersistenceInterface)(nil).BumpMutationVersion), tenantID, topic, at)
}
// Update mocks base method.
func (m *MockTopicsPersistenceInterface) Update(tenantID types.TenantID, topic string, update map[string]any) error {
m.ctrl.T.Helper()
@@ -1220,6 +1235,24 @@ func (mr *MockMessagesPersistenceInterfaceMockRecorder) Search(tenantID, topic,
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Search", reflect.TypeOf((*MockMessagesPersistenceInterface)(nil).Search), tenantID, topic, forUser, query, messageType, cursor, limit)
}
// Mutations mocks base method.
func (m *MockMessagesPersistenceInterface) Mutations(tenantID types.TenantID, topic string, forUser types.Uid, sinceVersion int64, limit int) ([]types.MessageMutation, string, bool, int64, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "Mutations", tenantID, topic, forUser, sinceVersion, limit)
ret0, _ := ret[0].([]types.MessageMutation)
ret1, _ := ret[1].(string)
ret2, _ := ret[2].(bool)
ret3, _ := ret[3].(int64)
ret4, _ := ret[4].(error)
return ret0, ret1, ret2, ret3, ret4
}
// Mutations indicates an expected call of Mutations.
func (mr *MockMessagesPersistenceInterfaceMockRecorder) Mutations(tenantID, topic, forUser, sinceVersion, limit interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Mutations", reflect.TypeOf((*MockMessagesPersistenceInterface)(nil).Mutations), tenantID, topic, forUser, sinceVersion, limit)
}
// GetOne mocks base method.
func (m *MockMessagesPersistenceInterface) GetOne(tenantID types.TenantID, topic string, seq int64) (*types.Message, error) {
m.ctrl.T.Helper()
server/store/store.go
+80 -6
View File
@@ -1079,6 +1079,7 @@ type TopicsPersistenceInterface interface {
GetSubs(tenantID types.TenantID, topic string, opts *types.QueryOpt) ([]types.Subscription, error)
GetSubsAny(tenantID types.TenantID, topic string, opts *types.QueryOpt) ([]types.Subscription, error)
Update(tenantID types.TenantID, topic string, update map[string]any) error
BumpMutationVersion(tenantID types.TenantID, topic string, at time.Time) (int64, error)
UpdateSubCnt(tenantID types.TenantID, topic string) error
OwnerChange(tenantID types.TenantID, topic string, newOwner types.Uid) error
Delete(tenantID types.TenantID, topic string, isChan, hard bool) error
@@ -1235,6 +1236,18 @@ func (topicsMapper) Update(tenantID types.TenantID, topic string, update map[str
return nil
}
func (topicsMapper) BumpMutationVersion(tenantID types.TenantID, topic string, at time.Time) (int64, error) {
tenantAdp, err := tenantBusinessAdapter(tenantID)
if err != nil {
return 0, err
}
version, err := tenantAdp.TenantTopicBumpMutationVersion(tenantID, topic, at)
if err == nil {
_ = DeleteCachedObject(tenantID, cacheNamespaceTopic, topic)
}
return version, err
}
// OwnerChange replaces the old topic owner with the new owner.
func (topicsMapper) OwnerChange(tenantID types.TenantID, topic string, newOwner types.Uid) error {
tenantAdp, err := tenantBusinessAdapter(tenantID)
@@ -1258,6 +1271,9 @@ func (topicsMapper) Delete(tenantID types.TenantID, topic string, isChan, hard b
return err
}
_ = DeleteCachedObject(tenantID, cacheNamespaceTopic, topic)
if hard && types.GetTopicCat(topic) == types.TopicCatGrp {
_ = DeleteGroupMessageShardRoutes(tenantID, topic)
}
return nil
}
@@ -1386,6 +1402,7 @@ type MessagesPersistenceInterface interface {
GetAll(tenantID types.TenantID, topic string, forUser types.Uid, opt *types.QueryOpt) ([]types.Message, error)
GetDeleted(tenantID types.TenantID, topic string, forUser types.Uid, opt *types.QueryOpt) ([]types.Range, int, error)
Search(tenantID types.TenantID, topic string, forUser types.Uid, query, messageType, cursor string, limit int) ([]types.Message, string, error)
Mutations(tenantID types.TenantID, topic string, forUser types.Uid, sinceVersion int64, limit int) ([]types.MessageMutation, string, bool, int64, error)
}
// messagesMapper is a concrete type implementing MessagesPersistenceInterface.
@@ -1635,6 +1652,7 @@ type GroupBusinessPersistenceInterface interface {
PutAnnouncement(types.TenantID, *types.GroupAnnouncement, int) (*types.GroupAnnouncement, error)
ReadAnnouncement(types.TenantID, string, string, types.Uid, time.Time) error
ListMutes(types.TenantID, string, string, int) ([]types.GroupMute, string, error)
GetMute(types.TenantID, string, types.Uid) (*types.GroupMute, error)
UpsertMute(types.TenantID, *types.GroupMute) (*types.GroupMute, error)
DeleteMute(types.TenantID, string, types.Uid, types.Uid, time.Time) error
ListFiles(types.TenantID, string, string, string, string, int) ([]types.GroupFile, string, error)
@@ -1838,6 +1856,16 @@ func invalidateGroupMemberPages(tenantID types.TenantID, topic string) {
}
}
func invalidateGroupSettings(tenantID types.TenantID, topic string) {
topic = strings.TrimSpace(topic)
if topic == "" {
return
}
if err := DeleteCachedObject(tenantID, cacheNamespaceGroupSettings, topic); err != nil {
logs.Warn.Printf("cache group settings[%s]: delete failed: %v", topic, err)
}
}
func invalidateGroupMemberObjects(tenantID types.TenantID, topic string) {
topic = strings.TrimSpace(topic)
if topic == "" {
@@ -1905,6 +1933,7 @@ func (groupBusinessMapper) AcceptInvitation(tenantID types.TenantID, invitationI
invitation, err := tenantAdp.TenantGroupInvitationAccept(tenantID, strings.TrimSpace(invitationID), uid, memberPrivate, now)
if err == nil && invitation != nil {
invalidateGroupMemberPages(tenantID, invitation.Topic)
invalidateGroupSettings(tenantID, invitation.Topic)
}
return invitation, err
}
@@ -1945,6 +1974,7 @@ func (groupBusinessMapper) ResolveJoinRequest(tenantID types.TenantID, topic, re
resolved, err := tenantAdp.TenantGroupJoinRequestResolve(tenantID, strings.TrimSpace(topic), strings.TrimSpace(requestID), actor, strings.TrimSpace(action), strings.TrimSpace(rejectReason), version, memberPrivate, now)
if err == nil && resolved != nil && resolved.Status == types.GroupJoinRequestStatusAccepted {
invalidateGroupMemberPages(tenantID, topic)
invalidateGroupSettings(tenantID, topic)
}
return resolved, err
}
@@ -2007,6 +2037,17 @@ func (groupBusinessMapper) ListMutes(tenantID types.TenantID, topic, cursor stri
return tenantAdp.TenantGroupMuteList(tenantID, strings.TrimSpace(topic), strings.TrimSpace(cursor), limit)
}
func (groupBusinessMapper) GetMute(tenantID types.TenantID, topic string, uid types.Uid) (*types.GroupMute, error) {
if strings.TrimSpace(topic) == "" || uid.IsZero() {
return nil, types.ErrMalformed
}
tenantAdp, err := tenantBusinessAdapter(tenantID)
if err != nil {
return nil, err
}
return tenantAdp.TenantGroupMuteGet(tenantID, strings.TrimSpace(topic), uid)
}
func (groupBusinessMapper) UpsertMute(tenantID types.TenantID, mute *types.GroupMute) (*types.GroupMute, error) {
if mute == nil || mute.Topic == "" || mute.UserID.IsZero() || mute.MutedBy.IsZero() {
return nil, types.ErrMalformed
@@ -2076,8 +2117,9 @@ func (groupBusinessMapper) InviteMember(tenantID types.TenantID, topic string, i
}
created, err := tenantAdp.TenantGroupInviteMember(tenantID, strings.TrimSpace(topic), inviter, target,
strings.TrimSpace(message), strings.TrimSpace(idempotencyKey), private, now)
if err == nil {
if err == nil && created {
invalidateGroupMemberPages(tenantID, topic)
invalidateGroupSettings(tenantID, topic)
}
return created, err
}
@@ -2097,6 +2139,7 @@ func (groupBusinessMapper) Leave(tenantID types.TenantID, topic string, uid type
_ = DeleteCachedObject(tenantID, cacheNamespaceSub, subscriptionCacheID(strings.TrimSpace(topic), uid, false))
_ = DeleteCachedObject(tenantID, cacheNamespaceSub, subscriptionCacheID(strings.TrimSpace(topic), uid, true))
invalidateGroupMemberPages(tenantID, topic)
invalidateGroupSettings(tenantID, topic)
return nil
}
@@ -2141,6 +2184,7 @@ func (groupBusinessMapper) TransferOwner(tenantID types.TenantID, topic string,
// object so a later Topics.Get sees the new owner.
_ = DeleteCachedObject(tenantID, cacheNamespaceTopic, topic)
invalidateGroupMemberPages(tenantID, topic)
invalidateGroupSettings(tenantID, topic)
return nil
}
@@ -3008,9 +3052,8 @@ func refreshUserProfileProjections(tenantID types.TenantID, uid types.Uid, profi
if item.Friendship.Topic == "" || item.Friendship.PeerID.IsZero() {
continue
}
if err := Subs.Update(tenantID, item.Friendship.Topic, item.Friendship.PeerID, map[string]any{"Public": public}); err != nil {
logs.Warn.Printf("profile %s: refresh subscription %s/%s public failed: %v", uid, item.Friendship.Topic, item.Friendship.PeerID, err)
}
_ = DeleteCachedObject(tenantID, cacheNamespaceSub, subscriptionCacheID(item.Friendship.Topic, item.Friendship.PeerID, false))
_ = DeleteCachedObject(tenantID, cacheNamespaceSub, subscriptionCacheID(item.Friendship.Topic, item.Friendship.PeerID, true))
}
if nextCursor == "" {
break
@@ -3589,6 +3632,7 @@ func (messagesMapper) PreparePersist(tenantID types.TenantID, msg *types.Message
return nil, err
}
msg.ClientMsgID = clientMsgID
msg.InitTimes()
fromUid := types.ParseUid(msg.From)
isGroupMessage := types.GetTopicCat(msg.Topic) == types.TopicCatGrp
if isGroupMessage && groupMsgRuntime != nil {
@@ -3605,7 +3649,7 @@ func (messagesMapper) PreparePersist(tenantID types.TenantID, msg *types.Message
return nil, types.ErrDuplicate
}
}
seq, err := nextMessageSeqWithFloor(tenantID, msg.Topic, msg.SeqId-1)
seq, shardMonth, err := nextMessageSeqWithFloor(tenantID, msg.Topic, msg.SeqId-1, msg.CreatedAt.UTC().Format("200601"))
if err != nil {
if clientMsgID != "" {
_ = groupMsgRuntime.MarkFailed(tenantID, msg.Topic, fromUid, clientMsgID)
@@ -3613,6 +3657,10 @@ func (messagesMapper) PreparePersist(tenantID types.TenantID, msg *types.Message
return nil, err
}
msg.SeqId = seq
msg.ShardMonth = shardMonth
}
if isGroupMessage && msg.ShardMonth == "" {
msg.ShardMonth = msg.CreatedAt.UTC().Format("200601")
}
if clientMsgID != "" {
var duplicate *types.Message
@@ -3631,6 +3679,7 @@ func (messagesMapper) PreparePersist(tenantID types.TenantID, msg *types.Message
msg.DeletedFor = duplicate.DeletedFor
msg.SeqId = duplicate.SeqId
msg.Topic = duplicate.Topic
msg.ShardMonth = duplicate.ShardMonth
msg.From = duplicate.From
msg.ClientMsgID = duplicate.ClientMsgID
msg.Head = duplicate.Head
@@ -3638,7 +3687,6 @@ func (messagesMapper) PreparePersist(tenantID types.TenantID, msg *types.Message
return &types.MessageSaveResult{SeqId: duplicate.SeqId, Duplicate: true}, nil
}
}
msg.InitTimes()
msg.SetUid(Store.GetUid())
// Reserve topic's or user's SeqId before optional async persistence.
err = tenantAdp.TenantTopicUpdateOnMessage(tenantID, msg.Topic, msg)
@@ -3677,6 +3725,7 @@ func (messagesMapper) PersistPrepared(tenantID types.TenantID, msg *types.Messag
}
if duplicate != nil {
msg.SeqId = duplicate.SeqId
msg.ShardMonth = duplicate.ShardMonth
if groupMsgRuntime != nil {
_ = groupMsgRuntime.MarkDone(tenantID, msg.Topic, fromUid, clientMsgID, duplicate.SeqId, duplicate.Uid().String())
}
@@ -3716,6 +3765,11 @@ func (messagesMapper) PersistPrepared(tenantID types.TenantID, msg *types.Messag
}
}
if isGroupMessage && groupMsgRuntime != nil {
if err := groupMsgRuntime.PutShardRoute(tenantID, msg.Topic, GroupMessageShardRoute{
Month: msg.ShardMonth, MinSeq: msg.SeqId, MaxSeq: msg.SeqId, Count: 1,
}); err != nil {
logs.Warn.Printf("topic[%s]: failed to update group message shard route: %v", msg.Topic, err)
}
if err := groupMsgRuntime.CacheHotMessage(tenantID, msg); err != nil {
logs.Warn.Printf("topic[%s]: failed to cache hot group message: %v", msg.Topic, err)
}
@@ -3778,6 +3832,7 @@ func copyPersistedMessage(dst, src *types.Message) {
dst.DeletedFor = src.DeletedFor
dst.SeqId = src.SeqId
dst.Topic = src.Topic
dst.ShardMonth = src.ShardMonth
dst.From = src.From
dst.ClientMsgID = src.ClientMsgID
dst.Head = src.Head
@@ -4026,6 +4081,11 @@ func (messagesMapper) DeleteList(tenantID types.TenantID, topic string, delID in
if cacheErr := groupMsgRuntime.InvalidateHotMessages(tenantID, topic, ranges); cacheErr != nil {
logs.Warn.Printf("topic[%s]: failed to invalidate deleted group message cache: %v", topic, cacheErr)
}
if delID == 0 {
if routeErr := groupMsgRuntime.DeleteShardRoutes(tenantID, topic); routeErr != nil {
logs.Warn.Printf("topic[%s]: failed to delete group message shard route: %v", topic, routeErr)
}
}
}
// TODO: move to adapter.
@@ -4080,6 +4140,20 @@ func (messagesMapper) Search(tenantID types.TenantID, topic string, forUser type
return tenantAdp.TenantMessageSearch(tenantID, topic, forUser, query, messageType, cursor, limit)
}
func (messagesMapper) Mutations(tenantID types.TenantID, topic string, forUser types.Uid, sinceVersion int64, limit int) ([]types.MessageMutation, string, bool, int64, error) {
if topic == "" || forUser.IsZero() || sinceVersion < 0 {
return nil, "", false, 0, types.ErrMalformed
}
if limit <= 0 || limit > 100 {
limit = 50
}
tenantAdp, err := tenantBusinessAdapter(tenantID)
if err != nil {
return nil, "", false, 0, err
}
return tenantAdp.TenantMessageMutations(tenantID, topic, forUser, sinceVersion, limit)
}
// GetDeleted returns the ranges of deleted messages and the largest DelId reported in the list.
func (messagesMapper) GetDeleted(tenantID types.TenantID, topic string, forUser types.Uid, opt *types.QueryOpt) ([]types.Range, int, error) {
tenantAdp, err := tenantBusinessAdapter(tenantID)
server/store/types/types.go
+18
View File
@@ -138,6 +138,10 @@ func (uid *Uid) MarshalJSON() ([]byte, error) {
// UnmarshalJSON reads Uid from a double quoted string.
func (uid *Uid) UnmarshalJSON(b []byte) error {
size := len(b)
if size == 2 && b[0] == '"' && b[1] == '"' {
*uid = ZeroUid
return nil
}
if size != (uidBase64Unpadded + 2) {
return errors.New("Uid.UnmarshalJSON: invalid length")
} else if b[0] != '"' || b[size-1] != '"' {
@@ -1151,6 +1155,9 @@ type Topic struct {
// Count of topic subscribers.
SubCnt int
// Conversation-level version for in-place message mutations such as recall and edit.
MutationVersion int64 `db:"mutation_version" json:"MutationVersion,omitempty" bson:"mutation_version,omitempty"`
Public any
Trusted any
@@ -1241,6 +1248,7 @@ type Message struct {
DeletedFor []SoftDelete `json:"DeletedFor,omitempty" bson:",omitempty"`
SeqId int64
Topic string
ShardMonth string `db:"shard_month" json:"-" bson:"shard_month,omitempty"`
// Sender's user ID as string (without 'usr' prefix), could be empty.
From string
ClientMsgID string `db:"client_msg_id" json:"ClientMsgId,omitempty" bson:"client_msg_id,omitempty"`
@@ -1264,6 +1272,16 @@ type MessageRevision struct {
Reason string
}
// MessageMutation describes an in-place message change used for offline compensation.
type MessageMutation struct {
SeqId int64
Version int
Event string
Operator string
MutatedAt time.Time
MutationVersion int64
}
// MessageSaveResult describes the outcome of a message persistence attempt.
type MessageSaveResult struct {
SeqId int64
server/store/types/uid_json_test.go
+23
View File
@@ -0,0 +1,23 @@
package types
import (
"encoding/json"
"testing"
)
func TestZeroUidJSONRoundTrip(t *testing.T) {
type wrapper struct {
UID Uid
}
raw, err := json.Marshal(&wrapper{UID: ZeroUid})
if err != nil {
t.Fatal(err)
}
var decoded wrapper
if err = json.Unmarshal(raw, &decoded); err != nil {
t.Fatal(err)
}
if decoded.UID != ZeroUid {
t.Fatalf("decoded uid=%v want zero", decoded.UID)
}
}
server/topic_test.go
+8
View File
@@ -1115,6 +1115,14 @@ func TestSaveMessageMaybeKafkaPublishesWithoutDBPersist(t *testing.T) {
if publishedEnvelope.IdempotencyKey != "client-1" {
t.Fatalf("unexpected idempotency key: %q", publishedEnvelope.IdempotencyKey)
}
var payload kafkaMessagePersistPayload
if err := json.Unmarshal(publishedEnvelope.Payload, &payload); err != nil {
t.Fatalf("decode kafka message payload: %v", err)
}
wantMonth := msg.CreatedAt.UTC().Format("200601")
if payload.ShardMonth != wantMonth {
t.Fatalf("group message shard month was not frozen: payload=%q want=%q", payload.ShardMonth, wantMonth)
}
}
func TestShouldTryHotMessageFallback(t *testing.T) {