pro/im_server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: shard group messages by month

Browse Source
This commit is contained in:
richjohnrambo
2026-09-15 23:53:51 +08:00
parent b7c2536527
commit 139681fadb
25 changed files with 1718 additions and 247 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/internal_admin_query_resources.go
+84 -16
View File
@@ -63,19 +63,20 @@ var internalTenantResourceSpecs = map[string]*internalQuerySpec{
"private-messages": privateMessagesInternalQuerySpec(),
"group-messages": groupMessagesInternalQuerySpec(),
"user-login-logs": {
Select: `SELECT id,CAST(user_id AS CHAR) AS user_id,username,account_type,account_mask,login_type,event_type,session_id,device_id,platform,app_version,ip,region,user_agent,risk_level,request_id,login_at,logout_at,metadata`,
Select: `SELECT id,user_id,username,account_type,account_mask,login_type,event_type,session_id,device_id,platform,app_version,ip,region,user_agent,risk_level,request_id,login_at,logout_at,metadata`,
From: "FROM im_user_login_logs", DefaultSort: "ORDER BY id DESC", DetailKey: "id",
Filters: []internalQueryFilter{{"user_id", "user_id", "=", "uid"}, {"account", "account_mask", "LIKE", "string"}, {"login_type", "login_type", "=", "string"}, {"event_type", "event_type", "=", "string"}, {"platform", "platform", "=", "string"}, {"ip", "ip", "LIKE", "string"}, {"login_at_start", "login_at", ">=", "time"}, {"login_at_end", "login_at", "<=", "time"}},
},
"login-failure-logs": {
Select: `SELECT id,CAST(user_id AS CHAR) AS user_id,account_type,account_mask,failure_type,reason,detail,ip,region,device_id,platform,app_version,user_agent,risk_level,risk_action,request_id,attempt_at,metadata`,
Select: `SELECT id,user_id,account_type,account_mask,failure_type,reason,detail,ip,region,device_id,platform,app_version,user_agent,risk_level,risk_action,request_id,attempt_at,metadata`,
From: "FROM im_login_failure_logs", DefaultSort: "ORDER BY id DESC", DetailKey: "id",
Filters: []internalQueryFilter{{"user_id", "user_id", "=", "uid"}, {"account", "account_mask", "LIKE", "string"}, {"failure_type", "failure_type", "=", "string"}, {"platform", "platform", "=", "string"}, {"ip", "ip", "LIKE", "string"}, {"attempt_at_start", "attempt_at", ">=", "time"}, {"attempt_at_end", "attempt_at", "<=", "time"}},
},
"audit-logs": {
Select: "SELECT id,request_id,actor_user_id,action,target_type,target_id,ip,user_agent,detail,created_at",
From: "FROM im_audit_logs", DefaultSort: "ORDER BY id DESC", DetailKey: "id",
Filters: []internalQueryFilter{{"actor_user_id", "actor_user_id", "=", "uid"}, {"action", "action", "=", "string"}, {"target_type", "target_type", "=", "string"}, {"target_id", "target_id", "=", "string"}, {"created_at_start", "created_at", ">=", "time"}, {"created_at_end", "created_at", "<=", "time"}},
Select: `SELECT id,request_id,actor_uid AS actor_user_id,action,target_type,target_id,ip,user_agent,
JSON_OBJECT('actor_role',actor_role,'before_snapshot',before_snapshot,'after_snapshot',after_snapshot,'reason',reason,'metadata',metadata) AS detail,created_at`,
From: "FROM im_audit_logs", DefaultSort: "ORDER BY id DESC", DetailKey: "id",
Filters: []internalQueryFilter{{"actor_user_id", "actor_uid", "=", "uid"}, {"action", "action", "=", "string"}, {"target_type", "target_type", "=", "string"}, {"target_id", "target_id", "=", "string"}, {"created_at_start", "created_at", ">=", "time"}, {"created_at_end", "created_at", "<=", "time"}},
},
"realname-verifications": {
Select: "SELECT id,user_id,submission_id,status,masked_name AS real_name,masked_id_number AS id_card_no_masked,review_reason AS reject_reason,verified_at AS reviewed_at,created_at,updated_at",
@@ -136,6 +137,14 @@ func serveInternalAdminResourceList(wrt http.ResponseWriter, req *http.Request,
writeIMError(wrt, req, http.StatusNotFound, 30004, imReasonMalformedRequest, "resource not found", nil)
return
}
if resource == "group-messages" {
var err error
spec, err = groupMessagesInternalQuerySpecForTenant(tenantID)
if err != nil {
writeInternalQueryError(wrt, req, err)
return
}
}
query, countQuery, args, page, perPage, err := buildInternalAdminListQuery(req, spec, scope)
if err != nil {
writeIMError(wrt, req, http.StatusBadRequest, 30000, imReasonMalformedRequest, "malformed request", nil)
@@ -256,7 +265,7 @@ func internalAdminQuerySpec(resource string) (*internalQuerySpec, bool) {
func usersInternalQuerySpec() *internalQuerySpec {
return &internalQuerySpec{
Select: `SELECT CAST(u.id AS CHAR) AS user_id,u.account_no AS display_no,u.username,u.phone,u.state,u.stateat AS state_at,u.access,u.lastseen AS last_seen,u.useragent AS user_agent,u.public,u.trusted,
Select: `SELECT u.id AS user_id,u.account_no AS display_no,u.username,u.phone,u.state,u.stateat AS state_at,u.access,u.lastseen AS last_seen,u.useragent AS user_agent,u.public,u.trusted,
p.nickname,p.public_account,p.gender,p.region,p.avatar_file_id,
COALESCE(
NULLIF(avatar.location,''),
@@ -283,7 +292,7 @@ func groupsInternalQuerySpec() *internalQuerySpec {
return &internalQuerySpec{
Select: `SELECT t.name AS topic,t.state,t.owner AS owner_id,t.seqid AS seq_id,t.delid AS del_id,t.subcnt AS sub_count,t.touchedat AS touched_at,t.createdat AS created_at,t.updatedat AS updated_at,t.public,t.trusted,t.access,t.tags,t.aux,
gs.name AS group_name,gs.avatar_file_id,gs.description,gs.group_status,gs.join_approval,gs.mute_all,gs.member_upload_files,gs.member_private_chat,gs.member_invite,gs.max_member_count,gs.max_admin_count,gs.member_count AS business_member_count,gs.version AS settings_version,gs.updated_at AS settings_updated_at`,
From: "FROM im_topics AS t LEFT JOIN im_group_settings AS gs ON gs.topic=t.name",
From: "FROM (SELECT * FROM im_topics WHERE name LIKE 'grp%') AS t LEFT JOIN im_group_settings AS gs ON gs.topic=t.name",
DefaultSort: "ORDER BY t.id DESC",
DetailKey: "t.name",
Filters: []internalQueryFilter{{"topic", "t.name", "LIKE", "string"}, {"owner_id", "t.owner", "=", "uid"}, {"state", "t.state", "=", "int"}, {"group_status", "gs.group_status", "=", "string"}, {"created_at_start", "t.createdat", ">=", "time"}, {"created_at_end", "t.createdat", "<=", "time"}, {"touched_at_start", "t.touchedat", ">=", "time"}, {"touched_at_end", "t.touchedat", "<=", "time"}},
@@ -296,9 +305,9 @@ func privateMessagesInternalQuerySpec() *internalQuerySpec {
contentExcerpt := `LEFT(COALESCE(NULLIF(JSON_UNQUOTE(JSON_EXTRACT(m.content,'$.txt')),''),CASE JSON_TYPE(m.content) WHEN 'STRING' THEN JSON_UNQUOTE(m.content) ELSE CONCAT('[',` + contentType + `,']') END,''),500)`
status := `CASE WHEN aa.action='delete' THEN 'deleted' WHEN aa.action='hide' THEN 'hidden' WHEN m.recalled_at IS NOT NULL THEN 'recalled' WHEN m.deletedat IS NOT NULL THEN 'deleted' ELSE 'normal' END`
return &internalQuerySpec{
Select: `SELECT m.topic,m.seqid AS seq_id,CAST(m.` + "`from`" + ` AS CHAR) AS from_user_id,
Select: `SELECT m.topic,m.seqid AS seq_id,m.` + "`from`" + ` AS from_user_id,
COALESCE(su.username,'') AS sender_username,COALESCE(sp.nickname,'') AS sender_nickname,
CAST(rs.userid AS CHAR) AS to_user_id,COALESCE(ru.username,'') AS receiver_username,COALESCE(rp.nickname,'') AS receiver_nickname,
rs.userid AS to_user_id,COALESCE(ru.username,'') AS receiver_username,COALESCE(rp.nickname,'') AS receiver_nickname,
` + contentType + ` AS content_type,` + contentExcerpt + ` AS content_excerpt,
` + status + ` AS status,m.createdat AS created_at,m.updatedat AS updated_at`,
From: `FROM (SELECT * FROM im_messages WHERE topic LIKE 'p2p%') AS m
@@ -318,7 +327,7 @@ func groupMessagesInternalQuerySpec() *internalQuerySpec {
contentType := `CASE m.msg_type WHEN 'call' THEN 'system' WHEN 'system_notice' THEN 'system' WHEN 'contact_card' THEN 'card' WHEN 'forward_bundle' THEN 'card' WHEN 'link' THEN 'text' ELSE m.msg_type END`
contentExcerpt := `LEFT(COALESCE(NULLIF(JSON_UNQUOTE(JSON_EXTRACT(m.content,'$.txt')),''),CASE JSON_TYPE(m.content) WHEN 'STRING' THEN JSON_UNQUOTE(m.content) ELSE CONCAT('[',` + contentType + `,']') END,''),500)`
return &internalQuerySpec{
Select: `SELECT m.topic,m.seqid AS seq_id,CAST(m.from_user AS CHAR) AS from_user_id,
Select: `SELECT m.topic,m.seqid AS seq_id,m.from_user AS from_user_id,
COALESCE(u.username,'') AS sender_username,COALESCE(p.nickname,'') AS sender_nickname,
COALESCE(gs.name,'') AS group_name,m.topic AS group_no,` + contentType + ` AS content_type,
` + contentExcerpt + ` AS content_excerpt,
@@ -346,14 +355,68 @@ func groupMessagesInternalQuerySpec() *internalQuerySpec {
}
}
func groupMessagesInternalQuerySpecForTenant(tenantID types.TenantID) (*internalQuerySpec, error) {
source, err := internalGroupMessageSource(tenantID)
if err != nil {
return nil, err
}
spec := groupMessagesInternalQuerySpec()
spec.From = strings.Replace(spec.From, "FROM im_group_messages AS m", "FROM "+source+" AS m", 1)
spec.DefaultSort = "ORDER BY m.created_at DESC,m.id DESC"
return spec, nil
}
func internalGroupMessageSource(tenantID types.TenantID) (string, error) {
result, err := store.InternalAdminQueries.Query(tenantID,
"SELECT table_name FROM im_group_message_shards WHERE status<>'archived' ORDER BY shard_month DESC",
"SELECT COUNT(*) FROM im_group_message_shards WHERE status<>'archived'", nil, 1, 200)
if err != nil {
return "", err
}
selects := make([]string, 0)
for _, item := range result.Items {
table := strings.TrimSpace(fmt.Sprint(item["table_name"]))
if !validInternalGroupMessageTable(table) {
continue
}
selects = append(selects, fmt.Sprintf(`SELECT message_id AS id,topic,seqid,from_user,msg_type,status,content,head,reply_seqid,mentioned_user_ids,created_at,updated_at FROM %s`, table))
}
selects = append(selects, `SELECT id,topic COLLATE utf8mb4_0900_ai_ci AS topic,seqid,from_user,
msg_type COLLATE utf8mb4_0900_ai_ci AS msg_type,status COLLATE utf8mb4_0900_ai_ci AS status,
content,head,reply_seqid,mentioned_user_ids,created_at,updated_at
FROM im_group_messages AS legacy WHERE NOT EXISTS (
SELECT 1 FROM im_group_message_topic_shards AS route
WHERE route.topic=legacy.topic COLLATE utf8mb4_0900_ai_ci AND legacy.seqid BETWEEN route.min_seq AND route.max_seq)`)
return "(" + strings.Join(selects, " UNION ALL ") + ")", nil
}
func validInternalGroupMessageTable(table string) bool {
const prefix = "im_group_messages_"
if len(table) != len(prefix)+6 || !strings.HasPrefix(table, prefix) {
return false
}
for _, ch := range table[len(prefix):] {
if ch < '0' || ch > '9' {
return false
}
}
return true
}
func serveInternalDashboardOverview(wrt http.ResponseWriter, req *http.Request, tenantID types.TenantID) {
row, err := store.InternalAdminQueries.QueryOne(tenantID, `SELECT
groupMessages, err := internalGroupMessageSource(tenantID)
if err != nil {
writeInternalQueryError(wrt, req, err)
return
}
query := fmt.Sprintf(`SELECT
(SELECT COUNT(*) FROM im_users WHERE state<>20) AS total_users,
(SELECT COUNT(*) FROM im_users WHERE state=0 AND lastseen>=UTC_DATE()) AS active_users,
(SELECT COUNT(*) FROM im_group_settings WHERE group_status='active') AS total_groups,
(SELECT COUNT(*) FROM im_topics WHERE state<>20 AND (name LIKE 'p2p%' OR name LIKE 'grp%')) AS total_conversations,
((SELECT COUNT(*) FROM im_messages WHERE topic LIKE 'p2p%')+(SELECT COUNT(*) FROM im_group_messages)) AS total_messages,
((SELECT COUNT(*) FROM im_messages WHERE topic LIKE 'p2p%' AND createdat>=UTC_DATE())+(SELECT COUNT(*) FROM im_group_messages WHERE created_at>=UTC_DATE())) AS today_messages`, nil)
(SELECT COUNT(*) FROM im_topics WHERE state<>20 AND (name LIKE 'p2p%%' OR name LIKE 'grp%%')) AS total_conversations,
((SELECT COUNT(*) FROM im_messages WHERE topic LIKE 'p2p%%')+(SELECT COUNT(*) FROM %s AS gm)) AS total_messages,
((SELECT COUNT(*) FROM im_messages WHERE topic LIKE 'p2p%%' AND createdat>=UTC_DATE())+(SELECT COUNT(*) FROM %s AS gm WHERE gm.created_at>=UTC_DATE())) AS today_messages`, groupMessages, groupMessages)
row, err := store.InternalAdminQueries.QueryOne(tenantID, query, nil)
if err != nil {
writeInternalQueryError(wrt, req, err)
return
@@ -366,14 +429,19 @@ func serveInternalDashboardTrends(wrt http.ResponseWriter, req *http.Request, te
if days > 31 {
days = 31
}
groupMessages, err := internalGroupMessageSource(tenantID)
if err != nil {
writeInternalQueryError(wrt, req, err)
return
}
query := fmt.Sprintf(`WITH RECURSIVE dates AS (
SELECT UTC_DATE()-INTERVAL %d DAY AS day
UNION ALL SELECT day+INTERVAL 1 DAY FROM dates WHERE day<UTC_DATE()
) SELECT DATE_FORMAT(d.day,'%%Y-%%m-%%d') AS date,
(SELECT COUNT(*) FROM im_users AS u WHERE u.createdat>=d.day AND u.createdat<d.day+INTERVAL 1 DAY) AS user_count,
((SELECT COUNT(*) FROM im_messages AS pm WHERE pm.topic LIKE 'p2p%%' AND pm.createdat>=d.day AND pm.createdat<d.day+INTERVAL 1 DAY)+
(SELECT COUNT(*) FROM im_group_messages AS gm WHERE gm.created_at>=d.day AND gm.created_at<d.day+INTERVAL 1 DAY)) AS message_count
FROM dates AS d ORDER BY d.day ASC`, days-1)
(SELECT COUNT(*) FROM %s AS gm WHERE gm.created_at>=d.day AND gm.created_at<d.day+INTERVAL 1 DAY)) AS message_count
FROM dates AS d ORDER BY d.day ASC`, days-1, groupMessages)
result, err := store.InternalAdminQueries.Query(tenantID, query, fmt.Sprintf("SELECT %d", days), nil, 1, days)
if err != nil {
writeInternalQueryError(wrt, req, err)