pro/im_server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: harden single-machine load handling

Browse Source
This commit is contained in:
richjohnrambo
2026-09-29 17:52:13 +08:00
parent ab73b69870
commit 4b00b5ae37
24 changed files with 1030 additions and 144 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/hub.go
+5 -1
View File
@@ -251,7 +251,11 @@ func (h *Hub) run() {
immetrics.ObserveInternalQueueEnqueue("topic_client", len(dst.clientMsg), cap(dst.clientMsg), true)
default:
immetrics.ObserveInternalQueueEnqueue("topic_client", len(dst.clientMsg), cap(dst.clientMsg), false)
logs.Err.Println("hub: topic's broadcast queue is full", dst.name)
if msg.sess != nil {
msg.sess.queueOut(ErrServiceUnavailableReply(msg, types.TimeNow()))
}
logs.Err.Printf("hub: topic's broadcast queue is full topic=%s tenant=%d len=%d cap=%d",
dst.name, dst.tenantID, len(dst.clientMsg), cap(dst.clientMsg))
}
} else {
logs.Warn.Println("hub: invalid topic category for broadcast", dst.name)
server/hub_tenant_test.go
+45
View File
@@ -102,3 +102,48 @@ func TestHubRejectsUnroutedClientMessageInsteadOfAcceptingWithoutSeq(t *testing.
t.Fatal("timeout waiting for unrouted message response")
}
}
func TestHubReturnsServiceUnavailableWhenTopicClientQueueIsFull(t *testing.T) {
h := &Hub{
topics: &sync.Map{},
routeCli: make(chan *ClientComMessage, 1),
shutdown: make(chan chan<- bool),
routeSrv: make(chan *ServerComMessage),
join: make(chan *ClientComMessage),
meta: make(chan *ClientComMessage),
unreg: make(chan *topicUnreg),
userStatus: make(chan *userStatusReq),
rehash: make(chan bool),
}
topic := &Topic{
tenantID: 7, name: "grpBusy", clientMsg: make(chan *ClientComMessage, 1),
exit: make(chan *shutDown, 1),
}
topic.clientMsg <- &ClientComMessage{Id: "occupy"}
h.topicPut(types.TopicKey{TenantID: 7, Topic: topic.name}, topic)
go h.run()
done := make(chan bool)
defer func() {
h.topicDel(types.TopicKey{TenantID: 7, Topic: topic.name})
h.shutdown <- done
<-done
}()
sess := &Session{tenantID: 7, send: make(chan any, 1)}
now := time.Now().UTC()
h.routeCli <- &ClientComMessage{
Id: "pub-overload", TenantID: 7, RcptTo: topic.name, Original: topic.name,
Timestamp: now, sess: sess,
}
select {
case raw := <-sess.send:
msg, ok := raw.(*ServerComMessage)
if !ok || msg.Ctrl == nil || msg.Ctrl.Code != http.StatusServiceUnavailable {
t.Fatalf("unexpected response: %#v", raw)
}
case <-time.After(time.Second):
t.Fatal("timeout waiting for overload response")
}
}
server/im_http.go
+12
View File
@@ -4079,6 +4079,12 @@ func serveIMFriendRequestCreate(wrt http.ResponseWriter, req *http.Request) {
writeIMRelationError(wrt, req, acceptErr)
return
}
if accepted == nil {
logs.Err.Printf("friend request auto-accept returned an empty result: tenant=%d request=%s actor=%s",
principal.TenantID, created.RequestID, target.String())
writeIMError(wrt, req, http.StatusInternalServerError, 90000, imReasonInternal, "internal error", nil)
return
}
created = accepted
}
if imReliableHTTPEventsEnabled(req) {
@@ -6391,6 +6397,12 @@ func serveIMFriendRequestAction(wrt http.ResponseWriter, req *http.Request) {
writeIMRelationError(wrt, req, err)
return
}
if updated == nil {
logs.Err.Printf("friend request resolution returned an empty result: tenant=%d request=%s actor=%s status=%s",
principal.TenantID, requestID, principal.Uid.String(), status)
writeIMError(wrt, req, http.StatusInternalServerError, 90000, imReasonInternal, "internal error", nil)
return
}
if status == types.FriendRequestAccepted && imReliableHTTPEventsEnabled(req) {
topic := updated.ApplicantID.P2PName(updated.TargetID)
publishIMConversationSystemNotice(principal.TenantID, topic,
server/im_http_test.go
+26
View File
@@ -158,6 +158,7 @@ type testRelationsStore struct {
deleteTenant types.TenantID
deleteUser types.Uid
deletePeer types.Uid
resolveNil bool
}
type testFavoritesStore struct {
@@ -962,6 +963,9 @@ func (s *testRelationsStore) CountPendingFriendRequests(tenantID types.TenantID,
}
func (s *testRelationsStore) ResolveFriendRequest(_ types.TenantID, requestID string, actor types.Uid, status string, handledAt time.Time) (*types.FriendRequest, error) {
if s.resolveNil {
return nil, nil
}
return &types.FriendRequest{
TenantID: 7,
RequestID: requestID,
@@ -975,6 +979,28 @@ func (s *testRelationsStore) ResolveFriendRequest(_ types.TenantID, requestID st
}, nil
}
func TestServeIMFriendRequestActionRejectsEmptyStoreResult(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
expectIMBearer(t, ctrl, "token-1", 7, 42)
oldRelations := store.Relations
store.Relations = &testRelationsStore{resolveNil: true}
t.Cleanup(func() { store.Relations = oldRelations })
req := httptest.NewRequest(http.MethodPost, "/v1/im/friend-requests/frq-1/accept", nil)
req.Header.Set("Authorization", "Bearer token-1")
rec := httptest.NewRecorder()
serveIMFriendRequestAction(rec, req)
if rec.Code != http.StatusInternalServerError {
t.Fatalf("status: got %d, want %d; body=%s", rec.Code, http.StatusInternalServerError, rec.Body.String())
}
if response := decodeIMHTTPResponse(t, rec); response.Reason != imReasonInternal {
t.Fatalf("reason: got %q, want %q", response.Reason, imReasonInternal)
}
}
func (s *testRelationsStore) ListFriends(_ types.TenantID, _ types.Uid, _, _, _ string, _ int) ([]types.FriendListItem, string, error) {
return s.friends, "", nil
}
server/store/group_message_runtime_test.go
+29
View File
@@ -3,6 +3,7 @@ package store
import (
"context"
"encoding/json"
"errors"
"strconv"
"strings"
"testing"
@@ -443,6 +444,34 @@ func TestInvalidateGroupSettingsUsesTopicCacheKey(t *testing.T) {
}
}
func TestGetGroupMuteUsesPositiveAndNegativeCache(t *testing.T) {
oldRuntime := groupMsgRuntime
runtime := newTestGroupMessageRuntime()
groupMsgRuntime = runtime
t.Cleanup(func() { groupMsgRuntime = oldRuntime })
tenantID := types.TenantID(7)
topic := "grpAlpha"
uid := types.Uid(42)
cacheID := groupMemberCacheID(topic, uid)
expires := time.Now().UTC().Add(time.Minute)
want := &types.GroupMute{TenantID: tenantID, Topic: topic, UserID: uid, Status: types.GroupMuteStatusActive, ExpiresAt: &expires}
if err := CacheObject(tenantID, cacheNamespaceGroupMute, cacheID, groupMuteCacheValue{Found: true, Mute: want}); err != nil {
t.Fatalf("cache positive mute: %v", err)
}
got, err := (groupBusinessMapper{}).GetMute(tenantID, topic, uid)
if err != nil || got == nil || got.UserID != uid || got.Status != types.GroupMuteStatusActive {
t.Fatalf("positive cache result: mute=%#v err=%v", got, err)
}
if err = CacheObject(tenantID, cacheNamespaceGroupMute, cacheID, groupMuteCacheValue{}); err != nil {
t.Fatalf("cache negative mute: %v", err)
}
if got, err = (groupBusinessMapper{}).GetMute(tenantID, topic, uid); !errors.Is(err, types.ErrNotFound) || got != nil {
t.Fatalf("negative cache result: mute=%#v err=%v", got, err)
}
}
func TestTopicAndSubscriptionCacheTTLIsShort(t *testing.T) {
runtime := &redisGroupMessageRuntime{cfg: redisConfig{CacheTTL: 300, ShortTTL: 600}}
for _, namespace := range []string{cacheNamespaceTopic, cacheNamespaceSub} {
server/store/store.go
+53 -3
View File
@@ -142,6 +142,7 @@ const (
cacheNamespaceSub = "sub"
cacheNamespaceGroupSettings = "group_settings"
cacheNamespaceGroupMember = "group_member:v2"
cacheNamespaceGroupMute = "group_mute:v1"
cacheNamespaceVerification = "verification"
cacheNamespaceActionToken = "action_token"
cacheNamespaceFriends = "friends"
@@ -2422,11 +2423,32 @@ func (groupBusinessMapper) GetMute(tenantID types.TenantID, topic string, uid ty
if strings.TrimSpace(topic) == "" || uid.IsZero() {
return nil, types.ErrMalformed
}
topic = strings.TrimSpace(topic)
cacheID := groupMemberCacheID(topic, uid)
var cached groupMuteCacheValue
if ok, err := GetCachedObject(tenantID, cacheNamespaceGroupMute, cacheID, &cached); err != nil {
logs.Warn.Printf("cache group mute[%s]: read failed: %v", cacheID, err)
_ = DeleteCachedObject(tenantID, cacheNamespaceGroupMute, cacheID)
} else if ok {
if !cached.Found || cached.Mute == nil {
return nil, types.ErrNotFound
}
if cached.Mute.ExpiresAt == nil || cached.Mute.ExpiresAt.After(time.Now().UTC()) {
return cached.Mute, nil
}
_ = DeleteCachedObject(tenantID, cacheNamespaceGroupMute, cacheID)
}
tenantAdp, err := tenantBusinessAdapter(tenantID)
if err != nil {
return nil, err
}
return tenantAdp.TenantGroupMuteGet(tenantID, strings.TrimSpace(topic), uid)
mute, err := tenantAdp.TenantGroupMuteGet(tenantID, topic, uid)
if err == nil && mute != nil {
_ = CacheObject(tenantID, cacheNamespaceGroupMute, cacheID, groupMuteCacheValue{Found: true, Mute: mute})
} else if errors.Is(err, types.ErrNotFound) {
_ = CacheObject(tenantID, cacheNamespaceGroupMute, cacheID, groupMuteCacheValue{})
}
return mute, err
}
func (groupBusinessMapper) UpsertMute(tenantID types.TenantID, mute *types.GroupMute) (*types.GroupMute, error) {
@@ -2440,6 +2462,12 @@ func (groupBusinessMapper) UpsertMute(tenantID types.TenantID, mute *types.Group
updated, err := tenantAdp.TenantGroupMuteUpsert(tenantID, mute)
if err == nil {
invalidateGroupMemberPages(tenantID, mute.Topic)
cacheID := groupMemberCacheID(mute.Topic, mute.UserID)
if updated != nil {
_ = CacheObject(tenantID, cacheNamespaceGroupMute, cacheID, groupMuteCacheValue{Found: true, Mute: updated})
} else {
_ = DeleteCachedObject(tenantID, cacheNamespaceGroupMute, cacheID)
}
}
return updated, err
}
@@ -2456,6 +2484,7 @@ func (groupBusinessMapper) DeleteMute(tenantID types.TenantID, topic string, uid
return err
}
invalidateGroupMemberPages(tenantID, topic)
_ = CacheObject(tenantID, cacheNamespaceGroupMute, groupMemberCacheID(topic, uid), groupMuteCacheValue{})
return nil
}
@@ -2581,11 +2610,32 @@ func (groupBusinessMapper) CanPost(tenantID types.TenantID, topic string, uid ty
if strings.TrimSpace(topic) == "" || uid.IsZero() || now.IsZero() {
return false, types.ErrMalformed
}
tenantAdp, err := tenantBusinessAdapter(tenantID)
if admin {
return true, nil
}
topic = strings.TrimSpace(topic)
mapper := groupBusinessMapper{}
settings, err := mapper.GetSettings(tenantID, topic)
if err != nil && !errors.Is(err, types.ErrNotFound) {
return false, err
}
if settings != nil && settings.GroupStatus == types.GroupStatusActive && settings.MuteAll {
return false, nil
}
mute, err := mapper.GetMute(tenantID, topic, uid)
if errors.Is(err, types.ErrNotFound) {
return true, nil
}
if err != nil {
return false, err
}
return tenantAdp.TenantGroupCanPost(tenantID, strings.TrimSpace(topic), uid, admin, now)
return mute == nil || mute.Status != types.GroupMuteStatusActive ||
(mute.ExpiresAt != nil && !mute.ExpiresAt.After(now)), nil
}
type groupMuteCacheValue struct {
Found bool `json:"found"`
Mute *types.GroupMute `json:"mute,omitempty"`
}
func groupMemberCacheID(topic string, uid types.Uid) string {
server/topic.go
+4
View File
@@ -1348,7 +1348,9 @@ func (t *Topic) saveAndBroadcastMessage(msg *ClientComMessage, asUid types.Uid,
if t.cat == types.TopicCatGrp && msg.sess != nil && msg.sess.authLvl != auth.LevelRoot &&
store.GroupBusiness != nil && store.Store != nil && store.Store.IsOpen() {
access := pud.modeWant & pud.modeGiven
started := time.Now()
canPost, err := store.GroupBusiness.CanPost(t.tenantID, t.name, asUid, access.IsAdmin(), msg.Timestamp)
immetrics.ObserveMessagePublishStage("group", "permission_check", started, err)
if err != nil {
logs.Warn.Printf("topic[%s]: failed to check group mute policy: %v", t.name, err)
msg.sess.queueOut(ErrUnknown(msg.Id, t.original(asUid), msg.Timestamp))
@@ -1457,8 +1459,10 @@ func (t *Topic) saveAndBroadcastMessage(msg *ClientComMessage, asUid types.Uid,
// Tell the plugins that a message was accepted for delivery
pluginMessage(t.tenantID, data.Data, plgActCreate)
fanoutStarted := time.Now()
t.broadcastToSessions(data)
publishRealtimeTopicMessage(t.name, data)
immetrics.ObserveMessagePublishStage(kind, "realtime_fanout", fanoutStarted, nil)
// sendPush will update unread message count and send push notification.
if pushRcpt := t.pushForData(asUid, data.Data, markedReadBySender); pushRcpt != nil {