pro/im_server
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: enforce tenant content moderation

Browse Source
This commit is contained in:
richjohnrambo
2026-09-16 22:21:25 +08:00
parent 0ae4af0c31
commit 0f8e3b82b7
26 changed files with 891 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/moderation/aliyun/text.go
+100
View File
@@ -0,0 +1,100 @@
package aliyun
import (
"context"
"encoding/json"
"errors"
"strings"
"time"
openapi "github.com/alibabacloud-go/darabonba-openapi/v2/client"
green "github.com/alibabacloud-go/green-20220302/v3/client"
"github.com/alibabacloud-go/tea/dara"
"github.com/tinode/chat/server/store/types"
)
type TextClient struct {
client *green.Client
timeout time.Duration
}
type Result struct {
ProviderRequestID string
RiskLevel string
RiskLabels []string
Summary string
Latency time.Duration
}
func NewTextClient(config *types.ModerationProviderConfig) (*TextClient, error) {
if config == nil || strings.TrimSpace(config.AccessKeyID) == "" || strings.TrimSpace(config.AccessKeySecret) == "" {
return nil, errors.New("missing aliyun moderation credentials")
}
clientConfig := &openapi.Config{AccessKeyId: dara.String(config.AccessKeyID), AccessKeySecret: dara.String(config.AccessKeySecret), RegionId: dara.String(config.RegionID), Protocol: dara.String("https")}
if endpoint := strings.TrimSpace(config.Endpoint); endpoint != "" {
clientConfig.Endpoint = dara.String(endpoint)
}
client, err := green.NewClient(clientConfig)
if err != nil {
return nil, err
}
timeout := time.Duration(config.TimeoutMS) * time.Millisecond
if timeout <= 0 {
timeout = 1500 * time.Millisecond
}
return &TextClient{client: client, timeout: timeout}, nil
}
func (c *TextClient) Review(ctx context.Context, service, text, dataID string) (*Result, error) {
ctx, cancel := context.WithTimeout(ctx, c.timeout)
defer cancel()
params, err := json.Marshal(map[string]string{"content": text, "dataId": dataID})
if err != nil {
return nil, err
}
started := time.Now()
resp, err := c.client.TextModerationPlusWithContext(ctx, &green.TextModerationPlusRequest{Service: dara.String(service), ServiceParameters: dara.String(string(params))}, &dara.RuntimeOptions{})
latency := time.Since(started)
if err != nil {
return nil, err
}
if resp == nil || resp.Body == nil || dara.Int32Value(resp.Body.Code) != 200 || resp.Body.Data == nil {
return nil, errors.New("aliyun text moderation failed")
}
result := &Result{ProviderRequestID: dara.StringValue(resp.Body.RequestId), RiskLevel: moderationRisk(dara.StringValue(resp.Body.Data.RiskLevel)), Latency: latency}
seen := map[string]bool{}
for _, item := range resp.Body.Data.Result {
if item == nil {
continue
}
label := strings.TrimSpace(dara.StringValue(item.Label))
if label != "" && !seen[label] {
seen[label] = true
result.RiskLabels = append(result.RiskLabels, label)
}
}
if ext := resp.Body.Data.Ext; ext != nil && ext.LlmContent != nil {
result.Summary = strings.TrimSpace(dara.StringValue(ext.LlmContent.OutputText))
if llmIndicatesRisk(result.Summary) && result.RiskLevel == "none" {
result.RiskLevel = "medium"
result.RiskLabels = append(result.RiskLabels, "llm_risk")
}
}
return result, nil
}
func moderationRisk(value string) string {
switch strings.ToLower(strings.TrimSpace(value)) {
case "high", "medium", "low":
return strings.ToLower(strings.TrimSpace(value))
default:
return "none"
}
}
func llmIndicatesRisk(value string) bool {
value = strings.TrimSpace(value)
if value == "" {
return false
}
return !strings.Contains(value, "无风险") && !strings.HasPrefix(value, "正常") && (strings.Contains(value, "风险") || strings.Contains(value, "违规") || strings.Contains(value, "诈骗") || strings.Contains(value, "引流"))
}
server/moderation/extractor.go
+34
View File
@@ -0,0 +1,34 @@
package moderation
import "strings"
var visibleTextKeys = map[string]bool{
"txt": true, "text": true, "title": true, "caption": true,
"description": true, "file_name": true, "filename": true, "name": true,
}
func ExtractText(content any) string {
parts := make([]string, 0, 4)
var walk func(any, int, bool)
walk = func(value any, depth int, includeString bool) {
if depth > 6 || value == nil {
return
}
switch typed := value.(type) {
case string:
if text := strings.TrimSpace(typed); includeString && text != "" {
parts = append(parts, text)
}
case map[string]any:
for key, item := range typed {
walk(item, depth+1, visibleTextKeys[strings.ToLower(strings.TrimSpace(key))])
}
case []any:
for _, item := range typed {
walk(item, depth+1, includeString)
}
}
}
walk(content, 0, true)
return strings.TrimSpace(strings.Join(parts, "\n"))
}
server/moderation/local.go
+74
View File
@@ -0,0 +1,74 @@
package moderation
import (
"regexp"
"strings"
"github.com/mozillazg/go-pinyin"
"github.com/tinode/chat/server/store/types"
)
func matchLocalWords(text, scene string, words *types.SensitiveWordSet) *ProviderResult {
normalized := strings.ToLower(strings.TrimSpace(text))
if normalized == "" || words == nil {
return nil
}
for _, rule := range words.Items {
if rule == nil || strings.TrimSpace(rule.Word) == "" || !sceneEnabled(rule.Scenes, scene) {
continue
}
matched := false
switch strings.ToLower(strings.TrimSpace(rule.MatchType)) {
case "exact":
matched = normalized == strings.ToLower(strings.TrimSpace(rule.Word))
case "regex":
pattern, err := regexp.Compile(rule.Word)
matched = err == nil && pattern.MatchString(text)
case "pinyin":
args := pinyin.NewArgs()
args.Style = pinyin.Normal
textPinyin := strings.Join(pinyin.LazyConvert(text, &args), "")
wordPinyin := strings.Join(pinyin.LazyConvert(rule.Word, &args), "")
if wordPinyin == "" {
wordPinyin = strings.ToLower(strings.TrimSpace(rule.Word))
}
matched = wordPinyin != "" && strings.Contains(strings.ToLower(textPinyin), strings.ToLower(wordPinyin))
default:
matched = strings.Contains(normalized, strings.ToLower(strings.TrimSpace(rule.Word)))
}
if matched {
return &ProviderResult{RiskLevel: normalizedRisk(rule.RiskLevel), RiskLabels: []string{"local_sensitive_word"}, Summary: normalizedAction(rule.Action)}
}
}
return nil
}
func sceneEnabled(scenes []string, scene string) bool {
if len(scenes) == 0 {
return true
}
for _, value := range scenes {
if value == "*" || value == scene {
return true
}
}
return false
}
func normalizedRisk(value string) string {
switch strings.ToLower(strings.TrimSpace(value)) {
case "high", "medium", "low":
return strings.ToLower(strings.TrimSpace(value))
default:
return "none"
}
}
func normalizedAction(value string) string {
switch strings.ToLower(strings.TrimSpace(value)) {
case ActionBlock, ActionAuditOnly:
return strings.ToLower(strings.TrimSpace(value))
default:
return ActionAllow
}
}
server/moderation/moderation_test.go
+67
View File
@@ -0,0 +1,67 @@
package moderation
import (
"context"
"testing"
"time"
"github.com/tinode/chat/server/store"
"github.com/tinode/chat/server/store/types"
)
type testConfig struct {
policy *types.ModerationPolicy
words *types.SensitiveWordSet
}
func (c testConfig) Provider() (*types.ModerationProviderConfig, error) {
return &types.ModerationProviderConfig{}, nil
}
func (c testConfig) Policy(string) (*types.ModerationPolicy, error) { return c.policy, nil }
func (c testConfig) Words(string) (*types.SensitiveWordSet, error) { return c.words, nil }
type testAudit struct{}
func (testAudit) RecordLogin(types.TenantID, *types.LoginAuditEvent) error { return nil }
func (testAudit) RecordLoginFailure(types.TenantID, *types.LoginFailureEvent) error { return nil }
func (testAudit) CloseLogin(types.TenantID, string, time.Time) error { return nil }
func (testAudit) ApplyMessageAction(types.TenantID, *types.MessageAdminAction) error { return nil }
func (testAudit) RecordContentModeration(types.TenantID, *types.ContentModerationLog) error {
return nil
}
func TestExtractText(t *testing.T) {
content := map[string]any{"txt": "正文", "ent": []any{map[string]any{"data": map[string]any{"title": "查看详情", "url": "https://example.com"}}}, "meta": "ignore"}
if got := ExtractText(content); got != "正文\n查看详情" && got != "查看详情\n正文" {
t.Fatalf("ExtractText()=%q", got)
}
}
func TestReviewLocalBlock(t *testing.T) {
oldConfig, oldAudit := store.ModerationConfig, store.SecurityAudit
defer func() { store.ModerationConfig, store.SecurityAudit = oldConfig, oldAudit }()
store.ModerationConfig = testConfig{policy: &types.ModerationPolicy{Enabled: true, LocalWordsEnabled: true, Scenes: map[string]bool{ScenePrivateMessage: true}, RiskActions: map[string]string{"high": ActionBlock}}, words: &types.SensitiveWordSet{Items: []*types.SensitiveWordRule{{Word: "违规词", MatchType: "contains", Scenes: []string{ScenePrivateMessage}, Action: ActionBlock, RiskLevel: "high"}}}}
store.SecurityAudit = testAudit{}
result, err := Review(context.Background(), Request{TenantID: 1, TenantCode: "default", Scene: ScenePrivateMessage, Text: "包含违规词的消息"})
if err != ErrRejected || result.Action != ActionBlock || result.Provider != "local" {
t.Fatalf("result=%+v err=%v", result, err)
}
}
func TestReviewOtherTenantUsesOwnPolicy(t *testing.T) {
oldConfig, oldAudit := store.ModerationConfig, store.SecurityAudit
defer func() { store.ModerationConfig, store.SecurityAudit = oldConfig, oldAudit }()
store.ModerationConfig = testConfig{policy: &types.ModerationPolicy{Enabled: true, LocalWordsEnabled: true, Scenes: map[string]bool{ScenePrivateMessage: true}, RiskActions: map[string]string{"high": ActionBlock}}, words: &types.SensitiveWordSet{Items: []*types.SensitiveWordRule{{Word: "违规词", MatchType: "contains", Scenes: []string{ScenePrivateMessage}, Action: ActionBlock, RiskLevel: "high"}}}}
store.SecurityAudit = testAudit{}
result, err := Review(context.Background(), Request{TenantID: 2, TenantCode: "team", Scene: ScenePrivateMessage, Text: "违规词"})
if err != ErrRejected || result.Action != ActionBlock {
t.Fatalf("result=%+v err=%v", result, err)
}
}
func TestMatchLocalWordsPinyin(t *testing.T) {
result := matchLocalWords("这里包含敏感内容", SceneGroupMessage, &types.SensitiveWordSet{Items: []*types.SensitiveWordRule{{Word: "mingan", MatchType: "pinyin", Scenes: []string{SceneGroupMessage}, RiskLevel: "medium"}}})
if result == nil || result.RiskLevel != "medium" {
t.Fatalf("result=%+v", result)
}
}
server/moderation/provider.go
+62
View File
@@ -0,0 +1,62 @@
package moderation
import (
"context"
"errors"
"time"
"github.com/tinode/chat/server/store/types"
)
const (
SceneUserNickname = "user_nickname"
SceneUserSignature = "user_signature"
SceneGroupName = "group_name"
SceneGroupNickname = "group_nickname"
ScenePrivateMessage = "private_message"
SceneGroupMessage = "group_message"
SceneGroupAnnouncement = "group_announcement"
ActionAllow = "allow"
ActionBlock = "block"
ActionAuditOnly = "audit_only"
)
var (
ErrRejected = errors.New("content rejected")
ErrUnavailable = errors.New("content review unavailable")
)
type Request struct {
TenantID types.TenantID
TenantCode string
Scene string
ActorID types.Uid
Topic string
Text string
TraceID string
}
type ProviderResult struct {
ProviderRequestID string
RiskLevel string
RiskLabels []string
Summary string
Latency time.Duration
}
type Result struct {
RequestID string
Action string
RiskLevel string
RiskLabels []string
Provider string
ProviderRequestID string
Latency time.Duration
BlockMessage string
Err error
}
type Provider interface {
Review(context.Context, string, string, string) (*ProviderResult, error)
}
server/moderation/service.go
+172
View File
@@ -0,0 +1,172 @@
package moderation
import (
"context"
"crypto/rand"
"crypto/sha256"
"encoding/hex"
"strings"
"sync"
"time"
"github.com/tinode/chat/server/logs"
"github.com/tinode/chat/server/moderation/aliyun"
"github.com/tinode/chat/server/store"
"github.com/tinode/chat/server/store/types"
)
type circuitState struct {
sync.Mutex
failures int
openUntil time.Time
}
var aliyunCircuit circuitState
func Review(ctx context.Context, req Request) (*Result, error) {
text := strings.TrimSpace(req.Text)
result := &Result{RequestID: moderationRequestID(), Action: ActionAllow, RiskLevel: "none"}
if text == "" || strings.TrimSpace(req.TenantCode) == "" {
return result, nil
}
policy, err := store.ModerationConfig.Policy(req.TenantCode)
if err != nil {
return failureResult(result, nil, err)
}
if policy == nil || !policy.Enabled || !policy.Scenes[req.Scene] {
return result, nil
}
defer func() { recordDecision(req, text, result) }()
result.BlockMessage = policy.BlockMessage
if policy.LocalWordsEnabled {
words, wordsErr := store.ModerationConfig.Words(req.TenantCode)
if wordsErr != nil {
if logs.Warn != nil {
logs.Warn.Printf("moderation local words unavailable: tenant=%s scene=%s err=%v", req.TenantCode, req.Scene, wordsErr)
}
} else if localResult := matchLocalWords(text, req.Scene, words); localResult != nil {
result.Provider, result.RiskLevel, result.RiskLabels = "local", localResult.RiskLevel, localResult.RiskLabels
result.Action = actionForRisk(policy, localResult.RiskLevel)
if localResult.Summary == ActionBlock {
result.Action = ActionBlock
}
if result.Action == ActionBlock {
result.Err = ErrRejected
return result, ErrRejected
}
}
}
if !policy.AliyunEnabled {
return result, nil
}
providerConfig, err := store.ModerationConfig.Provider()
if err != nil || providerConfig == nil || !providerConfig.Enabled {
return failureResult(result, policy, firstError(err, ErrUnavailable))
}
if circuitOpen() {
return failureResult(result, policy, ErrUnavailable)
}
provider, err := aliyun.NewTextClient(providerConfig)
if err != nil {
recordFailure()
return failureResult(result, policy, err)
}
service := providerConfig.ChatService
if req.Scene == SceneUserNickname || req.Scene == SceneUserSignature || req.Scene == SceneGroupName || req.Scene == SceneGroupNickname {
service = providerConfig.NicknameService
}
providerResult, err := provider.Review(ctx, service, text, result.RequestID)
if err != nil {
recordFailure()
return failureResult(result, policy, err)
}
recordSuccess()
result.Provider, result.ProviderRequestID, result.RiskLevel, result.RiskLabels, result.Latency = "aliyun_green", providerResult.ProviderRequestID, providerResult.RiskLevel, providerResult.RiskLabels, providerResult.Latency
result.Action = actionForRisk(policy, result.RiskLevel)
if result.Action == ActionBlock {
result.Err = ErrRejected
return result, ErrRejected
}
return result, nil
}
func failureResult(result *Result, policy *types.ModerationPolicy, err error) (*Result, error) {
result.Err = err
failureMode := "allow"
if policy != nil && policy.FailureMode == "block" {
failureMode = "block"
result.BlockMessage = policy.BlockMessage
}
if failureMode == "block" {
result.Action = ActionBlock
return result, ErrUnavailable
}
result.Action = ActionAuditOnly
return result, nil
}
func actionForRisk(policy *types.ModerationPolicy, risk string) string {
if policy != nil {
if action := normalizedAction(policy.RiskActions[normalizedRisk(risk)]); action != "" {
return action
}
}
return ActionAllow
}
func firstError(err, fallback error) error {
if err != nil {
return err
}
return fallback
}
func moderationRequestID() string {
var raw [12]byte
if _, err := rand.Read(raw[:]); err != nil {
return hex.EncodeToString([]byte(time.Now().Format(time.RFC3339Nano)))
}
return hex.EncodeToString(raw[:])
}
func circuitOpen() bool {
aliyunCircuit.Lock()
defer aliyunCircuit.Unlock()
return time.Now().Before(aliyunCircuit.openUntil)
}
func recordFailure() {
aliyunCircuit.Lock()
defer aliyunCircuit.Unlock()
aliyunCircuit.failures++
if aliyunCircuit.failures >= 5 {
aliyunCircuit.openUntil = time.Now().Add(30 * time.Second)
aliyunCircuit.failures = 0
}
}
func recordSuccess() {
aliyunCircuit.Lock()
aliyunCircuit.failures = 0
aliyunCircuit.openUntil = time.Time{}
aliyunCircuit.Unlock()
}
func logDecision(req Request, result *Result) {
if logs.Info != nil {
logs.Info.Printf("content moderation: request=%s tenant=%s scene=%s actor=%s provider=%s risk=%s action=%s latency_ms=%d", result.RequestID, req.TenantCode, req.Scene, req.ActorID.UserId(), result.Provider, result.RiskLevel, result.Action, result.Latency.Milliseconds())
}
}
func recordDecision(req Request, text string, result *Result) {
logDecision(req, result)
sum := sha256.Sum256([]byte(text))
errorCode := ""
if result.Err != nil {
errorCode = "PROVIDER_ERROR"
}
if err := store.SecurityAudit.RecordContentModeration(req.TenantID, &types.ContentModerationLog{
RequestID: result.RequestID, Scene: req.Scene, Topic: req.Topic, ActorID: req.ActorID,
Provider: result.Provider, ProviderRequestID: result.ProviderRequestID, RiskLevel: result.RiskLevel,
RiskLabels: result.RiskLabels, Action: result.Action, ContentHash: hex.EncodeToString(sum[:]),
LatencyMS: result.Latency.Milliseconds(), ErrorCode: errorCode, CreatedAt: time.Now().UTC().Round(time.Millisecond),
}); err != nil {
if logs.Warn != nil {
logs.Warn.Printf("content moderation audit write failed: request=%s tenant=%s err=%v", result.RequestID, req.TenantCode, err)
}
}
}