feat: enforce tenant content moderation

server/moderation/aliyun/text.go

@@ -0,0 +1,100 @@
+package aliyun
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"strings"
+	"time"
+
+	openapi "github.com/alibabacloud-go/darabonba-openapi/v2/client"
+	green "github.com/alibabacloud-go/green-20220302/v3/client"
+	"github.com/alibabacloud-go/tea/dara"
+	"github.com/tinode/chat/server/store/types"
+)
+
+type TextClient struct {
+	client  *green.Client
+	timeout time.Duration
+}
+
+type Result struct {
+	ProviderRequestID string
+	RiskLevel         string
+	RiskLabels        []string
+	Summary           string
+	Latency           time.Duration
+}
+
+func NewTextClient(config *types.ModerationProviderConfig) (*TextClient, error) {
+	if config == nil || strings.TrimSpace(config.AccessKeyID) == "" || strings.TrimSpace(config.AccessKeySecret) == "" {
+		return nil, errors.New("missing aliyun moderation credentials")
+	}
+	clientConfig := &openapi.Config{AccessKeyId: dara.String(config.AccessKeyID), AccessKeySecret: dara.String(config.AccessKeySecret), RegionId: dara.String(config.RegionID), Protocol: dara.String("https")}
+	if endpoint := strings.TrimSpace(config.Endpoint); endpoint != "" {
+		clientConfig.Endpoint = dara.String(endpoint)
+	}
+	client, err := green.NewClient(clientConfig)
+	if err != nil {
+		return nil, err
+	}
+	timeout := time.Duration(config.TimeoutMS) * time.Millisecond
+	if timeout <= 0 {
+		timeout = 1500 * time.Millisecond
+	}
+	return &TextClient{client: client, timeout: timeout}, nil
+}
+
+func (c *TextClient) Review(ctx context.Context, service, text, dataID string) (*Result, error) {
+	ctx, cancel := context.WithTimeout(ctx, c.timeout)
+	defer cancel()
+	params, err := json.Marshal(map[string]string{"content": text, "dataId": dataID})
+	if err != nil {
+		return nil, err
+	}
+	started := time.Now()
+	resp, err := c.client.TextModerationPlusWithContext(ctx, &green.TextModerationPlusRequest{Service: dara.String(service), ServiceParameters: dara.String(string(params))}, &dara.RuntimeOptions{})
+	latency := time.Since(started)
+	if err != nil {
+		return nil, err
+	}
+	if resp == nil || resp.Body == nil || dara.Int32Value(resp.Body.Code) != 200 || resp.Body.Data == nil {
+		return nil, errors.New("aliyun text moderation failed")
+	}
+	result := &Result{ProviderRequestID: dara.StringValue(resp.Body.RequestId), RiskLevel: moderationRisk(dara.StringValue(resp.Body.Data.RiskLevel)), Latency: latency}
+	seen := map[string]bool{}
+	for _, item := range resp.Body.Data.Result {
+		if item == nil {
+			continue
+		}
+		label := strings.TrimSpace(dara.StringValue(item.Label))
+		if label != "" && !seen[label] {
+			seen[label] = true
+			result.RiskLabels = append(result.RiskLabels, label)
+		}
+	}
+	if ext := resp.Body.Data.Ext; ext != nil && ext.LlmContent != nil {
+		result.Summary = strings.TrimSpace(dara.StringValue(ext.LlmContent.OutputText))
+		if llmIndicatesRisk(result.Summary) && result.RiskLevel == "none" {
+			result.RiskLevel = "medium"
+			result.RiskLabels = append(result.RiskLabels, "llm_risk")
+		}
+	}
+	return result, nil
+}
+
+func moderationRisk(value string) string {
+	switch strings.ToLower(strings.TrimSpace(value)) {
+	case "high", "medium", "low":
+		return strings.ToLower(strings.TrimSpace(value))
+	default:
+		return "none"
+	}
+}
+func llmIndicatesRisk(value string) bool {
+	value = strings.TrimSpace(value)
+	if value == "" {
+		return false
+	}
+	return !strings.Contains(value, "无风险") && !strings.HasPrefix(value, "正常") && (strings.Contains(value, "风险") || strings.Contains(value, "违规") || strings.Contains(value, "诈骗") || strings.Contains(value, "引流"))
+}