feat: add file safety config integration
This commit is contained in:
richjohnrambo
@@ -0,0 +1,243 @@
|
||||
// Package aliyun wraps Alibaba Cloud Green file moderation APIs.
|
||||
package aliyun
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
openapi "github.com/alibabacloud-go/darabonba-openapi/v2/client"
|
||||
green "github.com/alibabacloud-go/green-20220302/v3/client"
|
||||
"github.com/alibabacloud-go/tea/dara"
|
||||
|
||||
"github.com/tinode/chat/server/store/types"
|
||||
)
|
||||
|
||||
const defaultDocumentService = "document_detection"
|
||||
|
||||
type GreenConfig struct {
|
||||
AccessKeyID string
|
||||
AccessKeySecret string
|
||||
RegionID string
|
||||
Endpoint string
|
||||
DocumentService string
|
||||
}
|
||||
|
||||
type OSSObject struct {
|
||||
BucketName string
|
||||
ObjectName string
|
||||
RegionID string
|
||||
DataID string
|
||||
}
|
||||
|
||||
type SubmitResult struct {
|
||||
ProviderTaskID string
|
||||
RequestID string
|
||||
}
|
||||
|
||||
type PollResult struct {
|
||||
Status string
|
||||
RiskLevel string
|
||||
RiskLabels []string
|
||||
Raw map[string]any
|
||||
RequestID string
|
||||
}
|
||||
|
||||
type GreenClient struct {
|
||||
service string
|
||||
client *green.Client
|
||||
}
|
||||
|
||||
func NewGreenClient(conf GreenConfig) (*GreenClient, error) {
|
||||
if strings.TrimSpace(conf.AccessKeyID) == "" {
|
||||
return nil, errors.New("missing Access Key ID")
|
||||
}
|
||||
if strings.TrimSpace(conf.AccessKeySecret) == "" {
|
||||
return nil, errors.New("missing Access Key Secret")
|
||||
}
|
||||
if strings.TrimSpace(conf.RegionID) == "" {
|
||||
return nil, errors.New("missing Region ID")
|
||||
}
|
||||
service := strings.TrimSpace(conf.DocumentService)
|
||||
if service == "" {
|
||||
service = defaultDocumentService
|
||||
}
|
||||
clientConf := &openapi.Config{
|
||||
AccessKeyId: dara.String(conf.AccessKeyID),
|
||||
AccessKeySecret: dara.String(conf.AccessKeySecret),
|
||||
RegionId: dara.String(conf.RegionID),
|
||||
Protocol: dara.String("https"),
|
||||
}
|
||||
if endpoint := strings.TrimSpace(conf.Endpoint); endpoint != "" {
|
||||
clientConf.Endpoint = dara.String(endpoint)
|
||||
}
|
||||
client, err := green.NewClient(clientConf)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &GreenClient{service: service, client: client}, nil
|
||||
}
|
||||
|
||||
func (c *GreenClient) SubmitOSSFile(ctx context.Context, object OSSObject) (*SubmitResult, error) {
|
||||
params, err := buildOSSServiceParameters(object)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp, err := c.client.FileModerationWithContext(ctx, &green.FileModerationRequest{
|
||||
Service: dara.String(c.service),
|
||||
ServiceParameters: dara.String(params),
|
||||
}, &dara.RuntimeOptions{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resp == nil || resp.Body == nil || resp.Body.Code == nil || *resp.Body.Code != 200 || resp.Body.Data == nil || resp.Body.Data.TaskId == nil {
|
||||
return nil, responseError("file moderation submit failed", responseCode(resp), responseMessage(resp))
|
||||
}
|
||||
return &SubmitResult{ProviderTaskID: *resp.Body.Data.TaskId, RequestID: responseRequestID(resp)}, nil
|
||||
}
|
||||
|
||||
func (c *GreenClient) PollOSSFile(ctx context.Context, providerTaskID string) (*PollResult, error) {
|
||||
params, err := json.Marshal(map[string]string{"taskId": strings.TrimSpace(providerTaskID)})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp, err := c.client.DescribeFileModerationResultWithContext(ctx, &green.DescribeFileModerationResultRequest{
|
||||
Service: dara.String(c.service),
|
||||
ServiceParameters: dara.String(string(params)),
|
||||
}, &dara.RuntimeOptions{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resp == nil || resp.Body == nil || resp.Body.Code == nil || *resp.Body.Code != 200 {
|
||||
return nil, responseError("file moderation poll failed", describeResponseCode(resp), describeResponseMessage(resp))
|
||||
}
|
||||
raw := map[string]any{}
|
||||
if data := resp.Body.Data; data != nil {
|
||||
encoded, _ := json.Marshal(data)
|
||||
json.Unmarshal(encoded, &raw)
|
||||
riskLevel := strings.TrimSpace(dara.StringValue(data.RiskLevel))
|
||||
return &PollResult{
|
||||
Status: statusFromRiskLevel(riskLevel),
|
||||
RiskLevel: riskLevel,
|
||||
RiskLabels: labelsFromFileResult(data),
|
||||
Raw: raw,
|
||||
RequestID: describeResponseRequestID(resp),
|
||||
}, nil
|
||||
}
|
||||
return &PollResult{Status: types.FileReviewStatusRunning, Raw: raw, RequestID: describeResponseRequestID(resp)}, nil
|
||||
}
|
||||
|
||||
func buildOSSServiceParameters(object OSSObject) (string, error) {
|
||||
if strings.TrimSpace(object.BucketName) == "" || strings.TrimSpace(object.ObjectName) == "" || strings.TrimSpace(object.RegionID) == "" {
|
||||
return "", types.ErrMalformed
|
||||
}
|
||||
params := map[string]string{
|
||||
"ossBucketName": strings.TrimSpace(object.BucketName),
|
||||
"ossObjectName": strings.TrimSpace(object.ObjectName),
|
||||
"ossRegionId": strings.TrimSpace(object.RegionID),
|
||||
}
|
||||
if dataID := strings.TrimSpace(object.DataID); dataID != "" {
|
||||
params["dataId"] = dataID
|
||||
}
|
||||
encoded, err := json.Marshal(params)
|
||||
return string(encoded), err
|
||||
}
|
||||
|
||||
func statusFromRiskLevel(riskLevel string) string {
|
||||
switch strings.ToLower(strings.TrimSpace(riskLevel)) {
|
||||
case "":
|
||||
return types.FileReviewStatusRunning
|
||||
case "none", "low":
|
||||
return types.FileReviewStatusClean
|
||||
default:
|
||||
return types.FileReviewStatusRisky
|
||||
}
|
||||
}
|
||||
|
||||
func labelsFromFileResult(data *green.DescribeFileModerationResultResponseBodyData) []string {
|
||||
seen := map[string]bool{}
|
||||
var labels []string
|
||||
for _, page := range data.PageResult {
|
||||
if page == nil {
|
||||
continue
|
||||
}
|
||||
for _, image := range page.ImageResult {
|
||||
if image == nil {
|
||||
continue
|
||||
}
|
||||
for _, result := range image.LabelResult {
|
||||
if result == nil {
|
||||
continue
|
||||
}
|
||||
if label := strings.TrimSpace(dara.StringValue(result.Label)); label != "" && !seen[label] {
|
||||
seen[label] = true
|
||||
labels = append(labels, label)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, text := range page.TextResult {
|
||||
if text == nil {
|
||||
continue
|
||||
}
|
||||
for _, label := range strings.Split(dara.StringValue(text.Labels), ",") {
|
||||
label = strings.TrimSpace(label)
|
||||
if label != "" && !seen[label] {
|
||||
seen[label] = true
|
||||
labels = append(labels, label)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return labels
|
||||
}
|
||||
|
||||
func responseError(prefix string, code int32, msg string) error {
|
||||
if msg == "" {
|
||||
return errors.New(prefix)
|
||||
}
|
||||
return errors.New(prefix + ": " + msg + " (" + strconv.FormatInt(int64(code), 10) + ")")
|
||||
}
|
||||
|
||||
func responseCode(resp *green.FileModerationResponse) int32 {
|
||||
if resp != nil && resp.Body != nil && resp.Body.Code != nil {
|
||||
return *resp.Body.Code
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func responseMessage(resp *green.FileModerationResponse) string {
|
||||
if resp != nil && resp.Body != nil && resp.Body.Message != nil {
|
||||
return *resp.Body.Message
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func responseRequestID(resp *green.FileModerationResponse) string {
|
||||
if resp != nil && resp.Body != nil && resp.Body.RequestId != nil {
|
||||
return *resp.Body.RequestId
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func describeResponseCode(resp *green.DescribeFileModerationResultResponse) int32 {
|
||||
if resp != nil && resp.Body != nil && resp.Body.Code != nil {
|
||||
return *resp.Body.Code
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func describeResponseMessage(resp *green.DescribeFileModerationResultResponse) string {
|
||||
if resp != nil && resp.Body != nil && resp.Body.Message != nil {
|
||||
return *resp.Body.Message
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func describeResponseRequestID(resp *green.DescribeFileModerationResultResponse) string {
|
||||
if resp != nil && resp.Body != nil && resp.Body.RequestId != nil {
|
||||
return *resp.Body.RequestId
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
package aliyun
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
green "github.com/alibabacloud-go/green-20220302/v3/client"
|
||||
"github.com/alibabacloud-go/tea/dara"
|
||||
"github.com/tinode/chat/server/store/types"
|
||||
)
|
||||
|
||||
func TestBuildOSSServiceParameters(t *testing.T) {
|
||||
got, err := buildOSSServiceParameters(OSSObject{
|
||||
BucketName: "im-oss-t1",
|
||||
ObjectName: "tenants/1/2026/08/file",
|
||||
RegionID: "cn-shenzhen",
|
||||
DataID: "task-1",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var params map[string]string
|
||||
if err := json.Unmarshal([]byte(got), ¶ms); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for key, want := range map[string]string{
|
||||
"ossBucketName": "im-oss-t1",
|
||||
"ossObjectName": "tenants/1/2026/08/file",
|
||||
"ossRegionId": "cn-shenzhen",
|
||||
"dataId": "task-1",
|
||||
} {
|
||||
if params[key] != want {
|
||||
t.Fatalf("params[%s] = %q, want %q", key, params[key], want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatusFromRiskLevel(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"": types.FileReviewStatusRunning,
|
||||
"none": types.FileReviewStatusClean,
|
||||
"low": types.FileReviewStatusClean,
|
||||
"medium": types.FileReviewStatusRisky,
|
||||
"high": types.FileReviewStatusRisky,
|
||||
}
|
||||
for input, want := range cases {
|
||||
if got := statusFromRiskLevel(input); got != want {
|
||||
t.Fatalf("statusFromRiskLevel(%q) = %q, want %q", input, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLabelsFromFileResult(t *testing.T) {
|
||||
got := labelsFromFileResult(&green.DescribeFileModerationResultResponseBodyData{
|
||||
PageResult: []*green.DescribeFileModerationResultResponseBodyDataPageResult{
|
||||
{ImageResult: []*green.DescribeFileModerationResultResponseBodyDataPageResultImageResult{
|
||||
{LabelResult: []*green.DescribeFileModerationResultResponseBodyDataPageResultImageResultLabelResult{
|
||||
{Label: dara.String("porn")},
|
||||
{Label: dara.String("porn")},
|
||||
}},
|
||||
}},
|
||||
{TextResult: []*green.DescribeFileModerationResultResponseBodyDataPageResultTextResult{
|
||||
{Labels: dara.String("violence,ad")},
|
||||
}},
|
||||
},
|
||||
})
|
||||
if len(got) != 3 || got[0] != "porn" || got[1] != "violence" || got[2] != "ad" {
|
||||
t.Fatalf("labelsFromFileResult = %#v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
package filesafety
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/tinode/chat/server/filesafety/aliyun"
|
||||
"github.com/tinode/chat/server/store"
|
||||
"github.com/tinode/chat/server/store/types"
|
||||
)
|
||||
|
||||
const thirdPartyConfigGroup = "im_third_party"
|
||||
|
||||
type ContentReviewConfig struct {
|
||||
Enabled bool
|
||||
Provider string
|
||||
Green aliyun.GreenConfig
|
||||
}
|
||||
|
||||
type MalwareReviewConfig struct {
|
||||
Enabled bool
|
||||
Provider string
|
||||
RegionID string
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
Content ContentReviewConfig
|
||||
Malware MalwareReviewConfig
|
||||
}
|
||||
|
||||
func LoadConfig() (*Config, error) {
|
||||
values, err := store.ThirdPartyConfig.Get(thirdPartyConfigGroup,
|
||||
"file_review_content_enabled",
|
||||
"file_review_content_provider",
|
||||
"aliyun_green_region_id",
|
||||
"aliyun_green_endpoint",
|
||||
"aliyun_green_document_service",
|
||||
"file_review_malware_enabled",
|
||||
"file_review_malware_provider",
|
||||
"aliyun_sas_region_id",
|
||||
"aliyun_access_key_id",
|
||||
"aliyun_access_key_secret",
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
config := &Config{
|
||||
Content: ContentReviewConfig{
|
||||
Enabled: boolValue(values["file_review_content_enabled"]),
|
||||
Provider: stringDefault(values["file_review_content_provider"], types.FileReviewProviderAliyunGreen),
|
||||
Green: aliyun.GreenConfig{
|
||||
AccessKeyID: strings.TrimSpace(values["aliyun_access_key_id"]),
|
||||
AccessKeySecret: strings.TrimSpace(values["aliyun_access_key_secret"]),
|
||||
RegionID: strings.TrimSpace(values["aliyun_green_region_id"]),
|
||||
Endpoint: strings.TrimSpace(values["aliyun_green_endpoint"]),
|
||||
DocumentService: strings.TrimSpace(values["aliyun_green_document_service"]),
|
||||
},
|
||||
},
|
||||
Malware: MalwareReviewConfig{
|
||||
Enabled: boolValue(values["file_review_malware_enabled"]),
|
||||
Provider: stringDefault(values["file_review_malware_provider"], types.FileReviewProviderAliyunSAS),
|
||||
RegionID: strings.TrimSpace(values["aliyun_sas_region_id"]),
|
||||
},
|
||||
}
|
||||
if config.Content.Enabled && config.Content.Provider != types.FileReviewProviderAliyunGreen {
|
||||
return nil, errors.New("unsupported file content review provider: " + config.Content.Provider)
|
||||
}
|
||||
if config.Malware.Enabled && config.Malware.Provider != types.FileReviewProviderAliyunSAS {
|
||||
return nil, errors.New("unsupported malware review provider: " + config.Malware.Provider)
|
||||
}
|
||||
return config, nil
|
||||
}
|
||||
|
||||
func NewGreenClientFromConfig() (*aliyun.GreenClient, error) {
|
||||
config, err := LoadConfig()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !config.Content.Enabled {
|
||||
return nil, types.ErrUnsupported
|
||||
}
|
||||
return aliyun.NewGreenClient(config.Content.Green)
|
||||
}
|
||||
|
||||
func boolValue(value string) bool {
|
||||
if parsed, err := strconv.ParseBool(strings.TrimSpace(value)); err == nil {
|
||||
return parsed
|
||||
}
|
||||
switch strings.ToLower(strings.TrimSpace(value)) {
|
||||
case "1", "yes", "on":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func stringDefault(value, fallback string) string {
|
||||
if value = strings.TrimSpace(value); value != "" {
|
||||
return value
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
package filesafety
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/tinode/chat/server/store"
|
||||
"github.com/tinode/chat/server/store/types"
|
||||
)
|
||||
|
||||
type fakeThirdPartyConfig map[string]string
|
||||
|
||||
func (f fakeThirdPartyConfig) Get(group string, keys ...string) (map[string]string, error) {
|
||||
result := map[string]string{}
|
||||
for _, key := range keys {
|
||||
if value, ok := f[key]; ok {
|
||||
result[key] = value
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func TestLoadConfig(t *testing.T) {
|
||||
original := store.ThirdPartyConfig
|
||||
store.ThirdPartyConfig = fakeThirdPartyConfig{
|
||||
"file_review_content_enabled": "1",
|
||||
"file_review_content_provider": types.FileReviewProviderAliyunGreen,
|
||||
"aliyun_green_region_id": "cn-shanghai",
|
||||
"aliyun_green_endpoint": "green-cip.cn-shanghai.aliyuncs.com",
|
||||
"aliyun_green_document_service": "document_detection",
|
||||
"file_review_malware_enabled": "true",
|
||||
"file_review_malware_provider": types.FileReviewProviderAliyunSAS,
|
||||
"aliyun_sas_region_id": "cn-shanghai",
|
||||
"aliyun_access_key_id": "ak",
|
||||
"aliyun_access_key_secret": "sk",
|
||||
}
|
||||
defer func() { store.ThirdPartyConfig = original }()
|
||||
|
||||
config, err := LoadConfig()
|
||||
if err != nil {
|
||||
t.Fatalf("LoadConfig failed: %v", err)
|
||||
}
|
||||
if !config.Content.Enabled || config.Content.Provider != types.FileReviewProviderAliyunGreen {
|
||||
t.Fatalf("unexpected content config: %+v", config.Content)
|
||||
}
|
||||
if !config.Malware.Enabled || config.Malware.Provider != types.FileReviewProviderAliyunSAS {
|
||||
t.Fatalf("unexpected malware config: %+v", config.Malware)
|
||||
}
|
||||
if config.Content.Green.AccessKeyID != "ak" || config.Content.Green.RegionID != "cn-shanghai" {
|
||||
t.Fatalf("unexpected green config: %+v", config.Content.Green)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user