feat: add file safety config integration
This commit is contained in:
@@ -221,6 +221,12 @@ type RegistrationConfigAdapter interface {
|
||||
RegistrationSMSEnabled() (bool, error)
|
||||
}
|
||||
|
||||
// ThirdPartyConfigAdapter reads global third-party integration settings from
|
||||
// the control database.
|
||||
type ThirdPartyConfigAdapter interface {
|
||||
ThirdPartyConfig(group string, keys ...string) (map[string]string, error)
|
||||
}
|
||||
|
||||
// TenantRoutedAdapter can return a tenant-specific adapter, e.g. for the
|
||||
// "one tenant, one database" deployment model.
|
||||
type TenantRoutedAdapter interface {
|
||||
|
||||
@@ -255,6 +255,9 @@ func (a *adapter) Open(jsonconfig json.RawMessage) error {
|
||||
if err = ensureIMHTTPInfrastructureTables(a.db); err != nil {
|
||||
return errors.New("mysql adapter failed to ensure HTTP infrastructure schema: " + err.Error())
|
||||
}
|
||||
if err = ensureIMFileReviewTables(a.db); err != nil {
|
||||
return errors.New("mysql adapter failed to ensure file review schema: " + err.Error())
|
||||
}
|
||||
if err = ensureSnowflakePrimaryKeys(a.db); err != nil {
|
||||
return errors.New("mysql adapter failed to ensure snowflake primary keys: " + err.Error())
|
||||
}
|
||||
@@ -458,6 +461,10 @@ func (a *adapter) TenantAdapterFor(tenantID t.TenantID) (dbadapter.TenantBusines
|
||||
db.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
if err = ensureIMFileReviewTables(db); err != nil {
|
||||
db.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
if err = ensureSnowflakePrimaryKeys(db); err != nil {
|
||||
db.Close()
|
||||
return nil, nil, err
|
||||
@@ -877,6 +884,9 @@ func (a *adapter) CreateDb(reset bool) error {
|
||||
if err = ensureIMHTTPInfrastructureTables(tx); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = ensureIMFileReviewTables(tx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Deletion log
|
||||
if _, err = tx.Exec(
|
||||
@@ -4465,9 +4475,13 @@ func (a *adapter) FileFinishUpload(fd *t.FileDef, success bool, size int64) (*t.
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
fd.UpdatedAt = now
|
||||
fd.Size = size
|
||||
if err = insertFileReviewTasks(ctx, tx, fd); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
fd.Status = t.UploadCompleted
|
||||
fd.Size = size
|
||||
} else {
|
||||
// Deleting the record: there is no value in keeping it in the DB.
|
||||
_, err = tx.ExecContext(ctx, "DELETE FROM im_fileuploads WHERE id=?", store.DecodeUid(fd.Uid()))
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
//go:build mysql
|
||||
// +build mysql
|
||||
|
||||
package mysql
|
||||
|
||||
// ensureIMFileReviewTables installs async file safety review state in the
|
||||
// currently selected tenant database. Tenant databases intentionally do not
|
||||
// carry tenant_id columns.
|
||||
func ensureIMFileReviewTables(exec sqlExecutor) error {
|
||||
statements := []string{
|
||||
`CREATE TABLE IF NOT EXISTS im_file_review_tasks (
|
||||
id BIGINT NOT NULL, task_id VARCHAR(64) NOT NULL, file_id BIGINT NOT NULL,
|
||||
user_id BIGINT NOT NULL DEFAULT 0, task_type VARCHAR(32) NOT NULL,
|
||||
provider VARCHAR(64) NOT NULL, storage_provider VARCHAR(32) NOT NULL DEFAULT '',
|
||||
object_name VARCHAR(2048) NOT NULL DEFAULT '', mime_type VARCHAR(255) NOT NULL DEFAULT '',
|
||||
file_size BIGINT NOT NULL DEFAULT 0, provider_task_id VARCHAR(128) NOT NULL DEFAULT '',
|
||||
status VARCHAR(16) NOT NULL DEFAULT 'pending', risk_level VARCHAR(32) NOT NULL DEFAULT '',
|
||||
risk_labels JSON, result_json JSON, error_code VARCHAR(64) NOT NULL DEFAULT '',
|
||||
error_message VARCHAR(512) NOT NULL DEFAULT '', attempts INT NOT NULL DEFAULT 0,
|
||||
next_retry_at DATETIME(3), completed_at DATETIME(3),
|
||||
created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
|
||||
updated_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3),
|
||||
PRIMARY KEY(id), UNIQUE INDEX uk_im_file_review_tasks_task(task_id),
|
||||
UNIQUE INDEX uk_im_file_review_tasks_file_provider(file_id, task_type, provider),
|
||||
INDEX idx_im_file_review_tasks_pending(status, next_retry_at, id),
|
||||
INDEX idx_im_file_review_tasks_file(file_id),
|
||||
INDEX idx_im_file_review_tasks_object(storage_provider, object_name(191)),
|
||||
INDEX idx_im_file_review_tasks_provider_task(provider, provider_task_id)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci`,
|
||||
}
|
||||
for _, statement := range statements {
|
||||
if _, err := exec.Exec(statement); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
//go:build mysql
|
||||
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestEnsureIMFileReviewTablesUsesSelectedTenantDatabase(t *testing.T) {
|
||||
exec := &recordingGroupSchemaExecutor{}
|
||||
if err := ensureIMFileReviewTables(exec); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(exec.statements) != 1 {
|
||||
t.Fatalf("created %d file review tables, want 1", len(exec.statements))
|
||||
}
|
||||
joined := strings.ToLower(strings.Join(exec.statements, "\n"))
|
||||
for _, expected := range []string{
|
||||
"im_file_review_tasks",
|
||||
"unique index uk_im_file_review_tasks_file_provider(file_id, task_type, provider)",
|
||||
"index idx_im_file_review_tasks_pending(status, next_retry_at, id)",
|
||||
"index idx_im_file_review_tasks_object(storage_provider, object_name(191))",
|
||||
} {
|
||||
if !strings.Contains(joined, expected) {
|
||||
t.Fatalf("file review schema is missing %q", expected)
|
||||
}
|
||||
}
|
||||
if strings.Contains(joined, "tinode.") || strings.Contains(joined, "create database") || strings.Contains(joined, "tenant_id") {
|
||||
t.Fatalf("file review schema must stay inside the selected tenant database: %s", joined)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
//go:build mysql
|
||||
// +build mysql
|
||||
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
|
||||
"github.com/tinode/chat/server/store"
|
||||
t "github.com/tinode/chat/server/store/types"
|
||||
)
|
||||
|
||||
type fileReviewExecContext interface {
|
||||
ExecContext(context.Context, string, ...any) (sql.Result, error)
|
||||
}
|
||||
|
||||
func insertFileReviewTasks(ctx context.Context, exec fileReviewExecContext, fd *t.FileDef) error {
|
||||
fileID := decodeUID(fd.Uid())
|
||||
userID := int64(0)
|
||||
if fd.User != "" {
|
||||
userID = decodeUID(t.ParseUid(fd.User))
|
||||
}
|
||||
tasks := []struct {
|
||||
taskType string
|
||||
provider string
|
||||
}{
|
||||
{taskType: t.FileReviewTaskTypeContent, provider: t.FileReviewProviderAliyunGreen},
|
||||
{taskType: t.FileReviewTaskTypeMalware, provider: t.FileReviewProviderAliyunSAS},
|
||||
}
|
||||
for _, task := range tasks {
|
||||
id, err := nextSnowflakeID()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
taskID := store.EncodeUid(id).String()
|
||||
_, err = exec.ExecContext(ctx, `INSERT IGNORE INTO im_file_review_tasks
|
||||
(id,task_id,file_id,user_id,task_type,provider,storage_provider,object_name,mime_type,file_size,status,created_at,updated_at)
|
||||
VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)`,
|
||||
id, taskID, fileID, userID, task.taskType, task.provider, "", fd.Location, fd.MimeType, fd.Size,
|
||||
t.FileReviewStatusPending, fd.UpdatedAt, fd.UpdatedAt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -12,6 +12,7 @@
|
||||
# - Default tenant business database: tinode
|
||||
# - im_users, im_topics, im_subscriptions, im_messages and extension tables
|
||||
# - im_http_idempotency and im_business_events_outbox for reliable HTTP writes
|
||||
# - im_file_review_tasks for asynchronous content safety and malware checks
|
||||
# - business tables do not store tenant_id in independent-database mode
|
||||
#
|
||||
# The previous shared-table tenant_id draft has been removed from this file to
|
||||
|
||||
@@ -5439,8 +5439,12 @@ func (a *adapter) TenantFileFinishUpload(id t.TenantID, fd *t.FileDef, success b
|
||||
if success {
|
||||
_, err = tx.Exec("UPDATE im_fileuploads SET updatedat=?,status=?,size=?,etag=?,location=? WHERE tenant_id=? AND id=?",
|
||||
now, t.UploadCompleted, size, fd.ETag, fd.Location, id, decodeUID(fd.Uid()))
|
||||
if err == nil {
|
||||
fd.UpdatedAt = now
|
||||
fd.Size = size
|
||||
err = insertFileReviewTasks(ctx, tx, fd)
|
||||
}
|
||||
fd.Status = t.UploadCompleted
|
||||
fd.Size = size
|
||||
} else {
|
||||
_, err = tx.Exec("DELETE FROM im_fileuploads WHERE tenant_id=? AND id=?", id, decodeUID(fd.Uid()))
|
||||
fd.Status = t.UploadFailed
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
//go:build mysql
|
||||
// +build mysql
|
||||
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"strings"
|
||||
|
||||
"github.com/jmoiron/sqlx"
|
||||
t "github.com/tinode/chat/server/store/types"
|
||||
)
|
||||
|
||||
func (a *adapter) ThirdPartyConfig(group string, keys ...string) (map[string]string, error) {
|
||||
db := a.registryDB
|
||||
if db == nil {
|
||||
db = a.db
|
||||
}
|
||||
if db == nil {
|
||||
return nil, t.ErrInternal
|
||||
}
|
||||
group = strings.TrimSpace(group)
|
||||
if group == "" {
|
||||
return nil, t.ErrMalformed
|
||||
}
|
||||
|
||||
ctx, cancel := a.getContext()
|
||||
if cancel != nil {
|
||||
defer cancel()
|
||||
}
|
||||
|
||||
rows := []struct {
|
||||
Key string `db:"key"`
|
||||
Value string `db:"value"`
|
||||
}{}
|
||||
var err error
|
||||
if len(keys) == 0 {
|
||||
err = db.SelectContext(ctx, &rows, `SELECT `+"`key`"+`, value FROM hg_sys_config
|
||||
WHERE `+"`group`"+`=? AND status=1`, group)
|
||||
} else {
|
||||
normalized := make([]string, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
if key = strings.TrimSpace(key); key != "" {
|
||||
normalized = append(normalized, key)
|
||||
}
|
||||
}
|
||||
if len(normalized) == 0 {
|
||||
return map[string]string{}, nil
|
||||
}
|
||||
query, args, inErr := sqlx.In(`SELECT `+"`key`"+`, value FROM hg_sys_config
|
||||
WHERE `+"`group`"+`=? AND status=1 AND `+"`key`"+` IN (?)`, group, normalized)
|
||||
if inErr != nil {
|
||||
return nil, inErr
|
||||
}
|
||||
query = db.Rebind(query)
|
||||
err = db.SelectContext(ctx, &rows, query, args...)
|
||||
}
|
||||
if err != nil {
|
||||
if err == sql.ErrNoRows {
|
||||
return map[string]string{}, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
result := make(map[string]string, len(rows))
|
||||
for _, row := range rows {
|
||||
result[row.Key] = row.Value
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
// Package aliyun wraps Alibaba Cloud Green file moderation APIs.
|
||||
package aliyun
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
openapi "github.com/alibabacloud-go/darabonba-openapi/v2/client"
|
||||
green "github.com/alibabacloud-go/green-20220302/v3/client"
|
||||
"github.com/alibabacloud-go/tea/dara"
|
||||
|
||||
"github.com/tinode/chat/server/store/types"
|
||||
)
|
||||
|
||||
const defaultDocumentService = "document_detection"
|
||||
|
||||
type GreenConfig struct {
|
||||
AccessKeyID string
|
||||
AccessKeySecret string
|
||||
RegionID string
|
||||
Endpoint string
|
||||
DocumentService string
|
||||
}
|
||||
|
||||
type OSSObject struct {
|
||||
BucketName string
|
||||
ObjectName string
|
||||
RegionID string
|
||||
DataID string
|
||||
}
|
||||
|
||||
type SubmitResult struct {
|
||||
ProviderTaskID string
|
||||
RequestID string
|
||||
}
|
||||
|
||||
type PollResult struct {
|
||||
Status string
|
||||
RiskLevel string
|
||||
RiskLabels []string
|
||||
Raw map[string]any
|
||||
RequestID string
|
||||
}
|
||||
|
||||
type GreenClient struct {
|
||||
service string
|
||||
client *green.Client
|
||||
}
|
||||
|
||||
func NewGreenClient(conf GreenConfig) (*GreenClient, error) {
|
||||
if strings.TrimSpace(conf.AccessKeyID) == "" {
|
||||
return nil, errors.New("missing Access Key ID")
|
||||
}
|
||||
if strings.TrimSpace(conf.AccessKeySecret) == "" {
|
||||
return nil, errors.New("missing Access Key Secret")
|
||||
}
|
||||
if strings.TrimSpace(conf.RegionID) == "" {
|
||||
return nil, errors.New("missing Region ID")
|
||||
}
|
||||
service := strings.TrimSpace(conf.DocumentService)
|
||||
if service == "" {
|
||||
service = defaultDocumentService
|
||||
}
|
||||
clientConf := &openapi.Config{
|
||||
AccessKeyId: dara.String(conf.AccessKeyID),
|
||||
AccessKeySecret: dara.String(conf.AccessKeySecret),
|
||||
RegionId: dara.String(conf.RegionID),
|
||||
Protocol: dara.String("https"),
|
||||
}
|
||||
if endpoint := strings.TrimSpace(conf.Endpoint); endpoint != "" {
|
||||
clientConf.Endpoint = dara.String(endpoint)
|
||||
}
|
||||
client, err := green.NewClient(clientConf)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &GreenClient{service: service, client: client}, nil
|
||||
}
|
||||
|
||||
func (c *GreenClient) SubmitOSSFile(ctx context.Context, object OSSObject) (*SubmitResult, error) {
|
||||
params, err := buildOSSServiceParameters(object)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp, err := c.client.FileModerationWithContext(ctx, &green.FileModerationRequest{
|
||||
Service: dara.String(c.service),
|
||||
ServiceParameters: dara.String(params),
|
||||
}, &dara.RuntimeOptions{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resp == nil || resp.Body == nil || resp.Body.Code == nil || *resp.Body.Code != 200 || resp.Body.Data == nil || resp.Body.Data.TaskId == nil {
|
||||
return nil, responseError("file moderation submit failed", responseCode(resp), responseMessage(resp))
|
||||
}
|
||||
return &SubmitResult{ProviderTaskID: *resp.Body.Data.TaskId, RequestID: responseRequestID(resp)}, nil
|
||||
}
|
||||
|
||||
func (c *GreenClient) PollOSSFile(ctx context.Context, providerTaskID string) (*PollResult, error) {
|
||||
params, err := json.Marshal(map[string]string{"taskId": strings.TrimSpace(providerTaskID)})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp, err := c.client.DescribeFileModerationResultWithContext(ctx, &green.DescribeFileModerationResultRequest{
|
||||
Service: dara.String(c.service),
|
||||
ServiceParameters: dara.String(string(params)),
|
||||
}, &dara.RuntimeOptions{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resp == nil || resp.Body == nil || resp.Body.Code == nil || *resp.Body.Code != 200 {
|
||||
return nil, responseError("file moderation poll failed", describeResponseCode(resp), describeResponseMessage(resp))
|
||||
}
|
||||
raw := map[string]any{}
|
||||
if data := resp.Body.Data; data != nil {
|
||||
encoded, _ := json.Marshal(data)
|
||||
json.Unmarshal(encoded, &raw)
|
||||
riskLevel := strings.TrimSpace(dara.StringValue(data.RiskLevel))
|
||||
return &PollResult{
|
||||
Status: statusFromRiskLevel(riskLevel),
|
||||
RiskLevel: riskLevel,
|
||||
RiskLabels: labelsFromFileResult(data),
|
||||
Raw: raw,
|
||||
RequestID: describeResponseRequestID(resp),
|
||||
}, nil
|
||||
}
|
||||
return &PollResult{Status: types.FileReviewStatusRunning, Raw: raw, RequestID: describeResponseRequestID(resp)}, nil
|
||||
}
|
||||
|
||||
func buildOSSServiceParameters(object OSSObject) (string, error) {
|
||||
if strings.TrimSpace(object.BucketName) == "" || strings.TrimSpace(object.ObjectName) == "" || strings.TrimSpace(object.RegionID) == "" {
|
||||
return "", types.ErrMalformed
|
||||
}
|
||||
params := map[string]string{
|
||||
"ossBucketName": strings.TrimSpace(object.BucketName),
|
||||
"ossObjectName": strings.TrimSpace(object.ObjectName),
|
||||
"ossRegionId": strings.TrimSpace(object.RegionID),
|
||||
}
|
||||
if dataID := strings.TrimSpace(object.DataID); dataID != "" {
|
||||
params["dataId"] = dataID
|
||||
}
|
||||
encoded, err := json.Marshal(params)
|
||||
return string(encoded), err
|
||||
}
|
||||
|
||||
func statusFromRiskLevel(riskLevel string) string {
|
||||
switch strings.ToLower(strings.TrimSpace(riskLevel)) {
|
||||
case "":
|
||||
return types.FileReviewStatusRunning
|
||||
case "none", "low":
|
||||
return types.FileReviewStatusClean
|
||||
default:
|
||||
return types.FileReviewStatusRisky
|
||||
}
|
||||
}
|
||||
|
||||
func labelsFromFileResult(data *green.DescribeFileModerationResultResponseBodyData) []string {
|
||||
seen := map[string]bool{}
|
||||
var labels []string
|
||||
for _, page := range data.PageResult {
|
||||
if page == nil {
|
||||
continue
|
||||
}
|
||||
for _, image := range page.ImageResult {
|
||||
if image == nil {
|
||||
continue
|
||||
}
|
||||
for _, result := range image.LabelResult {
|
||||
if result == nil {
|
||||
continue
|
||||
}
|
||||
if label := strings.TrimSpace(dara.StringValue(result.Label)); label != "" && !seen[label] {
|
||||
seen[label] = true
|
||||
labels = append(labels, label)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, text := range page.TextResult {
|
||||
if text == nil {
|
||||
continue
|
||||
}
|
||||
for _, label := range strings.Split(dara.StringValue(text.Labels), ",") {
|
||||
label = strings.TrimSpace(label)
|
||||
if label != "" && !seen[label] {
|
||||
seen[label] = true
|
||||
labels = append(labels, label)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return labels
|
||||
}
|
||||
|
||||
func responseError(prefix string, code int32, msg string) error {
|
||||
if msg == "" {
|
||||
return errors.New(prefix)
|
||||
}
|
||||
return errors.New(prefix + ": " + msg + " (" + strconv.FormatInt(int64(code), 10) + ")")
|
||||
}
|
||||
|
||||
func responseCode(resp *green.FileModerationResponse) int32 {
|
||||
if resp != nil && resp.Body != nil && resp.Body.Code != nil {
|
||||
return *resp.Body.Code
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func responseMessage(resp *green.FileModerationResponse) string {
|
||||
if resp != nil && resp.Body != nil && resp.Body.Message != nil {
|
||||
return *resp.Body.Message
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func responseRequestID(resp *green.FileModerationResponse) string {
|
||||
if resp != nil && resp.Body != nil && resp.Body.RequestId != nil {
|
||||
return *resp.Body.RequestId
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func describeResponseCode(resp *green.DescribeFileModerationResultResponse) int32 {
|
||||
if resp != nil && resp.Body != nil && resp.Body.Code != nil {
|
||||
return *resp.Body.Code
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func describeResponseMessage(resp *green.DescribeFileModerationResultResponse) string {
|
||||
if resp != nil && resp.Body != nil && resp.Body.Message != nil {
|
||||
return *resp.Body.Message
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func describeResponseRequestID(resp *green.DescribeFileModerationResultResponse) string {
|
||||
if resp != nil && resp.Body != nil && resp.Body.RequestId != nil {
|
||||
return *resp.Body.RequestId
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
package aliyun
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
green "github.com/alibabacloud-go/green-20220302/v3/client"
|
||||
"github.com/alibabacloud-go/tea/dara"
|
||||
"github.com/tinode/chat/server/store/types"
|
||||
)
|
||||
|
||||
func TestBuildOSSServiceParameters(t *testing.T) {
|
||||
got, err := buildOSSServiceParameters(OSSObject{
|
||||
BucketName: "im-oss-t1",
|
||||
ObjectName: "tenants/1/2026/08/file",
|
||||
RegionID: "cn-shenzhen",
|
||||
DataID: "task-1",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var params map[string]string
|
||||
if err := json.Unmarshal([]byte(got), ¶ms); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for key, want := range map[string]string{
|
||||
"ossBucketName": "im-oss-t1",
|
||||
"ossObjectName": "tenants/1/2026/08/file",
|
||||
"ossRegionId": "cn-shenzhen",
|
||||
"dataId": "task-1",
|
||||
} {
|
||||
if params[key] != want {
|
||||
t.Fatalf("params[%s] = %q, want %q", key, params[key], want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatusFromRiskLevel(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"": types.FileReviewStatusRunning,
|
||||
"none": types.FileReviewStatusClean,
|
||||
"low": types.FileReviewStatusClean,
|
||||
"medium": types.FileReviewStatusRisky,
|
||||
"high": types.FileReviewStatusRisky,
|
||||
}
|
||||
for input, want := range cases {
|
||||
if got := statusFromRiskLevel(input); got != want {
|
||||
t.Fatalf("statusFromRiskLevel(%q) = %q, want %q", input, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLabelsFromFileResult(t *testing.T) {
|
||||
got := labelsFromFileResult(&green.DescribeFileModerationResultResponseBodyData{
|
||||
PageResult: []*green.DescribeFileModerationResultResponseBodyDataPageResult{
|
||||
{ImageResult: []*green.DescribeFileModerationResultResponseBodyDataPageResultImageResult{
|
||||
{LabelResult: []*green.DescribeFileModerationResultResponseBodyDataPageResultImageResultLabelResult{
|
||||
{Label: dara.String("porn")},
|
||||
{Label: dara.String("porn")},
|
||||
}},
|
||||
}},
|
||||
{TextResult: []*green.DescribeFileModerationResultResponseBodyDataPageResultTextResult{
|
||||
{Labels: dara.String("violence,ad")},
|
||||
}},
|
||||
},
|
||||
})
|
||||
if len(got) != 3 || got[0] != "porn" || got[1] != "violence" || got[2] != "ad" {
|
||||
t.Fatalf("labelsFromFileResult = %#v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
package filesafety
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/tinode/chat/server/filesafety/aliyun"
|
||||
"github.com/tinode/chat/server/store"
|
||||
"github.com/tinode/chat/server/store/types"
|
||||
)
|
||||
|
||||
const thirdPartyConfigGroup = "im_third_party"
|
||||
|
||||
type ContentReviewConfig struct {
|
||||
Enabled bool
|
||||
Provider string
|
||||
Green aliyun.GreenConfig
|
||||
}
|
||||
|
||||
type MalwareReviewConfig struct {
|
||||
Enabled bool
|
||||
Provider string
|
||||
RegionID string
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
Content ContentReviewConfig
|
||||
Malware MalwareReviewConfig
|
||||
}
|
||||
|
||||
func LoadConfig() (*Config, error) {
|
||||
values, err := store.ThirdPartyConfig.Get(thirdPartyConfigGroup,
|
||||
"file_review_content_enabled",
|
||||
"file_review_content_provider",
|
||||
"aliyun_green_region_id",
|
||||
"aliyun_green_endpoint",
|
||||
"aliyun_green_document_service",
|
||||
"file_review_malware_enabled",
|
||||
"file_review_malware_provider",
|
||||
"aliyun_sas_region_id",
|
||||
"aliyun_access_key_id",
|
||||
"aliyun_access_key_secret",
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
config := &Config{
|
||||
Content: ContentReviewConfig{
|
||||
Enabled: boolValue(values["file_review_content_enabled"]),
|
||||
Provider: stringDefault(values["file_review_content_provider"], types.FileReviewProviderAliyunGreen),
|
||||
Green: aliyun.GreenConfig{
|
||||
AccessKeyID: strings.TrimSpace(values["aliyun_access_key_id"]),
|
||||
AccessKeySecret: strings.TrimSpace(values["aliyun_access_key_secret"]),
|
||||
RegionID: strings.TrimSpace(values["aliyun_green_region_id"]),
|
||||
Endpoint: strings.TrimSpace(values["aliyun_green_endpoint"]),
|
||||
DocumentService: strings.TrimSpace(values["aliyun_green_document_service"]),
|
||||
},
|
||||
},
|
||||
Malware: MalwareReviewConfig{
|
||||
Enabled: boolValue(values["file_review_malware_enabled"]),
|
||||
Provider: stringDefault(values["file_review_malware_provider"], types.FileReviewProviderAliyunSAS),
|
||||
RegionID: strings.TrimSpace(values["aliyun_sas_region_id"]),
|
||||
},
|
||||
}
|
||||
if config.Content.Enabled && config.Content.Provider != types.FileReviewProviderAliyunGreen {
|
||||
return nil, errors.New("unsupported file content review provider: " + config.Content.Provider)
|
||||
}
|
||||
if config.Malware.Enabled && config.Malware.Provider != types.FileReviewProviderAliyunSAS {
|
||||
return nil, errors.New("unsupported malware review provider: " + config.Malware.Provider)
|
||||
}
|
||||
return config, nil
|
||||
}
|
||||
|
||||
func NewGreenClientFromConfig() (*aliyun.GreenClient, error) {
|
||||
config, err := LoadConfig()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !config.Content.Enabled {
|
||||
return nil, types.ErrUnsupported
|
||||
}
|
||||
return aliyun.NewGreenClient(config.Content.Green)
|
||||
}
|
||||
|
||||
func boolValue(value string) bool {
|
||||
if parsed, err := strconv.ParseBool(strings.TrimSpace(value)); err == nil {
|
||||
return parsed
|
||||
}
|
||||
switch strings.ToLower(strings.TrimSpace(value)) {
|
||||
case "1", "yes", "on":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func stringDefault(value, fallback string) string {
|
||||
if value = strings.TrimSpace(value); value != "" {
|
||||
return value
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
package filesafety
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/tinode/chat/server/store"
|
||||
"github.com/tinode/chat/server/store/types"
|
||||
)
|
||||
|
||||
type fakeThirdPartyConfig map[string]string
|
||||
|
||||
func (f fakeThirdPartyConfig) Get(group string, keys ...string) (map[string]string, error) {
|
||||
result := map[string]string{}
|
||||
for _, key := range keys {
|
||||
if value, ok := f[key]; ok {
|
||||
result[key] = value
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func TestLoadConfig(t *testing.T) {
|
||||
original := store.ThirdPartyConfig
|
||||
store.ThirdPartyConfig = fakeThirdPartyConfig{
|
||||
"file_review_content_enabled": "1",
|
||||
"file_review_content_provider": types.FileReviewProviderAliyunGreen,
|
||||
"aliyun_green_region_id": "cn-shanghai",
|
||||
"aliyun_green_endpoint": "green-cip.cn-shanghai.aliyuncs.com",
|
||||
"aliyun_green_document_service": "document_detection",
|
||||
"file_review_malware_enabled": "true",
|
||||
"file_review_malware_provider": types.FileReviewProviderAliyunSAS,
|
||||
"aliyun_sas_region_id": "cn-shanghai",
|
||||
"aliyun_access_key_id": "ak",
|
||||
"aliyun_access_key_secret": "sk",
|
||||
}
|
||||
defer func() { store.ThirdPartyConfig = original }()
|
||||
|
||||
config, err := LoadConfig()
|
||||
if err != nil {
|
||||
t.Fatalf("LoadConfig failed: %v", err)
|
||||
}
|
||||
if !config.Content.Enabled || config.Content.Provider != types.FileReviewProviderAliyunGreen {
|
||||
t.Fatalf("unexpected content config: %+v", config.Content)
|
||||
}
|
||||
if !config.Malware.Enabled || config.Malware.Provider != types.FileReviewProviderAliyunSAS {
|
||||
t.Fatalf("unexpected malware config: %+v", config.Malware)
|
||||
}
|
||||
if config.Content.Green.AccessKeyID != "ak" || config.Content.Green.RegionID != "cn-shanghai" {
|
||||
t.Fatalf("unexpected green config: %+v", config.Content.Green)
|
||||
}
|
||||
}
|
||||
@@ -38,11 +38,13 @@ type ossConfig struct {
|
||||
AccessKeySecret string `json:"access_key_secret"`
|
||||
Endpoint string `json:"endpoint"`
|
||||
BucketName string `json:"bucket"`
|
||||
RegionID string `json:"region_id"`
|
||||
CorsOrigins []string `json:"cors_origins"`
|
||||
ServeURL string `json:"serve_url"`
|
||||
PresignTTL int `json:"presign_ttl"`
|
||||
CacheControl string `json:"cache_control"`
|
||||
VerifyBucket bool `json:"verify_bucket"`
|
||||
UseSysConfig bool `json:"use_sys_config"`
|
||||
}
|
||||
|
||||
type ossHandler struct {
|
||||
@@ -69,6 +71,9 @@ func (oh *ossHandler) Init(jsconf string) error {
|
||||
if err = json.Unmarshal([]byte(jsconf), &oh.conf); err != nil {
|
||||
return errors.New("failed to parse config: " + err.Error())
|
||||
}
|
||||
if err = oh.loadSysConfig(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if oh.conf.AccessKeyID == "" {
|
||||
return errors.New("missing Access Key ID")
|
||||
@@ -114,6 +119,59 @@ func (oh *ossHandler) Init(jsconf string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
func (oh *ossHandler) loadSysConfig() error {
|
||||
if !oh.conf.UseSysConfig {
|
||||
return nil
|
||||
}
|
||||
values, err := store.ThirdPartyConfig.Get("im_third_party",
|
||||
"oss_enabled",
|
||||
"oss_provider",
|
||||
"oss_endpoint",
|
||||
"oss_bucket",
|
||||
"oss_region_id",
|
||||
"oss_access_key_id",
|
||||
"oss_access_key_secret",
|
||||
"oss_presign_ttl",
|
||||
)
|
||||
if err != nil {
|
||||
return errors.New("failed to load OSS config from sys config: " + err.Error())
|
||||
}
|
||||
if !truthy(values["oss_enabled"]) {
|
||||
return errors.New("OSS is disabled in sys config")
|
||||
}
|
||||
if provider := strings.TrimSpace(values["oss_provider"]); provider != "" && provider != handlerName {
|
||||
return errors.New("unsupported storage provider in sys config: " + provider)
|
||||
}
|
||||
applyString(&oh.conf.Endpoint, values["oss_endpoint"])
|
||||
applyString(&oh.conf.BucketName, values["oss_bucket"])
|
||||
applyString(&oh.conf.RegionID, values["oss_region_id"])
|
||||
applyString(&oh.conf.AccessKeyID, values["oss_access_key_id"])
|
||||
applyString(&oh.conf.AccessKeySecret, values["oss_access_key_secret"])
|
||||
if ttl := strings.TrimSpace(values["oss_presign_ttl"]); ttl != "" {
|
||||
parsed, err := strconv.Atoi(ttl)
|
||||
if err != nil {
|
||||
return errors.New("invalid OSS presign TTL in sys config: " + err.Error())
|
||||
}
|
||||
oh.conf.PresignTTL = parsed
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func applyString(target *string, value string) {
|
||||
if value = strings.TrimSpace(value); value != "" {
|
||||
*target = value
|
||||
}
|
||||
}
|
||||
|
||||
func truthy(value string) bool {
|
||||
switch strings.ToLower(strings.TrimSpace(value)) {
|
||||
case "1", "true", "yes", "on":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// Headers adds CORS headers and redirects GET and HEAD requests to OSS.
|
||||
func (oh *ossHandler) Headers(tenantID types.TenantID, method string, url *url.URL, headers http.Header, serve bool) (http.Header, int, error) {
|
||||
headers, status := media.CORSHandler(method, headers, oh.corsOrigins, serve)
|
||||
|
||||
@@ -4,9 +4,22 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/tinode/chat/server/store"
|
||||
"github.com/tinode/chat/server/store/types"
|
||||
)
|
||||
|
||||
type fakeThirdPartyConfig map[string]string
|
||||
|
||||
func (f fakeThirdPartyConfig) Get(group string, keys ...string) (map[string]string, error) {
|
||||
result := map[string]string{}
|
||||
for _, key := range keys {
|
||||
if value, ok := f[key]; ok {
|
||||
result[key] = value
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func TestTenantObjectKey(t *testing.T) {
|
||||
got, err := tenantObjectKey(types.TenantID(42), "abc123", time.Date(2026, 8, 11, 9, 0, 0, 0, time.UTC))
|
||||
if err != nil {
|
||||
@@ -32,3 +45,29 @@ func TestGetIdFromUrl(t *testing.T) {
|
||||
t.Fatalf("GetIdFromUrl = %v, want %v", got, fid)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadSysConfig(t *testing.T) {
|
||||
original := store.ThirdPartyConfig
|
||||
store.ThirdPartyConfig = fakeThirdPartyConfig{
|
||||
"oss_enabled": "1",
|
||||
"oss_provider": "oss",
|
||||
"oss_endpoint": "oss-cn-shenzhen.aliyuncs.com",
|
||||
"oss_bucket": "im-files",
|
||||
"oss_region_id": "cn-shenzhen",
|
||||
"oss_access_key_id": "ak",
|
||||
"oss_access_key_secret": "sk",
|
||||
"oss_presign_ttl": "300",
|
||||
}
|
||||
defer func() { store.ThirdPartyConfig = original }()
|
||||
|
||||
handler := ossHandler{conf: ossConfig{UseSysConfig: true}}
|
||||
if err := handler.loadSysConfig(); err != nil {
|
||||
t.Fatalf("loadSysConfig failed: %v", err)
|
||||
}
|
||||
if handler.conf.Endpoint != "oss-cn-shenzhen.aliyuncs.com" || handler.conf.BucketName != "im-files" {
|
||||
t.Fatalf("unexpected OSS config: %+v", handler.conf)
|
||||
}
|
||||
if handler.conf.RegionID != "cn-shenzhen" || handler.conf.PresignTTL != 300 {
|
||||
t.Fatalf("unexpected OSS region or ttl: %+v", handler.conf)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -155,6 +155,24 @@ func (registrationConfigMapper) SMSEnabled() (bool, error) {
|
||||
return configAdp.RegistrationSMSEnabled()
|
||||
}
|
||||
|
||||
// ThirdPartyConfigPersistenceInterface exposes global third-party integration settings.
|
||||
type ThirdPartyConfigPersistenceInterface interface {
|
||||
Get(group string, keys ...string) (map[string]string, error)
|
||||
}
|
||||
|
||||
type thirdPartyConfigMapper struct{}
|
||||
|
||||
// ThirdPartyConfig reads third-party settings from the control database.
|
||||
var ThirdPartyConfig ThirdPartyConfigPersistenceInterface = thirdPartyConfigMapper{}
|
||||
|
||||
func (thirdPartyConfigMapper) Get(group string, keys ...string) (map[string]string, error) {
|
||||
configAdp, ok := adp.(adapter.ThirdPartyConfigAdapter)
|
||||
if !ok {
|
||||
return nil, types.ErrUnsupported
|
||||
}
|
||||
return configAdp.ThirdPartyConfig(group, keys...)
|
||||
}
|
||||
|
||||
// GetByCode resolves a tenant by its public enterprise code.
|
||||
func (tenantsMapper) GetByCode(code string) (*types.Tenant, error) {
|
||||
tenantAdp, ok := adp.(adapter.TenantAdapter)
|
||||
|
||||
@@ -10,6 +10,18 @@ const (
|
||||
|
||||
AbuseReportStatusReceived = "received"
|
||||
FeedbackStatusReceived = "received"
|
||||
|
||||
FileReviewTaskTypeContent = "content"
|
||||
FileReviewTaskTypeMalware = "malware"
|
||||
|
||||
FileReviewProviderAliyunGreen = "aliyun_green"
|
||||
FileReviewProviderAliyunSAS = "aliyun_sas"
|
||||
|
||||
FileReviewStatusPending = "pending"
|
||||
FileReviewStatusRunning = "running"
|
||||
FileReviewStatusClean = "clean"
|
||||
FileReviewStatusRisky = "risky"
|
||||
FileReviewStatusFailed = "failed"
|
||||
)
|
||||
|
||||
type RiskChallenge struct {
|
||||
@@ -84,3 +96,28 @@ type FeedbackSubmission struct {
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
}
|
||||
|
||||
type FileReviewTask struct {
|
||||
ID int64
|
||||
TaskID string
|
||||
FileID Uid
|
||||
UserID Uid
|
||||
TaskType string
|
||||
Provider string
|
||||
StorageProvider string
|
||||
ObjectName string
|
||||
MimeType string
|
||||
FileSize int64
|
||||
ProviderTaskID string
|
||||
Status string
|
||||
RiskLevel string
|
||||
RiskLabels []string
|
||||
ResultJSON map[string]any
|
||||
ErrorCode string
|
||||
ErrorMessage string
|
||||
Attempts int
|
||||
NextRetryAt *time.Time
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
CompletedAt *time.Time
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user