diff --git a/docker/tinode/Dockerfile b/docker/tinode/Dockerfile index 637d6cc..b86a738 100644 --- a/docker/tinode/Dockerfile +++ b/docker/tinode/Dockerfile @@ -114,7 +114,7 @@ ENV POSTGRES_DSN='postgresql://postgres:postgres@localhost:5432/tinode?sslmode=d ENV PLUGIN_PYTHON_CHAT_BOT_ENABLED=false # Default handler for large files -ENV MEDIA_HANDLER=fs +ENV MEDIA_HANDLER=sys_config # Whitelisted domains for file and S3 large media handler. ENV FS_CORS_ORIGINS='["*"]' diff --git a/docker/tinode/Dockerfile.build b/docker/tinode/Dockerfile.build index a50589e..fea9e2a 100644 --- a/docker/tinode/Dockerfile.build +++ b/docker/tinode/Dockerfile.build @@ -91,7 +91,7 @@ ENV ADMIN_INTERNAL_API_TIMEOUT_MS=3000 # DB routing resolves to the same tinode DB for the seeded "default" tenant. ENV TENANT_REGISTRY_DB=hotgo -ENV MEDIA_HANDLER=fs +ENV MEDIA_HANDLER=sys_config ENV FS_CORS_ORIGINS='["*"]' ENV AWS_CORS_ORIGINS='["*"]' ENV SMTP_HOST_URL='http://localhost:6060' diff --git a/docker/tinode/config.template b/docker/tinode/config.template index 0f52852..af3def4 100644 --- a/docker/tinode/config.template +++ b/docker/tinode/config.template @@ -34,6 +34,9 @@ "presign_ttl": 3600, "cache_control": "max-age=86400", "cors_origins": $AWS_CORS_ORIGINS + }, + "oss": { + "use_sys_config": true } } }, diff --git a/docs/third/IM-第三方配置与文件安全SQL.sql b/docs/third/IM-第三方配置与文件安全SQL.sql index 1e52f40..b8b9374 100644 --- a/docs/third/IM-第三方配置与文件安全SQL.sql +++ b/docs/third/IM-第三方配置与文件安全SQL.sql @@ -125,6 +125,11 @@ SELECT '字符串 UID', 'string', 'string', 'im_rtc_uid_mode', 'info', 0, 20, ' WHERE NOT EXISTS (SELECT 1 FROM hg_sys_dict_data WHERE `type` = 'im_rtc_uid_mode' AND `value` = 'string'); -- 3. 第三方配置项。配置真值存 hg_sys_config；字典只提供 UI 选项。 +INSERT INTO hg_sys_config + (`group`, `name`, `type`, `key`, `value`, `default_value`, `sort`, `tip`, `is_default`, `status`, `created_at`, `updated_at`) +SELECT 'im_third_party', '媒体存储 Handler', 'string', 'media_handler', 'oss', 'oss', 9, 'IM 文件存储 handler；生产使用 oss，配置文件可使用 sys_config 间接读取', 1, 1, NOW(), NOW() +WHERE NOT EXISTS (SELECT 1 FROM hg_sys_config WHERE `group` = 'im_third_party' AND `key` = 'media_handler'); + INSERT INTO hg_sys_config (`group`, `name`, `type`, `key`, `value`, `default_value`, `sort`, `tip`, `is_default`, `status`, `created_at`, `updated_at`) SELECT 'im_third_party', 'OSS 启用状态', 'bool', 'oss_enabled', '0', '0', 10, '是否启用阿里云 OSS 作为 IM 文件存储', 1, 1, NOW(), NOW() @@ -165,6 +170,26 @@ INSERT INTO hg_sys_config SELECT 'im_third_party', 'OSS 签名 URL TTL', 'int', 'oss_presign_ttl', '120', '120', 17, '签名下载 URL 有效秒数', 1, 1, NOW(), NOW() WHERE NOT EXISTS (SELECT 1 FROM hg_sys_config WHERE `group` = 'im_third_party' AND `key` = 'oss_presign_ttl'); +INSERT INTO hg_sys_config + (`group`, `name`, `type`, `key`, `value`, `default_value`, `sort`, `tip`, `is_default`, `status`, `created_at`, `updated_at`) +SELECT 'im_third_party', 'OSS 文件服务 URL 前缀', 'string', 'oss_serve_url', '/v0/file/s/', '/v0/file/s/', 18, 'IM 文件下载接口前缀，OSS handler 用于解析文件 ID 和返回签名跳转', 1, 1, NOW(), NOW() +WHERE NOT EXISTS (SELECT 1 FROM hg_sys_config WHERE `group` = 'im_third_party' AND `key` = 'oss_serve_url'); + +INSERT INTO hg_sys_config + (`group`, `name`, `type`, `key`, `value`, `default_value`, `sort`, `tip`, `is_default`, `status`, `created_at`, `updated_at`) +SELECT 'im_third_party', 'OSS 下载缓存头', 'string', 'oss_cache_control', 'no-cache, must-revalidate', 'no-cache, must-revalidate', 19, 'OSS 签名下载响应 Cache-Control', 1, 1, NOW(), NOW() +WHERE NOT EXISTS (SELECT 1 FROM hg_sys_config WHERE `group` = 'im_third_party' AND `key` = 'oss_cache_control'); + +INSERT INTO hg_sys_config + (`group`, `name`, `type`, `key`, `value`, `default_value`, `sort`, `tip`, `is_default`, `status`, `created_at`, `updated_at`) +SELECT 'im_third_party', 'OSS CORS 来源', 'string', 'oss_cors_origins', '["*"]', '["*"]', 20, 'JSON 数组或逗号分隔列表', 1, 1, NOW(), NOW() +WHERE NOT EXISTS (SELECT 1 FROM hg_sys_config WHERE `group` = 'im_third_party' AND `key` = 'oss_cors_origins'); + +INSERT INTO hg_sys_config + (`group`, `name`, `type`, `key`, `value`, `default_value`, `sort`, `tip`, `is_default`, `status`, `created_at`, `updated_at`) +SELECT 'im_third_party', 'OSS 启动校验 Bucket', 'bool', 'oss_verify_bucket', '0', '0', 21, '启动时是否调用 OSS 校验 Bucket 存在；生产可按需开启', 1, 1, NOW(), NOW() +WHERE NOT EXISTS (SELECT 1 FROM hg_sys_config WHERE `group` = 'im_third_party' AND `key` = 'oss_verify_bucket'); + INSERT INTO hg_sys_config (`group`, `name`, `type`, `key`, `value`, `default_value`, `sort`, `tip`, `is_default`, `status`, `created_at`, `updated_at`) SELECT 'im_third_party', '文件内容安全启用状态', 'bool', 'file_review_content_enabled', '0', '0', 30, '是否启用阿里云内容安全增强版文件审核', 1, 1, NOW(), NOW() diff --git a/server/main.go b/server/main.go index d2fa7b3..2a4d986 100644 --- a/server/main.go +++ b/server/main.go @@ -13,6 +13,7 @@ package main import ( "context" "encoding/json" + "errors" "flag" "net/http" "os" @@ -254,6 +255,52 @@ type mediaConfig struct { Handlers map[string]json.RawMessage `json:"handlers"` } +func resolveMediaHandlerConfig(config *mediaConfig) (string, string, error) { + if config == nil { + return "", "", nil + } + handler := strings.TrimSpace(config.UseHandler) + if handler == "sys_config" { + values, err := store.ThirdPartyConfig.Get("im_third_party", "media_handler") + if err != nil { + return "", "", err + } + handler = strings.TrimSpace(values["media_handler"]) + } + if handler == "" { + return "", "", errors.New("empty media handler") + } + var conf string + if config.Handlers != nil { + if params := config.Handlers[handler]; params != nil { + conf = string(params) + } + } + if handler == "oss" { + ossConf, err := ensureOSSUsesSysConfig(conf) + if err != nil { + return "", "", err + } + return handler, ossConf, nil + } + return handler, conf, nil +} + +func ensureOSSUsesSysConfig(conf string) (string, error) { + params := map[string]any{} + if strings.TrimSpace(conf) != "" { + if err := json.Unmarshal([]byte(conf), ¶ms); err != nil { + return "", err + } + } + params["use_sys_config"] = true + encoded, err := json.Marshal(params) + if err != nil { + return "", err + } + return string(encoded), nil +} + // Contentx of the configuration file type configType struct { // HTTP(S) address:port to listen on for websocket and long polling clients. Either a @@ -640,12 +687,13 @@ func main() { } else { globals.maxFileUploadSize = config.Media.MaxFileUploadSize if config.Media.Handlers != nil { - var conf string - if params := config.Media.Handlers[config.Media.UseHandler]; params != nil { - conf = string(params) + handler, conf, err := resolveMediaHandlerConfig(config.Media) + if err != nil { + logs.Err.Fatalf("Failed to resolve media handler config: %s", err) } - if err = store.Store.UseMediaHandler(config.Media.UseHandler, conf); err != nil { - logs.Err.Fatalf("Failed to init media handler '%s': %s", config.Media.UseHandler, err) + config.Media.UseHandler = handler + if err = store.Store.UseMediaHandler(handler, conf); err != nil { + logs.Err.Fatalf("Failed to init media handler '%s': %s", handler, err) } } if config.Media.GcPeriod > 0 && config.Media.GcBlockSize > 0 { diff --git a/server/media/oss/oss.go b/server/media/oss/oss.go index be243eb..fb1dd27 100644 --- a/server/media/oss/oss.go +++ b/server/media/oss/oss.go @@ -131,6 +131,10 @@ func (oh *ossHandler) loadSysConfig() error { "oss_access_key_id", "oss_access_key_secret", "oss_presign_ttl", + "oss_serve_url", + "oss_cache_control", + "oss_cors_origins", + "oss_verify_bucket", ) if err != nil { return errors.New("failed to load OSS config from sys config: " + err.Error()) @@ -146,6 +150,14 @@ func (oh *ossHandler) loadSysConfig() error { applyString(&oh.conf.RegionID, values["oss_region_id"]) applyString(&oh.conf.AccessKeyID, values["oss_access_key_id"]) applyString(&oh.conf.AccessKeySecret, values["oss_access_key_secret"]) + applyString(&oh.conf.ServeURL, values["oss_serve_url"]) + applyString(&oh.conf.CacheControl, values["oss_cache_control"]) + if origins := parseStringList(values["oss_cors_origins"]); len(origins) > 0 { + oh.conf.CorsOrigins = origins + } + if verify := strings.TrimSpace(values["oss_verify_bucket"]); verify != "" { + oh.conf.VerifyBucket = truthy(verify) + } if ttl := strings.TrimSpace(values["oss_presign_ttl"]); ttl != "" { parsed, err := strconv.Atoi(ttl) if err != nil { @@ -156,6 +168,30 @@ func (oh *ossHandler) loadSysConfig() error { return nil } +func parseStringList(value string) []string { + value = strings.TrimSpace(value) + if value == "" { + return nil + } + var items []string + if strings.HasPrefix(value, "[") { + if err := json.Unmarshal([]byte(value), &items); err == nil { + return compactStrings(items) + } + } + return compactStrings(strings.Split(value, ",")) +} + +func compactStrings(values []string) []string { + items := make([]string, 0, len(values)) + for _, value := range values { + if value = strings.TrimSpace(value); value != "" { + items = append(items, value) + } + } + return items +} + func applyString(target *string, value string) { if value = strings.TrimSpace(value); value != "" { *target = value diff --git a/server/media/oss/oss_live_test.go b/server/media/oss/oss_live_test.go new file mode 100644 index 0000000..d56b6b9 --- /dev/null +++ b/server/media/oss/oss_live_test.go @@ -0,0 +1,87 @@ +package oss + +import ( + "bytes" + "encoding/json" + "io" + "net/http" + "os" + "testing" + "time" + + aliyunoss "github.com/aliyun/aliyun-oss-go-sdk/oss" + "github.com/tinode/chat/server/media" +) + +func TestLiveAliyunOSSUploadPublic(t *testing.T) { + endpoint := os.Getenv("LIVE_OSS_ENDPOINT") + bucket := os.Getenv("LIVE_OSS_BUCKET") + accessKeyID := os.Getenv("LIVE_OSS_ACCESS_KEY_ID") + accessKeySecret := os.Getenv("LIVE_OSS_ACCESS_KEY_SECRET") + if endpoint == "" || bucket == "" || accessKeyID == "" || accessKeySecret == "" { + t.Skip("LIVE_OSS_* env is not configured") + } + + conf, err := json.Marshal(ossConfig{ + Endpoint: endpoint, + BucketName: bucket, + AccessKeyID: accessKeyID, + AccessKeySecret: accessKeySecret, + PresignTTL: 60, + CacheControl: "no-cache", + CorsOrigins: []string{"*"}, + }) + if err != nil { + t.Fatal(err) + } + handler := &ossHandler{} + if err = handler.Init(string(conf)); err != nil { + t.Fatalf("oss init failed: %v", err) + } + + body := []byte{0x89, 'P', 'N', 'G', '\r', '\n', 0x1a, '\n'} + file := &media.PublicFileDef{ + ID: "codex-live-" + time.Now().UTC().Format("20060102150405"), + Purpose: "codex_live", + MimeType: "image/png", + CreatedAt: time.Now().UTC(), + } + location, size, err := handler.UploadPublic(file, bytes.NewReader(body)) + if err != nil { + t.Fatalf("oss upload failed: %v", err) + } + defer func() { + if err := handler.Delete([]string{file.Location}); err != nil { + t.Logf("oss cleanup failed: %v", err) + } + }() + if size != int64(len(body)) || location == "" || file.Location == "" || file.ETag == "" { + t.Fatalf("unexpected upload result: location=%q file=%+v size=%d", location, file, size) + } + + got, err := handler.bucket.GetObject(file.Location) + if err != nil { + t.Fatalf("oss get object failed: %v", err) + } + defer got.Close() + downloaded, err := io.ReadAll(got) + if err != nil { + t.Fatalf("oss read object failed: %v", err) + } + if !bytes.Equal(downloaded, body) { + t.Fatalf("oss downloaded bytes mismatch: got %x want %x", downloaded, body) + } + + signedURL, err := handler.bucket.SignURL(file.Location, aliyunoss.HTTPGet, 60) + if err != nil { + t.Fatalf("oss sign url failed: %v", err) + } + resp, err := http.Get(signedURL) + if err != nil { + t.Fatalf("oss signed url get failed: %v", err) + } + resp.Body.Close() + if resp.StatusCode >= http.StatusBadRequest { + t.Fatalf("oss signed url get status=%d", resp.StatusCode) + } +} diff --git a/server/media/oss/oss_tenant_test.go b/server/media/oss/oss_tenant_test.go index cb46e60..ab04b8b 100644 --- a/server/media/oss/oss_tenant_test.go +++ b/server/media/oss/oss_tenant_test.go @@ -57,6 +57,10 @@ func TestLoadSysConfig(t *testing.T) { "oss_access_key_id": "ak", "oss_access_key_secret": "sk", "oss_presign_ttl": "300", + "oss_serve_url": "/files/", + "oss_cache_control": "private", + "oss_cors_origins": `["https://example.com"]`, + "oss_verify_bucket": "true", } defer func() { store.ThirdPartyConfig = original }() @@ -70,4 +74,10 @@ func TestLoadSysConfig(t *testing.T) { if handler.conf.RegionID != "cn-shenzhen" || handler.conf.PresignTTL != 300 { t.Fatalf("unexpected OSS region or ttl: %+v", handler.conf) } + if handler.conf.ServeURL != "/files/" || handler.conf.CacheControl != "private" || !handler.conf.VerifyBucket { + t.Fatalf("unexpected OSS serve settings: %+v", handler.conf) + } + if len(handler.conf.CorsOrigins) != 1 || handler.conf.CorsOrigins[0] != "https://example.com" { + t.Fatalf("unexpected OSS cors origins: %+v", handler.conf.CorsOrigins) + } } diff --git a/server/meetingim.conf b/server/meetingim.conf index b273726..b1d3e88 100644 --- a/server/meetingim.conf +++ b/server/meetingim.conf @@ -23,7 +23,7 @@ "alias_tag": "alias", "media": { - "use_handler": "fs", + "use_handler": "sys_config", "max_size": 104857600, "gc_period": 3600, "gc_block_size": 100, @@ -32,6 +32,9 @@ "upload_dir": "uploads", "cache_control": "max-age=86400", "cors_origins": ["*"] + }, + "oss": { + "use_sys_config": true } } }, diff --git a/server/tinode.conf b/server/tinode.conf index b273726..b1d3e88 100644 --- a/server/tinode.conf +++ b/server/tinode.conf @@ -23,7 +23,7 @@ "alias_tag": "alias", "media": { - "use_handler": "fs", + "use_handler": "sys_config", "max_size": 104857600, "gc_period": 3600, "gc_block_size": 100, @@ -32,6 +32,9 @@ "upload_dir": "uploads", "cache_control": "max-age=86400", "cors_origins": ["*"] + }, + "oss": { + "use_sys_config": true } } },