feat(media): add MinIO storage switch and HTTPS OSS endpoint
This commit is contained in:
@@ -6,6 +6,7 @@ package mysql
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"strings"
|
||||
|
||||
"github.com/tinode/chat/server/store"
|
||||
t "github.com/tinode/chat/server/store/types"
|
||||
@@ -16,6 +17,10 @@ type fileReviewExecContext interface {
|
||||
}
|
||||
|
||||
func insertFileReviewTasks(ctx context.Context, exec fileReviewExecContext, fd *t.FileDef) error {
|
||||
storageProvider := ""
|
||||
if strings.HasPrefix(fd.Location, "minio://") {
|
||||
storageProvider = "minio"
|
||||
}
|
||||
fileID := decodeUID(fd.Uid())
|
||||
userID := int64(0)
|
||||
if fd.User != "" {
|
||||
@@ -37,7 +42,7 @@ func insertFileReviewTasks(ctx context.Context, exec fileReviewExecContext, fd *
|
||||
_, err = exec.ExecContext(ctx, `INSERT IGNORE INTO im_file_review_tasks
|
||||
(id,task_id,file_id,user_id,task_type,provider,storage_provider,object_name,mime_type,file_size,status,created_at,updated_at)
|
||||
VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)`,
|
||||
id, taskID, fileID, userID, task.taskType, task.provider, "", fd.Location, fd.MimeType, fd.Size,
|
||||
id, taskID, fileID, userID, task.taskType, task.provider, storageProvider, fd.Location, fd.MimeType, fd.Size,
|
||||
t.FileReviewStatusPending, fd.UpdatedAt, fd.UpdatedAt)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
+5
-4
@@ -281,17 +281,17 @@ func resolveMediaHandlerConfig(config *mediaConfig) (string, string, error) {
|
||||
conf = string(params)
|
||||
}
|
||||
}
|
||||
if handler == "oss" {
|
||||
ossConf, err := ensureOSSUsesSysConfig(conf)
|
||||
if handler == "oss" || handler == "minio" {
|
||||
storageConf, err := ensureObjectStorageUsesSysConfig(conf, handler)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
return handler, ossConf, nil
|
||||
return handler, storageConf, nil
|
||||
}
|
||||
return handler, conf, nil
|
||||
}
|
||||
|
||||
func ensureOSSUsesSysConfig(conf string) (string, error) {
|
||||
func ensureObjectStorageUsesSysConfig(conf, provider string) (string, error) {
|
||||
params := map[string]any{}
|
||||
if strings.TrimSpace(conf) != "" {
|
||||
if err := json.Unmarshal([]byte(conf), ¶ms); err != nil {
|
||||
@@ -299,6 +299,7 @@ func ensureOSSUsesSysConfig(conf string) (string, error) {
|
||||
}
|
||||
}
|
||||
params["use_sys_config"] = true
|
||||
params["provider"] = provider
|
||||
encoded, err := json.Marshal(params)
|
||||
if err != nil {
|
||||
return "", err
|
||||
|
||||
@@ -0,0 +1,161 @@
|
||||
package oss
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net/url"
|
||||
"strings"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
|
||||
"github.com/tinode/chat/server/media"
|
||||
"github.com/tinode/chat/server/store"
|
||||
"github.com/tinode/chat/server/store/types"
|
||||
)
|
||||
|
||||
const minioLocationPrefix = "minio://"
|
||||
|
||||
type minioConfig struct {
|
||||
Endpoint string `json:"endpoint"`
|
||||
Bucket string `json:"bucket"`
|
||||
Region string `json:"region"`
|
||||
AccessKeyID string `json:"access_key_id"`
|
||||
SecretAccessKey string `json:"secret_access_key"`
|
||||
}
|
||||
|
||||
type minioStorage struct {
|
||||
client *minio.Client
|
||||
bucket string
|
||||
}
|
||||
|
||||
func newMinIOStorage(conf minioConfig) (*minioStorage, error) {
|
||||
if conf.Endpoint == "" || conf.Bucket == "" || conf.AccessKeyID == "" || conf.SecretAccessKey == "" {
|
||||
return nil, nil
|
||||
}
|
||||
endpoint := strings.TrimSpace(conf.Endpoint)
|
||||
secure := true
|
||||
if strings.Contains(endpoint, "://") {
|
||||
parsed, err := url.Parse(endpoint)
|
||||
if err != nil || parsed.Host == "" || parsed.Path != "" || parsed.RawQuery != "" || parsed.Fragment != "" || (parsed.Scheme != "http" && parsed.Scheme != "https") {
|
||||
return nil, errors.New("invalid MinIO endpoint")
|
||||
}
|
||||
endpoint = parsed.Host
|
||||
secure = parsed.Scheme == "https"
|
||||
}
|
||||
client, err := minio.New(endpoint, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(conf.AccessKeyID, conf.SecretAccessKey, ""),
|
||||
Secure: secure,
|
||||
Region: conf.Region,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &minioStorage{client: client, bucket: conf.Bucket}, nil
|
||||
}
|
||||
|
||||
func (oh *ossHandler) uploadMinIO(fd *types.FileDef, file io.Reader) (string, int64, error) {
|
||||
key, err := tenantObjectKey(fd.TenantID, fd.Uid().String32(), fd.CreatedAt)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
if err := store.Files.StartUpload(fd.TenantID, fd); err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
result, err := oh.minio.client.PutObject(context.Background(), oh.minio.bucket, key, file, remainingSize(file), minio.PutObjectOptions{
|
||||
ContentType: fd.MimeType,
|
||||
CacheControl: oh.conf.CacheControl,
|
||||
})
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
fd.Location = minioLocation(oh.minio.bucket, key)
|
||||
fd.ETag = strings.Trim(result.ETag, "\"")
|
||||
return oh.conf.ServeURL + media.FileURLName(fd), result.Size, nil
|
||||
}
|
||||
|
||||
func (oh *ossHandler) uploadPublicMinIO(fd *media.PublicFileDef, file io.Reader) (string, int64, error) {
|
||||
key, err := publicObjectKey(fd.Purpose, fd.ID, fd.CreatedAt)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
result, err := oh.minio.client.PutObject(context.Background(), oh.minio.bucket, key, file, remainingSize(file), minio.PutObjectOptions{
|
||||
ContentType: fd.MimeType,
|
||||
CacheControl: oh.conf.CacheControl,
|
||||
})
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
fd.Location = minioLocation(oh.minio.bucket, key)
|
||||
fd.ETag = strings.Trim(result.ETag, "\"")
|
||||
return fd.Location, result.Size, nil
|
||||
}
|
||||
|
||||
func (oh *ossHandler) downloadMinIO(fd *types.FileDef) (*types.FileDef, media.ReadSeekCloser, error) {
|
||||
bucket, key, err := oh.minioObject(fd.Location)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if fd.Size <= 0 {
|
||||
meta, err := oh.minio.client.StatObject(context.Background(), bucket, key, minio.StatObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
fd.Size = meta.Size
|
||||
}
|
||||
reader := &ossReadSeekCloser{size: fd.Size}
|
||||
reader.open = func(offset int64) (io.ReadCloser, error) {
|
||||
options := minio.GetObjectOptions{}
|
||||
if offset > 0 {
|
||||
if err := options.SetRange(offset, fd.Size-1); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return oh.minio.client.GetObject(context.Background(), bucket, key, options)
|
||||
}
|
||||
return fd, reader, nil
|
||||
}
|
||||
|
||||
func (oh *ossHandler) deleteMinIO(location string) error {
|
||||
bucket, key, err := oh.minioObject(location)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return oh.minio.client.RemoveObject(context.Background(), bucket, key, minio.RemoveObjectOptions{})
|
||||
}
|
||||
|
||||
func (oh *ossHandler) minioObject(location string) (string, string, error) {
|
||||
if oh.minio == nil {
|
||||
return "", "", errors.New("MinIO is not configured for stored file")
|
||||
}
|
||||
bucket, key, ok := strings.Cut(strings.TrimPrefix(location, minioLocationPrefix), "/")
|
||||
if !strings.HasPrefix(location, minioLocationPrefix) || !ok || bucket != oh.minio.bucket || key == "" {
|
||||
return "", "", types.ErrMalformed
|
||||
}
|
||||
return bucket, key, nil
|
||||
}
|
||||
|
||||
func minioLocation(bucket, key string) string {
|
||||
return minioLocationPrefix + bucket + "/" + key
|
||||
}
|
||||
|
||||
func remainingSize(reader io.Reader) int64 {
|
||||
seeker, ok := reader.(io.Seeker)
|
||||
if !ok {
|
||||
return -1
|
||||
}
|
||||
current, err := seeker.Seek(0, io.SeekCurrent)
|
||||
if err != nil {
|
||||
return -1
|
||||
}
|
||||
end, err := seeker.Seek(0, io.SeekEnd)
|
||||
if err != nil {
|
||||
_, _ = seeker.Seek(current, io.SeekStart)
|
||||
return -1
|
||||
}
|
||||
if _, err := seeker.Seek(current, io.SeekStart); err != nil {
|
||||
return -1
|
||||
}
|
||||
return end - current
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
package oss
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
)
|
||||
|
||||
func TestLiveMinIOUploadDownload(t *testing.T) {
|
||||
endpoint := os.Getenv("LIVE_MINIO_ENDPOINT")
|
||||
accessKey := os.Getenv("LIVE_MINIO_ACCESS_KEY")
|
||||
secretKey := os.Getenv("LIVE_MINIO_SECRET_KEY")
|
||||
bucket := os.Getenv("LIVE_MINIO_BUCKET")
|
||||
if endpoint == "" || accessKey == "" || secretKey == "" || bucket == "" {
|
||||
t.Skip("live MinIO configuration is not set")
|
||||
}
|
||||
storage, err := newMinIOStorage(minioConfig{
|
||||
Endpoint: endpoint, Bucket: bucket, Region: "us-east-1",
|
||||
AccessKeyID: accessKey, SecretAccessKey: secretKey,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
key := fmt.Sprintf("im-smoke/%d.txt", time.Now().UnixNano())
|
||||
defer func() {
|
||||
if err := storage.client.RemoveObject(context.Background(), bucket, key, minio.RemoveObjectOptions{}); err != nil {
|
||||
t.Errorf("cleanup object: %v", err)
|
||||
}
|
||||
}()
|
||||
content := []byte("im-minio-smoke-test")
|
||||
if _, err := storage.client.PutObject(ctx, bucket, key, bytes.NewReader(content), int64(len(content)), minio.PutObjectOptions{ContentType: "text/plain"}); err != nil {
|
||||
t.Fatalf("upload: %v", err)
|
||||
}
|
||||
object, err := storage.client.GetObject(ctx, bucket, key, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("download: %v", err)
|
||||
}
|
||||
defer object.Close()
|
||||
got, err := io.ReadAll(object)
|
||||
if err != nil || !bytes.Equal(got, content) {
|
||||
t.Fatalf("downloaded content differs: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
package oss
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/tinode/chat/server/media"
|
||||
"github.com/tinode/chat/server/store"
|
||||
"github.com/tinode/chat/server/store/types"
|
||||
)
|
||||
|
||||
func TestMinIOUploadDownloadDelete(t *testing.T) {
|
||||
const body = "hello minio"
|
||||
var stored []byte
|
||||
var deleted bool
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if !strings.HasPrefix(r.URL.Path, "/xgjs/public/") {
|
||||
t.Errorf("unexpected request path: %s", r.URL.Path)
|
||||
http.Error(w, "wrong path", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
switch r.Method {
|
||||
case http.MethodPut:
|
||||
stored, _ = io.ReadAll(r.Body)
|
||||
if strings.Contains(r.Header.Get("Content-Encoding"), "aws-chunked") {
|
||||
separator := bytes.Index(stored, []byte("\r\n"))
|
||||
if separator < 0 || len(stored) < separator+2+len(body) {
|
||||
t.Errorf("invalid signed chunked request")
|
||||
http.Error(w, "invalid body", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
stored = stored[separator+2 : separator+2+len(body)]
|
||||
}
|
||||
w.Header().Set("ETag", `"test-etag"`)
|
||||
case http.MethodGet:
|
||||
w.Header().Set("Last-Modified", time.Now().UTC().Format(http.TimeFormat))
|
||||
w.Header().Set("ETag", `"test-etag"`)
|
||||
if r.Header.Get("Range") == "bytes=6-10" {
|
||||
w.WriteHeader(http.StatusPartialContent)
|
||||
_, _ = w.Write(stored[6:])
|
||||
return
|
||||
}
|
||||
_, _ = w.Write(stored)
|
||||
case http.MethodDelete:
|
||||
deleted = true
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
default:
|
||||
t.Errorf("unexpected request method: %s", r.Method)
|
||||
http.Error(w, "wrong method", http.StatusMethodNotAllowed)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
previous := store.ThirdPartyConfig
|
||||
store.ThirdPartyConfig = fakeThirdPartyConfig{
|
||||
"oss_enabled": "0",
|
||||
"minio_endpoint": server.URL,
|
||||
"minio_bucket": "xgjs",
|
||||
"minio_region": "us-east-1",
|
||||
"minio_access_key_id": "test-key",
|
||||
"minio_secret_access_key": "test-secret",
|
||||
}
|
||||
defer func() { store.ThirdPartyConfig = previous }()
|
||||
|
||||
handler := &ossHandler{}
|
||||
if err := handler.Init(`{"provider":"minio","use_sys_config":true}`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if handler.bucket != nil || handler.minio == nil {
|
||||
t.Fatal("MinIO provider should initialize without OSS credentials")
|
||||
}
|
||||
file := &media.PublicFileDef{ID: "file_test", Purpose: "avatar", MimeType: "text/plain", CreatedAt: time.Now()}
|
||||
location, size, err := handler.UploadPublic(file, bytes.NewReader([]byte(body)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if size != int64(len(body)) || string(stored) != body || !strings.HasPrefix(location, "minio://xgjs/public/avatar/") {
|
||||
t.Fatalf("upload: location=%q size=%d data=%q", location, size, stored)
|
||||
}
|
||||
fd := &types.FileDef{Location: location, Size: size}
|
||||
_, reader, err := handler.downloadMinIO(fd)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer reader.Close()
|
||||
buf := make([]byte, len(body))
|
||||
if _, err := io.ReadFull(reader, buf); err != nil || string(buf) != body {
|
||||
t.Fatalf("download: %q, %v", buf, err)
|
||||
}
|
||||
if _, err := reader.Seek(6, io.SeekStart); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, err := io.ReadAll(reader); err != nil || string(got) != "minio" {
|
||||
t.Fatalf("range download: %q, %v", got, err)
|
||||
}
|
||||
if err := handler.Delete([]string{location}); err != nil || !deleted {
|
||||
t.Fatalf("delete: deleted=%v err=%v", deleted, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMinIOLocationRejectsDifferentBucket(t *testing.T) {
|
||||
handler := &ossHandler{minio: &minioStorage{bucket: "xgjs"}}
|
||||
if _, _, err := handler.minioObject("minio://other/public/file"); err != types.ErrMalformed {
|
||||
t.Fatalf("wrong bucket should be rejected, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMinIOPublicEndpointUsesHTTPSPathStyle(t *testing.T) {
|
||||
storage, err := newMinIOStorage(minioConfig{
|
||||
Endpoint: "https://minio.118im.com", Bucket: "xgjs", Region: "us-east-1",
|
||||
AccessKeyID: "test-key", SecretAccessKey: "test-secret",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
url, err := storage.client.PresignedGetObject(context.Background(), storage.bucket, "images/test.jpg", time.Minute, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if url.Scheme != "https" || url.Host != "minio.118im.com" || url.Path != "/xgjs/images/test.jpg" {
|
||||
t.Fatalf("unexpected MinIO URL: %s", url)
|
||||
}
|
||||
}
|
||||
+102
-40
@@ -27,29 +27,33 @@ const (
|
||||
defaultServeURL = "/v0/file/s/"
|
||||
defaultCacheControl = "no-cache, must-revalidate"
|
||||
|
||||
handlerName = "oss"
|
||||
handlerName = "oss"
|
||||
minioHandlerName = "minio"
|
||||
// Presign GET URLs for this number of seconds.
|
||||
defaultPresignDuration = 120
|
||||
)
|
||||
|
||||
type ossConfig struct {
|
||||
AccessKeyID string `json:"access_key_id"`
|
||||
AccessKeySecret string `json:"access_key_secret"`
|
||||
Endpoint string `json:"endpoint"`
|
||||
BucketName string `json:"bucket"`
|
||||
RegionID string `json:"region_id"`
|
||||
CorsOrigins []string `json:"cors_origins"`
|
||||
ServeURL string `json:"serve_url"`
|
||||
PresignTTL int `json:"presign_ttl"`
|
||||
CacheControl string `json:"cache_control"`
|
||||
VerifyBucket bool `json:"verify_bucket"`
|
||||
UseSysConfig bool `json:"use_sys_config"`
|
||||
Provider string `json:"provider"`
|
||||
AccessKeyID string `json:"access_key_id"`
|
||||
AccessKeySecret string `json:"access_key_secret"`
|
||||
Endpoint string `json:"endpoint"`
|
||||
BucketName string `json:"bucket"`
|
||||
RegionID string `json:"region_id"`
|
||||
CorsOrigins []string `json:"cors_origins"`
|
||||
ServeURL string `json:"serve_url"`
|
||||
PresignTTL int `json:"presign_ttl"`
|
||||
CacheControl string `json:"cache_control"`
|
||||
VerifyBucket bool `json:"verify_bucket"`
|
||||
UseSysConfig bool `json:"use_sys_config"`
|
||||
MinIO minioConfig `json:"minio"`
|
||||
}
|
||||
|
||||
type ossHandler struct {
|
||||
client *aliyunoss.Client
|
||||
bucket *aliyunoss.Bucket
|
||||
conf ossConfig
|
||||
minio *minioStorage
|
||||
corsOrigins []media.AllowedOrigin
|
||||
}
|
||||
|
||||
@@ -138,21 +142,18 @@ func (oh *ossHandler) Init(jsconf string) error {
|
||||
if err = json.Unmarshal([]byte(jsconf), &oh.conf); err != nil {
|
||||
return errors.New("failed to parse config: " + err.Error())
|
||||
}
|
||||
if oh.conf.Provider == "" {
|
||||
oh.conf.Provider = handlerName
|
||||
}
|
||||
if oh.conf.Provider != handlerName && oh.conf.Provider != minioHandlerName {
|
||||
return errors.New("unsupported storage provider: " + oh.conf.Provider)
|
||||
}
|
||||
if err = oh.loadSysConfig(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if oh.conf.AccessKeyID == "" {
|
||||
return errors.New("missing Access Key ID")
|
||||
}
|
||||
if oh.conf.AccessKeySecret == "" {
|
||||
return errors.New("missing Access Key Secret")
|
||||
}
|
||||
if oh.conf.Endpoint == "" {
|
||||
return errors.New("missing Endpoint")
|
||||
}
|
||||
if oh.conf.BucketName == "" {
|
||||
return errors.New("missing Bucket")
|
||||
if oh.conf.Provider == handlerName && (oh.conf.AccessKeyID == "" || oh.conf.AccessKeySecret == "" || oh.conf.Endpoint == "" || oh.conf.BucketName == "") {
|
||||
return errors.New("incomplete OSS configuration")
|
||||
}
|
||||
if oh.conf.PresignTTL <= 0 {
|
||||
oh.conf.PresignTTL = defaultPresignDuration
|
||||
@@ -169,21 +170,46 @@ func (oh *ossHandler) Init(jsconf string) error {
|
||||
return errors.New("failed to parse CORS allowed origins: " + err.Error())
|
||||
}
|
||||
|
||||
oh.client, err = aliyunoss.New(oh.conf.Endpoint, oh.conf.AccessKeyID, oh.conf.AccessKeySecret)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if oh.conf.VerifyBucket {
|
||||
exists, err := oh.client.IsBucketExist(oh.conf.BucketName)
|
||||
if oh.conf.AccessKeyID != "" && oh.conf.AccessKeySecret != "" && oh.conf.Endpoint != "" && oh.conf.BucketName != "" {
|
||||
oh.conf.Endpoint = normalizeOSSEndpoint(oh.conf.Endpoint)
|
||||
oh.client, err = aliyunoss.New(oh.conf.Endpoint, oh.conf.AccessKeyID, oh.conf.AccessKeySecret)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !exists {
|
||||
return errors.New("bucket does not exist")
|
||||
if oh.conf.VerifyBucket && oh.conf.Provider == handlerName {
|
||||
exists, err := oh.client.IsBucketExist(oh.conf.BucketName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !exists {
|
||||
return errors.New("bucket does not exist")
|
||||
}
|
||||
}
|
||||
oh.bucket, err = oh.client.Bucket(oh.conf.BucketName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
oh.bucket, err = oh.client.Bucket(oh.conf.BucketName)
|
||||
return err
|
||||
oh.minio, err = newMinIOStorage(oh.conf.MinIO)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if oh.conf.Provider == minioHandlerName && oh.minio == nil {
|
||||
return errors.New("incomplete MinIO configuration")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func normalizeOSSEndpoint(endpoint string) string {
|
||||
endpoint = strings.TrimSpace(endpoint)
|
||||
if !strings.Contains(endpoint, "://") {
|
||||
return "https://" + endpoint
|
||||
}
|
||||
parsed, err := url.Parse(endpoint)
|
||||
if err == nil && parsed.Scheme == "http" && strings.HasSuffix(strings.ToLower(parsed.Hostname()), ".aliyuncs.com") {
|
||||
return "https://" + strings.TrimPrefix(endpoint, "http://")
|
||||
}
|
||||
return endpoint
|
||||
}
|
||||
|
||||
func (oh *ossHandler) loadSysConfig() error {
|
||||
@@ -191,6 +217,11 @@ func (oh *ossHandler) loadSysConfig() error {
|
||||
return nil
|
||||
}
|
||||
values, err := store.ThirdPartyConfig.Get("im_third_party",
|
||||
"minio_endpoint",
|
||||
"minio_bucket",
|
||||
"minio_region",
|
||||
"minio_access_key_id",
|
||||
"minio_secret_access_key",
|
||||
"oss_enabled",
|
||||
"oss_provider",
|
||||
"oss_endpoint",
|
||||
@@ -207,12 +238,17 @@ func (oh *ossHandler) loadSysConfig() error {
|
||||
if err != nil {
|
||||
return errors.New("failed to load OSS config from sys config: " + err.Error())
|
||||
}
|
||||
if !truthy(values["oss_enabled"]) {
|
||||
if oh.conf.Provider == handlerName && !truthy(values["oss_enabled"]) {
|
||||
return errors.New("OSS is disabled in sys config")
|
||||
}
|
||||
if provider := strings.TrimSpace(values["oss_provider"]); provider != "" && provider != handlerName {
|
||||
if provider := strings.TrimSpace(values["oss_provider"]); oh.conf.Provider == handlerName && provider != "" && provider != handlerName {
|
||||
return errors.New("unsupported storage provider in sys config: " + provider)
|
||||
}
|
||||
applyString(&oh.conf.MinIO.Endpoint, values["minio_endpoint"])
|
||||
applyString(&oh.conf.MinIO.Bucket, values["minio_bucket"])
|
||||
applyString(&oh.conf.MinIO.Region, values["minio_region"])
|
||||
applyString(&oh.conf.MinIO.AccessKeyID, values["minio_access_key_id"])
|
||||
applyString(&oh.conf.MinIO.SecretAccessKey, values["minio_secret_access_key"])
|
||||
applyString(&oh.conf.Endpoint, values["oss_endpoint"])
|
||||
applyString(&oh.conf.BucketName, values["oss_bucket"])
|
||||
applyString(&oh.conf.RegionID, values["oss_region_id"])
|
||||
@@ -308,6 +344,9 @@ func (oh *ossHandler) Headers(tenantID types.TenantID, method string, url *url.U
|
||||
|
||||
// Upload processes request for a file upload.
|
||||
func (oh *ossHandler) Upload(fdef *types.FileDef, file io.Reader) (string, int64, error) {
|
||||
if oh.conf.Provider == minioHandlerName {
|
||||
return oh.uploadMinIO(fdef, file)
|
||||
}
|
||||
key, err := tenantObjectKey(fdef.TenantID, fdef.Uid().String32(), fdef.CreatedAt)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
@@ -342,6 +381,9 @@ func (oh *ossHandler) Upload(fdef *types.FileDef, file io.Reader) (string, int64
|
||||
}
|
||||
|
||||
func (oh *ossHandler) UploadPublic(fdef *media.PublicFileDef, file io.Reader) (string, int64, error) {
|
||||
if oh.conf.Provider == minioHandlerName {
|
||||
return oh.uploadPublicMinIO(fdef, file)
|
||||
}
|
||||
key, err := publicObjectKey(fdef.Purpose, fdef.ID, fdef.CreatedAt)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
@@ -378,6 +420,12 @@ func (oh *ossHandler) Download(tenantID types.TenantID, url string) (*types.File
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if strings.HasPrefix(fd.Location, minioLocationPrefix) {
|
||||
return oh.downloadMinIO(fd)
|
||||
}
|
||||
if oh.bucket == nil {
|
||||
return nil, nil, errors.New("OSS is not configured for legacy file download")
|
||||
}
|
||||
objectKey, err := oh.resolveObjectKey(fd.Location)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
@@ -405,12 +453,25 @@ func (oh *ossHandler) Download(tenantID types.TenantID, url string) (*types.File
|
||||
|
||||
// Delete deletes files from OSS by provided object keys.
|
||||
func (oh *ossHandler) Delete(locations []string) error {
|
||||
for i := 0; i < len(locations); i += 1000 {
|
||||
end := i + 1000
|
||||
if end > len(locations) {
|
||||
end = len(locations)
|
||||
ossLocations := make([]string, 0, len(locations))
|
||||
for _, location := range locations {
|
||||
if strings.HasPrefix(location, minioLocationPrefix) {
|
||||
if err := oh.deleteMinIO(location); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
if _, err := oh.bucket.DeleteObjects(locations[i:end], aliyunoss.DeleteObjectsQuiet(true)); err != nil {
|
||||
ossLocations = append(ossLocations, location)
|
||||
}
|
||||
if len(ossLocations) > 0 && oh.bucket == nil {
|
||||
return errors.New("OSS is not configured for legacy file deletion")
|
||||
}
|
||||
for i := 0; i < len(ossLocations); i += 1000 {
|
||||
end := i + 1000
|
||||
if end > len(ossLocations) {
|
||||
end = len(ossLocations)
|
||||
}
|
||||
if _, err := oh.bucket.DeleteObjects(ossLocations[i:end], aliyunoss.DeleteObjectsQuiet(true)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -504,4 +565,5 @@ func etagFromLocation(location string) string {
|
||||
|
||||
func init() {
|
||||
store.RegisterMediaHandler(handlerName, &ossHandler{})
|
||||
store.RegisterMediaHandler(minioHandlerName, &ossHandler{})
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"bytes"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -123,3 +124,24 @@ func TestLoadSysConfig(t *testing.T) {
|
||||
t.Fatalf("unexpected OSS cors origins: %+v", handler.conf.CorsOrigins)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOSSEndpointUsesHTTPS(t *testing.T) {
|
||||
for _, endpoint := range []string{
|
||||
"oss-cn-shenzhen.aliyuncs.com",
|
||||
"http://oss-cn-shenzhen.aliyuncs.com",
|
||||
"https://oss-cn-shenzhen.aliyuncs.com",
|
||||
} {
|
||||
handler := &ossHandler{}
|
||||
config := `{"endpoint":"` + endpoint + `","access_key_id":"ak","access_key_secret":"sk","bucket":"im-files"}`
|
||||
if err := handler.Init(config); err != nil {
|
||||
t.Fatalf("Init(%q): %v", endpoint, err)
|
||||
}
|
||||
signedURL, err := handler.bucket.SignURL("test.txt", aliyunoss.HTTPGet, 60)
|
||||
if err != nil {
|
||||
t.Fatalf("SignURL(%q): %v", endpoint, err)
|
||||
}
|
||||
if !strings.HasPrefix(signedURL, "https://") {
|
||||
t.Fatalf("endpoint %q produced non-HTTPS URL %q", endpoint, signedURL)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/tinode/chat/server/store"
|
||||
)
|
||||
|
||||
type testMediaConfigProvider struct{ handler string }
|
||||
|
||||
func (p testMediaConfigProvider) Get(string, ...string) (map[string]string, error) {
|
||||
return map[string]string{"media_handler": p.handler}, nil
|
||||
}
|
||||
|
||||
func TestResolveObjectStorageMediaHandler(t *testing.T) {
|
||||
for _, provider := range []string{"oss", "minio"} {
|
||||
t.Run(provider, func(t *testing.T) {
|
||||
handler, raw, err := resolveMediaHandlerConfig(&mediaConfig{UseHandler: provider})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if handler != provider {
|
||||
t.Fatalf("handler = %q", handler)
|
||||
}
|
||||
var config struct {
|
||||
Provider string `json:"provider"`
|
||||
UseSysConfig bool `json:"use_sys_config"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(raw), &config); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if config.Provider != provider || !config.UseSysConfig {
|
||||
t.Fatalf("resolved config = %+v", config)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveMediaHandlerFromSysConfig(t *testing.T) {
|
||||
previous := store.ThirdPartyConfig
|
||||
store.ThirdPartyConfig = testMediaConfigProvider{handler: "minio"}
|
||||
defer func() { store.ThirdPartyConfig = previous }()
|
||||
|
||||
handler, raw, err := resolveMediaHandlerConfig(&mediaConfig{UseHandler: "sys_config"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var config struct {
|
||||
Provider string `json:"provider"`
|
||||
UseSysConfig bool `json:"use_sys_config"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(raw), &config); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if handler != "minio" || config.Provider != "minio" || !config.UseSysConfig {
|
||||
t.Fatalf("handler=%q config=%+v", handler, config)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user