385 lines
11 KiB
Go
385 lines
11 KiB
Go
// Package s3 implements media interface by storing media objects in Amazon S3 bucket.
|
|
package s3
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"io"
|
|
"net/http"
|
|
"net/url"
|
|
"path"
|
|
"strconv"
|
|
"strings"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"github.com/aws/aws-sdk-go-v2/aws"
|
|
"github.com/aws/aws-sdk-go-v2/config"
|
|
"github.com/aws/aws-sdk-go-v2/credentials"
|
|
"github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager"
|
|
"github.com/aws/aws-sdk-go-v2/service/s3"
|
|
s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
|
|
"github.com/aws/smithy-go"
|
|
|
|
"github.com/tinode/chat/server/logs"
|
|
"github.com/tinode/chat/server/media"
|
|
"github.com/tinode/chat/server/store"
|
|
"github.com/tinode/chat/server/store/types"
|
|
)
|
|
|
|
const (
|
|
defaultServeURL = "/v0/file/s/"
|
|
defaultCacheControl = "no-cache, must-revalidate"
|
|
|
|
handlerName = "s3"
|
|
// Presign GET URLs for this number of seconds.
|
|
defaultPresignDuration = 120
|
|
)
|
|
|
|
type awsconfig struct {
|
|
AccessKeyId string `json:"access_key_id"`
|
|
SecretAccessKey string `json:"secret_access_key"`
|
|
Region string `json:"region"`
|
|
DisableSSL bool `json:"disable_ssl"`
|
|
ForcePathStyle bool `json:"force_path_style"`
|
|
Endpoint string `json:"endpoint"`
|
|
BucketName string `json:"bucket"`
|
|
CorsOrigins []string `json:"cors_origins"`
|
|
ServeURL string `json:"serve_url"`
|
|
PresignTTL int `json:"presign_ttl"`
|
|
CacheControl string `json:"cache_control"`
|
|
}
|
|
|
|
type awshandler struct {
|
|
svc *s3.Client
|
|
presign *s3.PresignClient
|
|
conf awsconfig
|
|
corsOrigins []media.AllowedOrigin
|
|
}
|
|
|
|
// readerCounter is a byte counter for bytes read through the io.Reader
|
|
type readerCounter struct {
|
|
io.Reader
|
|
count int64
|
|
reader io.Reader
|
|
}
|
|
|
|
// Read reads the bytes and records the number of read bytes.
|
|
func (rc *readerCounter) Read(buf []byte) (int, error) {
|
|
n, err := rc.reader.Read(buf)
|
|
atomic.AddInt64(&rc.count, int64(n))
|
|
return n, err
|
|
}
|
|
|
|
// Init initializes the media handler.
|
|
func (ah *awshandler) Init(jsconf string) error {
|
|
var err error
|
|
if err = json.Unmarshal([]byte(jsconf), &ah.conf); err != nil {
|
|
return errors.New("failed to parse config: " + err.Error())
|
|
}
|
|
|
|
if ah.conf.AccessKeyId == "" {
|
|
return errors.New("missing Access Key ID")
|
|
}
|
|
if ah.conf.SecretAccessKey == "" {
|
|
return errors.New("missing Secret Access Key")
|
|
}
|
|
if ah.conf.Region == "" {
|
|
return errors.New("missing Region")
|
|
}
|
|
if ah.conf.BucketName == "" {
|
|
return errors.New("missing Bucket")
|
|
}
|
|
if ah.conf.PresignTTL <= 0 {
|
|
ah.conf.PresignTTL = defaultPresignDuration
|
|
}
|
|
if ah.conf.CacheControl == "" {
|
|
ah.conf.CacheControl = defaultCacheControl
|
|
}
|
|
if ah.conf.ServeURL == "" {
|
|
ah.conf.ServeURL = defaultServeURL
|
|
}
|
|
ah.corsOrigins, err = media.ParseCORSAllow(ah.conf.CorsOrigins)
|
|
if err != nil {
|
|
return errors.New("failed to parse CORS allowed origins: " + err.Error())
|
|
}
|
|
|
|
cfgOpts := []func(*config.LoadOptions) error{
|
|
config.WithRegion(ah.conf.Region),
|
|
config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(
|
|
ah.conf.AccessKeyId,
|
|
ah.conf.SecretAccessKey,
|
|
"",
|
|
)),
|
|
}
|
|
|
|
var cfg aws.Config
|
|
if cfg, err = config.LoadDefaultConfig(context.Background(), cfgOpts...); err != nil {
|
|
return err
|
|
}
|
|
|
|
// Create S3 service client
|
|
clientOpts := []func(*s3.Options){
|
|
func(o *s3.Options) {
|
|
o.UsePathStyle = ah.conf.ForcePathStyle
|
|
},
|
|
}
|
|
if ah.conf.Endpoint != "" {
|
|
endpoint := ah.conf.Endpoint
|
|
if !strings.Contains(endpoint, "://") {
|
|
if ah.conf.DisableSSL {
|
|
endpoint = "http://" + endpoint
|
|
} else {
|
|
endpoint = "https://" + endpoint
|
|
}
|
|
}
|
|
clientOpts = append(clientOpts, func(o *s3.Options) {
|
|
o.BaseEndpoint = aws.String(endpoint)
|
|
})
|
|
}
|
|
ah.svc = s3.NewFromConfig(cfg, clientOpts...)
|
|
ah.presign = s3.NewPresignClient(ah.svc)
|
|
|
|
// Check if bucket already exists.
|
|
_, err = ah.svc.HeadBucket(context.Background(), &s3.HeadBucketInput{Bucket: aws.String(ah.conf.BucketName)})
|
|
if err == nil {
|
|
// Bucket exists
|
|
return nil
|
|
}
|
|
|
|
if !isAPIError(err, "NoSuchBucket", "NotFound") {
|
|
// Hard error.
|
|
return err
|
|
}
|
|
|
|
// Bucket does not exist. Create one.
|
|
_, err = ah.svc.CreateBucket(context.Background(), &s3.CreateBucketInput{Bucket: aws.String(ah.conf.BucketName)})
|
|
if err != nil {
|
|
if isAPIError(err, "BucketAlreadyExists", "BucketAlreadyOwnedByYou", "OperationAborted") {
|
|
// Check if someone has already created a bucket (possible in a cluster).
|
|
err = nil
|
|
}
|
|
} else {
|
|
// This is a new bucket.
|
|
|
|
// The following serves two purposes:
|
|
// 1. Setup CORS policy to be able to serve media directly from S3.
|
|
// 2. Verify that the bucket is accessible to the current user.
|
|
origins := ah.conf.CorsOrigins
|
|
if len(origins) == 0 {
|
|
origins = append(origins, "*")
|
|
}
|
|
_, err = ah.svc.PutBucketCors(context.Background(), &s3.PutBucketCorsInput{
|
|
Bucket: aws.String(ah.conf.BucketName),
|
|
CORSConfiguration: &s3types.CORSConfiguration{
|
|
CORSRules: []s3types.CORSRule{{
|
|
AllowedMethods: []string{http.MethodGet, http.MethodHead},
|
|
AllowedOrigins: origins,
|
|
AllowedHeaders: []string{"*"},
|
|
}},
|
|
},
|
|
})
|
|
}
|
|
return err
|
|
}
|
|
|
|
// Headers adds CORS headers and redirects GET and HEAD requests to the AWS server.
|
|
func (ah *awshandler) Headers(tenantID types.TenantID, method string, url *url.URL, headers http.Header, serve bool) (http.Header, int, error) {
|
|
// Add CORS headers, if necessary.
|
|
headers, status := media.CORSHandler(method, headers, ah.corsOrigins, serve)
|
|
if status != 0 || method == http.MethodPost || method == http.MethodPut {
|
|
return headers, status, nil
|
|
}
|
|
|
|
fid := ah.GetIdFromUrl(url.String())
|
|
if fid.IsZero() {
|
|
return nil, 0, types.ErrNotFound
|
|
}
|
|
|
|
fdef, err := ah.getFileRecord(tenantID, fid)
|
|
if err != nil {
|
|
return nil, 0, err
|
|
}
|
|
|
|
if fdef.ETag != "" && headers.Get("If-None-Match") == `"`+fdef.ETag+`"` {
|
|
return http.Header{
|
|
"ETag": {`"` + fdef.ETag + `"`},
|
|
"Cache-Control": {ah.conf.CacheControl},
|
|
},
|
|
http.StatusNotModified, nil
|
|
}
|
|
|
|
ctx := context.Background()
|
|
var redirURL string
|
|
switch method {
|
|
case http.MethodGet:
|
|
// If the query parameter "asatt" is set to a true, set Content-Disposition to attachment.
|
|
// This will cause browsers to download the file rather than attempt to display it.
|
|
// This closes an XSS vulnerability when users upload HTML files.
|
|
var contentDisposition *string
|
|
if isAttachment, _ := strconv.ParseBool(url.Query().Get("asatt")); isAttachment {
|
|
contentDisposition = aws.String("attachment")
|
|
}
|
|
presigned, err := ah.presign.PresignGetObject(ctx, &s3.GetObjectInput{
|
|
Bucket: aws.String(ah.conf.BucketName),
|
|
Key: aws.String(fid.String32()),
|
|
ResponseCacheControl: aws.String(ah.conf.CacheControl),
|
|
ResponseContentType: aws.String(fdef.MimeType),
|
|
ResponseContentDisposition: contentDisposition,
|
|
}, func(opts *s3.PresignOptions) {
|
|
opts.Expires = time.Second * time.Duration(ah.conf.PresignTTL)
|
|
})
|
|
if err != nil {
|
|
return nil, 0, err
|
|
}
|
|
redirURL = presigned.URL
|
|
case http.MethodHead:
|
|
presigned, err := ah.presign.PresignHeadObject(ctx, &s3.HeadObjectInput{
|
|
Bucket: aws.String(ah.conf.BucketName),
|
|
Key: aws.String(fid.String32()),
|
|
}, func(opts *s3.PresignOptions) {
|
|
opts.Expires = time.Second * time.Duration(ah.conf.PresignTTL)
|
|
})
|
|
if err != nil {
|
|
return nil, 0, err
|
|
}
|
|
redirURL = presigned.URL
|
|
}
|
|
|
|
if redirURL != "" {
|
|
// Return presigned URL with 308 Permanent redirect. Let the client cache the response.
|
|
// The original URL will stop working after a short period of time to prevent use of Tinode
|
|
// as a free file server.
|
|
return http.Header{
|
|
"Location": {redirURL},
|
|
"ETag": {`"` + fdef.ETag + `"`},
|
|
"Content-Type": {"application/json; charset=utf-8"},
|
|
"Cache-Control": {ah.conf.CacheControl},
|
|
},
|
|
http.StatusPermanentRedirect, nil
|
|
}
|
|
return nil, 0, nil
|
|
}
|
|
|
|
// Upload processes request for a file upload. The file is given as io.Reader.
|
|
func (ah *awshandler) Upload(fdef *types.FileDef, file io.Reader) (string, int64, error) {
|
|
var err error
|
|
|
|
key, err := tenantObjectKey(fdef.TenantID, fdef.Uid().String32(), fdef.CreatedAt)
|
|
if err != nil {
|
|
return "", 0, err
|
|
}
|
|
|
|
tmClient := transfermanager.New(ah.svc)
|
|
|
|
if err = store.Files.StartUpload(fdef.TenantID, fdef); err != nil {
|
|
logs.Warn.Println("failed to create file record", fdef.Id, err)
|
|
return "", 0, err
|
|
}
|
|
|
|
rc := readerCounter{reader: file}
|
|
input := &transfermanager.UploadObjectInput{
|
|
CacheControl: aws.String(ah.conf.CacheControl),
|
|
Bucket: aws.String(ah.conf.BucketName),
|
|
Key: aws.String(key),
|
|
Body: &rc,
|
|
}
|
|
if fdef.MimeType != "" {
|
|
input.ContentType = aws.String(fdef.MimeType)
|
|
}
|
|
result, err := tmClient.UploadObject(context.Background(), input)
|
|
|
|
if err != nil {
|
|
return "", 0, err
|
|
}
|
|
|
|
fname := fdef.Id
|
|
fname += media.ExtensionByMimeType(fdef.MimeType)
|
|
|
|
fdef.Location = key
|
|
if result.ETag != nil {
|
|
fdef.ETag = strings.Trim(*result.ETag, "\"")
|
|
}
|
|
return ah.conf.ServeURL + fname, rc.count, nil
|
|
}
|
|
|
|
// Download processes request for file download.
|
|
// The returned ReadSeekCloser must be closed after use.
|
|
func (ah *awshandler) Download(tenantID types.TenantID, url string) (*types.FileDef, media.ReadSeekCloser, error) {
|
|
return nil, nil, types.ErrUnsupported
|
|
}
|
|
|
|
// Delete deletes files from aws by provided slice of locations.
|
|
func (ah *awshandler) Delete(locations []string) error {
|
|
ctx := context.Background()
|
|
for i := 0; i < len(locations); i += 1000 {
|
|
end := i + 1000
|
|
if end > len(locations) {
|
|
end = len(locations)
|
|
}
|
|
|
|
objects := make([]s3types.ObjectIdentifier, end-i)
|
|
for j, key := range locations[i:end] {
|
|
objects[j] = s3types.ObjectIdentifier{Key: aws.String(key)}
|
|
}
|
|
|
|
_, err := ah.svc.DeleteObjects(ctx, &s3.DeleteObjectsInput{
|
|
Bucket: aws.String(ah.conf.BucketName),
|
|
Delete: &s3types.Delete{
|
|
Objects: objects,
|
|
Quiet: aws.Bool(true),
|
|
},
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func isAPIError(err error, codes ...string) bool {
|
|
var apiErr smithy.APIError
|
|
if !errors.As(err, &apiErr) {
|
|
return false
|
|
}
|
|
for _, code := range codes {
|
|
if apiErr.ErrorCode() == code {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// GetIdFromUrl converts an attahment URL to a file UID.
|
|
func (ah *awshandler) GetIdFromUrl(url string) types.Uid {
|
|
return media.GetIdFromUrl(url, ah.conf.ServeURL)
|
|
}
|
|
|
|
// getFileRecord given file ID reads file record from the database.
|
|
func (ah *awshandler) getFileRecord(tenantID types.TenantID, fid types.Uid) (*types.FileDef, error) {
|
|
fd, err := store.Files.Get(tenantID, fid.String())
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if fd == nil {
|
|
return nil, types.ErrNotFound
|
|
}
|
|
return fd, nil
|
|
}
|
|
|
|
func tenantObjectKey(tenantID types.TenantID, fileID string, createdAt time.Time) (string, error) {
|
|
if !tenantID.IsValid() || fileID == "" {
|
|
return "", types.ErrMalformed
|
|
}
|
|
if createdAt.IsZero() {
|
|
createdAt = time.Now().UTC()
|
|
}
|
|
return path.Join("tenants", strconv.FormatInt(int64(tenantID), 10),
|
|
createdAt.Format("2006"), createdAt.Format("01"), fileID), nil
|
|
}
|
|
|
|
func init() {
|
|
store.RegisterMediaHandler(handlerName, &awshandler{})
|
|
}
|