多租户更改
This commit is contained in:
@@ -0,0 +1,231 @@
|
||||
// Package fs implements github.com/tinode/chat/server/media interface by storing media objects in a single
|
||||
// directory in the file system.
|
||||
// This module won't perform well with tens of thousand of files because it stores all files in a single directory.
|
||||
package fs
|
||||
|
||||
import (
|
||||
"encoding/base32"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"hash/fnv"
|
||||
"io"
|
||||
"mime"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/tinode/chat/server/logs"
|
||||
"github.com/tinode/chat/server/media"
|
||||
"github.com/tinode/chat/server/store"
|
||||
"github.com/tinode/chat/server/store/types"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultServeURL = "/v0/file/s/"
|
||||
defaultCacheControl = "max-age=86400"
|
||||
|
||||
handlerName = "fs"
|
||||
)
|
||||
|
||||
type fileConfig struct {
|
||||
// FileUploadDirectory: In case of a cluster fileUploadLocation must be accessible to all cluster members.
|
||||
FileUploadDirectory string `json:"upload_dir"`
|
||||
ServeURL string `json:"serve_url"`
|
||||
CorsOrigins []string `json:"cors_origins"`
|
||||
CacheControl string `json:"cache_control"`
|
||||
}
|
||||
|
||||
type fshandler struct {
|
||||
fileConfig
|
||||
// corsOrigins parsed allowed origins.
|
||||
corsOrigins []media.AllowedOrigin
|
||||
}
|
||||
|
||||
func (fh *fshandler) Init(jsconf string) error {
|
||||
var err error
|
||||
|
||||
if err = json.Unmarshal([]byte(jsconf), &fh.fileConfig); err != nil {
|
||||
return errors.New("failed to parse config: " + err.Error())
|
||||
}
|
||||
|
||||
if fh.FileUploadDirectory == "" {
|
||||
return errors.New("missing upload location")
|
||||
}
|
||||
|
||||
if fh.ServeURL == "" {
|
||||
fh.ServeURL = defaultServeURL
|
||||
}
|
||||
|
||||
if fh.CacheControl == "" {
|
||||
fh.CacheControl = defaultCacheControl
|
||||
}
|
||||
|
||||
fh.corsOrigins, err = media.ParseCORSAllow(fh.CorsOrigins)
|
||||
if err != nil {
|
||||
return errors.New("failed to parse CORS allowed origins: " + err.Error())
|
||||
}
|
||||
// Make sure the upload directory exists.
|
||||
return os.MkdirAll(fh.FileUploadDirectory, 0777)
|
||||
}
|
||||
|
||||
// Headers is used for cache management and serving CORS headers.
|
||||
func (fh *fshandler) Headers(tenantID types.TenantID, method string, url *url.URL, headers http.Header, serve bool) (http.Header, int, error) {
|
||||
if method == http.MethodGet {
|
||||
|
||||
fid := fh.GetIdFromUrl(url.String())
|
||||
if fid.IsZero() {
|
||||
return nil, 0, types.ErrNotFound
|
||||
}
|
||||
|
||||
fdef, err := fh.getFileRecord(tenantID, fid)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
if etag := strings.Trim(headers.Get("If-None-Match"), "\""); etag != "" && etag == fdef.ETag {
|
||||
return http.Header{
|
||||
"Last-Modified": {fdef.UpdatedAt.Format(http.TimeFormat)},
|
||||
"ETag": {`"` + fdef.ETag + `"`},
|
||||
"Cache-Control": {fh.CacheControl},
|
||||
},
|
||||
http.StatusNotModified, nil
|
||||
}
|
||||
|
||||
return http.Header{
|
||||
"Content-Type": {fdef.MimeType},
|
||||
"Cache-Control": {fh.CacheControl},
|
||||
"ETag": {`"` + fdef.ETag + `"`},
|
||||
}, 0, nil
|
||||
}
|
||||
|
||||
if method != http.MethodOptions {
|
||||
// Not an OPTIONS request. No special handling for all other requests.
|
||||
return nil, 0, nil
|
||||
}
|
||||
header, status := media.CORSHandler(method, headers, fh.corsOrigins, serve)
|
||||
return header, status, nil
|
||||
}
|
||||
|
||||
// Upload processes request for file upload. The file is given as io.Reader.
|
||||
func (fh *fshandler) Upload(fdef *types.FileDef, file io.Reader) (string, int64, error) {
|
||||
location, err := tenantFileLocation(fh.FileUploadDirectory, fdef.TenantID, fdef.Uid().String32(), fdef.CreatedAt)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
if err = os.MkdirAll(filepath.Dir(location), 0777); err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
|
||||
outfile, err := os.Create(location)
|
||||
if err != nil {
|
||||
logs.Warn.Println("Upload: failed to create file", fdef.Location, err)
|
||||
return "", 0, err
|
||||
}
|
||||
|
||||
if err = store.Files.StartUpload(fdef.TenantID, fdef); err != nil {
|
||||
outfile.Close()
|
||||
os.Remove(location)
|
||||
logs.Warn.Println("failed to create file record", fdef.Id, err)
|
||||
return "", 0, err
|
||||
}
|
||||
|
||||
size, err := io.Copy(outfile, file)
|
||||
outfile.Close()
|
||||
if err != nil {
|
||||
os.Remove(location)
|
||||
return "", 0, err
|
||||
}
|
||||
|
||||
fname := fdef.Id
|
||||
ext, _ := mime.ExtensionsByType(fdef.MimeType)
|
||||
if len(ext) > 0 {
|
||||
fname += ext[0]
|
||||
}
|
||||
|
||||
fdef.Location = location
|
||||
// Use file path to create ETag. File paths are unique so will be the ETag.
|
||||
fdef.ETag = etagFromPath(fdef.Location)
|
||||
|
||||
return fh.ServeURL + fname, size, nil
|
||||
}
|
||||
|
||||
// Download processes request for file download.
|
||||
// The returned ReadSeekCloser must be closed after use.
|
||||
func (fh *fshandler) Download(tenantID types.TenantID, url string) (*types.FileDef, media.ReadSeekCloser, error) {
|
||||
fid := fh.GetIdFromUrl(url)
|
||||
if fid.IsZero() {
|
||||
return nil, nil, types.ErrNotFound
|
||||
}
|
||||
|
||||
fd, err := fh.getFileRecord(tenantID, fid)
|
||||
if err != nil {
|
||||
logs.Warn.Println("Download: file not found", fid)
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
file, err := os.Open(fd.Location)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
// If the file is not found, send 404 instead of the default 500
|
||||
err = types.ErrNotFound
|
||||
}
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
return fd, file, nil
|
||||
}
|
||||
|
||||
// Delete deletes files from storage by provided slice of locations.
|
||||
func (fh *fshandler) Delete(locations []string) error {
|
||||
for _, loc := range locations {
|
||||
if err, _ := os.Remove(loc).(*os.PathError); err != nil {
|
||||
if err != os.ErrNotExist {
|
||||
logs.Warn.Println("fs: error deleting file", loc, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetIdFromUrl converts an attahment URL to a file UID.
|
||||
func (fh *fshandler) GetIdFromUrl(url string) types.Uid {
|
||||
return media.GetIdFromUrl(url, fh.ServeURL)
|
||||
}
|
||||
|
||||
// getFileRecord given file ID reads file record from the database.
|
||||
func (fh *fshandler) getFileRecord(tenantID types.TenantID, fid types.Uid) (*types.FileDef, error) {
|
||||
fd, err := store.Files.Get(tenantID, fid.String())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if fd == nil {
|
||||
return nil, types.ErrNotFound
|
||||
}
|
||||
return fd, nil
|
||||
}
|
||||
|
||||
func etagFromPath(path string) string {
|
||||
hasher := fnv.New128()
|
||||
hasher.Write([]byte(path))
|
||||
return strings.ToLower(base32.StdEncoding.WithPadding(base32.NoPadding).
|
||||
EncodeToString(hasher.Sum(make([]byte, 0, hasher.Size()))))
|
||||
}
|
||||
|
||||
func tenantFileLocation(root string, tenantID types.TenantID, fileID string, createdAt time.Time) (string, error) {
|
||||
if !tenantID.IsValid() || fileID == "" {
|
||||
return "", types.ErrMalformed
|
||||
}
|
||||
if createdAt.IsZero() {
|
||||
createdAt = time.Now().UTC()
|
||||
}
|
||||
return filepath.Join(root, "tenants", strconv.FormatInt(int64(tenantID), 10),
|
||||
createdAt.Format("2006"), createdAt.Format("01"), fileID), nil
|
||||
}
|
||||
|
||||
func init() {
|
||||
store.RegisterMediaHandler(handlerName, &fshandler{})
|
||||
}
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
package fs
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/tinode/chat/server/store/types"
|
||||
)
|
||||
|
||||
func TestTenantFileLocation(t *testing.T) {
|
||||
got, err := tenantFileLocation("/tmp/uploads", types.TenantID(42), "abc123", time.Date(2026, 8, 11, 9, 0, 0, 0, time.UTC))
|
||||
if err != nil {
|
||||
t.Fatalf("tenantFileLocation failed: %v", err)
|
||||
}
|
||||
want := filepath.Join("/tmp/uploads", "tenants", "42", "2026", "08", "abc123")
|
||||
if got != want {
|
||||
t.Fatalf("tenantFileLocation = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTenantFileLocationRejectsZeroTenant(t *testing.T) {
|
||||
if _, err := tenantFileLocation("/tmp/uploads", types.ZeroTenantID, "abc123", time.Time{}); err != types.ErrMalformed {
|
||||
t.Fatalf("tenantFileLocation err = %v, want malformed", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,220 @@
|
||||
// Package media defines an interface which must be implemented by media upload/download handlers.
|
||||
package media
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"slices"
|
||||
|
||||
"github.com/tinode/chat/server/store/types"
|
||||
)
|
||||
|
||||
// ReadSeekCloser must be implemented by the media being downloaded.
|
||||
type ReadSeekCloser interface {
|
||||
io.Reader
|
||||
io.Seeker
|
||||
io.Closer
|
||||
}
|
||||
|
||||
// Handler is an interface which must be implemented by media handlers (uploaders-downloaders).
|
||||
type Handler interface {
|
||||
// Init initializes the media upload handler.
|
||||
Init(jsconf string) error
|
||||
|
||||
// Headers checks if the handler wants to provide additional HTTP headers for the request.
|
||||
// It could be CORS headers, redirect to serve files from another URL, cache-control headers.
|
||||
// It returns headers as a map, HTTP status code to stop processing or 0 to continue, error.
|
||||
Headers(tenantID types.TenantID, method string, url *url.URL, headers http.Header, serve bool) (http.Header, int, error)
|
||||
|
||||
// Upload processes request for file upload. Returns file URL, file size, error.
|
||||
Upload(fdef *types.FileDef, file io.Reader) (string, int64, error)
|
||||
|
||||
// Download processes request for file download.
|
||||
Download(tenantID types.TenantID, url string) (*types.FileDef, ReadSeekCloser, error)
|
||||
|
||||
// Delete deletes file from storage.
|
||||
Delete(locations []string) error
|
||||
|
||||
// GetIdFromUrl extracts file ID from download URL.
|
||||
GetIdFromUrl(url string) types.Uid
|
||||
}
|
||||
|
||||
type AllowedOrigin struct {
|
||||
Origin string
|
||||
URL url.URL
|
||||
HostParts []string
|
||||
HasWildcard bool
|
||||
}
|
||||
|
||||
var fileNamePattern = regexp.MustCompile(`^[-_A-Za-z0-9]+`)
|
||||
|
||||
// GetIdFromUrl is a helper method for extracting file ID from a URL.
|
||||
func GetIdFromUrl(url, serveUrl string) types.Uid {
|
||||
dir, fname := path.Split(path.Clean(url))
|
||||
|
||||
if dir != "" && dir != serveUrl {
|
||||
return types.ZeroUid
|
||||
}
|
||||
|
||||
return types.ParseUid(fileNamePattern.FindString(fname))
|
||||
}
|
||||
|
||||
// ParseCORSAllow pre-parses allowed origins from the configuration.
|
||||
func ParseCORSAllow(allowed []string) ([]AllowedOrigin, error) {
|
||||
if len(allowed) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
result := make([]AllowedOrigin, 0, len(allowed))
|
||||
for _, val := range allowed {
|
||||
parsed := AllowedOrigin{Origin: val}
|
||||
switch val {
|
||||
case "*":
|
||||
if len(allowed) > 1 {
|
||||
return nil, errors.New("wildcard origin '*' must be the only entry")
|
||||
}
|
||||
parsed.HasWildcard = true
|
||||
case "":
|
||||
if len(allowed) > 1 {
|
||||
return nil, errors.New("empty allowed origin '' must be the only entry")
|
||||
}
|
||||
// Empty string means no origin allowed - no URL parsing needed
|
||||
parsed.HasWildcard = false
|
||||
default:
|
||||
u, err := url.ParseRequestURI(val)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
parsed.HostParts = strings.Split(u.Hostname(), ".")
|
||||
parsed.URL = *u
|
||||
parsed.HasWildcard = strings.Contains(u.Hostname(), "*")
|
||||
}
|
||||
result = append(result, parsed)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// matchCORSOrigin compares origin from the HTTP request to a list of allowed origins.
|
||||
func matchCORSOrigin(allowed []AllowedOrigin, origin string) string {
|
||||
if len(allowed) == 0 {
|
||||
// Not configured
|
||||
return ""
|
||||
}
|
||||
|
||||
if origin == "" && allowed[0].Origin != "*" {
|
||||
// Request has no Origin header and "*" (any origin) not allowed.
|
||||
return ""
|
||||
}
|
||||
|
||||
if allowed[0].Origin == "*" {
|
||||
if origin == "" {
|
||||
return "*"
|
||||
}
|
||||
return origin
|
||||
}
|
||||
|
||||
// Check for empty string in allowed origins - this means no origin is allowed.
|
||||
if allowed[0].Origin == "" {
|
||||
return ""
|
||||
}
|
||||
|
||||
originUrl, err := url.ParseRequestURI(origin)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
originParts := strings.Split(originUrl.Hostname(), ".")
|
||||
|
||||
for _, val := range allowed {
|
||||
if val.Origin == origin {
|
||||
return origin
|
||||
}
|
||||
|
||||
if !val.HasWildcard ||
|
||||
originUrl.Scheme != val.URL.Scheme ||
|
||||
originUrl.Port() != val.URL.Port() ||
|
||||
len(originParts) != len(val.HostParts) {
|
||||
continue
|
||||
}
|
||||
|
||||
matched := true
|
||||
for i, part := range val.HostParts {
|
||||
if part == "*" {
|
||||
continue
|
||||
}
|
||||
if part != originParts[i] {
|
||||
matched = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if matched {
|
||||
return origin
|
||||
}
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
// allowMethods must be in UPPERCASE.
|
||||
func matchCORSMethod(allowMethods []string, method string) bool {
|
||||
if method == "" {
|
||||
// Request has no Method header.
|
||||
return false
|
||||
}
|
||||
|
||||
return slices.Contains(allowMethods, strings.ToUpper(method))
|
||||
}
|
||||
|
||||
// CORSHandler is the default CORS processor for use by media handlers. It adds CORS headers to
|
||||
// preflight OPTIONS requests, Vary & Access-Control-Allow-Origin headers to all responses.
|
||||
func CORSHandler(method string, reqHeader http.Header, allowedOrigins []AllowedOrigin, serve bool) (http.Header, int) {
|
||||
respHeader := map[string][]string{
|
||||
// Always add Vary because of possible intermediate caches.
|
||||
"Vary": {"Origin", "Access-Control-Request-Method, Access-Control-Request-Headers"},
|
||||
}
|
||||
|
||||
origin := reqHeader.Get("Origin")
|
||||
|
||||
allowedOrigin := matchCORSOrigin(allowedOrigins, origin)
|
||||
if acMethod := reqHeader.Get("Access-Control-Request-Method"); method == http.MethodOptions && acMethod != "" {
|
||||
// Preflight request.
|
||||
|
||||
if allowedOrigin == "" {
|
||||
return respHeader, http.StatusNoContent
|
||||
}
|
||||
|
||||
var allowMethods []string
|
||||
if serve {
|
||||
allowMethods = []string{http.MethodGet, http.MethodHead, http.MethodOptions}
|
||||
} else {
|
||||
allowMethods = []string{http.MethodPost, http.MethodPut, http.MethodHead, http.MethodOptions}
|
||||
}
|
||||
|
||||
if !matchCORSMethod(allowMethods, acMethod) {
|
||||
// CORS policy does not allow this method.
|
||||
return respHeader, http.StatusNoContent
|
||||
}
|
||||
|
||||
respHeader["Access-Control-Allow-Headers"] = []string{"*"}
|
||||
respHeader["Access-Control-Allow-Credentials"] = []string{"true"}
|
||||
respHeader["Access-Control-Allow-Methods"] = []string{strings.Join(allowMethods, ", ")}
|
||||
respHeader["Access-Control-Max-Age"] = []string{"86400"}
|
||||
respHeader["Access-Control-Allow-Origin"] = []string{allowedOrigin}
|
||||
|
||||
return respHeader, http.StatusNoContent
|
||||
}
|
||||
|
||||
// Regular request, not a preflight.
|
||||
|
||||
if allowedOrigin != "" {
|
||||
// Returning Origin from the actual request instead of '*', otherwise there could be an issue with Credentials.
|
||||
respHeader["Access-Control-Allow-Origin"] = []string{origin}
|
||||
}
|
||||
|
||||
return respHeader, 0
|
||||
}
|
||||
|
||||
@@ -0,0 +1,221 @@
|
||||
package media
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestMatchCORSOrigin(t *testing.T) {
|
||||
cases := []struct {
|
||||
allowed []string
|
||||
origin string
|
||||
expected string
|
||||
expectError bool
|
||||
errorMessage string
|
||||
}{
|
||||
{
|
||||
allowed: []string{"https://example.com"},
|
||||
origin: "https://example.com",
|
||||
expected: "https://example.com",
|
||||
},
|
||||
{
|
||||
allowed: []string{"https://example2.com", "https://example.com"},
|
||||
origin: "https://example.com",
|
||||
expected: "https://example.com",
|
||||
},
|
||||
{
|
||||
allowed: []string{"*"},
|
||||
origin: "https://example.com",
|
||||
expected: "https://example.com",
|
||||
},
|
||||
{
|
||||
allowed: []string{""},
|
||||
origin: "https://example.com",
|
||||
expected: "",
|
||||
},
|
||||
{
|
||||
allowed: []string{},
|
||||
origin: "",
|
||||
expected: "",
|
||||
},
|
||||
{
|
||||
allowed: []string{"https://example.com"},
|
||||
origin: "",
|
||||
expected: "",
|
||||
},
|
||||
{
|
||||
allowed: []string{},
|
||||
origin: "https://example.com",
|
||||
expected: "",
|
||||
},
|
||||
{
|
||||
allowed: []string{"http://example.com"},
|
||||
origin: "https://example.com",
|
||||
expected: "",
|
||||
},
|
||||
{
|
||||
allowed: []string{"https://example.com"},
|
||||
origin: "http://example.com",
|
||||
expected: "",
|
||||
},
|
||||
{
|
||||
allowed: []string{"http://example.com:8000"},
|
||||
origin: "http://example.com:8000",
|
||||
expected: "http://example.com:8000",
|
||||
},
|
||||
{
|
||||
allowed: []string{"http://localhost:8090"},
|
||||
origin: "http://localhost:8090",
|
||||
expected: "http://localhost:8090",
|
||||
},
|
||||
{
|
||||
allowed: []string{"http://localhost:8090"},
|
||||
origin: "http://localhost:8080",
|
||||
expected: "",
|
||||
},
|
||||
{
|
||||
allowed: []string{"https://example.com"},
|
||||
origin: "https://sub.example.com",
|
||||
expected: "",
|
||||
},
|
||||
{
|
||||
allowed: []string{"https://*.example.com"},
|
||||
origin: "https://sub.example.com",
|
||||
expected: "https://sub.example.com",
|
||||
},
|
||||
{
|
||||
allowed: []string{"https://*.example.com"},
|
||||
origin: "https://pre.sub.example.com",
|
||||
expected: "",
|
||||
},
|
||||
{
|
||||
allowed: []string{"https://*.example.com", "https://*.sub.example.com"},
|
||||
origin: "https://pre.sub.example.com",
|
||||
expected: "https://pre.sub.example.com",
|
||||
},
|
||||
{
|
||||
allowed: []string{"https://*.*.example.com"},
|
||||
origin: "https://pre.sub.example.com",
|
||||
expected: "https://pre.sub.example.com",
|
||||
},
|
||||
{
|
||||
allowed: []string{"https://*.sub.example.com"},
|
||||
origin: "https://pre.asd.example.com",
|
||||
expected: "",
|
||||
},
|
||||
{
|
||||
allowed: []string{"https://pre.*.example.com"},
|
||||
origin: "https://pre.sub.example.com",
|
||||
expected: "https://pre.sub.example.com",
|
||||
},
|
||||
{
|
||||
allowed: []string{"https://*.*.*.example.com"},
|
||||
origin: "https://www.pre.sub.example.com",
|
||||
expected: "https://www.pre.sub.example.com",
|
||||
},
|
||||
{
|
||||
allowed: []string{"*"},
|
||||
origin: "",
|
||||
expected: "*", // Should allow any origin, including empty
|
||||
},
|
||||
// Error cases - these should fail during ParseCORSAllow
|
||||
{
|
||||
allowed: []string{"*", "https://example.com"},
|
||||
origin: "https://example.com",
|
||||
expectError: true,
|
||||
errorMessage: "wildcard origin '*' must be the only entry",
|
||||
},
|
||||
{
|
||||
allowed: []string{"not-a-valid-url"},
|
||||
origin: "https://example.com",
|
||||
expectError: true,
|
||||
errorMessage: "invalid URI for request",
|
||||
},
|
||||
{
|
||||
allowed: []string{"://invalid-url"},
|
||||
origin: "https://example.com",
|
||||
expectError: true,
|
||||
errorMessage: "missing protocol scheme",
|
||||
},
|
||||
{
|
||||
allowed: []string{"https://", "example.com"},
|
||||
origin: "https://example.com",
|
||||
expectError: true,
|
||||
errorMessage: "invalid URI for request",
|
||||
},
|
||||
// Valid cases that should not error
|
||||
{
|
||||
allowed: []string{"", "https://example.com"},
|
||||
origin: "https://example.com",
|
||||
expectError: true,
|
||||
errorMessage: "empty allowed origin '' must be the only entry", // Empty string should make all origins disallowed
|
||||
},
|
||||
{
|
||||
allowed: []string{"https://Example.com"},
|
||||
origin: "https://example.com",
|
||||
expected: "", // Should not match due to case sensitivity
|
||||
},
|
||||
{
|
||||
allowed: []string{"https://example.com/"},
|
||||
origin: "https://example.com",
|
||||
expected: "", // Should not match due to trailing slash
|
||||
},
|
||||
{
|
||||
allowed: []string{"https://example.com"},
|
||||
origin: "not-a-valid-url",
|
||||
expected: "", // Should handle malformed origin gracefully
|
||||
},
|
||||
{
|
||||
allowed: []string{"https://example.*.com"},
|
||||
origin: "https://example.sub.com",
|
||||
expected: "https://example.sub.com",
|
||||
},
|
||||
{
|
||||
allowed: []string{"https://*.com"},
|
||||
origin: "https://example.com",
|
||||
expected: "https://example.com",
|
||||
},
|
||||
{
|
||||
allowed: []string{"http://*.example.com"},
|
||||
origin: "https://sub.example.com",
|
||||
expected: "", // Should not match due to protocol difference
|
||||
},
|
||||
{
|
||||
allowed: []string{"https://example.com", "https://example.com"},
|
||||
origin: "https://example.com",
|
||||
expected: "https://example.com", // Should still work with duplicates
|
||||
},
|
||||
}
|
||||
|
||||
for i, tc := range cases {
|
||||
allowedOrigins, err := ParseCORSAllow(tc.allowed)
|
||||
|
||||
if tc.expectError {
|
||||
if err == nil {
|
||||
t.Errorf("Test case %d: Expected error but got none. Allowed: %v", i, tc.allowed)
|
||||
continue
|
||||
}
|
||||
if tc.errorMessage != "" && !containsSubstring(err.Error(), tc.errorMessage) {
|
||||
t.Errorf("Test case %d: Expected error containing '%s', got '%s'", i, tc.errorMessage, err.Error())
|
||||
}
|
||||
// For error cases, we don't test the matching logic
|
||||
continue
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
t.Errorf("Test case %d: Unexpected error parsing allowed origins: %v. Allowed: %v", i, err, tc.allowed)
|
||||
continue
|
||||
}
|
||||
|
||||
result := matchCORSOrigin(allowedOrigins, tc.origin)
|
||||
if result != tc.expected {
|
||||
t.Errorf("Test case %d: Match CORS origin got wrong result. Expected '%s', got '%s'. Allowed: %v, Origin: '%s'",
|
||||
i, tc.expected, result, tc.allowed, tc.origin)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Helper function to check if a string contains a substring (case-insensitive)
|
||||
func containsSubstring(str, substr string) bool {
|
||||
return strings.Contains(strings.ToLower(str), strings.ToLower(substr))
|
||||
}
|
||||
|
||||
@@ -0,0 +1,384 @@
|
||||
// Package s3 implements media interface by storing media objects in Amazon S3 bucket.
|
||||
package s3
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"mime"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/aws/smithy-go"
|
||||
|
||||
"github.com/tinode/chat/server/logs"
|
||||
"github.com/tinode/chat/server/media"
|
||||
"github.com/tinode/chat/server/store"
|
||||
"github.com/tinode/chat/server/store/types"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultServeURL = "/v0/file/s/"
|
||||
defaultCacheControl = "no-cache, must-revalidate"
|
||||
|
||||
handlerName = "s3"
|
||||
// Presign GET URLs for this number of seconds.
|
||||
defaultPresignDuration = 120
|
||||
)
|
||||
|
||||
type awsconfig struct {
|
||||
AccessKeyId string `json:"access_key_id"`
|
||||
SecretAccessKey string `json:"secret_access_key"`
|
||||
Region string `json:"region"`
|
||||
DisableSSL bool `json:"disable_ssl"`
|
||||
ForcePathStyle bool `json:"force_path_style"`
|
||||
Endpoint string `json:"endpoint"`
|
||||
BucketName string `json:"bucket"`
|
||||
CorsOrigins []string `json:"cors_origins"`
|
||||
ServeURL string `json:"serve_url"`
|
||||
PresignTTL int `json:"presign_ttl"`
|
||||
CacheControl string `json:"cache_control"`
|
||||
}
|
||||
|
||||
type awshandler struct {
|
||||
svc *s3.Client
|
||||
presign *s3.PresignClient
|
||||
conf awsconfig
|
||||
corsOrigins []media.AllowedOrigin
|
||||
}
|
||||
|
||||
// readerCounter is a byte counter for bytes read through the io.Reader
|
||||
type readerCounter struct {
|
||||
io.Reader
|
||||
count int64
|
||||
reader io.Reader
|
||||
}
|
||||
|
||||
// Read reads the bytes and records the number of read bytes.
|
||||
func (rc *readerCounter) Read(buf []byte) (int, error) {
|
||||
n, err := rc.reader.Read(buf)
|
||||
atomic.AddInt64(&rc.count, int64(n))
|
||||
return n, err
|
||||
}
|
||||
|
||||
// Init initializes the media handler.
|
||||
func (ah *awshandler) Init(jsconf string) error {
|
||||
var err error
|
||||
if err = json.Unmarshal([]byte(jsconf), &ah.conf); err != nil {
|
||||
return errors.New("failed to parse config: " + err.Error())
|
||||
}
|
||||
|
||||
if ah.conf.AccessKeyId == "" {
|
||||
return errors.New("missing Access Key ID")
|
||||
}
|
||||
if ah.conf.SecretAccessKey == "" {
|
||||
return errors.New("missing Secret Access Key")
|
||||
}
|
||||
if ah.conf.Region == "" {
|
||||
return errors.New("missing Region")
|
||||
}
|
||||
if ah.conf.BucketName == "" {
|
||||
return errors.New("missing Bucket")
|
||||
}
|
||||
if ah.conf.PresignTTL <= 0 {
|
||||
ah.conf.PresignTTL = defaultPresignDuration
|
||||
}
|
||||
if ah.conf.CacheControl == "" {
|
||||
ah.conf.CacheControl = defaultCacheControl
|
||||
}
|
||||
if ah.conf.ServeURL == "" {
|
||||
ah.conf.ServeURL = defaultServeURL
|
||||
}
|
||||
ah.corsOrigins, err = media.ParseCORSAllow(ah.conf.CorsOrigins)
|
||||
if err != nil {
|
||||
return errors.New("failed to parse CORS allowed origins: " + err.Error())
|
||||
}
|
||||
|
||||
cfgOpts := []func(*config.LoadOptions) error{
|
||||
config.WithRegion(ah.conf.Region),
|
||||
config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(
|
||||
ah.conf.AccessKeyId,
|
||||
ah.conf.SecretAccessKey,
|
||||
"",
|
||||
)),
|
||||
}
|
||||
|
||||
var cfg aws.Config
|
||||
if cfg, err = config.LoadDefaultConfig(context.Background(), cfgOpts...); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Create S3 service client
|
||||
clientOpts := []func(*s3.Options){
|
||||
func(o *s3.Options) {
|
||||
o.UsePathStyle = ah.conf.ForcePathStyle
|
||||
},
|
||||
}
|
||||
if ah.conf.Endpoint != "" {
|
||||
endpoint := ah.conf.Endpoint
|
||||
if !strings.Contains(endpoint, "://") {
|
||||
if ah.conf.DisableSSL {
|
||||
endpoint = "http://" + endpoint
|
||||
} else {
|
||||
endpoint = "https://" + endpoint
|
||||
}
|
||||
}
|
||||
clientOpts = append(clientOpts, func(o *s3.Options) {
|
||||
o.BaseEndpoint = aws.String(endpoint)
|
||||
})
|
||||
}
|
||||
ah.svc = s3.NewFromConfig(cfg, clientOpts...)
|
||||
ah.presign = s3.NewPresignClient(ah.svc)
|
||||
|
||||
// Check if bucket already exists.
|
||||
_, err = ah.svc.HeadBucket(context.Background(), &s3.HeadBucketInput{Bucket: aws.String(ah.conf.BucketName)})
|
||||
if err == nil {
|
||||
// Bucket exists
|
||||
return nil
|
||||
}
|
||||
|
||||
if !isAPIError(err, "NoSuchBucket", "NotFound") {
|
||||
// Hard error.
|
||||
return err
|
||||
}
|
||||
|
||||
// Bucket does not exist. Create one.
|
||||
_, err = ah.svc.CreateBucket(context.Background(), &s3.CreateBucketInput{Bucket: aws.String(ah.conf.BucketName)})
|
||||
if err != nil {
|
||||
if isAPIError(err, "BucketAlreadyExists", "BucketAlreadyOwnedByYou", "OperationAborted") {
|
||||
// Check if someone has already created a bucket (possible in a cluster).
|
||||
err = nil
|
||||
}
|
||||
} else {
|
||||
// This is a new bucket.
|
||||
|
||||
// The following serves two purposes:
|
||||
// 1. Setup CORS policy to be able to serve media directly from S3.
|
||||
// 2. Verify that the bucket is accessible to the current user.
|
||||
origins := ah.conf.CorsOrigins
|
||||
if len(origins) == 0 {
|
||||
origins = append(origins, "*")
|
||||
}
|
||||
_, err = ah.svc.PutBucketCors(context.Background(), &s3.PutBucketCorsInput{
|
||||
Bucket: aws.String(ah.conf.BucketName),
|
||||
CORSConfiguration: &s3types.CORSConfiguration{
|
||||
CORSRules: []s3types.CORSRule{{
|
||||
AllowedMethods: []string{http.MethodGet, http.MethodHead},
|
||||
AllowedOrigins: origins,
|
||||
AllowedHeaders: []string{"*"},
|
||||
}},
|
||||
},
|
||||
})
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Headers adds CORS headers and redirects GET and HEAD requests to the AWS server.
|
||||
func (ah *awshandler) Headers(tenantID types.TenantID, method string, url *url.URL, headers http.Header, serve bool) (http.Header, int, error) {
|
||||
// Add CORS headers, if necessary.
|
||||
headers, status := media.CORSHandler(method, headers, ah.corsOrigins, serve)
|
||||
if status != 0 || method == http.MethodPost || method == http.MethodPut {
|
||||
return headers, status, nil
|
||||
}
|
||||
|
||||
fid := ah.GetIdFromUrl(url.String())
|
||||
if fid.IsZero() {
|
||||
return nil, 0, types.ErrNotFound
|
||||
}
|
||||
|
||||
fdef, err := ah.getFileRecord(tenantID, fid)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
if fdef.ETag != "" && headers.Get("If-None-Match") == `"`+fdef.ETag+`"` {
|
||||
return http.Header{
|
||||
"ETag": {`"` + fdef.ETag + `"`},
|
||||
"Cache-Control": {ah.conf.CacheControl},
|
||||
},
|
||||
http.StatusNotModified, nil
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
var redirURL string
|
||||
switch method {
|
||||
case http.MethodGet:
|
||||
// If the query parameter "asatt" is set to a true, set Content-Disposition to attachment.
|
||||
// This will cause browsers to download the file rather than attempt to display it.
|
||||
// This closes an XSS vulnerability when users upload HTML files.
|
||||
var contentDisposition *string
|
||||
if isAttachment, _ := strconv.ParseBool(url.Query().Get("asatt")); isAttachment {
|
||||
contentDisposition = aws.String("attachment")
|
||||
}
|
||||
presigned, err := ah.presign.PresignGetObject(ctx, &s3.GetObjectInput{
|
||||
Bucket: aws.String(ah.conf.BucketName),
|
||||
Key: aws.String(fid.String32()),
|
||||
ResponseCacheControl: aws.String(ah.conf.CacheControl),
|
||||
ResponseContentType: aws.String(fdef.MimeType),
|
||||
ResponseContentDisposition: contentDisposition,
|
||||
}, func(opts *s3.PresignOptions) {
|
||||
opts.Expires = time.Second * time.Duration(ah.conf.PresignTTL)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
redirURL = presigned.URL
|
||||
case http.MethodHead:
|
||||
presigned, err := ah.presign.PresignHeadObject(ctx, &s3.HeadObjectInput{
|
||||
Bucket: aws.String(ah.conf.BucketName),
|
||||
Key: aws.String(fid.String32()),
|
||||
}, func(opts *s3.PresignOptions) {
|
||||
opts.Expires = time.Second * time.Duration(ah.conf.PresignTTL)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
redirURL = presigned.URL
|
||||
}
|
||||
|
||||
if redirURL != "" {
|
||||
// Return presigned URL with 308 Permanent redirect. Let the client cache the response.
|
||||
// The original URL will stop working after a short period of time to prevent use of Tinode
|
||||
// as a free file server.
|
||||
return http.Header{
|
||||
"Location": {redirURL},
|
||||
"ETag": {`"` + fdef.ETag + `"`},
|
||||
"Content-Type": {"application/json; charset=utf-8"},
|
||||
"Cache-Control": {ah.conf.CacheControl},
|
||||
},
|
||||
http.StatusPermanentRedirect, nil
|
||||
}
|
||||
return nil, 0, nil
|
||||
}
|
||||
|
||||
// Upload processes request for a file upload. The file is given as io.Reader.
|
||||
func (ah *awshandler) Upload(fdef *types.FileDef, file io.Reader) (string, int64, error) {
|
||||
var err error
|
||||
|
||||
key, err := tenantObjectKey(fdef.TenantID, fdef.Uid().String32(), fdef.CreatedAt)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
|
||||
tmClient := transfermanager.New(ah.svc)
|
||||
|
||||
if err = store.Files.StartUpload(fdef.TenantID, fdef); err != nil {
|
||||
logs.Warn.Println("failed to create file record", fdef.Id, err)
|
||||
return "", 0, err
|
||||
}
|
||||
|
||||
rc := readerCounter{reader: file}
|
||||
result, err := tmClient.UploadObject(context.Background(), &transfermanager.UploadObjectInput{
|
||||
CacheControl: aws.String(ah.conf.CacheControl),
|
||||
Bucket: aws.String(ah.conf.BucketName),
|
||||
Key: aws.String(key),
|
||||
Body: &rc,
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
|
||||
fname := fdef.Id
|
||||
ext, _ := mime.ExtensionsByType(fdef.MimeType)
|
||||
if len(ext) > 0 {
|
||||
fname += ext[0]
|
||||
}
|
||||
|
||||
fdef.Location = key
|
||||
if result.ETag != nil {
|
||||
fdef.ETag = strings.Trim(*result.ETag, "\"")
|
||||
}
|
||||
return ah.conf.ServeURL + fname, rc.count, nil
|
||||
}
|
||||
|
||||
// Download processes request for file download.
|
||||
// The returned ReadSeekCloser must be closed after use.
|
||||
func (ah *awshandler) Download(tenantID types.TenantID, url string) (*types.FileDef, media.ReadSeekCloser, error) {
|
||||
return nil, nil, types.ErrUnsupported
|
||||
}
|
||||
|
||||
// Delete deletes files from aws by provided slice of locations.
|
||||
func (ah *awshandler) Delete(locations []string) error {
|
||||
ctx := context.Background()
|
||||
for i := 0; i < len(locations); i += 1000 {
|
||||
end := i + 1000
|
||||
if end > len(locations) {
|
||||
end = len(locations)
|
||||
}
|
||||
|
||||
objects := make([]s3types.ObjectIdentifier, end-i)
|
||||
for j, key := range locations[i:end] {
|
||||
objects[j] = s3types.ObjectIdentifier{Key: aws.String(key)}
|
||||
}
|
||||
|
||||
_, err := ah.svc.DeleteObjects(ctx, &s3.DeleteObjectsInput{
|
||||
Bucket: aws.String(ah.conf.BucketName),
|
||||
Delete: &s3types.Delete{
|
||||
Objects: objects,
|
||||
Quiet: aws.Bool(true),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func isAPIError(err error, codes ...string) bool {
|
||||
var apiErr smithy.APIError
|
||||
if !errors.As(err, &apiErr) {
|
||||
return false
|
||||
}
|
||||
for _, code := range codes {
|
||||
if apiErr.ErrorCode() == code {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// GetIdFromUrl converts an attahment URL to a file UID.
|
||||
func (ah *awshandler) GetIdFromUrl(url string) types.Uid {
|
||||
return media.GetIdFromUrl(url, ah.conf.ServeURL)
|
||||
}
|
||||
|
||||
// getFileRecord given file ID reads file record from the database.
|
||||
func (ah *awshandler) getFileRecord(tenantID types.TenantID, fid types.Uid) (*types.FileDef, error) {
|
||||
fd, err := store.Files.Get(tenantID, fid.String())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if fd == nil {
|
||||
return nil, types.ErrNotFound
|
||||
}
|
||||
return fd, nil
|
||||
}
|
||||
|
||||
func tenantObjectKey(tenantID types.TenantID, fileID string, createdAt time.Time) (string, error) {
|
||||
if !tenantID.IsValid() || fileID == "" {
|
||||
return "", types.ErrMalformed
|
||||
}
|
||||
if createdAt.IsZero() {
|
||||
createdAt = time.Now().UTC()
|
||||
}
|
||||
return path.Join("tenants", strconv.FormatInt(int64(tenantID), 10),
|
||||
createdAt.Format("2006"), createdAt.Format("01"), fileID), nil
|
||||
}
|
||||
|
||||
func init() {
|
||||
store.RegisterMediaHandler(handlerName, &awshandler{})
|
||||
}
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
package s3
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/tinode/chat/server/store/types"
|
||||
)
|
||||
|
||||
func TestTenantObjectKey(t *testing.T) {
|
||||
got, err := tenantObjectKey(types.TenantID(42), "abc123", time.Date(2026, 8, 11, 9, 0, 0, 0, time.UTC))
|
||||
if err != nil {
|
||||
t.Fatalf("tenantObjectKey failed: %v", err)
|
||||
}
|
||||
if want := "tenants/42/2026/08/abc123"; got != want {
|
||||
t.Fatalf("tenantObjectKey = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTenantObjectKeyRejectsZeroTenant(t *testing.T) {
|
||||
if _, err := tenantObjectKey(types.ZeroTenantID, "abc123", time.Time{}); err != types.ErrMalformed {
|
||||
t.Fatalf("tenantObjectKey err = %v, want malformed", err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user