im_server/server/media/fs/filesys.go

// Package fs implements github.com/tinode/chat/server/media interface by storing media objects in a single
// directory in the file system.
// This module won't perform well with tens of thousand of files because it stores all files in a single directory.
package fs

import (
	"encoding/base32"
	"encoding/json"
	"errors"
	"hash/fnv"
	"io"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/tinode/chat/server/logs"
	"github.com/tinode/chat/server/media"
	"github.com/tinode/chat/server/store"
	"github.com/tinode/chat/server/store/types"
)

const (
	defaultServeURL     = "/v0/file/s/"
	defaultCacheControl = "max-age=86400"

	handlerName = "fs"
)

type fileConfig struct {
	// FileUploadDirectory: In case of a cluster fileUploadLocation must be accessible to all cluster members.
	FileUploadDirectory string   `json:"upload_dir"`
	ServeURL            string   `json:"serve_url"`
	CorsOrigins         []string `json:"cors_origins"`
	CacheControl        string   `json:"cache_control"`
}

type fshandler struct {
	fileConfig
	// corsOrigins parsed allowed origins.
	corsOrigins []media.AllowedOrigin
}

func (fh *fshandler) Init(jsconf string) error {
	var err error

	if err = json.Unmarshal([]byte(jsconf), &fh.fileConfig); err != nil {
		return errors.New("failed to parse config: " + err.Error())
	}

	if fh.FileUploadDirectory == "" {
		return errors.New("missing upload location")
	}

	if fh.ServeURL == "" {
		fh.ServeURL = defaultServeURL
	}

	if fh.CacheControl == "" {
		fh.CacheControl = defaultCacheControl
	}

	fh.corsOrigins, err = media.ParseCORSAllow(fh.CorsOrigins)
	if err != nil {
		return errors.New("failed to parse CORS allowed origins: " + err.Error())
	}
	// Make sure the upload directory exists.
	return os.MkdirAll(fh.FileUploadDirectory, 0777)
}

// Headers is used for cache management and serving CORS headers.
func (fh *fshandler) Headers(tenantID types.TenantID, method string, url *url.URL, headers http.Header, serve bool) (http.Header, int, error) {
	if method == http.MethodGet {

		fid := fh.GetIdFromUrl(url.String())
		if fid.IsZero() {
			return nil, 0, types.ErrNotFound
		}

		fdef, err := fh.getFileRecord(tenantID, fid)
		if err != nil {
			return nil, 0, err
		}

		if etag := strings.Trim(headers.Get("If-None-Match"), "\""); etag != "" && etag == fdef.ETag {
			return http.Header{
					"Last-Modified": {fdef.UpdatedAt.Format(http.TimeFormat)},
					"ETag":          {`"` + fdef.ETag + `"`},
					"Cache-Control": {fh.CacheControl},
				},
				http.StatusNotModified, nil
		}

		return http.Header{
			"Content-Type":  {fdef.MimeType},
			"Cache-Control": {fh.CacheControl},
			"ETag":          {`"` + fdef.ETag + `"`},
		}, 0, nil
	}

	if method != http.MethodOptions {
		// Not an OPTIONS request. No special handling for all other requests.
		return nil, 0, nil
	}
	header, status := media.CORSHandler(method, headers, fh.corsOrigins, serve)
	return header, status, nil
}

// Upload processes request for file upload. The file is given as io.Reader.
func (fh *fshandler) Upload(fdef *types.FileDef, file io.Reader) (string, int64, error) {
	location, err := tenantFileLocation(fh.FileUploadDirectory, fdef.TenantID, fdef.Uid().String32(), fdef.CreatedAt)
	if err != nil {
		return "", 0, err
	}
	if err = os.MkdirAll(filepath.Dir(location), 0777); err != nil {
		return "", 0, err
	}

	outfile, err := os.Create(location)
	if err != nil {
		logs.Warn.Println("Upload: failed to create file", fdef.Location, err)
		return "", 0, err
	}

	if err = store.Files.StartUpload(fdef.TenantID, fdef); err != nil {
		outfile.Close()
		os.Remove(location)
		logs.Warn.Println("failed to create file record", fdef.Id, err)
		return "", 0, err
	}

	size, err := io.Copy(outfile, file)
	outfile.Close()
	if err != nil {
		os.Remove(location)
		return "", 0, err
	}

	fname := media.FileURLName(fdef)

	fdef.Location = location
	// Use file path to create ETag. File paths are unique so will be the ETag.
	fdef.ETag = etagFromPath(fdef.Location)

	return fh.ServeURL + fname, size, nil
}

func (fh *fshandler) UploadPublic(fdef *media.PublicFileDef, file io.Reader) (string, int64, error) {
	location, err := publicFileLocation(fh.FileUploadDirectory, fdef.Purpose, fdef.ID, fdef.CreatedAt)
	if err != nil {
		return "", 0, err
	}
	if err = os.MkdirAll(filepath.Dir(location), 0777); err != nil {
		return "", 0, err
	}
	outfile, err := os.Create(location)
	if err != nil {
		return "", 0, err
	}
	size, err := io.Copy(outfile, file)
	outfile.Close()
	if err != nil {
		os.Remove(location)
		return "", 0, err
	}
	fdef.Location = location
	fdef.ETag = etagFromPath(location)
	return location, size, nil
}

// Download processes request for file download.
// The returned ReadSeekCloser must be closed after use.
func (fh *fshandler) Download(tenantID types.TenantID, url string) (*types.FileDef, media.ReadSeekCloser, error) {
	fid := fh.GetIdFromUrl(url)
	if fid.IsZero() {
		return nil, nil, types.ErrNotFound
	}

	fd, err := fh.getFileRecord(tenantID, fid)
	if err != nil {
		logs.Warn.Println("Download: file not found", fid)
		return nil, nil, err
	}

	file, err := os.Open(fd.Location)
	if err != nil {
		if os.IsNotExist(err) {
			// If the file is not found, send 404 instead of the default 500
			err = types.ErrNotFound
		}
		return nil, nil, err
	}

	return fd, file, nil
}

// Delete deletes files from storage by provided slice of locations.
func (fh *fshandler) Delete(locations []string) error {
	for _, loc := range locations {
		if err, _ := os.Remove(loc).(*os.PathError); err != nil {
			if err != os.ErrNotExist {
				logs.Warn.Println("fs: error deleting file", loc, err)
			}
		}
	}
	return nil
}

// GetIdFromUrl converts an attahment URL to a file UID.
func (fh *fshandler) GetIdFromUrl(url string) types.Uid {
	return media.GetIdFromUrl(url, fh.ServeURL)
}

// getFileRecord given file ID reads file record from the database.
func (fh *fshandler) getFileRecord(tenantID types.TenantID, fid types.Uid) (*types.FileDef, error) {
	fd, err := store.Files.Get(tenantID, fid.String())
	if err != nil {
		return nil, err
	}
	if fd == nil {
		return nil, types.ErrNotFound
	}
	return fd, nil
}

func etagFromPath(path string) string {
	hasher := fnv.New128()
	hasher.Write([]byte(path))
	return strings.ToLower(base32.StdEncoding.WithPadding(base32.NoPadding).
		EncodeToString(hasher.Sum(make([]byte, 0, hasher.Size()))))
}

func tenantFileLocation(root string, tenantID types.TenantID, fileID string, createdAt time.Time) (string, error) {
	if !tenantID.IsValid() || fileID == "" {
		return "", types.ErrMalformed
	}
	if createdAt.IsZero() {
		createdAt = time.Now().UTC()
	}
	return filepath.Join(root, "tenants", strconv.FormatInt(int64(tenantID), 10),
		createdAt.Format("2006"), createdAt.Format("01"), fileID), nil
}

func publicFileLocation(root, purpose, fileID string, createdAt time.Time) (string, error) {
	purpose = cleanPublicPurpose(purpose)
	fileID = strings.Trim(fileID, `/\`)
	if root == "" || purpose == "" || fileID == "" {
		return "", types.ErrMalformed
	}
	if createdAt.IsZero() {
		createdAt = time.Now().UTC()
	}
	return filepath.Join(root, "public", purpose, createdAt.Format("2006"), createdAt.Format("01"), fileID), nil
}

func cleanPublicPurpose(purpose string) string {
	purpose = strings.Trim(strings.ToLower(strings.TrimSpace(purpose)), `/\`)
	return strings.Map(func(r rune) rune {
		if r >= 'a' && r <= 'z' || r >= '0' && r <= '9' || r == '_' || r == '-' {
			return r
		}
		return -1
	}, purpose)
}

func init() {
	store.RegisterMediaHandler(handlerName, &fshandler{})
}