pro/olons_admin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add Go backend

Browse Source
This commit is contained in:
richjohnrambo
2026-10-10 20:56:17 +08:00
parent 50fcfd2d3d
commit f6024f0964
1040 changed files with 107169 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/internal/library/storager/config.go
+28
View File
@@ -0,0 +1,28 @@
// Package storager
// @Link https://github.com/bufanyun/hotgo
// @Copyright Copyright (c) 2023 HotGo CLI
// @Author Ms <133814250@qq.com>
// @License https://github.com/bufanyun/hotgo/blob/master/LICENSE
package storager
import (
"context"
"hotgo/internal/dao"
"hotgo/internal/model"
"github.com/gogf/gf/v2/database/gdb"
)
var config *model.UploadConfig
func SetConfig(c *model.UploadConfig) {
config = c
}
func GetConfig() *model.UploadConfig {
return config
}
func GetModel(ctx context.Context) *gdb.Model {
return dao.SysAttachment.Ctx(ctx)
}
server/internal/library/storager/mime.go
+232
View File
@@ -0,0 +1,232 @@
// Package storager
// @Link https://github.com/bufanyun/hotgo
// @Copyright Copyright (c) 2023 HotGo CLI
// @Author Ms <133814250@qq.com>
// @License https://github.com/bufanyun/hotgo/blob/master/LICENSE
package storager
import (
"crypto/md5"
"fmt"
"github.com/gogf/gf/v2/errors/gerror"
"github.com/gogf/gf/v2/frame/g"
"github.com/gogf/gf/v2/net/ghttp"
"github.com/gogf/gf/v2/text/gstr"
"io"
"path"
)
// 文件归属分类
const (
KindImg = "image" // 图片
KindDoc = "doc" // 文档
KindAudio = "audio" // 音频
KindVideo = "video" // 视频
KindZip = "zip" // 压缩包
KindOther = "other" // 其他
)
var KindSlice = []string{KindImg, KindDoc, KindAudio, KindVideo, KindZip, KindOther}
var (
// 图片类型
imgType = g.MapStrStr{
"jpeg": "image/jpeg",
"jpg": "image/jpeg",
"png": "image/png",
"gif": "image/gif",
"webp": "image/webp",
"cr2": "image/x-canon-cr2",
"tif": "image/tiff",
"bmp": "image/bmp",
"heif": "image/heif",
"jxr": "image/vnd.ms-photo",
"psd": "image/vnd.adobe.photoshop",
"ico": "image/vnd.microsoft.icon",
"dwg": "image/vnd.dwg",
}
// 文档类型
docType = g.MapStrStr{
"doc": "application/msword",
"docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
"dot": "application/msword",
"xls": "application/vnd.ms-excel",
"xlt": "application/vnd.ms-excel",
"xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
"xltx": "application/vnd.openxmlformats-officedocument.spreadsheetml.template",
"ppt": "application/vnd.ms-powerpoint",
"pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation",
"pdf": "application/pdf",
"txt": "text/plain",
"csv": "text/csv",
"html": "text/html",
"xml": "text/xml",
"pptm": "application/vnd.ms-powerpoint.presentation.macroEnabled.12",
"pot": "application/vnd.ms-powerpoint",
"wpd": "application/wordperfect",
"md": "text/markdown",
"json": "application/json",
"yaml": "application/x-yaml",
"markdown": "text/markdown",
"asciidoc": "text/asciidoc",
"xsl": "application/xml",
"wps": "application/vnd.ms-works",
"sxi": "application/vnd.sun.xml.impress",
"sti": "application/vnd.sun.xml.impress.template",
"odp": "application/vnd.oasis.opendocument.presentation-template",
}
// 音频类型
audioType = g.MapStrStr{
"mid": "audio/midi",
"mp3": "audio/mpeg",
"m4a": "audio/mp4",
"ogg": "audio/ogg",
"flac": "audio/x-flac",
"wav": "audio/x-wav",
"amr": "audio/amr",
"aac": "audio/aac",
"aiff": "audio/x-aiff",
}
// 视频类型
videoType = g.MapStrStr{
"mp4": "video/mp4",
"m4v": "video/x-m4v",
"mkv": "video/x-matroska",
"webm": "video/webm",
"mov": "video/quicktime",
"avi": "video/x-msvideo",
"wmv": "video/x-ms-wmv",
"mpg": "video/mpeg",
"flv": "video/x-flv",
"3gp": "video/3gpp",
}
// 压缩包
zipType = g.MapStrStr{
"zip": "application/zip",
"rar": "application/x-rar-compressed",
"tar": "application/x-tar",
"gz": "application/gzip",
"7z": "application/octet-stream",
"tar.gz": "application/octet-stream",
}
// 其他
otherType = g.MapStrStr{
"dwf": "model/vnd.dwf",
"ics": "text/calendar",
"vcard": "text/vcard",
"apk": "application/vnd.android.package-archive",
"ipa": "application/octet-stream",
}
)
// IsImgType 判断是否为图片
func IsImgType(ext string) bool {
_, ok := imgType[ext]
return ok
}
// IsDocType 判断是否为文档
func IsDocType(ext string) bool {
_, ok := docType[ext]
return ok
}
// IsAudioType 判断是否为音频
func IsAudioType(ext string) bool {
_, ok := audioType[ext]
return ok
}
// IsVideoType 判断是否为视频
func IsVideoType(ext string) bool {
_, ok := videoType[ext]
return ok
}
// IsZipType 判断是否为压缩包
func IsZipType(ext string) bool {
_, ok := zipType[ext]
return ok
}
// GetFileMimeType 获取文件扩展类型
// 如果文件类型没有加入系统映射类型，默认认为不是合法的文件类型。建议将常用的上传文件类型加入映射关系。
// 当然你也可以不做限制，可以上传任意文件。但需要谨慎处理和设置相应的安全措施。
// 获取任意扩展名的扩展类型：mime.TypeByExtension(".xls")
func GetFileMimeType(ext string) string {
if mime, ok := imgType[ext]; ok {
return mime
}
if mime, ok := docType[ext]; ok {
return mime
}
if mime, ok := audioType[ext]; ok {
return mime
}
if mime, ok := videoType[ext]; ok {
return mime
}
if mime, ok := zipType[ext]; ok {
return mime
}
if mime, ok := otherType[ext]; ok {
return mime
}
return "application/octet-stream"
}
// GetFileKind 获取文件上传类型
func GetFileKind(ext string) string {
if _, ok := imgType[ext]; ok {
return KindImg
}
if _, ok := docType[ext]; ok {
return KindDoc
}
if _, ok := audioType[ext]; ok {
return KindAudio
}
if _, ok := videoType[ext]; ok {
return KindVideo
}
if _, ok := zipType[ext]; ok {
return KindZip
}
return KindOther
}
// Ext 获取文件后缀
func Ext(baseName string) string {
return gstr.ToLower(gstr.StrEx(path.Ext(baseName), "."))
}
// UploadFileByte 获取上传文件的byte
func UploadFileByte(file *ghttp.UploadFile) ([]byte, error) {
open, err := file.Open()
if err != nil {
return nil, err
}
return io.ReadAll(open)
}
// CalcFileMd5 计算文件md5值
func CalcFileMd5(file *ghttp.UploadFile) (string, error) {
f, err := file.Open()
if err != nil {
err = gerror.Wrapf(err, `os.Open failed for name "%s"`, file.Filename)
return "", err
}
defer f.Close()
h := md5.New()
_, err = io.Copy(h, f)
if err != nil {
err = gerror.Wrap(err, `io.Copy failed`)
return "", err
}
return fmt.Sprintf("%x", h.Sum(nil)), nil
}
server/internal/library/storager/model.go
+69
View File
@@ -0,0 +1,69 @@
// Package storager
// @Link https://github.com/bufanyun/hotgo
// @Copyright Copyright (c) 2023 HotGo CLI
// @Author Ms <133814250@qq.com>
// @License https://github.com/bufanyun/hotgo/blob/master/LICENSE
package storager
import (
"github.com/gogf/gf/v2/net/ghttp"
"github.com/gogf/gf/v2/os/gtime"
"hotgo/internal/model/entity"
)
// FileMeta 文件元数据
type FileMeta struct {
Filename string `json:"filename"` // 文件名称
Size int64 `json:"size"` // 文件大小
Kind string `json:"kind"` // 文件上传类型
MimeType string `json:"mimeType"` // 文件扩展类型
NaiveType string `json:"naiveType"` // NaiveUI类型
Ext string `json:"ext"` // 文件扩展名
Md5 string `json:"md5"` // 文件hash
}
// MultipartProgress 分片进度
type MultipartProgress struct {
UploadId string `json:"uploadId"` // 上传事件ID
ThirdUploadId string `json:"thirdUploadId"` // 第三方上传事件ID
Meta *FileMeta `json:"meta"` // 文件元数据
ShardCount int `json:"shardCount"` // 分片数量
UploadedIndex []int `json:"uploadedIndex"` // 已上传的分片索引
CreatedAt *gtime.Time `json:"createdAt"` // 创建时间
}
// CheckMultipartParams 检查文件分片
type CheckMultipartParams struct {
UploadType string `json:"uploadType" dc:"文件类型"`
FileName string `json:"fileName" dc:"文件名称"`
Size int64 `json:"size" dc:"文件大小"`
Md5 string `json:"md5" dc:"文件md5值"`
ShardCount int `json:"shardCount" dc:"分片数量"`
meta *FileMeta
}
type CheckMultipartModel struct {
UploadId string `json:"uploadId" dc:"上传事件ID"`
Attachment *entity.SysAttachment `json:"attachment" dc:"附件"`
WaitUploadIndex []int `json:"waitUploadIndex" dc:"等待上传的分片索引"`
Progress float64 `json:"progress" dc:"上传进度"`
SizeFormat string `json:"sizeFormat" dc:"文件大小"`
}
// UploadPartParams 分片上传
type UploadPartParams struct {
UploadId string `json:"uploadId" dc:"上传事件ID"`
UploadType string `json:"uploadType" dc:"文件类型"`
FileName string `json:"fileName" dc:"文件名称"`
Size int64 `json:"size" dc:"文件大小"`
Md5 string `json:"md5" dc:"文件md5值"`
Index int `json:"index" dc:"分片索引"`
File *ghttp.UploadFile `json:"file" type:"file" dc:"分片文件"`
mp *MultipartProgress
}
type UploadPartModel struct {
Attachment *entity.SysAttachment `json:"attachment" dc:"附件"`
Progress float64 `json:"progress" dc:"上传进度"`
Finish bool `json:"finish" dc:"是否完成"`
}
server/internal/library/storager/upload.go
+392
View File
@@ -0,0 +1,392 @@
// Package storager
// @Link https://github.com/bufanyun/hotgo
// @Copyright Copyright (c) 2023 HotGo CLI
// @Author Ms <133814250@qq.com>
// @License https://github.com/bufanyun/hotgo/blob/master/LICENSE
package storager
import (
"context"
"fmt"
"hotgo/internal/consts"
"hotgo/internal/dao"
"hotgo/internal/library/cache"
"hotgo/internal/library/contexts"
"hotgo/internal/model/entity"
"hotgo/utility/convert"
"hotgo/utility/format"
"hotgo/utility/url"
"hotgo/utility/validate"
"strconv"
"strings"
"time"
"github.com/gogf/gf/v2/errors/gerror"
"github.com/gogf/gf/v2/frame/g"
"github.com/gogf/gf/v2/net/ghttp"
"github.com/gogf/gf/v2/os/gtime"
"github.com/gogf/gf/v2/util/grand"
)
// UploadDrive 存储驱动
type UploadDrive interface {
// Upload 上传
Upload(ctx context.Context, file *ghttp.UploadFile) (fullPath string, err error)
// CreateMultipart 创建分片事件
CreateMultipart(ctx context.Context, in *CheckMultipartParams) (res *MultipartProgress, err error)
// UploadPart 上传分片
UploadPart(ctx context.Context, in *UploadPartParams) (res *UploadPartModel, err error)
}
// New 初始化存储驱动
func New(name ...string) UploadDrive {
var (
driveType = consts.UploadDriveLocal
drive UploadDrive
)
if len(name) > 0 && name[0] != "" {
driveType = name[0]
}
switch driveType {
case consts.UploadDriveLocal:
drive = &LocalDrive{}
case consts.UploadDriveUCloud:
drive = &UCloudDrive{}
case consts.UploadDriveCos:
drive = &CosDrive{}
case consts.UploadDriveOss:
drive = &OssDrive{}
case consts.UploadDriveQiNiu:
drive = &QiNiuDrive{}
case consts.UploadDriveMinio:
drive = &MinioDrive{}
default:
panic(fmt.Sprintf("暂不支持的存储驱动:%v", driveType))
}
return drive
}
// DoUpload 上传入口
func DoUpload(ctx context.Context, typ string, file *ghttp.UploadFile) (result *entity.SysAttachment, err error) {
if file == nil {
err = gerror.New("文件必须!")
return
}
meta, err := GetFileMeta(file)
if err != nil {
return
}
if err = ValidateFileMeta(typ, meta); err != nil {
return
}
result, err = HasFile(ctx, meta.Md5)
if err != nil {
return
}
// 相同存储相同身份才复用
if result != nil && result.Drive == config.Drive && result.MemberId == contexts.GetUserId(ctx) && result.AppId == contexts.GetModule(ctx) {
return
}
// 上传到驱动
fullPath, err := New(config.Drive).Upload(ctx, file)
if err != nil {
return
}
// 写入附件记录
return write(ctx, meta, fullPath)
}
// ValidateFileMeta 验证文件元数据
func ValidateFileMeta(typ string, meta *FileMeta) (err error) {
switch typ {
case KindImg:
if !IsImgType(meta.Ext) {
err = gerror.New("上传的文件不是图片")
return
}
if config.ImageSize > 0 && meta.Size > config.ImageSize*1024*1024 {
err = gerror.Newf("图片大小不能超过%vMB", config.ImageSize)
return
}
if len(config.ImageType) > 0 && !validate.InSlice(strings.Split(config.ImageType, `,`), meta.Ext) {
err = gerror.New("上传图片类型未经允许")
return
}
case KindDoc:
if !IsDocType(meta.Ext) {
err = gerror.New("上传的文件不是文档")
return
}
case KindAudio:
if !IsAudioType(meta.Ext) {
err = gerror.New("上传的文件不是音频")
return
}
case KindVideo:
if !IsVideoType(meta.Ext) {
err = gerror.New("上传的文件不是视频")
return
}
case KindZip:
if !IsZipType(meta.Ext) {
err = gerror.New("上传的文件不是压缩文件")
return
}
case KindOther:
fallthrough
default:
// 默认为通用的文件上传
if config.FileSize > 0 && meta.Size > config.FileSize*1024*1024 {
err = gerror.Newf("文件大小不能超过%vMB", config.FileSize)
return
}
if len(config.FileType) > 0 && !validate.InSlice(strings.Split(config.FileType, `,`), meta.Ext) {
err = gerror.Newf("上传文件类型未经允许:%v", meta.Ext)
return
}
}
return
}
// LastUrl 根据驱动获取最终文件访问地址
func LastUrl(ctx context.Context, fullPath, drive string) string {
if validate.IsURL(fullPath) {
return fullPath
}
switch drive {
case consts.UploadDriveLocal:
return url.GetAddr(ctx) + "/" + fullPath
case consts.UploadDriveUCloud:
return config.UCloudEndpoint + "/" + fullPath
case consts.UploadDriveCos:
return config.CosBucketURL + "/" + fullPath
case consts.UploadDriveOss:
return config.OssBucketURL + "/" + fullPath
case consts.UploadDriveQiNiu:
return config.QiNiuDomain + "/" + fullPath
case consts.UploadDriveMinio:
return fmt.Sprintf("%s/%s/%s", config.MinioDomain, config.MinioBucket, fullPath)
default:
return fullPath
}
}
// GetFileMeta 获取上传文件元数据
func GetFileMeta(file *ghttp.UploadFile) (meta *FileMeta, err error) {
meta = new(FileMeta)
meta.Filename = file.Filename
meta.Size = file.Size
meta.Ext = Ext(file.Filename)
meta.Kind = GetFileKind(meta.Ext)
meta.MimeType = GetFileMimeType(meta.Ext)
// 兼容naiveUI
naiveType := meta.MimeType
if len(naiveType) == 0 {
naiveType = "text/plain"
}
meta.NaiveType = naiveType
// 计算md5值
meta.Md5, err = CalcFileMd5(file)
return
}
// GenFullPath 根据目录和文件类型生成一个绝对地址
func GenFullPath(basePath, ext string) string {
fileName := strconv.FormatInt(gtime.TimestampNano(), 36) + grand.S(6)
fileName = fileName + ext
return basePath + gtime.Date() + "/" + strings.ToLower(fileName)
}
// write 写入附件记录
func write(ctx context.Context, meta *FileMeta, fullPath string) (models *entity.SysAttachment, err error) {
models = &entity.SysAttachment{
Id: 0,
AppId: contexts.GetModule(ctx),
MemberId: contexts.GetUserId(ctx),
Drive: config.Drive,
Size: meta.Size,
Path: fullPath,
FileUrl: fullPath,
Name: meta.Filename,
Kind: meta.Kind,
MimeType: meta.MimeType,
NaiveType: meta.NaiveType,
Ext: meta.Ext,
Md5: meta.Md5,
Status: consts.StatusEnabled,
}
id, err := GetModel(ctx).Data(models).OmitEmptyData().InsertAndGetId()
if err != nil {
return
}
models.Id = id
return
}
// HasFile 检查附件是否存在
func HasFile(ctx context.Context, md5 string) (res *entity.SysAttachment, err error) {
if err = GetModel(ctx).Where(dao.SysAttachment.Columns().Md5, md5).Scan(&res); err != nil {
err = gerror.Wrap(err, "检查文件hash时出现错误")
return
}
if res == nil {
return
}
// 只有在上传时才会检查md5值，如果附件存在则更新最后上传时间，保证上传列表更新显示在最前面
if res.Id > 0 {
update := g.Map{
dao.SysAttachment.Columns().Status: consts.StatusEnabled,
dao.SysAttachment.Columns().UpdatedAt: gtime.Now(),
}
_, _ = GetModel(ctx).WherePri(res.Id).Data(update).Update()
}
return
}
// CheckMultipart 检查文件分片
func CheckMultipart(ctx context.Context, in *CheckMultipartParams) (res *CheckMultipartModel, err error) {
res = new(CheckMultipartModel)
meta := new(FileMeta)
meta.Filename = in.FileName
meta.Size = in.Size
meta.Ext = Ext(in.FileName)
meta.Kind = GetFileKind(meta.Ext)
meta.MimeType = GetFileMimeType(meta.Ext)
// 兼容naiveUI
naiveType := "text/plain"
if IsImgType(Ext(in.FileName)) {
naiveType = ""
}
meta.NaiveType = naiveType
meta.Md5 = in.Md5
if err = ValidateFileMeta(in.UploadType, meta); err != nil {
return
}
result, err := HasFile(ctx, in.Md5)
if err != nil {
return nil, err
}
// 文件已存在，直接返回。相同存储相同身份才复用
if result != nil && result.Drive == config.Drive && result.MemberId == contexts.GetUserId(ctx) && result.AppId == contexts.GetModule(ctx) {
res.Attachment = result
return
}
for i := 0; i < in.ShardCount; i++ {
res.WaitUploadIndex = append(res.WaitUploadIndex, i+1)
}
in.meta = meta
progress, err := GetOrCreateMultipartProgress(ctx, in)
if err != nil {
return nil, err
}
if len(progress.UploadedIndex) > 0 {
res.WaitUploadIndex = convert.DifferenceSlice(progress.UploadedIndex, res.WaitUploadIndex)
}
if len(res.WaitUploadIndex) == 0 {
res.WaitUploadIndex = make([]int, 0)
}
res.UploadId = progress.UploadId
res.Progress = CalcUploadProgress(progress.UploadedIndex, progress.ShardCount)
res.SizeFormat = format.FileSize(progress.Meta.Size)
return
}
// CalcUploadProgress 计算上传进度
func CalcUploadProgress(uploadedIndex []int, shardCount int) float64 {
return format.Round2Float64(float64(len(uploadedIndex)) / float64(shardCount) * 100)
}
// GenUploadId 生成上传ID
func GenUploadId(ctx context.Context, md5 string) string {
return fmt.Sprintf("%v:%v:%v@%v", md5, contexts.GetUserId(ctx), contexts.GetModule(ctx), config.Drive)
}
// GetOrCreateMultipartProgress 获取或创建分片上传事件进度
func GetOrCreateMultipartProgress(ctx context.Context, in *CheckMultipartParams) (res *MultipartProgress, err error) {
uploadId := GenUploadId(ctx, in.Md5)
res, err = GetMultipartProgress(ctx, uploadId)
if err != nil {
return nil, err
}
if res != nil {
return res, nil
}
return New(config.Drive).CreateMultipart(ctx, in)
}
// GetMultipartProgress 获取分片上传事件进度
func GetMultipartProgress(ctx context.Context, uploadId string) (res *MultipartProgress, err error) {
key := fmt.Sprintf("%v:%v", consts.CacheMultipartUpload, uploadId)
get, err := cache.Instance().Get(ctx, key)
if err != nil {
return nil, err
}
err = get.Scan(&res)
return
}
// CreateMultipartProgress 创建分片上传事件进度
func CreateMultipartProgress(ctx context.Context, in *MultipartProgress) (err error) {
key := fmt.Sprintf("%v:%v", consts.CacheMultipartUpload, in.UploadId)
return cache.Instance().Set(ctx, key, in, time.Hour*24*7)
}
// UpdateMultipartProgress 更新分片上传事件进度
func UpdateMultipartProgress(ctx context.Context, in *MultipartProgress) (err error) {
key := fmt.Sprintf("%v:%v", consts.CacheMultipartUpload, in.UploadId)
return cache.Instance().Set(ctx, key, in, time.Hour*24*7)
}
// DelMultipartProgress 删除分片上传事件进度
func DelMultipartProgress(ctx context.Context, in *MultipartProgress) (err error) {
key := fmt.Sprintf("%v:%v", consts.CacheMultipartUpload, in.UploadId)
_, err = cache.Instance().Remove(ctx, key)
return
}
// UploadPart 上传分片
func UploadPart(ctx context.Context, in *UploadPartParams) (res *UploadPartModel, err error) {
in.mp, err = GetMultipartProgress(ctx, in.UploadId)
if err != nil {
return nil, err
}
if in.mp == nil {
err = gerror.New("分片事件不存在，请重新上传！")
return
}
if validate.InSlice(in.mp.UploadedIndex, in.Index) {
err = gerror.New("该分片已上传过了")
return
}
res, err = New(config.Drive).UploadPart(ctx, in)
if err != nil {
return nil, err
}
return
}
server/internal/library/storager/upload_cos.go
+59
View File
@@ -0,0 +1,59 @@
// Package storager
// @Link https://github.com/bufanyun/hotgo
// @Copyright Copyright (c) 2023 HotGo CLI
// @Author Ms <133814250@qq.com>
// @License https://github.com/bufanyun/hotgo/blob/master/LICENSE
package storager
import (
"context"
"github.com/gogf/gf/v2/errors/gerror"
"github.com/gogf/gf/v2/net/ghttp"
"github.com/gogf/gf/v2/os/gfile"
"github.com/tencentyun/cos-go-sdk-v5"
"net/http"
"net/url"
)
// CosDrive 腾讯云cos驱动
type CosDrive struct {
}
// Upload 上传到腾讯云cos对象存储
func (d *CosDrive) Upload(ctx context.Context, file *ghttp.UploadFile) (fullPath string, err error) {
if config.CosPath == "" {
err = gerror.New("COS存储驱动必须配置存储路径!")
return
}
// 流式上传本地小文件
f2, err := file.Open()
defer func() { _ = f2.Close() }()
if err != nil {
return
}
URL, _ := url.Parse(config.CosBucketURL)
client := cos.NewClient(&cos.BaseURL{BucketURL: URL}, &http.Client{
Transport: &cos.AuthorizationTransport{
SecretID: config.CosSecretId,
SecretKey: config.CosSecretKey,
},
})
fullPath = GenFullPath(config.CosPath, gfile.Ext(file.Filename))
_, err = client.Object.Put(ctx, fullPath, f2, nil)
return
}
// CreateMultipart 创建分片事件
func (d *CosDrive) CreateMultipart(ctx context.Context, in *CheckMultipartParams) (res *MultipartProgress, err error) {
err = gerror.New("当前驱动暂不支持分片上传！")
return
}
// UploadPart 上传分片
func (d *CosDrive) UploadPart(ctx context.Context, in *UploadPartParams) (res *UploadPartModel, err error) {
err = gerror.New("当前驱动暂不支持分片上传！")
return
}
server/internal/library/storager/upload_local.go
+156
View File
@@ -0,0 +1,156 @@
// Package storager
// @Link https://github.com/bufanyun/hotgo
// @Copyright Copyright (c) 2023 HotGo CLI
// @Author Ms <133814250@qq.com>
// @License https://github.com/bufanyun/hotgo/blob/master/LICENSE
package storager
import (
"context"
"github.com/gogf/gf/v2/errors/gerror"
"github.com/gogf/gf/v2/frame/g"
"github.com/gogf/gf/v2/net/ghttp"
"github.com/gogf/gf/v2/os/gfile"
"github.com/gogf/gf/v2/os/gtime"
"github.com/gogf/gf/v2/util/gconv"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
)
// LocalDrive 本地驱动
type LocalDrive struct {
}
// Upload 上传到本地
func (d *LocalDrive) Upload(ctx context.Context, file *ghttp.UploadFile) (fullPath string, err error) {
var (
sp = g.Cfg().MustGet(ctx, "server.serverRoot")
nowDate = gtime.Date()
)
if sp.IsEmpty() {
err = gerror.New("本地上传驱动必须配置静态路径!")
return
}
if config.LocalPath == "" {
err = gerror.New("本地上传驱动必须配置本地存储路径!")
return
}
// 包含静态文件夹的路径
fullDirPath := strings.Trim(sp.String(), "/") + "/" + config.LocalPath + nowDate
fileName, err := file.Save(fullDirPath, true)
if err != nil {
return
}
// 不含静态文件夹的路径
fullPath = config.LocalPath + nowDate + "/" + fileName
return
}
// CreateMultipart 创建分片事件
func (d *LocalDrive) CreateMultipart(ctx context.Context, in *CheckMultipartParams) (mp *MultipartProgress, err error) {
mp = new(MultipartProgress)
mp.UploadId = GenUploadId(ctx, in.Md5)
mp.Meta = in.meta
mp.ShardCount = in.ShardCount
mp.UploadedIndex = make([]int, 0)
mp.CreatedAt = gtime.Now()
if err = CreateMultipartProgress(ctx, mp); err != nil {
return nil, err
}
return
}
// UploadPart 上传分片
func (d *LocalDrive) UploadPart(ctx context.Context, in *UploadPartParams) (res *UploadPartModel, err error) {
sp := g.Cfg().MustGet(ctx, "server.serverRoot")
if sp.IsEmpty() {
err = gerror.New("本地上传驱动必须配置静态路径!")
return
}
spStr := strings.Trim(sp.String(), "/") + "/"
if config.LocalPath == "" {
err = gerror.New("本地上传驱动必须配置本地存储路径!")
return
}
// 分片文件存放路径
partFilePath := spStr + config.LocalPath + "tmp/" + in.Md5
// 写入文件
in.File.Filename = gconv.String(in.Index)
if _, err = in.File.Save(partFilePath, false); err != nil {
return
}
// 更新上传进度
in.mp.UploadedIndex = append(in.mp.UploadedIndex, in.Index)
if err = UpdateMultipartProgress(ctx, in.mp); err != nil {
return nil, err
}
res = new(UploadPartModel)
// 已全部上传完毕
if len(in.mp.UploadedIndex) == in.mp.ShardCount {
// 删除进度统计
if err = DelMultipartProgress(ctx, in.mp); err != nil {
return nil, err
}
// 合并文件
finalDirPath := GenFullPath(config.LocalPath, gfile.Ext(in.mp.Meta.Filename))
if err = MergePartFile(partFilePath, spStr+finalDirPath); err != nil {
err = gerror.Newf("合并分片文件出错:%v", err.Error())
return nil, err
}
// 删除临时分片
if err = os.RemoveAll(partFilePath); err != nil {
err = gerror.Newf("删除临时分片文件出错:%v", err.Error())
return nil, err
}
// 写入附件记录
attachment, err := write(ctx, in.mp.Meta, finalDirPath)
if err != nil {
return nil, err
}
res.Finish = true
res.Progress = 100
res.Attachment = attachment
return res, nil
}
// 计算上传进度
res.Progress = CalcUploadProgress(in.mp.UploadedIndex, in.mp.ShardCount)
return
}
// MergePartFile 合并分片文件
func MergePartFile(srcPath, dstPath string) (err error) {
dir, err := os.ReadDir(srcPath)
if err != nil {
return err
}
sort.Slice(dir, func(i, j int) bool {
fiIndex, _ := strconv.Atoi(dir[i].Name())
fjIndex, _ := strconv.Atoi(dir[j].Name())
return fiIndex < fjIndex
})
for _, file := range dir {
filePath := filepath.Join(srcPath, file.Name())
if err = gfile.PutBytesAppend(dstPath, gfile.GetBytes(filePath)); err != nil {
return err
}
}
return
}
server/internal/library/storager/upload_local_test.go
+61
View File
@@ -0,0 +1,61 @@
package storager
import (
"crypto/md5"
"errors"
"fmt"
"io"
"os"
"testing"
)
func TestMergePartFile(t *testing.T) {
srcFile, _ := os.Open("1.zip")
defer srcFile.Close()
fileInfo, err := srcFile.Stat()
if err != nil {
t.Error(err)
}
hash := md5.New()
_, err = io.Copy(hash, srcFile)
sum := hash.Sum(nil)
fileHash := fmt.Sprintf("%x", sum)
var singleSize = int64(1024 * 1024 * 100)
chunks := int(fileInfo.Size() / singleSize)
if fileInfo.Size()%singleSize != 0 {
chunks += 1
}
srcFile.Seek(0, io.SeekStart)
for j := 0; j < chunks; j++ {
partSize := singleSize
if j == chunks-1 {
partSize = fileInfo.Size() - int64(j)*singleSize
}
partData := make([]byte, partSize)
_, err = io.ReadFull(srcFile, partData)
pf, _ := os.Create(fmt.Sprintf("tmp/%d", j+1))
_, err = pf.Write(partData)
if err != nil {
t.Error(err)
return
}
pf.Close()
}
err = MergePartFile("tmp/", "2.zip")
if err != nil {
t.Error(err)
return
}
f2, _ := os.Open("2.zip")
hash2 := md5.New()
_, err = io.Copy(hash2, f2)
sum2 := hash.Sum(nil)
fileHash2 := fmt.Sprintf("%x", sum2)
if fileHash != fileHash2 {
t.Error(errors.New("hash mismatch"))
}
}
server/internal/library/storager/upload_minio.go
+76
View File
@@ -0,0 +1,76 @@
// Package storager
// @Link https://github.com/bufanyun/hotgo
// @Copyright Copyright (c) 2023 HotGo CLI
// @Author Ms <133814250@qq.com>
// @License https://github.com/bufanyun/hotgo/blob/master/LICENSE
package storager
import (
"context"
"github.com/gogf/gf/v2/errors/gerror"
"github.com/gogf/gf/v2/net/ghttp"
"github.com/gogf/gf/v2/os/gfile"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
"github.com/minio/minio-go/v7/pkg/s3utils"
"mime"
"path/filepath"
)
// MinioDrive minio对象存储驱动
type MinioDrive struct {
}
// Upload 上传到minio对象存储
func (d *MinioDrive) Upload(ctx context.Context, file *ghttp.UploadFile) (fullPath string, err error) {
if config.MinioPath == "" {
err = gerror.New("minio存储驱动必须配置存储路径!")
return
}
client, err := minio.New(config.MinioEndpoint, &minio.Options{
Creds: credentials.NewStaticV4(config.MinioAccessKey, config.MinioSecretKey, ""),
Secure: config.MinioUseSSL == 1,
})
if err != nil {
return "", err
}
if err = s3utils.CheckValidBucketName(config.MinioBucket); err != nil {
return
}
fullPath = GenFullPath(config.MinioPath, gfile.Ext(file.Filename))
if err = s3utils.CheckValidObjectName(fullPath); err != nil {
return
}
reader, err := file.Open()
if err != nil {
return "", err
}
defer reader.Close()
opts := minio.PutObjectOptions{
ContentType: mime.TypeByExtension(filepath.Ext(file.Filename)),
}
if opts.ContentType == "" {
opts.ContentType = "application/octet-stream"
}
_, err = client.PutObject(ctx, config.MinioBucket, fullPath, reader, file.Size, opts)
return
}
// CreateMultipart 创建分片事件
func (d *MinioDrive) CreateMultipart(ctx context.Context, in *CheckMultipartParams) (res *MultipartProgress, err error) {
err = gerror.New("当前驱动暂不支持分片上传！")
return
}
// UploadPart 上传分片
func (d *MinioDrive) UploadPart(ctx context.Context, in *UploadPartParams) (res *UploadPartModel, err error) {
err = gerror.New("当前驱动暂不支持分片上传！")
return
}
server/internal/library/storager/upload_oss.go
+59
View File
@@ -0,0 +1,59 @@
// Package storager
// @Link https://github.com/bufanyun/hotgo
// @Copyright Copyright (c) 2023 HotGo CLI
// @Author Ms <133814250@qq.com>
// @License https://github.com/bufanyun/hotgo/blob/master/LICENSE
package storager
import (
"context"
"github.com/aliyun/aliyun-oss-go-sdk/oss"
"github.com/gogf/gf/v2/errors/gerror"
"github.com/gogf/gf/v2/net/ghttp"
"github.com/gogf/gf/v2/os/gfile"
)
// OssDrive 阿里云oss驱动
type OssDrive struct {
}
// Upload 上传到阿里云oss
func (d *OssDrive) Upload(ctx context.Context, file *ghttp.UploadFile) (fullPath string, err error) {
if config.OssPath == "" {
err = gerror.New("OSS存储驱动必须配置存储路径!")
return
}
// 流式上传本地小文件
f2, err := file.Open()
defer func() { _ = f2.Close() }()
if err != nil {
return
}
client, err := oss.New(config.OssEndpoint, config.OssSecretId, config.OssSecretKey)
if err != nil {
return
}
bucket, err := client.Bucket(config.OssBucket)
if err != nil {
return
}
fullPath = GenFullPath(config.OssPath, gfile.Ext(file.Filename))
err = bucket.PutObject(fullPath, f2)
return
}
// CreateMultipart 创建分片事件
func (d *OssDrive) CreateMultipart(ctx context.Context, in *CheckMultipartParams) (res *MultipartProgress, err error) {
err = gerror.New("当前驱动暂不支持分片上传！")
return
}
// UploadPart 上传分片
func (d *OssDrive) UploadPart(ctx context.Context, in *UploadPartParams) (res *UploadPartModel, err error) {
err = gerror.New("当前驱动暂不支持分片上传！")
return
}
server/internal/library/storager/upload_qiniu.go
+69
View File
@@ -0,0 +1,69 @@
// Package storager
// @Link https://github.com/bufanyun/hotgo
// @Copyright Copyright (c) 2023 HotGo CLI
// @Author Ms <133814250@qq.com>
// @License https://github.com/bufanyun/hotgo/blob/master/LICENSE
package storager
import (
"context"
"github.com/gogf/gf/v2/errors/gerror"
"github.com/gogf/gf/v2/net/ghttp"
"github.com/gogf/gf/v2/os/gfile"
"github.com/qiniu/go-sdk/v7/auth/qbox"
"github.com/qiniu/go-sdk/v7/storage"
)
// QiNiuDrive 七牛云对象存储驱动
type QiNiuDrive struct {
}
// Upload 上传到七牛云对象存储
func (d *QiNiuDrive) Upload(ctx context.Context, file *ghttp.UploadFile) (fullPath string, err error) {
if config.QiNiuPath == "" {
err = gerror.New("七牛云存储驱动必须配置存储路径!")
return
}
// 流式上传本地小文件
f2, err := file.Open()
defer func() { _ = f2.Close() }()
if err != nil {
return
}
putPolicy := storage.PutPolicy{
Scope: config.QiNiuBucket,
}
token := putPolicy.UploadToken(qbox.NewMac(config.QiNiuAccessKey, config.QiNiuSecretKey))
cfg := storage.Config{}
// 是否使用https域名
cfg.UseHTTPS = true
// 上传是否使用CDN上传加速
cfg.UseCdnDomains = false
// 空间对应的机房
cfg.Region, err = storage.GetRegion(config.QiNiuAccessKey, config.QiNiuBucket)
if err != nil {
return
}
fullPath = GenFullPath(config.QiNiuPath, gfile.Ext(file.Filename))
err = storage.NewFormUploader(&cfg).Put(ctx, &storage.PutRet{}, token, fullPath, f2, file.Size, &storage.PutExtra{})
return
}
// CreateMultipart 创建分片事件
func (d *QiNiuDrive) CreateMultipart(ctx context.Context, in *CheckMultipartParams) (res *MultipartProgress, err error) {
err = gerror.New("当前驱动暂不支持分片上传！")
return
}
// UploadPart 上传分片
func (d *QiNiuDrive) UploadPart(ctx context.Context, in *UploadPartParams) (res *UploadPartModel, err error) {
err = gerror.New("当前驱动暂不支持分片上传！")
return
}
server/internal/library/storager/upload_ucloud.go
+62
View File
@@ -0,0 +1,62 @@
// Package storager
// @Link https://github.com/bufanyun/hotgo
// @Copyright Copyright (c) 2023 HotGo CLI
// @Author Ms <133814250@qq.com>
// @License https://github.com/bufanyun/hotgo/blob/master/LICENSE
package storager
import (
"context"
"github.com/gogf/gf/v2/errors/gerror"
"github.com/gogf/gf/v2/net/ghttp"
"github.com/gogf/gf/v2/os/gfile"
upload "github.com/ufilesdk-dev/ufile-gosdk"
)
// UCloudDrive UCloud对象存储驱动
type UCloudDrive struct {
}
// Upload 上传到UCloud对象存储
func (d *UCloudDrive) Upload(ctx context.Context, file *ghttp.UploadFile) (fullPath string, err error) {
if config.UCloudPath == "" {
err = gerror.New("UCloud存储驱动必须配置存储路径!")
return
}
client, err := upload.NewFileRequest(&upload.Config{
PublicKey: config.UCloudPublicKey,
PrivateKey: config.UCloudPrivateKey,
BucketHost: config.UCloudBucketHost,
BucketName: config.UCloudBucketName,
FileHost: config.UCloudFileHost,
Endpoint: config.UCloudEndpoint,
VerifyUploadMD5: false,
}, nil)
if err != nil {
return
}
// 流式上传本地小文件
f2, err := file.Open()
defer func() { _ = f2.Close() }()
if err != nil {
return
}
fullPath = GenFullPath(config.UCloudPath, gfile.Ext(file.Filename))
err = client.IOPut(f2, fullPath, "")
return
}
// CreateMultipart 创建分片事件
func (d *UCloudDrive) CreateMultipart(ctx context.Context, in *CheckMultipartParams) (res *MultipartProgress, err error) {
err = gerror.New("当前驱动暂不支持分片上传！")
return
}
// UploadPart 上传分片
func (d *UCloudDrive) UploadPart(ctx context.Context, in *UploadPartParams) (res *UploadPartModel, err error) {
err = gerror.New("当前驱动暂不支持分片上传！")
return
}