From 192ab342f6e9e7656c06822da21896f217dc2eda Mon Sep 17 00:00:00 2001 From: richjohnrambo Date: Tue, 29 Sep 2026 16:50:36 +0800 Subject: [PATCH] fix: upload ai table exports to oss --- go.mod | 4 +- go.sum | 8 +- internal/app/app.go | 2 +- internal/httpapi/ai.go | 12 ++- internal/httpapi/ai_export_oss.go | 159 ++++++++++++++++++++++++++++++ internal/httpapi/api.go | 5 +- internal/store/ai.go | 40 ++++++++ 7 files changed, 220 insertions(+), 10 deletions(-) create mode 100644 internal/httpapi/ai_export_oss.go diff --git a/go.mod b/go.mod index d9a931a..5fb923e 100644 --- a/go.mod +++ b/go.mod @@ -3,11 +3,13 @@ module git.118im.com/im_team/im_workbench go 1.26.0 require ( + github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible github.com/go-sql-driver/mysql v1.10.0 github.com/jmoiron/sqlx v1.4.0 github.com/prometheus/client_golang v1.23.2 github.com/redis/go-redis/v9 v9.12.1 github.com/segmentio/kafka-go v0.4.51 + github.com/xuri/excelize/v2 v2.11.0 golang.org/x/crypto v0.53.0 ) @@ -30,11 +32,11 @@ require ( github.com/xdg-go/scram v1.1.2 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect github.com/xuri/efp v0.0.1 // indirect - github.com/xuri/excelize/v2 v2.11.0 // indirect github.com/xuri/nfp v0.0.2-0.20250530014748-2ddeb826f9a9 // indirect go.yaml.in/yaml/v2 v2.4.2 // indirect golang.org/x/net v0.56.0 // indirect golang.org/x/sys v0.46.0 // indirect golang.org/x/text v0.38.0 // indirect + golang.org/x/time v0.16.0 // indirect google.golang.org/protobuf v1.36.8 // indirect ) diff --git a/go.sum b/go.sum index 690c346..98a31f9 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,8 @@ filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo= filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc= +github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F7Y6wUM49Lcha2FMXt4UM8g= +github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= @@ -82,12 +84,12 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= +golang.org/x/image v0.38.0 h1:5l+q+Y9JDC7mBOMjo4/aPhMDcxEptsX+Tt3GgRQRPuE= +golang.org/x/image v0.38.0/go.mod h1:/3f6vaXC+6CEanU4KJxbcUZyEePbyKbaLoDOe4ehFYY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= -golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -107,6 +109,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= +golang.org/x/time v0.16.0 h1:vMb6ptszcQMkcwiRTAuNNU50gom6++Q/6gY2hDM6VDE= +golang.org/x/time v0.16.0/go.mod h1:rVKOqvZeKvrDKTQiAHJ7wmwP0RzleSphoEA9RcdLA0s= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= diff --git a/internal/app/app.go b/internal/app/app.go index 236c990..cf71aee 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -64,7 +64,7 @@ func New(cfg *config.Config) (*App, error) { store: st, redis: redisClient, events: publisher, - api: httpapi.New(st, auth.New(cfg.Auth, redisClient), publisher, cfg.Auth, aiService), + api: httpapi.New(st, auth.New(cfg.Auth, redisClient), publisher, cfg.Auth, cfg.Manager, aiService), }, nil } diff --git a/internal/httpapi/ai.go b/internal/httpapi/ai.go index e251b58..b42ed9f 100644 --- a/internal/httpapi/ai.go +++ b/internal/httpapi/ai.go @@ -443,11 +443,15 @@ func (a *API) aiTableAnalysisResource(w http.ResponseWriter, r *http.Request, p writeStoreError(w, r, err) return } - fileName := aiTableExportFileName(item) + file, err := a.uploadAITableAnalysisExport(r.Context(), r, st, p, item) + if err != nil { + writeError(w, r, http.StatusServiceUnavailable, 30053, reasonAIProviderFailed, "ai export storage failed", nil) + return + } writeSuccess(w, r, map[string]any{ - "file_id": "ai_export_" + recordID, - "file_name": fileName, - "download_url": absoluteURL(r, "/v1/workbench/ai/table-analyses/"+recordID+"/export/download?format=docx"), + "file_id": file.FileID, + "file_name": file.FileName, + "download_url": file.DownloadURL, "expires_at": time.Now().UTC().Add(24 * time.Hour).Format(time.RFC3339Nano), }) return diff --git a/internal/httpapi/ai_export_oss.go b/internal/httpapi/ai_export_oss.go new file mode 100644 index 0000000..e46fff6 --- /dev/null +++ b/internal/httpapi/ai_export_oss.go @@ -0,0 +1,159 @@ +package httpapi + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/base32" + "encoding/base64" + "encoding/binary" + "fmt" + "net/http" + "path" + "strconv" + "strings" + "time" + + "git.118im.com/im_team/im_workbench/internal/auth" + "git.118im.com/im_team/im_workbench/internal/managerconfig" + "git.118im.com/im_team/im_workbench/internal/store" + aliyunoss "github.com/aliyun/aliyun-oss-go-sdk/oss" +) + +const aiTableExportDocxMIME = "application/vnd.openxmlformats-officedocument.wordprocessingml.document" + +type aiExportedFile struct { + FileID string + FileName string + DownloadURL string +} + +type ossUploadConfig struct { + Enabled bool + Endpoint string + Bucket string + AccessKeyID string + AccessKeySecret string + ServeURL string + CacheControl string +} + +func (a *API) uploadAITableAnalysisExport(ctx context.Context, r *http.Request, st *store.Store, p *auth.Principal, item *store.AITableAnalysis) (*aiExportedFile, error) { + cfg, err := a.ossUploadConfig(ctx) + if err != nil { + return nil, err + } + data, err := aiTableAnalysisDocx(item) + if err != nil { + return nil, err + } + fileID := newFileUploadID() + publicID, storageID := formatFileUploadID(fileID) + createdAt := time.Now().UTC() + objectKey := path.Join("tenants", strconv.FormatInt(p.TenantID, 10), createdAt.Format("2006"), createdAt.Format("01"), storageID) + client, err := aliyunoss.New(cfg.Endpoint, cfg.AccessKeyID, cfg.AccessKeySecret) + if err != nil { + return nil, err + } + bucket, err := client.Bucket(cfg.Bucket) + if err != nil { + return nil, err + } + options := []aliyunoss.Option{aliyunoss.ContentType(aiTableExportDocxMIME)} + if cfg.CacheControl != "" { + options = append(options, aliyunoss.CacheControl(cfg.CacheControl)) + } + if err := bucket.PutObject(objectKey, bytes.NewReader(data), options...); err != nil { + return nil, err + } + etag := "" + if meta, err := bucket.GetObjectDetailedMeta(objectKey); err == nil { + etag = strings.Trim(meta.Get("ETag"), `"`) + } + fileName := aiTableExportFileName(item) + fileURLName := publicID + ".docx" + if err := st.SaveCompletedFileUpload(ctx, store.CompletedFileUpload{ + ID: fileID, UserID: p.UserID, MIMEType: aiTableExportDocxMIME, Size: int64(len(data)), + ETag: etag, Location: objectKey, CreatedAt: createdAt, + }); err != nil { + return nil, err + } + if err := st.UpdateAITableAnalysisExportFileID(ctx, p.UserID, item.RecordID, fileURLName); err != nil { + return nil, err + } + return &aiExportedFile{ + FileID: fileURLName, + FileName: fileName, + DownloadURL: publicFileDownloadURL(r, cfg.ServeURL, fileURLName), + }, nil +} + +func (a *API) ossUploadConfig(ctx context.Context) (ossUploadConfig, error) { + values, err := managerconfig.New(a.manager).ThirdPartyConfig(ctx, "im_third_party", + "oss_enabled", + "oss_endpoint", + "oss_bucket", + "oss_access_key_id", + "oss_access_key_secret", + "oss_serve_url", + "oss_cache_control", + ) + if err != nil { + return ossUploadConfig{}, err + } + cfg := ossUploadConfig{ + Enabled: parseConfigBool(values["oss_enabled"]), + Endpoint: strings.TrimSpace(values["oss_endpoint"]), + Bucket: strings.TrimSpace(values["oss_bucket"]), + AccessKeyID: strings.TrimSpace(values["oss_access_key_id"]), + AccessKeySecret: strings.TrimSpace(values["oss_access_key_secret"]), + ServeURL: strings.TrimSpace(values["oss_serve_url"]), + CacheControl: strings.TrimSpace(values["oss_cache_control"]), + } + if cfg.ServeURL == "" { + cfg.ServeURL = "/v0/file/s/" + } + if !cfg.Enabled || cfg.Endpoint == "" || cfg.Bucket == "" || cfg.AccessKeyID == "" || cfg.AccessKeySecret == "" { + return ossUploadConfig{}, fmt.Errorf("oss config incomplete") + } + return cfg, nil +} + +func parseConfigBool(value string) bool { + switch strings.ToLower(strings.TrimSpace(value)) { + case "1", "true", "yes", "on", "enabled": + return true + default: + return false + } +} + +func newFileUploadID() int64 { + var random [8]byte + if _, err := rand.Read(random[:]); err != nil { + return time.Now().UnixNano() + } + suffix := int64(binary.LittleEndian.Uint64(random[:]) & ((1 << 20) - 1)) + return (time.Now().UnixMilli() << 20) | suffix +} + +func formatFileUploadID(id int64) (publicID string, storageID string) { + encrypted := encryptUID(id) + publicID = base64.URLEncoding.WithPadding(base64.NoPadding).EncodeToString(encrypted) + storageID = strings.ToLower(base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(encrypted)) + return publicID, storageID +} + +func publicFileDownloadURL(r *http.Request, serveURL, fileName string) string { + serveURL = strings.TrimSpace(serveURL) + if strings.HasPrefix(serveURL, "http://") || strings.HasPrefix(serveURL, "https://") { + return strings.TrimRight(serveURL, "/") + "/" + fileName + } + if serveURL == "" { + serveURL = "/v0/file/s/" + } + if !strings.HasPrefix(serveURL, "/") { + serveURL = "/" + serveURL + } + return absoluteURL(r, strings.TrimRight(serveURL, "/")+"/"+fileName) +} diff --git a/internal/httpapi/api.go b/internal/httpapi/api.go index 9ed2ddb..54a6549 100644 --- a/internal/httpapi/api.go +++ b/internal/httpapi/api.go @@ -24,6 +24,7 @@ type API struct { auth *auth.Authenticator events events.Publisher authCfg config.AuthConfig + manager config.ManagerConfig ai *ai.Service } @@ -34,11 +35,11 @@ const ( maxMailboxContentLength = 5000 ) -func New(st *store.Store, authenticator *auth.Authenticator, publisher events.Publisher, authCfg config.AuthConfig, aiService *ai.Service) *API { +func New(st *store.Store, authenticator *auth.Authenticator, publisher events.Publisher, authCfg config.AuthConfig, managerCfg config.ManagerConfig, aiService *ai.Service) *API { if publisher == nil { publisher = events.NoopPublisher{} } - return &API{store: st, auth: authenticator, events: publisher, authCfg: authCfg, ai: aiService} + return &API{store: st, auth: authenticator, events: publisher, authCfg: authCfg, manager: managerCfg, ai: aiService} } func (a *API) Register(mux *http.ServeMux) { diff --git a/internal/store/ai.go b/internal/store/ai.go index 4c1584d..c3a70f8 100644 --- a/internal/store/ai.go +++ b/internal/store/ai.go @@ -135,6 +135,16 @@ type AIFileMetadata struct { Location string } +type CompletedFileUpload struct { + ID int64 + UserID int64 + MIMEType string + Size int64 + ETag string + Location string + CreatedAt time.Time +} + func (s *Store) AICapabilities(ctx context.Context) ([]AICapability, error) { var items []AICapability err := s.db.SelectContext(ctx, &items, `SELECT feature,enabled,model,daily_limit,concurrency_limit, @@ -315,6 +325,36 @@ func (s *Store) SaveAITableAnalysis(ctx context.Context, tenantCode string, user return tx.Commit() } +func (s *Store) SaveCompletedFileUpload(ctx context.Context, file CompletedFileUpload) error { + now := file.CreatedAt.UTC().Round(time.Millisecond) + if now.IsZero() { + now = time.Now().UTC().Round(time.Millisecond) + } + _, err := s.db.ExecContext(ctx, `INSERT INTO im_fileuploads + (tenant_id,id,createdat,updatedat,userid,status,mimetype,size,etag,location) + VALUES(?,?,?,?,?,?,?,?,?,?)`, + s.tenant(), file.ID, now, now, formatInt64(file.UserID), 1, file.MIMEType, file.Size, file.ETag, file.Location) + return err +} + +func (s *Store) UpdateAITableAnalysisExportFileID(ctx context.Context, userID int64, recordID, fileID string) error { + now := time.Now().UTC().Round(time.Millisecond) + res, err := s.db.ExecContext(ctx, `UPDATE wb_ai_table_analysis SET export_file_id=?,updated_at=? + WHERE tenant_id=? AND user_id=? AND record_id=?`, + fileID, now, s.tenant(), formatInt64(userID), recordID) + if err != nil { + return err + } + affected, err := res.RowsAffected() + if err != nil { + return err + } + if affected == 0 { + return ErrNotFound + } + return nil +} + func (s *Store) AITableAnalyses(ctx context.Context, userID int64, queryText, cursor string, limit int) ([]AIRecord, string, error) { return s.aiRecords(ctx, userID, "smart_table", queryText, cursor, limit) }