pro/olons_server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: upload ai table exports to oss

Browse Source
This commit is contained in:
richjohnrambo
2026-09-29 16:50:36 +08:00
parent 9cb8c4d93a
commit 192ab342f6
7 changed files with 220 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
go.mod
+3 -1
View File
@@ -3,11 +3,13 @@ module git.118im.com/im_team/im_workbench
go 1.26.0
require (
github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible
github.com/go-sql-driver/mysql v1.10.0
github.com/jmoiron/sqlx v1.4.0
github.com/prometheus/client_golang v1.23.2
github.com/redis/go-redis/v9 v9.12.1
github.com/segmentio/kafka-go v0.4.51
github.com/xuri/excelize/v2 v2.11.0
golang.org/x/crypto v0.53.0
)
@@ -30,11 +32,11 @@ require (
github.com/xdg-go/scram v1.1.2 // indirect
github.com/xdg-go/stringprep v1.0.4 // indirect
github.com/xuri/efp v0.0.1 // indirect
github.com/xuri/excelize/v2 v2.11.0 // indirect
github.com/xuri/nfp v0.0.2-0.20250530014748-2ddeb826f9a9 // indirect
go.yaml.in/yaml/v2 v2.4.2 // indirect
golang.org/x/net v0.56.0 // indirect
golang.org/x/sys v0.46.0 // indirect
golang.org/x/text v0.38.0 // indirect
golang.org/x/time v0.16.0 // indirect
google.golang.org/protobuf v1.36.8 // indirect
)
go.sum
+6 -2
View File
@@ -1,6 +1,8 @@
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo=
filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc=
github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F7Y6wUM49Lcha2FMXt4UM8g=
github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
@@ -82,12 +84,12 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto=
golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio=
golang.org/x/image v0.38.0 h1:5l+q+Y9JDC7mBOMjo4/aPhMDcxEptsX+Tt3GgRQRPuE=
golang.org/x/image v0.38.0/go.mod h1:/3f6vaXC+6CEanU4KJxbcUZyEePbyKbaLoDOe4ehFYY=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8=
golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww=
golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o=
golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -107,6 +109,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE=
golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4=
golang.org/x/time v0.16.0 h1:vMb6ptszcQMkcwiRTAuNNU50gom6++Q/6gY2hDM6VDE=
golang.org/x/time v0.16.0/go.mod h1:rVKOqvZeKvrDKTQiAHJ7wmwP0RzleSphoEA9RcdLA0s=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
internal/app/app.go
+1 -1
View File
@@ -64,7 +64,7 @@ func New(cfg *config.Config) (*App, error) {
store: st,
redis: redisClient,
events: publisher,
api: httpapi.New(st, auth.New(cfg.Auth, redisClient), publisher, cfg.Auth, aiService),
api: httpapi.New(st, auth.New(cfg.Auth, redisClient), publisher, cfg.Auth, cfg.Manager, aiService),
}, nil
}
internal/httpapi/ai.go
+8 -4
View File
@@ -443,11 +443,15 @@ func (a *API) aiTableAnalysisResource(w http.ResponseWriter, r *http.Request, p
writeStoreError(w, r, err)
return
}
fileName := aiTableExportFileName(item)
file, err := a.uploadAITableAnalysisExport(r.Context(), r, st, p, item)
if err != nil {
writeError(w, r, http.StatusServiceUnavailable, 30053, reasonAIProviderFailed, "ai export storage failed", nil)
return
}
writeSuccess(w, r, map[string]any{
"file_id": "ai_export_" + recordID,
"file_name": fileName,
"download_url": absoluteURL(r, "/v1/workbench/ai/table-analyses/"+recordID+"/export/download?format=docx"),
"file_id": file.FileID,
"file_name": file.FileName,
"download_url": file.DownloadURL,
"expires_at": time.Now().UTC().Add(24 * time.Hour).Format(time.RFC3339Nano),
})
return
internal/httpapi/ai_export_oss.go
+159
View File
@@ -0,0 +1,159 @@
package httpapi
import (
"bytes"
"context"
"crypto/rand"
"encoding/base32"
"encoding/base64"
"encoding/binary"
"fmt"
"net/http"
"path"
"strconv"
"strings"
"time"
"git.118im.com/im_team/im_workbench/internal/auth"
"git.118im.com/im_team/im_workbench/internal/managerconfig"
"git.118im.com/im_team/im_workbench/internal/store"
aliyunoss "github.com/aliyun/aliyun-oss-go-sdk/oss"
)
const aiTableExportDocxMIME = "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
type aiExportedFile struct {
FileID string
FileName string
DownloadURL string
}
type ossUploadConfig struct {
Enabled bool
Endpoint string
Bucket string
AccessKeyID string
AccessKeySecret string
ServeURL string
CacheControl string
}
func (a *API) uploadAITableAnalysisExport(ctx context.Context, r *http.Request, st *store.Store, p *auth.Principal, item *store.AITableAnalysis) (*aiExportedFile, error) {
cfg, err := a.ossUploadConfig(ctx)
if err != nil {
return nil, err
}
data, err := aiTableAnalysisDocx(item)
if err != nil {
return nil, err
}
fileID := newFileUploadID()
publicID, storageID := formatFileUploadID(fileID)
createdAt := time.Now().UTC()
objectKey := path.Join("tenants", strconv.FormatInt(p.TenantID, 10), createdAt.Format("2006"), createdAt.Format("01"), storageID)
client, err := aliyunoss.New(cfg.Endpoint, cfg.AccessKeyID, cfg.AccessKeySecret)
if err != nil {
return nil, err
}
bucket, err := client.Bucket(cfg.Bucket)
if err != nil {
return nil, err
}
options := []aliyunoss.Option{aliyunoss.ContentType(aiTableExportDocxMIME)}
if cfg.CacheControl != "" {
options = append(options, aliyunoss.CacheControl(cfg.CacheControl))
}
if err := bucket.PutObject(objectKey, bytes.NewReader(data), options...); err != nil {
return nil, err
}
etag := ""
if meta, err := bucket.GetObjectDetailedMeta(objectKey); err == nil {
etag = strings.Trim(meta.Get("ETag"), `"`)
}
fileName := aiTableExportFileName(item)
fileURLName := publicID + ".docx"
if err := st.SaveCompletedFileUpload(ctx, store.CompletedFileUpload{
ID: fileID, UserID: p.UserID, MIMEType: aiTableExportDocxMIME, Size: int64(len(data)),
ETag: etag, Location: objectKey, CreatedAt: createdAt,
}); err != nil {
return nil, err
}
if err := st.UpdateAITableAnalysisExportFileID(ctx, p.UserID, item.RecordID, fileURLName); err != nil {
return nil, err
}
return &aiExportedFile{
FileID: fileURLName,
FileName: fileName,
DownloadURL: publicFileDownloadURL(r, cfg.ServeURL, fileURLName),
}, nil
}
func (a *API) ossUploadConfig(ctx context.Context) (ossUploadConfig, error) {
values, err := managerconfig.New(a.manager).ThirdPartyConfig(ctx, "im_third_party",
"oss_enabled",
"oss_endpoint",
"oss_bucket",
"oss_access_key_id",
"oss_access_key_secret",
"oss_serve_url",
"oss_cache_control",
)
if err != nil {
return ossUploadConfig{}, err
}
cfg := ossUploadConfig{
Enabled: parseConfigBool(values["oss_enabled"]),
Endpoint: strings.TrimSpace(values["oss_endpoint"]),
Bucket: strings.TrimSpace(values["oss_bucket"]),
AccessKeyID: strings.TrimSpace(values["oss_access_key_id"]),
AccessKeySecret: strings.TrimSpace(values["oss_access_key_secret"]),
ServeURL: strings.TrimSpace(values["oss_serve_url"]),
CacheControl: strings.TrimSpace(values["oss_cache_control"]),
}
if cfg.ServeURL == "" {
cfg.ServeURL = "/v0/file/s/"
}
if !cfg.Enabled || cfg.Endpoint == "" || cfg.Bucket == "" || cfg.AccessKeyID == "" || cfg.AccessKeySecret == "" {
return ossUploadConfig{}, fmt.Errorf("oss config incomplete")
}
return cfg, nil
}
func parseConfigBool(value string) bool {
switch strings.ToLower(strings.TrimSpace(value)) {
case "1", "true", "yes", "on", "enabled":
return true
default:
return false
}
}
func newFileUploadID() int64 {
var random [8]byte
if _, err := rand.Read(random[:]); err != nil {
return time.Now().UnixNano()
}
suffix := int64(binary.LittleEndian.Uint64(random[:]) & ((1 << 20) - 1))
return (time.Now().UnixMilli() << 20) | suffix
}
func formatFileUploadID(id int64) (publicID string, storageID string) {
encrypted := encryptUID(id)
publicID = base64.URLEncoding.WithPadding(base64.NoPadding).EncodeToString(encrypted)
storageID = strings.ToLower(base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(encrypted))
return publicID, storageID
}
func publicFileDownloadURL(r *http.Request, serveURL, fileName string) string {
serveURL = strings.TrimSpace(serveURL)
if strings.HasPrefix(serveURL, "http://") || strings.HasPrefix(serveURL, "https://") {
return strings.TrimRight(serveURL, "/") + "/" + fileName
}
if serveURL == "" {
serveURL = "/v0/file/s/"
}
if !strings.HasPrefix(serveURL, "/") {
serveURL = "/" + serveURL
}
return absoluteURL(r, strings.TrimRight(serveURL, "/")+"/"+fileName)
}
internal/httpapi/api.go
+3 -2
View File
@@ -24,6 +24,7 @@ type API struct {
auth *auth.Authenticator
events events.Publisher
authCfg config.AuthConfig
manager config.ManagerConfig
ai *ai.Service
}
@@ -34,11 +35,11 @@ const (
maxMailboxContentLength = 5000
)
func New(st *store.Store, authenticator *auth.Authenticator, publisher events.Publisher, authCfg config.AuthConfig, aiService *ai.Service) *API {
func New(st *store.Store, authenticator *auth.Authenticator, publisher events.Publisher, authCfg config.AuthConfig, managerCfg config.ManagerConfig, aiService *ai.Service) *API {
if publisher == nil {
publisher = events.NoopPublisher{}
}
return &API{store: st, auth: authenticator, events: publisher, authCfg: authCfg, ai: aiService}
return &API{store: st, auth: authenticator, events: publisher, authCfg: authCfg, manager: managerCfg, ai: aiService}
}
func (a *API) Register(mux *http.ServeMux) {
internal/store/ai.go
+40
View File
@@ -135,6 +135,16 @@ type AIFileMetadata struct {
Location string
}
type CompletedFileUpload struct {
ID int64
UserID int64
MIMEType string
Size int64
ETag string
Location string
CreatedAt time.Time
}
func (s *Store) AICapabilities(ctx context.Context) ([]AICapability, error) {
var items []AICapability
err := s.db.SelectContext(ctx, &items, `SELECT feature,enabled,model,daily_limit,concurrency_limit,
@@ -315,6 +325,36 @@ func (s *Store) SaveAITableAnalysis(ctx context.Context, tenantCode string, user
return tx.Commit()
}
func (s *Store) SaveCompletedFileUpload(ctx context.Context, file CompletedFileUpload) error {
now := file.CreatedAt.UTC().Round(time.Millisecond)
if now.IsZero() {
now = time.Now().UTC().Round(time.Millisecond)
}
_, err := s.db.ExecContext(ctx, `INSERT INTO im_fileuploads
(tenant_id,id,createdat,updatedat,userid,status,mimetype,size,etag,location)
VALUES(?,?,?,?,?,?,?,?,?,?)`,
s.tenant(), file.ID, now, now, formatInt64(file.UserID), 1, file.MIMEType, file.Size, file.ETag, file.Location)
return err
}
func (s *Store) UpdateAITableAnalysisExportFileID(ctx context.Context, userID int64, recordID, fileID string) error {
now := time.Now().UTC().Round(time.Millisecond)
res, err := s.db.ExecContext(ctx, `UPDATE wb_ai_table_analysis SET export_file_id=?,updated_at=?
WHERE tenant_id=? AND user_id=? AND record_id=?`,
fileID, now, s.tenant(), formatInt64(userID), recordID)
if err != nil {
return err
}
affected, err := res.RowsAffected()
if err != nil {
return err
}
if affected == 0 {
return ErrNotFound
}
return nil
}
func (s *Store) AITableAnalyses(ctx context.Context, userID int64, queryText, cursor string, limit int) ([]AIRecord, string, error) {
return s.aiRecords(ctx, userID, "smart_table", queryText, cursor, limit)
}