pro/olons_server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: improve ai table file handling

Browse Source
This commit is contained in:
richjohnrambo
2026-09-30 10:02:00 +08:00
parent 1bcce3cdde
commit 47a10c93be
5 changed files with 334 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/httpapi/ai.go
+26 -2
View File
@@ -379,7 +379,7 @@ func (a *API) aiTableAnalysisCreate(w http.ResponseWriter, r *http.Request, p *a
writeStoreError(w, r, err)
return
}
profile := tableProfileForAI(resolveTableFilePath(fileID, fileMeta.Location), fileType)
profile := a.tableProfileForAI(r, fileID, fileMeta, fileType)
result, err := a.ai.AnalyzeTable(r.Context(), ai.TableAnalysisInput{
Topic: topic, FileID: fileID, FileName: strings.TrimSpace(body.FileName), FileType: fileType, FileSize: fileSize, Profile: profile,
})
@@ -831,10 +831,15 @@ func aiRecordPayloads(records []store.AIRecord) []map[string]any {
func aiTableRecordPayloads(records []store.AIRecord) []map[string]any {
items := make([]map[string]any, 0, len(records))
for _, record := range records {
fileType := strings.ToLower(strings.TrimSpace(record.FileType))
items = append(items, map[string]any{
"record_id": record.RecordID,
"title": record.Title,
"summary": record.Summary,
"format": fileType,
"file_type": fileType,
"file_name": record.FileName,
"file_size": record.FileSize,
"status": record.Status,
"created_at": record.CreatedAt.UTC().Format(time.RFC3339Nano),
"updated_at": record.UpdatedAt.UTC().Format(time.RFC3339Nano),
@@ -976,7 +981,26 @@ func resolveTableFilePath(fileID, location string) string {
return ""
}
func tableProfileForAI(path, fileType string) ai.TableProfile {
func (a *API) tableProfileForAI(r *http.Request, fileID string, meta store.AIFileMetadata, fileType string) ai.TableProfile {
if path := resolveTableFilePath(fileID, meta.Location); path != "" {
return tableProfileFromPath(path, fileType)
}
if path, cleanup, err := downloadWorkbenchFileToTemp(r, fileID, fileType); err == nil {
defer cleanup()
return tableProfileFromPath(path, fileType)
}
if strings.TrimSpace(meta.Location) == "" {
return ai.TableProfile{Parsed: false, Message: "source file metadata is not available"}
}
path, cleanup, err := a.downloadOSSObjectToTemp(r.Context(), meta.Location, fileType)
if err != nil {
return ai.TableProfile{Parsed: false, Message: "source file is not available: " + err.Error()}
}
defer cleanup()
return tableProfileFromPath(path, fileType)
}
func tableProfileFromPath(path, fileType string) ai.TableProfile {
if strings.TrimSpace(path) == "" {
return ai.TableProfile{Parsed: false, Message: "source file is not available locally"}
}
internal/httpapi/ai_export_oss.go
+95
View File
@@ -8,7 +8,10 @@ import (
"encoding/base64"
"encoding/binary"
"fmt"
"io"
"net/http"
"net/url"
"os"
"path"
"strconv"
"strings"
@@ -88,6 +91,98 @@ func (a *API) uploadAITableAnalysisExport(ctx context.Context, r *http.Request,
}, nil
}
func downloadWorkbenchFileToTemp(r *http.Request, fileID, fileType string) (string, func(), error) {
fileID = strings.TrimSpace(fileID)
if fileID == "" {
return "", func() {}, fmt.Errorf("empty file id")
}
req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, absoluteURL(r, "/v0/file/s/"+url.PathEscape(fileID)), nil)
if err != nil {
return "", func() {}, err
}
copyHeader(req, r, "Authorization")
copyHeader(req, r, "X-MeetingIM-APIKey")
copyHeader(req, r, "X-Tenant-Code")
copyHeader(req, r, "X-IM-Schema")
copyHeader(req, r, "X-Request-ID")
if auth := strings.TrimSpace(r.Header.Get("X-MeetingIM-Auth")); auth != "" {
req.Header.Set("X-MeetingIM-Auth", auth)
} else if token := bearerTokenFromHeader(r.Header.Get("Authorization")); token != "" {
req.Header.Set("X-MeetingIM-Auth", "token "+token)
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return "", func() {}, err
}
defer resp.Body.Close()
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return "", func() {}, fmt.Errorf("file service returned %d", resp.StatusCode)
}
return responseBodyToTemp(resp.Body, fileType)
}
func (a *API) downloadOSSObjectToTemp(ctx context.Context, objectKey, fileType string) (string, func(), error) {
objectKey = strings.Trim(strings.TrimSpace(objectKey), "/")
if objectKey == "" {
return "", func() {}, fmt.Errorf("empty object key")
}
cfg, err := a.ossUploadConfig(ctx)
if err != nil {
return "", func() {}, err
}
client, err := aliyunoss.New(cfg.Endpoint, cfg.AccessKeyID, cfg.AccessKeySecret)
if err != nil {
return "", func() {}, err
}
bucket, err := client.Bucket(cfg.Bucket)
if err != nil {
return "", func() {}, err
}
reader, err := bucket.GetObject(objectKey)
if err != nil {
return "", func() {}, err
}
defer reader.Close()
return responseBodyToTemp(reader, fileType)
}
func responseBodyToTemp(reader io.Reader, fileType string) (string, func(), error) {
suffix := "." + strings.TrimLeft(strings.ToLower(strings.TrimSpace(fileType)), ".")
if suffix == "." {
suffix = ".table"
}
tmp, err := os.CreateTemp("", "im-workbench-ai-table-*"+suffix)
if err != nil {
return "", func() {}, err
}
tmpPath := tmp.Name()
cleanup := func() { _ = os.Remove(tmpPath) }
if _, err := io.Copy(tmp, reader); err != nil {
_ = tmp.Close()
cleanup()
return "", func() {}, err
}
if err := tmp.Close(); err != nil {
cleanup()
return "", func() {}, err
}
return tmpPath, cleanup, nil
}
func copyHeader(dst *http.Request, src *http.Request, key string) {
if value := strings.TrimSpace(src.Header.Get(key)); value != "" {
dst.Header.Set(key, value)
}
}
func bearerTokenFromHeader(value string) string {
token, ok := strings.CutPrefix(strings.TrimSpace(value), "Bearer ")
if !ok {
return ""
}
return strings.TrimSpace(token)
}
func (a *API) ossUploadConfig(ctx context.Context) (ossUploadConfig, error) {
values, err := managerconfig.New(a.manager).ThirdPartyConfig(ctx, "im_third_party",
"oss_enabled",
internal/store/ai.go
+31 -1
View File
@@ -53,6 +53,9 @@ type AIRecord struct {
Status string `db:"status"`
DetailID string `db:"detail_id"`
TaskID string `db:"task_id"`
FileName string `db:"file_name"`
FileType string `db:"file_type"`
FileSize int64 `db:"file_size"`
CreatedAt time.Time `db:"created_at"`
UpdatedAt time.Time `db:"updated_at"`
DeletedAt *time.Time `db:"deleted_at"`
@@ -356,7 +359,34 @@ func (s *Store) UpdateAITableAnalysisExportFileID(ctx context.Context, userID in
}
func (s *Store) AITableAnalyses(ctx context.Context, userID int64, queryText, cursor string, limit int) ([]AIRecord, string, error) {
return s.aiRecords(ctx, userID, "smart_table", queryText, cursor, limit)
where := `r.tenant_id=? AND r.user_id=? AND r.record_type='smart_table' AND r.deleted_at IS NULL`
args := []any{s.tenant(), formatInt64(userID)}
if queryText != "" {
where += ` AND (r.title LIKE ? OR r.summary LIKE ? OR a.file_name LIKE ?)`
like := "%" + queryText + "%"
args = append(args, like, like, like)
}
if cursor != "" {
where += ` AND r.record_id<?`
args = append(args, cursor)
}
args = append(args, limit+1)
var items []AIRecord
err := s.db.SelectContext(ctx, &items, `SELECT r.record_id,r.tenant_code,r.user_id,r.record_type,r.title,r.summary,r.status,r.detail_id,r.task_id,
COALESCE(a.file_name, '') AS file_name,COALESCE(a.file_type, '') AS file_type,COALESCE(a.file_size, 0) AS file_size,
r.created_at,r.updated_at,r.deleted_at
FROM wb_ai_record r
LEFT JOIN wb_ai_table_analysis a ON a.tenant_id=r.tenant_id AND a.record_id=r.detail_id
WHERE `+where+` ORDER BY r.created_at DESC,r.record_id DESC LIMIT ?`, args...)
if err != nil {
return nil, "", err
}
next := ""
if len(items) > limit {
next = items[limit-1].RecordID
items = items[:limit]
}
return items, next, nil
}
func (s *Store) AIRecords(ctx context.Context, userID int64, recordType, queryText, cursor string, limit int) ([]AIRecord, string, error) {
internal/tableparse/tableparse.go
+11 -3
View File
@@ -92,12 +92,17 @@ func parseXLSX(path string) (Profile, error) {
}
defer f.Close()
for _, sheet := range f.GetSheetList() {
rows, err := f.GetRows(sheet)
iter, err := f.Rows(sheet)
if err != nil {
continue
}
cleaned := make([][]string, 0, min(len(rows), maxRows))
for _, row := range rows {
cleaned := make([][]string, 0, maxRows)
for iter.Next() {
row, err := iter.Columns()
if err != nil {
_ = iter.Close()
return Profile{}, err
}
if emptyRow(row) {
continue
}
@@ -106,6 +111,9 @@ func parseXLSX(path string) (Profile, error) {
break
}
}
if err := iter.Close(); err != nil {
return Profile{}, err
}
if len(cleaned) > 0 {
return profileRows(cleaned), nil
}