feat: add workbench AI integration
This commit is contained in:
richjohnrambo
@@ -0,0 +1,254 @@
|
||||
package tableparse
|
||||
|
||||
import (
|
||||
"encoding/csv"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/xuri/excelize/v2"
|
||||
)
|
||||
|
||||
const (
|
||||
maxSampleRows = 20
|
||||
maxRows = 5000
|
||||
)
|
||||
|
||||
type Profile struct {
|
||||
Columns []ColumnStat `json:"columns"`
|
||||
SampleRows [][]string `json:"sample_rows"`
|
||||
RowCount int `json:"row_count"`
|
||||
Stats map[string]NumericStat `json:"stats,omitempty"`
|
||||
Outliers []Outlier `json:"outliers,omitempty"`
|
||||
Parsed bool `json:"parsed"`
|
||||
Message string `json:"message,omitempty"`
|
||||
}
|
||||
|
||||
type ColumnStat struct {
|
||||
Name string `json:"name"`
|
||||
Type string `json:"type"`
|
||||
Empty int `json:"empty"`
|
||||
}
|
||||
|
||||
type NumericStat struct {
|
||||
Count int `json:"count"`
|
||||
Sum float64 `json:"sum"`
|
||||
Avg float64 `json:"avg"`
|
||||
Min float64 `json:"min"`
|
||||
Max float64 `json:"max"`
|
||||
}
|
||||
|
||||
type Outlier struct {
|
||||
Row int `json:"row"`
|
||||
Column string `json:"column"`
|
||||
Description string `json:"description"`
|
||||
}
|
||||
|
||||
func Parse(path, fileType string) (Profile, error) {
|
||||
switch strings.ToLower(strings.TrimSpace(fileType)) {
|
||||
case "csv":
|
||||
return parseCSV(path)
|
||||
case "xlsx":
|
||||
return parseXLSX(path)
|
||||
case "xls":
|
||||
return Profile{Parsed: false, Message: "legacy xls parsing is not supported yet"}, nil
|
||||
default:
|
||||
return Profile{}, fmt.Errorf("unsupported table file type: %s", fileType)
|
||||
}
|
||||
}
|
||||
|
||||
func parseCSV(path string) (Profile, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return Profile{}, err
|
||||
}
|
||||
defer f.Close()
|
||||
reader := csv.NewReader(f)
|
||||
reader.FieldsPerRecord = -1
|
||||
rows := make([][]string, 0, maxRows)
|
||||
for len(rows) < maxRows {
|
||||
row, err := reader.Read()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return Profile{}, err
|
||||
}
|
||||
if emptyRow(row) {
|
||||
continue
|
||||
}
|
||||
rows = append(rows, trimRow(row))
|
||||
}
|
||||
return profileRows(rows), nil
|
||||
}
|
||||
|
||||
func parseXLSX(path string) (Profile, error) {
|
||||
f, err := excelize.OpenFile(path)
|
||||
if err != nil {
|
||||
return Profile{}, err
|
||||
}
|
||||
defer f.Close()
|
||||
for _, sheet := range f.GetSheetList() {
|
||||
rows, err := f.GetRows(sheet)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
cleaned := make([][]string, 0, min(len(rows), maxRows))
|
||||
for _, row := range rows {
|
||||
if emptyRow(row) {
|
||||
continue
|
||||
}
|
||||
cleaned = append(cleaned, trimRow(row))
|
||||
if len(cleaned) >= maxRows {
|
||||
break
|
||||
}
|
||||
}
|
||||
if len(cleaned) > 0 {
|
||||
return profileRows(cleaned), nil
|
||||
}
|
||||
}
|
||||
return Profile{Parsed: false, Message: "no non-empty sheet found"}, nil
|
||||
}
|
||||
|
||||
func profileRows(rows [][]string) Profile {
|
||||
if len(rows) == 0 {
|
||||
return Profile{Parsed: false, Message: "empty table"}
|
||||
}
|
||||
headers := rows[0]
|
||||
if len(headers) == 0 {
|
||||
return Profile{Parsed: false, Message: "empty header"}
|
||||
}
|
||||
dataRows := rows[1:]
|
||||
columnCount := len(headers)
|
||||
columns := make([]ColumnStat, columnCount)
|
||||
statsWork := make([]numericWork, columnCount)
|
||||
for i, name := range headers {
|
||||
name = strings.TrimSpace(name)
|
||||
if name == "" {
|
||||
name = fmt.Sprintf("列%d", i+1)
|
||||
}
|
||||
columns[i] = ColumnStat{Name: name, Type: "text"}
|
||||
}
|
||||
sample := make([][]string, 0, min(len(dataRows), maxSampleRows))
|
||||
for rowIndex, row := range dataRows {
|
||||
normalized := normalizeWidth(row, columnCount)
|
||||
if len(sample) < maxSampleRows {
|
||||
sample = append(sample, normalized)
|
||||
}
|
||||
for col := 0; col < columnCount; col++ {
|
||||
value := strings.TrimSpace(normalized[col])
|
||||
if value == "" {
|
||||
columns[col].Empty++
|
||||
continue
|
||||
}
|
||||
if parsed, ok := parseNumber(value); ok {
|
||||
statsWork[col].add(parsed, rowIndex+2)
|
||||
}
|
||||
}
|
||||
}
|
||||
stats := map[string]NumericStat{}
|
||||
outliers := []Outlier{}
|
||||
for i := range columns {
|
||||
work := statsWork[i]
|
||||
nonEmpty := len(dataRows) - columns[i].Empty
|
||||
if work.Count > 0 && work.Count*2 >= max(1, nonEmpty) {
|
||||
columns[i].Type = "number"
|
||||
stat := work.stat()
|
||||
stats[columns[i].Name] = stat
|
||||
if stat.Count >= 3 {
|
||||
for _, point := range work.Points {
|
||||
if point.Value == stat.Max && stat.Avg > 0 && point.Value > stat.Avg*2 {
|
||||
outliers = append(outliers, Outlier{Row: point.Row, Column: columns[i].Name, Description: "数值明显高于平均值"})
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return Profile{Columns: columns, SampleRows: sample, RowCount: len(dataRows), Stats: stats, Outliers: outliers, Parsed: true}
|
||||
}
|
||||
|
||||
type numericWork struct {
|
||||
Count int
|
||||
Sum float64
|
||||
Min float64
|
||||
Max float64
|
||||
Points []numericPoint
|
||||
}
|
||||
|
||||
type numericPoint struct {
|
||||
Row int
|
||||
Value float64
|
||||
}
|
||||
|
||||
func (w *numericWork) add(value float64, row int) {
|
||||
if w.Count == 0 {
|
||||
w.Min = value
|
||||
w.Max = value
|
||||
} else {
|
||||
w.Min = math.Min(w.Min, value)
|
||||
w.Max = math.Max(w.Max, value)
|
||||
}
|
||||
w.Count++
|
||||
w.Sum += value
|
||||
if len(w.Points) < 200 {
|
||||
w.Points = append(w.Points, numericPoint{Row: row, Value: value})
|
||||
}
|
||||
}
|
||||
|
||||
func (w numericWork) stat() NumericStat {
|
||||
avg := 0.0
|
||||
if w.Count > 0 {
|
||||
avg = w.Sum / float64(w.Count)
|
||||
}
|
||||
return NumericStat{Count: w.Count, Sum: w.Sum, Avg: avg, Min: w.Min, Max: w.Max}
|
||||
}
|
||||
|
||||
func parseNumber(value string) (float64, bool) {
|
||||
value = strings.TrimSpace(strings.ReplaceAll(value, ",", ""))
|
||||
value = strings.TrimSuffix(value, "%")
|
||||
parsed, err := strconv.ParseFloat(value, 64)
|
||||
return parsed, err == nil
|
||||
}
|
||||
|
||||
func normalizeWidth(row []string, width int) []string {
|
||||
out := make([]string, width)
|
||||
for i := 0; i < width && i < len(row); i++ {
|
||||
out[i] = strings.TrimSpace(row[i])
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func trimRow(row []string) []string {
|
||||
out := make([]string, len(row))
|
||||
for i := range row {
|
||||
out[i] = strings.TrimSpace(row[i])
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func emptyRow(row []string) bool {
|
||||
for _, value := range row {
|
||||
if strings.TrimSpace(value) != "" {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func min(a, b int) int {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func max(a, b int) int {
|
||||
if a > b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
Reference in New Issue
Block a user