olons_server/internal/tableparse/tableparse.go

package tableparse

import (
	"encoding/csv"
	"fmt"
	"io"
	"math"
	"os"
	"strconv"
	"strings"

	"github.com/xuri/excelize/v2"
)

const (
	maxSampleRows = 20
	maxRows       = 5000
)

type Profile struct {
	Columns    []ColumnStat           `json:"columns"`
	SampleRows [][]string             `json:"sample_rows"`
	RowCount   int                    `json:"row_count"`
	Stats      map[string]NumericStat `json:"stats,omitempty"`
	Outliers   []Outlier              `json:"outliers,omitempty"`
	Parsed     bool                   `json:"parsed"`
	Message    string                 `json:"message,omitempty"`
}

type ColumnStat struct {
	Name  string `json:"name"`
	Type  string `json:"type"`
	Empty int    `json:"empty"`
}

type NumericStat struct {
	Count int     `json:"count"`
	Sum   float64 `json:"sum"`
	Avg   float64 `json:"avg"`
	Min   float64 `json:"min"`
	Max   float64 `json:"max"`
}

type Outlier struct {
	Row         int    `json:"row"`
	Column      string `json:"column"`
	Description string `json:"description"`
}

func Parse(path, fileType string) (Profile, error) {
	switch strings.ToLower(strings.TrimSpace(fileType)) {
	case "csv":
		return parseCSV(path)
	case "xlsx":
		return parseXLSX(path)
	case "xls":
		return Profile{Parsed: false, Message: "legacy xls parsing is not supported yet"}, nil
	default:
		return Profile{}, fmt.Errorf("unsupported table file type: %s", fileType)
	}
}

func parseCSV(path string) (Profile, error) {
	f, err := os.Open(path)
	if err != nil {
		return Profile{}, err
	}
	defer f.Close()
	reader := csv.NewReader(f)
	reader.FieldsPerRecord = -1
	rows := make([][]string, 0, maxRows)
	for len(rows) < maxRows {
		row, err := reader.Read()
		if err == io.EOF {
			break
		}
		if err != nil {
			return Profile{}, err
		}
		if emptyRow(row) {
			continue
		}
		rows = append(rows, trimRow(row))
	}
	return profileRows(rows), nil
}

func parseXLSX(path string) (Profile, error) {
	f, err := excelize.OpenFile(path)
	if err != nil {
		return Profile{}, err
	}
	defer f.Close()
	for _, sheet := range f.GetSheetList() {
		iter, err := f.Rows(sheet)
		if err != nil {
			continue
		}
		cleaned := make([][]string, 0, maxRows)
		for iter.Next() {
			row, err := iter.Columns()
			if err != nil {
				_ = iter.Close()
				return Profile{}, err
			}
			if emptyRow(row) {
				continue
			}
			cleaned = append(cleaned, trimRow(row))
			if len(cleaned) >= maxRows {
				break
			}
		}
		if err := iter.Close(); err != nil {
			return Profile{}, err
		}
		if len(cleaned) > 0 {
			return profileRows(cleaned), nil
		}
	}
	return Profile{Parsed: false, Message: "no non-empty sheet found"}, nil
}

func profileRows(rows [][]string) Profile {
	if len(rows) == 0 {
		return Profile{Parsed: false, Message: "empty table"}
	}
	headers := rows[0]
	if len(headers) == 0 {
		return Profile{Parsed: false, Message: "empty header"}
	}
	dataRows := rows[1:]
	columnCount := len(headers)
	columns := make([]ColumnStat, columnCount)
	statsWork := make([]numericWork, columnCount)
	for i, name := range headers {
		name = strings.TrimSpace(name)
		if name == "" {
			name = fmt.Sprintf("列%d", i+1)
		}
		columns[i] = ColumnStat{Name: name, Type: "text"}
	}
	sample := make([][]string, 0, min(len(dataRows), maxSampleRows))
	for rowIndex, row := range dataRows {
		normalized := normalizeWidth(row, columnCount)
		if len(sample) < maxSampleRows {
			sample = append(sample, normalized)
		}
		for col := 0; col < columnCount; col++ {
			value := strings.TrimSpace(normalized[col])
			if value == "" {
				columns[col].Empty++
				continue
			}
			if parsed, ok := parseNumber(value); ok {
				statsWork[col].add(parsed, rowIndex+2)
			}
		}
	}
	stats := map[string]NumericStat{}
	outliers := []Outlier{}
	for i := range columns {
		work := statsWork[i]
		nonEmpty := len(dataRows) - columns[i].Empty
		if work.Count > 0 && work.Count*2 >= max(1, nonEmpty) {
			columns[i].Type = "number"
			stat := work.stat()
			stats[columns[i].Name] = stat
			if stat.Count >= 3 {
				for _, point := range work.Points {
					if point.Value == stat.Max && stat.Avg > 0 && point.Value > stat.Avg*2 {
						outliers = append(outliers, Outlier{Row: point.Row, Column: columns[i].Name, Description: "数值明显高于平均值"})
						break
					}
				}
			}
		}
	}
	return Profile{Columns: columns, SampleRows: sample, RowCount: len(dataRows), Stats: stats, Outliers: outliers, Parsed: true}
}

type numericWork struct {
	Count  int
	Sum    float64
	Min    float64
	Max    float64
	Points []numericPoint
}

type numericPoint struct {
	Row   int
	Value float64
}

func (w *numericWork) add(value float64, row int) {
	if w.Count == 0 {
		w.Min = value
		w.Max = value
	} else {
		w.Min = math.Min(w.Min, value)
		w.Max = math.Max(w.Max, value)
	}
	w.Count++
	w.Sum += value
	if len(w.Points) < 200 {
		w.Points = append(w.Points, numericPoint{Row: row, Value: value})
	}
}

func (w numericWork) stat() NumericStat {
	avg := 0.0
	if w.Count > 0 {
		avg = w.Sum / float64(w.Count)
	}
	return NumericStat{Count: w.Count, Sum: w.Sum, Avg: avg, Min: w.Min, Max: w.Max}
}

func parseNumber(value string) (float64, bool) {
	value = strings.TrimSpace(strings.ReplaceAll(value, ",", ""))
	value = strings.TrimSuffix(value, "%")
	parsed, err := strconv.ParseFloat(value, 64)
	return parsed, err == nil
}

func normalizeWidth(row []string, width int) []string {
	out := make([]string, width)
	for i := 0; i < width && i < len(row); i++ {
		out[i] = strings.TrimSpace(row[i])
	}
	return out
}

func trimRow(row []string) []string {
	out := make([]string, len(row))
	for i := range row {
		out[i] = strings.TrimSpace(row[i])
	}
	return out
}

func emptyRow(row []string) bool {
	for _, value := range row {
		if strings.TrimSpace(value) != "" {
			return false
		}
	}
	return true
}

func min(a, b int) int {
	if a < b {
		return a
	}
	return b
}

func max(a, b int) int {
	if a > b {
		return a
	}
	return b
}