package tableparse import ( "encoding/csv" "fmt" "io" "math" "os" "strconv" "strings" "github.com/xuri/excelize/v2" ) const ( maxSampleRows = 20 maxRows = 5000 ) type Profile struct { Columns []ColumnStat `json:"columns"` SampleRows [][]string `json:"sample_rows"` RowCount int `json:"row_count"` Stats map[string]NumericStat `json:"stats,omitempty"` Outliers []Outlier `json:"outliers,omitempty"` Parsed bool `json:"parsed"` Message string `json:"message,omitempty"` } type ColumnStat struct { Name string `json:"name"` Type string `json:"type"` Empty int `json:"empty"` } type NumericStat struct { Count int `json:"count"` Sum float64 `json:"sum"` Avg float64 `json:"avg"` Min float64 `json:"min"` Max float64 `json:"max"` } type Outlier struct { Row int `json:"row"` Column string `json:"column"` Description string `json:"description"` } func Parse(path, fileType string) (Profile, error) { switch strings.ToLower(strings.TrimSpace(fileType)) { case "csv": return parseCSV(path) case "xlsx": return parseXLSX(path) case "xls": return Profile{Parsed: false, Message: "legacy xls parsing is not supported yet"}, nil default: return Profile{}, fmt.Errorf("unsupported table file type: %s", fileType) } } func parseCSV(path string) (Profile, error) { f, err := os.Open(path) if err != nil { return Profile{}, err } defer f.Close() reader := csv.NewReader(f) reader.FieldsPerRecord = -1 rows := make([][]string, 0, maxRows) for len(rows) < maxRows { row, err := reader.Read() if err == io.EOF { break } if err != nil { return Profile{}, err } if emptyRow(row) { continue } rows = append(rows, trimRow(row)) } return profileRows(rows), nil } func parseXLSX(path string) (Profile, error) { f, err := excelize.OpenFile(path) if err != nil { return Profile{}, err } defer f.Close() for _, sheet := range f.GetSheetList() { iter, err := f.Rows(sheet) if err != nil { continue } cleaned := make([][]string, 0, maxRows) for iter.Next() { row, err := iter.Columns() if err != nil { _ = iter.Close() return Profile{}, err } if emptyRow(row) { continue } cleaned = append(cleaned, trimRow(row)) if len(cleaned) >= maxRows { break } } if err := iter.Close(); err != nil { return Profile{}, err } if len(cleaned) > 0 { return profileRows(cleaned), nil } } return Profile{Parsed: false, Message: "no non-empty sheet found"}, nil } func profileRows(rows [][]string) Profile { if len(rows) == 0 { return Profile{Parsed: false, Message: "empty table"} } headers := rows[0] if len(headers) == 0 { return Profile{Parsed: false, Message: "empty header"} } dataRows := rows[1:] columnCount := len(headers) columns := make([]ColumnStat, columnCount) statsWork := make([]numericWork, columnCount) for i, name := range headers { name = strings.TrimSpace(name) if name == "" { name = fmt.Sprintf("列%d", i+1) } columns[i] = ColumnStat{Name: name, Type: "text"} } sample := make([][]string, 0, min(len(dataRows), maxSampleRows)) for rowIndex, row := range dataRows { normalized := normalizeWidth(row, columnCount) if len(sample) < maxSampleRows { sample = append(sample, normalized) } for col := 0; col < columnCount; col++ { value := strings.TrimSpace(normalized[col]) if value == "" { columns[col].Empty++ continue } if parsed, ok := parseNumber(value); ok { statsWork[col].add(parsed, rowIndex+2) } } } stats := map[string]NumericStat{} outliers := []Outlier{} for i := range columns { work := statsWork[i] nonEmpty := len(dataRows) - columns[i].Empty if work.Count > 0 && work.Count*2 >= max(1, nonEmpty) { columns[i].Type = "number" stat := work.stat() stats[columns[i].Name] = stat if stat.Count >= 3 { for _, point := range work.Points { if point.Value == stat.Max && stat.Avg > 0 && point.Value > stat.Avg*2 { outliers = append(outliers, Outlier{Row: point.Row, Column: columns[i].Name, Description: "数值明显高于平均值"}) break } } } } } return Profile{Columns: columns, SampleRows: sample, RowCount: len(dataRows), Stats: stats, Outliers: outliers, Parsed: true} } type numericWork struct { Count int Sum float64 Min float64 Max float64 Points []numericPoint } type numericPoint struct { Row int Value float64 } func (w *numericWork) add(value float64, row int) { if w.Count == 0 { w.Min = value w.Max = value } else { w.Min = math.Min(w.Min, value) w.Max = math.Max(w.Max, value) } w.Count++ w.Sum += value if len(w.Points) < 200 { w.Points = append(w.Points, numericPoint{Row: row, Value: value}) } } func (w numericWork) stat() NumericStat { avg := 0.0 if w.Count > 0 { avg = w.Sum / float64(w.Count) } return NumericStat{Count: w.Count, Sum: w.Sum, Avg: avg, Min: w.Min, Max: w.Max} } func parseNumber(value string) (float64, bool) { value = strings.TrimSpace(strings.ReplaceAll(value, ",", "")) value = strings.TrimSuffix(value, "%") parsed, err := strconv.ParseFloat(value, 64) return parsed, err == nil } func normalizeWidth(row []string, width int) []string { out := make([]string, width) for i := 0; i < width && i < len(row); i++ { out[i] = strings.TrimSpace(row[i]) } return out } func trimRow(row []string) []string { out := make([]string, len(row)) for i := range row { out[i] = strings.TrimSpace(row[i]) } return out } func emptyRow(row []string) bool { for _, value := range row { if strings.TrimSpace(value) != "" { return false } } return true } func min(a, b int) int { if a < b { return a } return b } func max(a, b int) int { if a > b { return a } return b }