fix: harden multi-pod delivery and load verification

This commit is contained in:
richjohnrambo
2026-09-27 03:37:32 +08:00
parent a2de100f2b
commit ce3250be64
33 changed files with 1340 additions and 94 deletions
+37 -4
View File
@@ -27,6 +27,7 @@ func (n *namedURLs) Set(value string) error {
type metricSnapshot struct {
Timestamp time.Time `json:"timestamp"`
Phase string `json:"phase"`
Name string `json:"name"`
URL string `json:"url"`
Status int `json:"status"`
@@ -35,7 +36,8 @@ type metricSnapshot struct {
Error string `json:"error,omitempty"`
}
func runCollector(ctx context.Context, sources namedURLs, output string, interval, duration time.Duration) error {
func runCollector(ctx context.Context, sources namedURLs, output string, interval, duration time.Duration,
coordinatorURL, coordinatorOutput string) error {
if len(sources) == 0 {
return fmt.Errorf("at least one --url name=url is required")
}
@@ -56,8 +58,10 @@ func runCollector(ctx context.Context, sources namedURLs, output string, interva
defer writer.Flush()
client := &http.Client{Timeout: 10 * time.Second}
deadline := time.Now().Add(duration)
phase := "before"
for {
snapshots := collectOnce(ctx, client, sources)
phase = detectCollectorPhase(ctx, client, phase, coordinatorURL, coordinatorOutput)
snapshots := collectOnce(ctx, client, sources, phase)
for _, snapshot := range snapshots {
raw, _ := json.Marshal(snapshot)
if _, err = writer.Write(append(raw, '\n')); err != nil {
@@ -76,7 +80,36 @@ func runCollector(ctx context.Context, sources namedURLs, output string, interva
}
}
func collectOnce(ctx context.Context, client *http.Client, sources namedURLs) []metricSnapshot {
func detectCollectorPhase(ctx context.Context, client *http.Client, current, coordinatorURL, coordinatorOutput string) string {
if coordinatorOutput != "" {
if _, err := os.Stat(coordinatorOutput); err == nil {
return "after"
}
}
if coordinatorURL == "" {
return current
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet,
strings.TrimRight(coordinatorURL, "/")+"/v1/status", nil)
if err != nil {
return current
}
response, err := client.Do(req)
if err != nil {
return current
}
defer response.Body.Close()
var status struct {
Start StartInfo `json:"start"`
}
if response.StatusCode != http.StatusOK || json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&status) != nil ||
status.Start.StartAt.IsZero() || time.Now().Before(status.Start.StartAt) {
return current
}
return "during"
}
func collectOnce(ctx context.Context, client *http.Client, sources namedURLs, phase string) []metricSnapshot {
out := make([]metricSnapshot, len(sources))
var wg sync.WaitGroup
for i, source := range sources {
@@ -85,7 +118,7 @@ func collectOnce(ctx context.Context, client *http.Client, sources namedURLs) []
go func() {
defer wg.Done()
parts := strings.SplitN(source, "=", 2)
snapshot := metricSnapshot{Timestamp: time.Now().UTC(), Name: parts[0], URL: parts[1]}
snapshot := metricSnapshot{Timestamp: time.Now().UTC(), Phase: phase, Name: parts[0], URL: parts[1]}
started := time.Now()
req, err := http.NewRequestWithContext(ctx, http.MethodGet, parts[1], nil)
if err == nil {