fix(group): bound push pressure and synchronize load agents

2026-10-04 00:49:27 +08:00
parent 746f75ab0f
commit 7b25365ec3
14 changed files with 476 additions and 47 deletions
@@ -46,7 +46,7 @@
 - 两台机器必须使用 NTP/chrony 同步时间。报告会分别输出同机时钟延迟和跨机器单向延迟；跨机器出现负延迟样本时，后者仅用于诊断，不参与门禁。
 - agent 默认每 20 秒发送 WebSocket Ping，覆盖等待另一台机器、发压、drain 和历史核验阶段；`ws_ping_interval` 必须小于公网入口的空闲连接超时。
 - 正式开始前 agent 会持续检查实际存活连接；连接断开后自动重新登录并订阅 P2P、群 Topic。初次连接不齐或等待另一台机器期间掉线不会退出。
- coordinator 使用两阶段屏障：两边实时连接数都达到配置值后进入验证倒计时；倒计时内任一方掉线就撤销屏障。两边再次确认满连接后才提交最终开始时间。
+- coordinator 使用三阶段屏障：两边实时连接数都达到配置值后进入验证倒计时；倒计时内任一方掉线就撤销屏障；提交最终开始时间后，两边必须在该时间前后 3 秒内再次到达起跑屏障，且双方都到达后才允许发送。任一方迟到会让两边共同终止本轮，避免生成错时压测结果。
 - 压测机如保持 OpenVPN 等全局 TUN，仅在另一套代理中设置 `DIRECT` 无法绕过它。起压前必须确认 `http_base` 和 WS 域名的每个当前 IP 都走物理网卡；需保留 VPN 访问内网时，可为这些 IP 配置经局域网关的 `/32` 主机路由。VPN 重连、DNS 变更或机器重启后需重新核验。

 ## 统计口径
@@ -186,6 +186,9 @@ func runAgent(ctx context.Context, cfg *Config, manifest *Manifest, machine, coo
 	if err = waitUntil(ctx, start.StartAt); err != nil {
 		return err
 	}
+	if err = coord.started(ctx, machine, start.StartAt); err != nil {
+		return fmt.Errorf("coordinated start rejected: %w", err)
+	}
 	if alive := run.aliveConnections(); alive != len(accounts) {
 		return fmt.Errorf("connections alive at scheduled start=%d expected=%d", alive, len(accounts))
 	}
@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"log"
@@ -29,6 +30,20 @@ type AgentReady struct {
 	Armed               bool   `json:"armed"`
 }

+type AgentStarted struct {
+	Machine     string    `json:"machine"`
+	ScheduledAt time.Time `json:"scheduled_at"`
+}
+
+type StartGate struct {
+	Proceed        bool   `json:"proceed"`
+	Error          string `json:"error,omitempty"`
+	ArrivedAgents  int    `json:"arrived_agents"`
+	ExpectedAgents int    `json:"expected_agents"`
+}
+
+const maxAgentStartSkew = 3 * time.Second
+
 type coordinatorState struct {
 	mu             sync.Mutex
 	start          StartInfo
@@ -37,6 +52,8 @@ type coordinatorState struct {
 	expectedAgents int
 	ready          map[string]AgentReady
 	armed          map[string]bool
+	started        map[string]time.Time
+	startError     string
 	ledgers        map[string][]AcceptedMessage
 	observed       map[string][]AcceptedMessage
 	history        map[string][]string
@@ -60,11 +77,13 @@ func runCoordinator(ctx context.Context, listen, runID, scenario, output string,
 		expectedAgents: expectedAgents,
 		ready:          make(map[string]AgentReady), ledgers: make(map[string][]AcceptedMessage),
 		observed: make(map[string][]AcceptedMessage), history: make(map[string][]string),
-		armed: make(map[string]bool), summaries: make(map[string]AgentSummary), complete: make(chan struct{}),
+		armed: make(map[string]bool), started: make(map[string]time.Time),
+		summaries: make(map[string]AgentSummary), complete: make(chan struct{}),
 	}
 	mux := http.NewServeMux()
 	mux.HandleFunc("/v1/ready", state.handleReady)
 	mux.HandleFunc("/v1/start", state.handleStart)
+	mux.HandleFunc("/v1/started", state.handleStarted)
 	mux.HandleFunc("/v1/ledger", state.handleLedger)
 	mux.HandleFunc("/v1/observed", state.handleObserved)
 	mux.HandleFunc("/v1/history", state.handleHistory)
@@ -210,11 +229,15 @@ func (s *coordinatorState) handleReady(w http.ResponseWriter, r *http.Request) {
 		s.start.StartAt = time.Time{}
 		s.start.Committed = false
 		clear(s.armed)
+		clear(s.started)
+		s.startError = ""
 	}
 	if allStable && s.start.StartAt.IsZero() {
 		s.start.StartAt = now.Add(s.startDelay)
 		s.start.Committed = false
 		clear(s.armed)
+		clear(s.started)
+		s.startError = ""
 		log.Printf("all agents ready; validation_at=%s", s.start.StartAt.Format(time.RFC3339Nano))
 	}
 	if allStable && !s.start.Committed && ready.Armed && !now.Before(s.start.StartAt) {
@@ -234,6 +257,43 @@ func (s *coordinatorState) handleReady(w http.ResponseWriter, r *http.Request) {
 	writeJSON(w, http.StatusOK, start)
 }

+func (s *coordinatorState) handleStarted(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodPost {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	var arrival AgentStarted
+	if !decodeLimitedJSON(w, r, &arrival) || (arrival.Machine != "a" && arrival.Machine != "b") || arrival.ScheduledAt.IsZero() {
+		http.Error(w, "invalid start arrival", http.StatusBadRequest)
+		return
+	}
+
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if !s.start.Committed || !arrival.ScheduledAt.Equal(s.start.StartAt) {
+		writeJSON(w, http.StatusOK, StartGate{Error: "start schedule changed", ExpectedAgents: s.expectedAgents})
+		return
+	}
+	if s.started == nil {
+		s.started = make(map[string]time.Time)
+	}
+	if _, exists := s.started[arrival.Machine]; !exists {
+		now := time.Now().UTC()
+		s.started[arrival.Machine] = now
+		skew := now.Sub(s.start.StartAt)
+		if skew < -maxAgentStartSkew || skew > maxAgentStartSkew {
+			s.startError = fmt.Sprintf("agent %s reached start barrier %s from schedule (limit %s)",
+				arrival.Machine, skew.Round(time.Millisecond), maxAgentStartSkew)
+			log.Printf("start barrier rejected: %s", s.startError)
+		}
+	}
+	gate := StartGate{
+		Proceed: len(s.started) == s.expectedAgents && s.startError == "", Error: s.startError,
+		ArrivedAgents: len(s.started), ExpectedAgents: s.expectedAgents,
+	}
+	writeJSON(w, http.StatusOK, gate)
+}
+
 func (s *coordinatorState) handleStart(w http.ResponseWriter, r *http.Request) {
 	s.mu.Lock()
 	defer s.mu.Unlock()
@@ -311,7 +371,8 @@ func (s *coordinatorState) handleStatus(w http.ResponseWriter, _ *http.Request)
 	s.mu.Lock()
 	defer s.mu.Unlock()
 	writeJSON(w, http.StatusOK, map[string]any{
-		"start": s.start, "ready": s.ready, "armed": s.armed,
+		"start": s.start, "ready": s.ready, "armed": s.armed, "started": s.started,
+		"start_error":   s.startError,
 		"ledger_agents": len(s.ledgers), "observed_agents": len(s.observed),
 		"history_agents": len(s.history), "summary_agents": len(s.summaries),
 	})
@@ -320,14 +381,17 @@ func (s *coordinatorState) handleStatus(w http.ResponseWriter, _ *http.Request)
 func (s *coordinatorState) writeOutput(path string) error {
 	s.mu.Lock()
 	data := struct {
-		Start     StartInfo                    `json:"start"`
-		Ready     map[string]AgentReady        `json:"ready"`
-		Ledgers   map[string][]AcceptedMessage `json:"ledgers"`
-		Observed  map[string][]AcceptedMessage `json:"observed"`
-		History   map[string][]string          `json:"history"`
-		Summaries map[string]AgentSummary      `json:"summaries"`
+		Start      StartInfo                    `json:"start"`
+		Ready      map[string]AgentReady        `json:"ready"`
+		Started    map[string]time.Time         `json:"started"`
+		StartError string                       `json:"start_error,omitempty"`
+		Ledgers    map[string][]AcceptedMessage `json:"ledgers"`
+		Observed   map[string][]AcceptedMessage `json:"observed"`
+		History    map[string][]string          `json:"history"`
+		Summaries  map[string]AgentSummary      `json:"summaries"`
 	}{
-		Start: s.start, Ready: s.ready, Ledgers: s.ledgers, Observed: s.observed,
+		Start: s.start, Ready: s.ready, Started: s.started, StartError: s.startError,
+		Ledgers: s.ledgers, Observed: s.observed,
 		History: s.history, Summaries: s.summaries,
 	}
 	s.mu.Unlock()
@@ -403,6 +467,25 @@ func (c *coordinatorClient) submitLedger(ctx context.Context, machine string, me
 	return err
 }

+func (c *coordinatorClient) started(ctx context.Context, machine string, scheduledAt time.Time) error {
+	arrival := AgentStarted{Machine: machine, ScheduledAt: scheduledAt}
+	for {
+		var gate StartGate
+		if _, err := c.request(ctx, http.MethodPost, "/v1/started", arrival, &gate); err != nil {
+			return err
+		}
+		if gate.Error != "" {
+			return errors.New(gate.Error)
+		}
+		if gate.Proceed {
+			return nil
+		}
+		if err := waitContext(ctx, 100*time.Millisecond); err != nil {
+			return err
+		}
+	}
+}
+
 func (c *coordinatorClient) ledger(ctx context.Context) ([]AcceptedMessage, error) {
 	for {
 		var messages []AcceptedMessage
@@ -15,6 +15,7 @@ func TestCoordinatorResetsBarrierWhenConnectionCountDrops(t *testing.T) {
 		start: StartInfo{RunID: "run-reset", Scenario: "group"}, startDelay: time.Minute,
 		commitDelay: time.Second, expectedAgents: 2, ready: make(map[string]AgentReady),
 		armed: make(map[string]bool), ledgers: make(map[string][]AcceptedMessage),
+		started:   make(map[string]time.Time),
 		summaries: make(map[string]AgentSummary), complete: make(chan struct{}),
 	}
 	postReady := func(ready AgentReady) StartInfo {
@@ -52,6 +53,7 @@ func TestCoordinatorClientWaitsForBothAgentsAndMergesLedger(t *testing.T) {
 		commitDelay: 50 * time.Millisecond, expectedAgents: 2,
 		ready: make(map[string]AgentReady), ledgers: make(map[string][]AcceptedMessage),
 		armed: make(map[string]bool), summaries: make(map[string]AgentSummary), complete: make(chan struct{}),
+		started: make(map[string]time.Time),
 	}
 	server := httptest.NewServer(testCoordinatorMux(state))
 	defer server.Close()
@@ -116,6 +118,7 @@ func testCoordinatorMux(state *coordinatorState) *http.ServeMux {
 	mux := http.NewServeMux()
 	mux.HandleFunc("/v1/ready", state.handleReady)
 	mux.HandleFunc("/v1/start", state.handleStart)
+	mux.HandleFunc("/v1/started", state.handleStarted)
 	mux.HandleFunc("/v1/ledger", state.handleLedger)
 	mux.HandleFunc("/v1/observed", state.handleObserved)
 	mux.HandleFunc("/v1/history", state.handleHistory)
@@ -123,3 +126,50 @@ func testCoordinatorMux(state *coordinatorState) *http.ServeMux {
 	mux.HandleFunc("/v1/status", state.handleStatus)
 	return mux
 }
+
+func TestCoordinatorRejectsLateAgentForBothMachines(t *testing.T) {
+	startAt := time.Now().Add(-maxAgentStartSkew - time.Second).UTC()
+	state := &coordinatorState{
+		start:          StartInfo{RunID: "run-late", Scenario: "group", StartAt: startAt, Committed: true},
+		expectedAgents: 2, started: make(map[string]time.Time),
+	}
+	server := httptest.NewServer(testCoordinatorMux(state))
+	defer server.Close()
+	client := newCoordinatorClient(server.URL)
+	if err := client.started(context.Background(), "b", startAt); err == nil {
+		t.Fatal("late agent must be rejected")
+	}
+	if err := client.started(context.Background(), "a", startAt); err == nil {
+		t.Fatal("the other agent must observe the same rejected run")
+	}
+}
+
+func TestCoordinatorReleasesAgentsOnlyAfterBothReachStart(t *testing.T) {
+	startAt := time.Now().UTC()
+	state := &coordinatorState{
+		start:          StartInfo{RunID: "run-synced", Scenario: "group", StartAt: startAt, Committed: true},
+		expectedAgents: 2, started: make(map[string]time.Time),
+	}
+	server := httptest.NewServer(testCoordinatorMux(state))
+	defer server.Close()
+	client := newCoordinatorClient(server.URL)
+	aDone := make(chan error, 1)
+	go func() { aDone <- client.started(context.Background(), "a", startAt) }()
+	time.Sleep(50 * time.Millisecond)
+	select {
+	case err := <-aDone:
+		t.Fatalf("machine A passed before B arrived: %v", err)
+	default:
+	}
+	if err := client.started(context.Background(), "b", startAt); err != nil {
+		t.Fatalf("machine B start: %v", err)
+	}
+	select {
+	case err := <-aDone:
+		if err != nil {
+			t.Fatalf("machine A start: %v", err)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("machine A was not released")
+	}
+}