pro/im_server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(group): bound push pressure and synchronize load agents

Browse Source
This commit is contained in:
richjohnrambo
2026-10-04 00:49:27 +08:00
parent 746f75ab0f
commit 7b25365ec3
14 changed files with 476 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
loadtest/im-pressure/coordinator.go
+92 -9
View File
@@ -3,6 +3,7 @@ package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"log"
@@ -29,6 +30,20 @@ type AgentReady struct {
Armed bool `json:"armed"`
}
type AgentStarted struct {
Machine string `json:"machine"`
ScheduledAt time.Time `json:"scheduled_at"`
}
type StartGate struct {
Proceed bool `json:"proceed"`
Error string `json:"error,omitempty"`
ArrivedAgents int `json:"arrived_agents"`
ExpectedAgents int `json:"expected_agents"`
}
const maxAgentStartSkew = 3 * time.Second
type coordinatorState struct {
mu sync.Mutex
start StartInfo
@@ -37,6 +52,8 @@ type coordinatorState struct {
expectedAgents int
ready map[string]AgentReady
armed map[string]bool
started map[string]time.Time
startError string
ledgers map[string][]AcceptedMessage
observed map[string][]AcceptedMessage
history map[string][]string
@@ -60,11 +77,13 @@ func runCoordinator(ctx context.Context, listen, runID, scenario, output string,
expectedAgents: expectedAgents,
ready: make(map[string]AgentReady), ledgers: make(map[string][]AcceptedMessage),
observed: make(map[string][]AcceptedMessage), history: make(map[string][]string),
armed: make(map[string]bool), summaries: make(map[string]AgentSummary), complete: make(chan struct{}),
armed: make(map[string]bool), started: make(map[string]time.Time),
summaries: make(map[string]AgentSummary), complete: make(chan struct{}),
}
mux := http.NewServeMux()
mux.HandleFunc("/v1/ready", state.handleReady)
mux.HandleFunc("/v1/start", state.handleStart)
mux.HandleFunc("/v1/started", state.handleStarted)
mux.HandleFunc("/v1/ledger", state.handleLedger)
mux.HandleFunc("/v1/observed", state.handleObserved)
mux.HandleFunc("/v1/history", state.handleHistory)
@@ -210,11 +229,15 @@ func (s *coordinatorState) handleReady(w http.ResponseWriter, r *http.Request) {
s.start.StartAt = time.Time{}
s.start.Committed = false
clear(s.armed)
clear(s.started)
s.startError = ""
}
if allStable && s.start.StartAt.IsZero() {
s.start.StartAt = now.Add(s.startDelay)
s.start.Committed = false
clear(s.armed)
clear(s.started)
s.startError = ""
log.Printf("all agents ready; validation_at=%s", s.start.StartAt.Format(time.RFC3339Nano))
}
if allStable && !s.start.Committed && ready.Armed && !now.Before(s.start.StartAt) {
@@ -234,6 +257,43 @@ func (s *coordinatorState) handleReady(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusOK, start)
}
func (s *coordinatorState) handleStarted(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
return
}
var arrival AgentStarted
if !decodeLimitedJSON(w, r, &arrival) || (arrival.Machine != "a" && arrival.Machine != "b") || arrival.ScheduledAt.IsZero() {
http.Error(w, "invalid start arrival", http.StatusBadRequest)
return
}
s.mu.Lock()
defer s.mu.Unlock()
if !s.start.Committed || !arrival.ScheduledAt.Equal(s.start.StartAt) {
writeJSON(w, http.StatusOK, StartGate{Error: "start schedule changed", ExpectedAgents: s.expectedAgents})
return
}
if s.started == nil {
s.started = make(map[string]time.Time)
}
if _, exists := s.started[arrival.Machine]; !exists {
now := time.Now().UTC()
s.started[arrival.Machine] = now
skew := now.Sub(s.start.StartAt)
if skew < -maxAgentStartSkew || skew > maxAgentStartSkew {
s.startError = fmt.Sprintf("agent %s reached start barrier %s from schedule (limit %s)",
arrival.Machine, skew.Round(time.Millisecond), maxAgentStartSkew)
log.Printf("start barrier rejected: %s", s.startError)
}
}
gate := StartGate{
Proceed: len(s.started) == s.expectedAgents && s.startError == "", Error: s.startError,
ArrivedAgents: len(s.started), ExpectedAgents: s.expectedAgents,
}
writeJSON(w, http.StatusOK, gate)
}
func (s *coordinatorState) handleStart(w http.ResponseWriter, r *http.Request) {
s.mu.Lock()
defer s.mu.Unlock()
@@ -311,7 +371,8 @@ func (s *coordinatorState) handleStatus(w http.ResponseWriter, _ *http.Request)
s.mu.Lock()
defer s.mu.Unlock()
writeJSON(w, http.StatusOK, map[string]any{
"start": s.start, "ready": s.ready, "armed": s.armed,
"start": s.start, "ready": s.ready, "armed": s.armed, "started": s.started,
"start_error": s.startError,
"ledger_agents": len(s.ledgers), "observed_agents": len(s.observed),
"history_agents": len(s.history), "summary_agents": len(s.summaries),
})
@@ -320,14 +381,17 @@ func (s *coordinatorState) handleStatus(w http.ResponseWriter, _ *http.Request)
func (s *coordinatorState) writeOutput(path string) error {
s.mu.Lock()
data := struct {
Start StartInfo `json:"start"`
Ready map[string]AgentReady `json:"ready"`
Ledgers map[string][]AcceptedMessage `json:"ledgers"`
Observed map[string][]AcceptedMessage `json:"observed"`
History map[string][]string `json:"history"`
Summaries map[string]AgentSummary `json:"summaries"`
Start StartInfo `json:"start"`
Ready map[string]AgentReady `json:"ready"`
Started map[string]time.Time `json:"started"`
StartError string `json:"start_error,omitempty"`
Ledgers map[string][]AcceptedMessage `json:"ledgers"`
Observed map[string][]AcceptedMessage `json:"observed"`
History map[string][]string `json:"history"`
Summaries map[string]AgentSummary `json:"summaries"`
}{
Start: s.start, Ready: s.ready, Ledgers: s.ledgers, Observed: s.observed,
Start: s.start, Ready: s.ready, Started: s.started, StartError: s.startError,
Ledgers: s.ledgers, Observed: s.observed,
History: s.history, Summaries: s.summaries,
}
s.mu.Unlock()
@@ -403,6 +467,25 @@ func (c *coordinatorClient) submitLedger(ctx context.Context, machine string, me
return err
}
func (c *coordinatorClient) started(ctx context.Context, machine string, scheduledAt time.Time) error {
arrival := AgentStarted{Machine: machine, ScheduledAt: scheduledAt}
for {
var gate StartGate
if _, err := c.request(ctx, http.MethodPost, "/v1/started", arrival, &gate); err != nil {
return err
}
if gate.Error != "" {
return errors.New(gate.Error)
}
if gate.Proceed {
return nil
}
if err := waitContext(ctx, 100*time.Millisecond); err != nil {
return err
}
}
}
func (c *coordinatorClient) ledger(ctx context.Context) ([]AcceptedMessage, error) {
for {
var messages []AcceptedMessage