pro/im_server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test: stabilize distributed load verification

Browse Source
This commit is contained in:
richjohnrambo
2026-09-28 09:50:33 +08:00
parent d3c679878d
commit a974572f72
12 changed files with 344 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
loadtest/im-pressure/agent.go
+92 -29
View File
@@ -234,13 +234,20 @@ func runAgent(ctx context.Context, cfg *Config, manifest *Manifest, machine, coo
return err
}
history := run.reconcileHistory(ctx, verificationLedger, ledger)
if err = coord.submitHistory(ctx, machine, history.foundIDs); err != nil {
return err
}
storedMessageIDs, err := coord.history(ctx)
if err != nil {
return err
}
summary.HistoryChecked = int64(history.checked)
summary.HistoryFound = int64(history.found)
summary.HistoryUnackedChecked = int64(history.unackedChecked)
summary.HistoryUnackedFound = int64(history.unackedFound)
persistenceMissing := history.missing
summary.PersistenceLost = int64(len(persistenceMissing))
summary.RealtimeLostStored = run.recorder.countRealtimeLostStored(ledger)
summary.RealtimeLostStored = run.recorder.countRealtimeLostStored(ledger, storedMessageIDs)
if len(persistenceMissing) > 10000 {
summary.PersistenceMissing = append([]string(nil), persistenceMissing[:10000]...)
summary.Errors = append(summary.Errors, "persistence missing details truncated at 10000")
@@ -261,11 +268,12 @@ func runAgent(ctx context.Context, cfg *Config, manifest *Manifest, machine, coo
}
type historyQuery struct {
client *WSClient
topic string
since int64
before int64
limit int
client *WSClient
topic string
since int64
before int64
limit int
expected []string
}
type historyResult struct {
@@ -274,6 +282,7 @@ type historyResult struct {
unackedChecked int
unackedFound int
missing []string
foundIDs []string
}
func (r *agentRun) reconcileHistory(ctx context.Context, ledger, ackLedger []AcceptedMessage) historyResult {
@@ -316,8 +325,7 @@ func (r *agentRun) reconcileHistory(ctx context.Context, ledger, ackLedger []Acc
if end > len(messages) {
end = len(messages)
}
queries = append(queries, historyQuery{client: client, topic: topic,
since: messages[start].Seq, before: messages[end-1].Seq + 1, limit: pageSize})
queries = append(queries, newHistoryQuery(client, topic, messages[start:end], pageSize))
}
}
historyTimeout, _ := configDuration(r.cfg.HistoryTimeout)
@@ -340,8 +348,12 @@ func (r *agentRun) reconcileHistory(ctx context.Context, ledger, ackLedger []Acc
missingIDs = append(missingIDs, message.ClientMid)
}
sort.Strings(missingIDs)
result := historyResult{checked: len(expected), found: len(expected) - len(missing), missing: missingIDs}
result := historyResult{checked: len(expected), found: len(expected) - len(missing), missing: missingIDs,
foundIDs: make([]string, 0, len(expected)-len(missing))}
for mid := range expected {
if r.recorder.historyFound(mid) {
result.foundIDs = append(result.foundIDs, mid)
}
if acked[mid] {
continue
}
@@ -350,6 +362,7 @@ func (r *agentRun) reconcileHistory(ctx context.Context, ledger, ackLedger []Acc
result.unackedFound++
}
}
sort.Strings(result.foundIDs)
return result
}
@@ -374,48 +387,62 @@ func (r *agentRun) historyQueriesForMessages(messages []AcceptedMessage) []histo
if end > len(topicMessages) {
end = len(topicMessages)
}
queries = append(queries, historyQuery{client: client, topic: topic,
since: topicMessages[start].Seq, before: topicMessages[end-1].Seq + 1, limit: pageSize})
queries = append(queries, newHistoryQuery(client, topic, topicMessages[start:end], pageSize))
}
}
return queries
}
func (r *agentRun) issueHistoryQueries(ctx context.Context, queries []historyQuery) {
byTopic := make(map[string][]historyQuery)
for _, query := range queries {
byTopic[query.topic] = append(byTopic[query.topic], query)
}
workers := r.cfg.HistoryWorkers
if workers > len(queries) {
workers = len(queries)
if workers > len(byTopic) {
workers = len(byTopic)
}
if workers == 0 {
return
}
jobs := make(chan historyQuery)
jobs := make(chan []historyQuery)
interval := 500 * time.Millisecond
if r.cfg != nil && r.cfg.HistoryInterval != "" {
interval = mustDuration(r.cfg.HistoryInterval)
}
limiter := time.NewTicker(interval)
defer limiter.Stop()
pageWait := 3 * time.Second
if r.cfg != nil && r.cfg.HistoryPageWait != "" {
pageWait = mustDuration(r.cfg.HistoryPageWait)
}
var wg sync.WaitGroup
for i := 0; i < workers; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for query := range jobs {
select {
case <-limiter.C:
case <-ctx.Done():
return
}
if err := query.client.GetData(ctx, query.topic, query.since, query.before, query.limit); err != nil && ctx.Err() == nil {
r.addError(fmt.Sprintf("history get topic=%s range=%d-%d: %v", topicHash(query.topic), query.since, query.before, err))
for topicQueries := range jobs {
for index, query := range topicQueries {
if index > 0 {
if err := waitContext(ctx, interval); err != nil {
return
}
}
if err := query.client.GetData(ctx, query.topic, query.since, query.before, query.limit); err != nil && ctx.Err() == nil {
r.addError(fmt.Sprintf("history get topic=%s range=%d-%d: %v", topicHash(query.topic), query.since, query.before, err))
continue
}
r.waitHistoryPage(ctx, query.expected, pageWait)
}
}
}()
}
for _, query := range queries {
topics := make([]string, 0, len(byTopic))
for topic := range byTopic {
topics = append(topics, topic)
}
sort.Strings(topics)
for _, topic := range topics {
select {
case jobs <- query:
case jobs <- byTopic[topic]:
case <-ctx.Done():
close(jobs)
wg.Wait()
@@ -426,11 +453,39 @@ func (r *agentRun) issueHistoryQueries(ctx context.Context, queries []historyQue
wg.Wait()
}
func newHistoryQuery(client *WSClient, topic string, messages []AcceptedMessage, limit int) historyQuery {
expected := make([]string, 0, len(messages))
for _, message := range messages {
expected = append(expected, message.ClientMid)
}
return historyQuery{client: client, topic: topic, since: messages[0].Seq,
before: messages[len(messages)-1].Seq + 1, limit: limit, expected: expected}
}
func (r *agentRun) waitHistoryPage(ctx context.Context, expected []string, timeout time.Duration) {
deadline := time.NewTimer(timeout)
defer deadline.Stop()
ticker := time.NewTicker(25 * time.Millisecond)
defer ticker.Stop()
for {
if r.recorder.historyFoundAll(expected) {
return
}
select {
case <-ctx.Done():
return
case <-deadline.C:
return
case <-ticker.C:
}
}
}
func (r *agentRun) historyPageSize() int {
if r.cfg != nil && r.cfg.HistoryPageSize > 0 {
return r.cfg.HistoryPageSize
}
return 20
return 10
}
func missingHistory(recorder *deliveryRecorder, expected map[string]AcceptedMessage) []AcceptedMessage {
@@ -720,7 +775,7 @@ func (r *agentRun) classifyPublishError(err error) {
case strings.Contains(err.Error(), "ctrl timeout") || errors.Is(err, context.DeadlineExceeded):
r.publishTimeout.Add(1)
case strings.Contains(err.Error(), "websocket") || strings.Contains(err.Error(), "broken pipe") ||
strings.Contains(err.Error(), "connection reset"):
strings.Contains(err.Error(), "connection reset") || strings.Contains(err.Error(), "closed network connection"):
r.publishSocket.Add(1)
default:
r.publishOther.Add(1)
@@ -802,8 +857,16 @@ func (r *agentRun) drainDuration() (time.Duration, error) {
}
func (r *agentRun) close() {
for _, client := range r.clientsSnapshot() {
clients := r.clientsSnapshot()
var interval time.Duration
if r.cfg != nil && r.cfg.CloseRamp != "" && len(clients) > 1 {
interval = mustDuration(r.cfg.CloseRamp) / time.Duration(len(clients)-1)
}
for index, client := range clients {
client.Close()
if interval > 0 && index < len(clients)-1 {
time.Sleep(interval)
}
}
}