im_server/loadtest/im-pressure/agent.go

package main

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strings"
	"sync"
	"sync/atomic"
	"time"
)

var runIDPattern = regexp.MustCompile(`^[a-zA-Z0-9_-]{1,24}$`)

type agentRun struct {
	cfg            *Config
	machine        string
	runID          string
	scenario       string
	clients        map[int]*WSClient
	recorder       *deliveryRecorder
	ack            *DurationHistogram
	attempted      atomic.Int64
	rejected       atomic.Int64
	ready          atomic.Int64
	publish403     atomic.Int64
	publish502     atomic.Int64
	publish503     atomic.Int64
	publishTimeout atomic.Int64
	publishSocket  atomic.Int64
	publishOther   atomic.Int64

	mu       sync.Mutex
	accepted []AcceptedMessage
	errors   []string
}

func validScenario(scenario string) bool {
	return scenario == "p2p" || scenario == "group" || scenario == "mixed"
}

func preflightP2P(ctx context.Context, cfg *Config, manifest *Manifest, local []AccountRecord, apiKey string) error {
	accounts := manifestAccountMap(manifest)
	api := newAPIClient(cfg, apiKey)
	jobs := make(chan AccountRecord)
	errCh := make(chan error, len(local))
	var wg sync.WaitGroup
	workers := cfg.PreflightWorkers
	if workers > len(local) {
		workers = len(local)
	}
	for i := 0; i < workers; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for source := range jobs {
				peerMachine := "a"
				if source.Machine == "a" {
					peerMachine = "b"
				}
				peer := accounts[accountKey(peerMachine, source.Index)]
				if peer == nil {
					errCh <- fmt.Errorf("p2p preflight %s-%04d: peer is missing", source.Machine, source.Index)
					continue
				}
				conversation, err := api.ensureP2P(ctx, source, *peer)
				if err == nil && (conversation.Relationship != "friend" || conversation.Topic != source.P2PTopic) {
					err = fmt.Errorf("relationship=%s topic_match=%t", conversation.Relationship, conversation.Topic == source.P2PTopic)
				}
				if err != nil {
					errCh <- fmt.Errorf("p2p preflight %s-%04d: %w", source.Machine, source.Index, err)
				}
			}
		}()
	}
	for _, account := range local {
		jobs <- account
	}
	close(jobs)
	wg.Wait()
	close(errCh)
	failures := make([]string, 0, len(errCh))
	for err := range errCh {
		if len(failures) < 20 {
			failures = append(failures, err.Error())
		}
	}
	if len(failures) > 0 {
		return fmt.Errorf("p2p permission preflight failed: %s", strings.Join(failures, "; "))
	}
	return nil
}

func runAgent(ctx context.Context, cfg *Config, manifest *Manifest, machine, coordinatorURL, output, metricsListen string) error {
	if machine != "a" && machine != "b" {
		return fmt.Errorf("machine must be a or b")
	}
	apiKey, err := cfg.apiKey()
	if err != nil {
		return err
	}
	password := ""
	if name := cfg.PasswordEnv; name != "" {
		password = os.Getenv(name)
	} else {
		password = os.Getenv("IM_LOAD_PASSWORD")
	}
	accounts := accountsForMachine(manifest, machine)
	if len(accounts) != cfg.UsersPerMachine {
		return fmt.Errorf("machine %s accounts=%d expected=%d", machine, len(accounts), cfg.UsersPerMachine)
	}
	for i := range accounts {
		accounts[i].Endpoint = cfg.Endpoints[machine][(accounts[i].Index-1)%len(cfg.Endpoints[machine])]
	}
	if !cfg.SkipP2PPreflight {
		if err = preflightP2P(ctx, cfg, manifest, accounts, apiKey); err != nil {
			return err
		}
	}
	coord := newCoordinatorClient(coordinatorURL)
	placeholderRunID := "pending"
	run := &agentRun{cfg: cfg, machine: machine, clients: make(map[int]*WSClient), ack: NewDurationHistogram()}
	run.recorder = newDeliveryRecorder(placeholderRunID, machine, len(accounts))
	stopMetrics, err := startAgentMetrics(metricsListen, run)
	if err != nil {
		return err
	}
	defer stopMetrics()
	if err = run.connect(ctx, accounts, manifest.GroupTopic, apiKey, password); err != nil {
		run.close()
		return err
	}
	defer run.close()
	start, err := coord.ready(ctx, AgentReady{Machine: machine, ReadyConnections: len(run.clients)})
	if err != nil {
		return err
	}
	if !runIDPattern.MatchString(start.RunID) || !validScenario(start.Scenario) {
		return fmt.Errorf("invalid coordinator start: %+v", start)
	}
	run.runID = start.RunID
	run.scenario = start.Scenario
	run.recorder.setRunID(start.RunID)
	if err = waitUntil(ctx, start.StartAt); err != nil {
		return err
	}
	startedAt := time.Now().UTC()
	log.Printf("agent=%s scenario=%s started", machine, start.Scenario)
	if err = run.execute(ctx, start.StartAt, manifest.GroupTopic); err != nil {
		return err
	}
	loadEndedAt := time.Now().UTC()
	local := run.acceptedCopy()
	if err = coord.submitLedger(ctx, machine, local); err != nil {
		return err
	}
	ledger, err := coord.ledger(ctx)
	if err != nil {
		return err
	}
	drain, err := run.drainDuration()
	if err != nil {
		return err
	}
	if err = waitContext(ctx, drain); err != nil {
		return err
	}
	base := AgentSummary{
		RunID: start.RunID, Scenario: start.Scenario, Machine: machine, StartedAt: startedAt,
		LoadEndedAt: loadEndedAt, FinishedAt: time.Now().UTC(), ReadyConnections: len(run.clients),
		PublishAttempted: run.attempted.Load(), PublishAccepted: int64(len(local)), PublishRejected: run.rejected.Load(),
		PublishErrors: run.publishErrorCounts(),
		AckLatency:    run.ack.Snapshot(), AcceptedLocal: local,
	}
	run.mu.Lock()
	base.Errors = append(base.Errors, run.errors...)
	run.mu.Unlock()
	summary := run.recorder.summarize(base, ledger)
	checked, found, persistenceMissing := run.reconcileHistory(ctx, ledger)
	summary.HistoryChecked = int64(checked)
	summary.HistoryFound = int64(found)
	summary.PersistenceLost = int64(len(persistenceMissing))
	summary.RealtimeLostStored = run.recorder.countRealtimeLostStored(ledger)
	if len(persistenceMissing) > 10000 {
		summary.PersistenceMissing = append([]string(nil), persistenceMissing[:10000]...)
		summary.Errors = append(summary.Errors, "persistence missing details truncated at 10000")
	} else {
		summary.PersistenceMissing = persistenceMissing
	}
	summary.FinishedAt = time.Now().UTC()
	if err = writeAgentSummary(output, summary); err != nil {
		return err
	}
	if err = coord.submitSummary(ctx, summary); err != nil {
		return err
	}
	log.Printf("agent=%s accepted=%d expected_peer=%d received_peer=%d expected_group=%d received_group=%d lost=%d duplicates=%d",
		machine, summary.PublishAccepted, summary.ExpectedPeer, summary.ReceivedPeer,
		summary.ExpectedGroup, summary.ReceivedGroup, summary.RealtimeLost, summary.Duplicates)
	return nil
}

type historyQuery struct {
	client *WSClient
	topic  string
	since  int64
	before int64
	limit  int
}

func (r *agentRun) reconcileHistory(ctx context.Context, ledger []AcceptedMessage) (int, int, []string) {
	expected := make(map[string]AcceptedMessage)
	byTopic := make(map[string][]AcceptedMessage)
	for _, message := range ledger {
		if message.RunID != r.runID {
			continue
		}
		if message.Kind == "p2p" && message.PeerMachine == r.machine {
			expected[message.ClientMid] = message
			byTopic[message.Topic] = append(byTopic[message.Topic], message)
		} else if message.Kind == "group" && r.machine == "a" {
			expected[message.ClientMid] = message
			byTopic[message.Topic] = append(byTopic[message.Topic], message)
		}
	}
	if len(expected) == 0 {
		return 0, 0, nil
	}
	r.recorder.beginHistory()
	queries := make([]historyQuery, 0, len(byTopic))
	for topic, messages := range byTopic {
		sort.Slice(messages, func(i, j int) bool { return messages[i].Seq < messages[j].Seq })
		client := r.clients[messages[0].PeerIndex]
		if messages[0].Kind == "group" {
			client = r.clients[1]
		}
		if client == nil {
			r.addError(fmt.Sprintf("history topic=%s has no verifier client", topicHash(topic)))
			continue
		}
		for start := 0; start < len(messages); start += 200 {
			end := start + 200
			if end > len(messages) {
				end = len(messages)
			}
			queries = append(queries, historyQuery{client: client, topic: topic,
				since: messages[start].Seq, before: messages[end-1].Seq + 1, limit: 256})
		}
	}
	historyTimeout, _ := configDuration(r.cfg.HistoryTimeout)
	historyCtx, cancel := context.WithTimeout(ctx, historyTimeout)
	defer cancel()
	r.issueHistoryQueries(historyCtx, queries)
	_ = waitContext(historyCtx, 2*time.Second)
	for round := 0; round < 2; round++ {
		missing := missingHistory(r.recorder, expected)
		if len(missing) == 0 || historyCtx.Err() != nil {
			break
		}
		retryQueries := make([]historyQuery, 0, len(missing))
		for _, message := range missing {
			client := r.clients[message.PeerIndex]
			if message.Kind == "group" {
				client = r.clients[1]
			}
			if client != nil {
				retryQueries = append(retryQueries, historyQuery{client: client, topic: message.Topic,
					since: message.Seq, before: message.Seq + 1, limit: 1})
			}
		}
		r.issueHistoryQueries(historyCtx, retryQueries)
		_ = waitContext(historyCtx, 2*time.Second)
	}
	missing := missingHistory(r.recorder, expected)
	missingIDs := make([]string, 0, len(missing))
	for _, message := range missing {
		missingIDs = append(missingIDs, message.ClientMid)
	}
	sort.Strings(missingIDs)
	return len(expected), len(expected) - len(missing), missingIDs
}

func (r *agentRun) issueHistoryQueries(ctx context.Context, queries []historyQuery) {
	workers := r.cfg.HistoryWorkers
	if workers > len(queries) {
		workers = len(queries)
	}
	if workers == 0 {
		return
	}
	jobs := make(chan historyQuery)
	var wg sync.WaitGroup
	for i := 0; i < workers; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for query := range jobs {
				if err := query.client.GetData(ctx, query.topic, query.since, query.before, query.limit); err != nil && ctx.Err() == nil {
					r.addError(fmt.Sprintf("history get topic=%s range=%d-%d: %v", topicHash(query.topic), query.since, query.before, err))
				}
			}
		}()
	}
	for _, query := range queries {
		select {
		case jobs <- query:
		case <-ctx.Done():
			close(jobs)
			wg.Wait()
			return
		}
	}
	close(jobs)
	wg.Wait()
}

func missingHistory(recorder *deliveryRecorder, expected map[string]AcceptedMessage) []AcceptedMessage {
	missing := make([]AcceptedMessage, 0)
	for mid, message := range expected {
		if !recorder.historyFound(mid) {
			missing = append(missing, message)
		}
	}
	return missing
}

func (r *agentRun) addError(message string) {
	r.mu.Lock()
	defer r.mu.Unlock()
	if len(r.errors) < 1000 {
		r.errors = append(r.errors, message)
	}
}

func (r *agentRun) connect(ctx context.Context, accounts []AccountRecord, groupTopic, apiKey, password string) error {
	ramp, err := configDuration(r.cfg.ConnectRamp)
	if err != nil {
		return err
	}
	interval := ramp / time.Duration(len(accounts))
	var wg sync.WaitGroup
	errCh := make(chan error, len(accounts))
	var mu sync.Mutex
	for _, account := range accounts {
		account := account
		wg.Add(1)
		go func() {
			defer wg.Done()
			client, connectErr := r.connectAccountWithRetry(ctx, account, groupTopic, apiKey, password)
			if connectErr != nil {
				errCh <- connectErr
				return
			}
			mu.Lock()
			r.clients[account.Index] = client
			mu.Unlock()
			r.ready.Add(1)
		}()
		if interval > 0 {
			if err = waitContext(ctx, interval); err != nil {
				break
			}
		}
	}
	wg.Wait()
	close(errCh)
	var failures []string
	for connectErr := range errCh {
		if len(failures) < 20 {
			failures = append(failures, connectErr.Error())
		}
	}
	if len(r.clients) != len(accounts) {
		return fmt.Errorf("ready connections=%d expected=%d failures=%v", len(r.clients), len(accounts), failures)
	}
	return nil
}

func (r *agentRun) connectAccountWithRetry(ctx context.Context, account AccountRecord, groupTopic, apiKey, password string) (*WSClient, error) {
	var lastErr error
	for attempt := 1; attempt <= 6; attempt++ {
		client, err := connectWSClient(ctx, r.cfg, account, apiKey, password, r.recorder.onData, r.recorder.onDisconnect)
		if err == nil {
			if err = client.Subscribe(ctx, account.P2PTopic); err == nil {
				err = client.Subscribe(ctx, groupTopic)
			}
		}
		if err == nil {
			return client, nil
		}
		if client != nil {
			client.Close()
		}
		lastErr = err
		if ctx.Err() != nil {
			return nil, ctx.Err()
		}
		delay := time.Duration(attempt*2) * time.Second
		log.Printf("account=%s connect attempt=%d failed; retry_in=%s: %v", account.Username, attempt, delay, err)
		if err = waitContext(ctx, delay); err != nil {
			return nil, err
		}
	}
	return nil, fmt.Errorf("connect %s failed after retries: %w", account.Username, lastErr)
}

func (r *agentRun) execute(ctx context.Context, start time.Time, groupTopic string) error {
	if groupTopic == "" {
		return fmt.Errorf("group topic is empty")
	}
	switch r.scenario {
	case "p2p":
		return r.scheduleP2P(ctx, start, r.cfg.UsersPerMachine, r.cfg.P2PMessages, mustDuration(r.cfg.P2PInterval))
	case "group":
		return r.scheduleGroup(ctx, start, groupTopic, 1, 500, 2)
	case "mixed":
		var wg sync.WaitGroup
		errCh := make(chan error, 2)
		wg.Add(2)
		go func() { defer wg.Done(); errCh <- r.scheduleP2P(ctx, start, 300, 5, 20*time.Second) }()
		go func() { defer wg.Done(); errCh <- r.scheduleGroup(ctx, start, groupTopic, 301, 550, 2) }()
		wg.Wait()
		close(errCh)
		for err := range errCh {
			if err != nil {
				return err
			}
		}
		return nil
	default:
		return fmt.Errorf("unknown scenario %s", r.scenario)
	}
}

func (r *agentRun) scheduleP2P(ctx context.Context, start time.Time, users, messages int, interval time.Duration) error {
	var wg sync.WaitGroup
	for index := 1; index <= users; index++ {
		client := r.clients[index]
		if client == nil {
			return fmt.Errorf("missing client %d", index)
		}
		wg.Add(1)
		go func(index int, client *WSClient) {
			defer wg.Done()
			jitter := time.Duration((index*37)%200) * time.Millisecond
			for messageIndex := 1; messageIndex <= messages; messageIndex++ {
				target := start.Add(time.Duration(messageIndex-1)*interval + jitter)
				if waitUntil(ctx, target) != nil {
					return
				}
				mid := fmt.Sprintf("%s-p-%s-%04d-%02d", r.runID, r.machine, index, messageIndex)
				accepted, err := r.publish(ctx, client, client.account.P2PTopic, mid, messageIndex)
				if err == nil {
					accepted.PeerIndex = index
					if r.machine == "a" {
						accepted.PeerMachine = "b"
					} else {
						accepted.PeerMachine = "a"
					}
					r.addAccepted(accepted)
				}
			}
		}(index, client)
	}
	wg.Wait()
	return ctx.Err()
}

func (r *agentRun) scheduleGroup(ctx context.Context, start time.Time, topic string, first, last, rounds int) error {
	if first < 1 || last > len(r.clients) || first > last {
		return fmt.Errorf("invalid group publisher range %d-%d", first, last)
	}
	count := last - first + 1
	roundDuration := time.Duration(count) * 200 * time.Millisecond
	offset := time.Duration(0)
	if r.machine == "b" {
		offset = 100 * time.Millisecond
	}
	var wg sync.WaitGroup
	for index := first; index <= last; index++ {
		client := r.clients[index]
		position := index - first
		wg.Add(1)
		go func(index, position int, client *WSClient) {
			defer wg.Done()
			for round := 1; round <= rounds; round++ {
				target := start.Add(time.Duration(round-1)*roundDuration + time.Duration(position)*200*time.Millisecond + offset)
				if waitUntil(ctx, target) != nil {
					return
				}
				mid := fmt.Sprintf("%s-g-%s-%04d-%02d", r.runID, r.machine, index, round)
				accepted, err := r.publish(ctx, client, topic, mid, round)
				if err == nil {
					r.addAccepted(accepted)
				}
			}
		}(index, position, client)
	}
	wg.Wait()
	return ctx.Err()
}

func (r *agentRun) publish(ctx context.Context, client *WSClient, topic, mid string, messageIndex int) (AcceptedMessage, error) {
	r.attempted.Add(1)
	accepted, err := client.Publish(ctx, topic, mid, r.runID, r.scenario, messageIndex)
	if err != nil {
		r.rejected.Add(1)
		r.classifyPublishError(err)
		r.mu.Lock()
		if len(r.errors) < 1000 {
			r.errors = append(r.errors, fmt.Sprintf("publish %s: %v", mid, err))
		}
		r.mu.Unlock()
		return accepted, err
	}
	r.ack.Observe(accepted.AckAt.Sub(accepted.SentAt))
	return accepted, nil
}

func (r *agentRun) classifyPublishError(err error) {
	var ctrlErr *ctrlError
	switch {
	case errors.As(err, &ctrlErr) && ctrlErr.Code == 403:
		r.publish403.Add(1)
	case errors.As(err, &ctrlErr) && ctrlErr.Code == 502:
		r.publish502.Add(1)
	case errors.As(err, &ctrlErr) && ctrlErr.Code == 503:
		r.publish503.Add(1)
	case strings.Contains(err.Error(), "ctrl timeout") || errors.Is(err, context.DeadlineExceeded):
		r.publishTimeout.Add(1)
	case strings.Contains(err.Error(), "websocket") || strings.Contains(err.Error(), "broken pipe") ||
		strings.Contains(err.Error(), "connection reset"):
		r.publishSocket.Add(1)
	default:
		r.publishOther.Add(1)
	}
}

func (r *agentRun) publishErrorCounts() map[string]int64 {
	return map[string]int64{
		"403": r.publish403.Load(), "502": r.publish502.Load(), "503": r.publish503.Load(),
		"ctrl_timeout": r.publishTimeout.Load(), "socket": r.publishSocket.Load(), "other": r.publishOther.Load(),
	}
}

func (r *agentRun) addAccepted(message AcceptedMessage) {
	r.mu.Lock()
	r.accepted = append(r.accepted, message)
	r.mu.Unlock()
}

func (r *agentRun) acceptedCopy() []AcceptedMessage {
	r.mu.Lock()
	defer r.mu.Unlock()
	out := append([]AcceptedMessage(nil), r.accepted...)
	sort.Slice(out, func(i, j int) bool { return out[i].ClientMid < out[j].ClientMid })
	return out
}

func (r *agentRun) drainDuration() (time.Duration, error) {
	if r.scenario == "p2p" {
		return configDuration(r.cfg.DrainP2P)
	}
	return configDuration(r.cfg.DrainGroup)
}

func (r *agentRun) close() {
	for _, client := range r.clients {
		client.Close()
	}
}

func waitUntil(ctx context.Context, target time.Time) error {
	delay := time.Until(target)
	if delay <= 0 {
		return nil
	}
	return waitContext(ctx, delay)
}

func mustDuration(value string) time.Duration {
	d, err := time.ParseDuration(value)
	if err != nil || d <= 0 {
		return time.Second
	}
	return d
}

func writeAgentSummary(path string, summary AgentSummary) error {
	raw, err := json.MarshalIndent(summary, "", "  ")
	if err != nil {
		return err
	}
	if dir := filepath.Dir(path); dir != "." {
		if err = os.MkdirAll(dir, 0o750); err != nil {
			return err
		}
	}
	return os.WriteFile(path, append(raw, '\n'), 0o600)
}