fix: harden multi-pod delivery and load verification

loadtest/im-pressure/README.md

@@ -48,6 +48,9 @@
 - 实时到达只统计同时匹配本轮 `run_id` 和 `x-im-client-mid` 的 `{data}`。
 - 群消息设置 `noecho=false`，每条服务端接受的群消息期望 2000 个连接各收到一次。
 - 实时 drain 结束后通过 `{get what=data}` 按 ACK 的 Topic/Seq 回查历史，区分跨 Pod 实时漏投与数据库持久化缺失。
+- 每轮建立 WS 后、正式计时前，以 32 个 worker 对本机 1000 个账号做好友关系、P2P Topic 和发布权限预检；任一异常会终止本轮。
+- 历史回查默认使用 48 个 worker、总超时 3 分钟，并对缺失项再并发重试两轮。
+- publish 错误完整分类为 `403`、`502`、`503`、`ctrl_timeout`、`socket_error` 和 `other`，不受文本错误样本上限影响。
 - `manifest.json` 含登录 token，文件权限为 `0600`，不得提交到 Git 或放入结果包。
 
 ## 两台机器初始化
@@ -216,6 +219,8 @@ scp "${MACHINE_B_SSH}:${MACHINE_B_REPO}/loadtest/im-pressure/results/${RUN_ID}/a
 ```bash
 ./loadtest/im-pressure/im-pressure collect \
   -duration 20m -interval 5s \
+  -coordinator "http://${MACHINE_A_IP}:8090" \
+  -coordinator-output "loadtest/im-pressure/results/${RUN_ID}/coordinator.json" \
   -url agent-a=http://127.0.0.1:9105/metrics \
   -url agent-b=http://${MACHINE_B_IP}:9105/metrics \
   -url im=http://PROMETHEUS_HOST/api/v1/query?query=up \
@@ -227,7 +232,23 @@ scp "${MACHINE_B_SSH}:${MACHINE_B_REPO}/loadtest/im-pressure/results/${RUN_ID}/a
   -output "loadtest/im-pressure/results/${RUN_ID}/metrics.jsonl"
 ```
 
-机器 A 终端 A3 运行采集器。URL 含 `&` 时需用单引号包住整个 `name=url` 参数。建议起压前 2 分钟开始采集，结束后继续 5 分钟，以观察 Kafka lag、数据库落库追平时间及资源恢复。如果机器 A 无法访问机器 B 的 `9105`，就在机器 B 单独运行一份 `collect`，结束后把 JSONL 一并传回机器 A。
+机器 A 终端 A3 运行采集器。每条 JSONL 快照会自动标记 `phase=before|during|after`：coordinator 下发开始时间后进入 `during`，coordinator 结果文件生成后进入 `after`。URL 含 `&` 时需用单引号包住整个 `name=url` 参数。建议起压前 2 分钟开始采集，结束后继续 5 分钟，以观察 Kafka lag、数据库落库追平时间及资源恢复。如果机器 A 无法访问机器 B 的 `9105`，就在机器 B 单独运行一份 `collect`，结束后把 JSONL 一并传回机器 A。
+
+## MySQL 独立落库核验
+
+合并报告后在机器 A 使用只读 DSN 对 ACK ledger 做独立校验。工具会根据租户和消息时间自动查询 `im_NNN_messages` 及涉及的群消息月表，精确比较 `client_mid + topic + seq`，并统计缺失、错配、重复和非 ACK 消息：
+
+```bash
+export MYSQL_DSN='readonly:password@tcp(mysql-host:3306)/meeting_im?parseTime=true'
+./loadtest/im-pressure/im-pressure verify-db \
+  -coordinator "loadtest/im-pressure/results/${RUN_ID}/coordinator.json" \
+  -tenant allrich \
+  -dsn-env MYSQL_DSN \
+  -registry-db meeting_im \
+  -output "loadtest/im-pressure/results/${RUN_ID}/db-verification.json"
+```
+
+DSN 只能通过环境变量传入，不要写入配置、命令脚本或结果包。命令在 missing、mismatch、duplicate 或 unexpected 任一计数非零时返回失败。
 
 ## 运行前检查
 

loadtest/im-pressure/agent.go

@@ -3,12 +3,14 @@ package main
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"log"
 	"os"
 	"path/filepath"
 	"regexp"
 	"sort"
+	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -17,16 +19,22 @@ import (
 var runIDPattern = regexp.MustCompile(`^[a-zA-Z0-9_-]{1,24}$`)
 
 type agentRun struct {
-	cfg       *Config
-	machine   string
-	runID     string
-	scenario  string
-	clients   map[int]*WSClient
-	recorder  *deliveryRecorder
-	ack       *DurationHistogram
-	attempted atomic.Int64
-	rejected  atomic.Int64
-	ready     atomic.Int64
+	cfg            *Config
+	machine        string
+	runID          string
+	scenario       string
+	clients        map[int]*WSClient
+	recorder       *deliveryRecorder
+	ack            *DurationHistogram
+	attempted      atomic.Int64
+	rejected       atomic.Int64
+	ready          atomic.Int64
+	publish403     atomic.Int64
+	publish502     atomic.Int64
+	publish503     atomic.Int64
+	publishTimeout atomic.Int64
+	publishSocket  atomic.Int64
+	publishOther   atomic.Int64
 
 	mu       sync.Mutex
 	accepted []AcceptedMessage
@@ -37,6 +45,58 @@ func validScenario(scenario string) bool {
 	return scenario == "p2p" || scenario == "group" || scenario == "mixed"
 }
 
+func preflightP2P(ctx context.Context, cfg *Config, manifest *Manifest, local []AccountRecord, apiKey string) error {
+	accounts := manifestAccountMap(manifest)
+	api := newAPIClient(cfg, apiKey)
+	jobs := make(chan AccountRecord)
+	errCh := make(chan error, len(local))
+	var wg sync.WaitGroup
+	workers := cfg.PreflightWorkers
+	if workers > len(local) {
+		workers = len(local)
+	}
+	for i := 0; i < workers; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for source := range jobs {
+				peerMachine := "a"
+				if source.Machine == "a" {
+					peerMachine = "b"
+				}
+				peer := accounts[accountKey(peerMachine, source.Index)]
+				if peer == nil {
+					errCh <- fmt.Errorf("p2p preflight %s-%04d: peer is missing", source.Machine, source.Index)
+					continue
+				}
+				conversation, err := api.ensureP2P(ctx, source, *peer)
+				if err == nil && (conversation.Relationship != "friend" || conversation.Topic != source.P2PTopic) {
+					err = fmt.Errorf("relationship=%s topic_match=%t", conversation.Relationship, conversation.Topic == source.P2PTopic)
+				}
+				if err != nil {
+					errCh <- fmt.Errorf("p2p preflight %s-%04d: %w", source.Machine, source.Index, err)
+				}
+			}
+		}()
+	}
+	for _, account := range local {
+		jobs <- account
+	}
+	close(jobs)
+	wg.Wait()
+	close(errCh)
+	failures := make([]string, 0, len(errCh))
+	for err := range errCh {
+		if len(failures) < 20 {
+			failures = append(failures, err.Error())
+		}
+	}
+	if len(failures) > 0 {
+		return fmt.Errorf("p2p permission preflight failed: %s", strings.Join(failures, "; "))
+	}
+	return nil
+}
+
 func runAgent(ctx context.Context, cfg *Config, manifest *Manifest, machine, coordinatorURL, output, metricsListen string) error {
 	if machine != "a" && machine != "b" {
 		return fmt.Errorf("machine must be a or b")
@@ -58,6 +118,11 @@ func runAgent(ctx context.Context, cfg *Config, manifest *Manifest, machine, coo
 	for i := range accounts {
 		accounts[i].Endpoint = cfg.Endpoints[machine][(accounts[i].Index-1)%len(cfg.Endpoints[machine])]
 	}
+	if !cfg.SkipP2PPreflight {
+		if err = preflightP2P(ctx, cfg, manifest, accounts, apiKey); err != nil {
+			return err
+		}
+	}
 	coord := newCoordinatorClient(coordinatorURL)
 	placeholderRunID := "pending"
 	run := &agentRun{cfg: cfg, machine: machine, clients: make(map[int]*WSClient), ack: NewDurationHistogram()}
@@ -110,7 +175,8 @@ func runAgent(ctx context.Context, cfg *Config, manifest *Manifest, machine, coo
 		RunID: start.RunID, Scenario: start.Scenario, Machine: machine, StartedAt: startedAt,
 		LoadEndedAt: loadEndedAt, FinishedAt: time.Now().UTC(), ReadyConnections: len(run.clients),
 		PublishAttempted: run.attempted.Load(), PublishAccepted: int64(len(local)), PublishRejected: run.rejected.Load(),
-		AckLatency: run.ack.Snapshot(), AcceptedLocal: local,
+		PublishErrors: run.publishErrorCounts(),
+		AckLatency:    run.ack.Snapshot(), AcceptedLocal: local,
 	}
 	run.mu.Lock()
 	base.Errors = append(base.Errors, run.errors...)
@@ -187,31 +253,29 @@ func (r *agentRun) reconcileHistory(ctx context.Context, ledger []AcceptedMessag
 				since: messages[start].Seq, before: messages[end-1].Seq + 1, limit: 256})
 		}
 	}
-	for _, query := range queries {
-		if err := query.client.GetData(ctx, query.topic, query.since, query.before, query.limit); err != nil {
-			r.addError(fmt.Sprintf("history get topic=%s range=%d-%d: %v", topicHash(query.topic), query.since, query.before, err))
-		}
-	}
-	_ = waitContext(ctx, 2*time.Second)
-	deadline := time.Now().Add(120 * time.Second)
-	for {
+	historyTimeout, _ := configDuration(r.cfg.HistoryTimeout)
+	historyCtx, cancel := context.WithTimeout(ctx, historyTimeout)
+	defer cancel()
+	r.issueHistoryQueries(historyCtx, queries)
+	_ = waitContext(historyCtx, 2*time.Second)
+	for round := 0; round < 2; round++ {
 		missing := missingHistory(r.recorder, expected)
-		if len(missing) == 0 || time.Now().After(deadline) {
-			break
-		}
-		if waitContext(ctx, 5*time.Second) != nil {
+		if len(missing) == 0 || historyCtx.Err() != nil {
 			break
 		}
+		retryQueries := make([]historyQuery, 0, len(missing))
 		for _, message := range missing {
 			client := r.clients[message.PeerIndex]
 			if message.Kind == "group" {
 				client = r.clients[1]
 			}
 			if client != nil {
-				_ = client.GetData(ctx, message.Topic, message.Seq, message.Seq+1, 1)
+				retryQueries = append(retryQueries, historyQuery{client: client, topic: message.Topic,
+					since: message.Seq, before: message.Seq + 1, limit: 1})
 			}
 		}
-		_ = waitContext(ctx, 500*time.Millisecond)
+		r.issueHistoryQueries(historyCtx, retryQueries)
+		_ = waitContext(historyCtx, 2*time.Second)
 	}
 	missing := missingHistory(r.recorder, expected)
 	missingIDs := make([]string, 0, len(missing))
@@ -222,6 +286,40 @@ func (r *agentRun) reconcileHistory(ctx context.Context, ledger []AcceptedMessag
 	return len(expected), len(expected) - len(missing), missingIDs
 }
 
+func (r *agentRun) issueHistoryQueries(ctx context.Context, queries []historyQuery) {
+	workers := r.cfg.HistoryWorkers
+	if workers > len(queries) {
+		workers = len(queries)
+	}
+	if workers == 0 {
+		return
+	}
+	jobs := make(chan historyQuery)
+	var wg sync.WaitGroup
+	for i := 0; i < workers; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for query := range jobs {
+				if err := query.client.GetData(ctx, query.topic, query.since, query.before, query.limit); err != nil && ctx.Err() == nil {
+					r.addError(fmt.Sprintf("history get topic=%s range=%d-%d: %v", topicHash(query.topic), query.since, query.before, err))
+				}
+			}
+		}()
+	}
+	for _, query := range queries {
+		select {
+		case jobs <- query:
+		case <-ctx.Done():
+			close(jobs)
+			wg.Wait()
+			return
+		}
+	}
+	close(jobs)
+	wg.Wait()
+}
+
 func missingHistory(recorder *deliveryRecorder, expected map[string]AcceptedMessage) []AcceptedMessage {
 	missing := make([]AcceptedMessage, 0)
 	for mid, message := range expected {
@@ -413,6 +511,7 @@ func (r *agentRun) publish(ctx context.Context, client *WSClient, topic, mid str
 	accepted, err := client.Publish(ctx, topic, mid, r.runID, r.scenario, messageIndex)
 	if err != nil {
 		r.rejected.Add(1)
+		r.classifyPublishError(err)
 		r.mu.Lock()
 		if len(r.errors) < 1000 {
 			r.errors = append(r.errors, fmt.Sprintf("publish %s: %v", mid, err))
@@ -424,6 +523,32 @@ func (r *agentRun) publish(ctx context.Context, client *WSClient, topic, mid str
 	return accepted, nil
 }
 
+func (r *agentRun) classifyPublishError(err error) {
+	var ctrlErr *ctrlError
+	switch {
+	case errors.As(err, &ctrlErr) && ctrlErr.Code == 403:
+		r.publish403.Add(1)
+	case errors.As(err, &ctrlErr) && ctrlErr.Code == 502:
+		r.publish502.Add(1)
+	case errors.As(err, &ctrlErr) && ctrlErr.Code == 503:
+		r.publish503.Add(1)
+	case strings.Contains(err.Error(), "ctrl timeout") || errors.Is(err, context.DeadlineExceeded):
+		r.publishTimeout.Add(1)
+	case strings.Contains(err.Error(), "websocket") || strings.Contains(err.Error(), "broken pipe") ||
+		strings.Contains(err.Error(), "connection reset"):
+		r.publishSocket.Add(1)
+	default:
+		r.publishOther.Add(1)
+	}
+}
+
+func (r *agentRun) publishErrorCounts() map[string]int64 {
+	return map[string]int64{
+		"403": r.publish403.Load(), "502": r.publish502.Load(), "503": r.publish503.Load(),
+		"ctrl_timeout": r.publishTimeout.Load(), "socket": r.publishSocket.Load(), "other": r.publishOther.Load(),
+	}
+}
+
 func (r *agentRun) addAccepted(message AcceptedMessage) {
 	r.mu.Lock()
 	r.accepted = append(r.accepted, message)

loadtest/im-pressure/agent_metrics.go

@@ -39,6 +39,11 @@ func startAgentMetrics(address string, run *agentRun) (func(), error) {
 		prometheus.NewCounterFunc(prometheus.CounterOpts{Name: "im_load_agent_publish_rejected_total", Help: "Publish commands rejected or timed out.", ConstLabels: labels}, func() float64 {
 			return float64(run.rejected.Load())
 		}),
+		prometheus.NewCounterFunc(prometheus.CounterOpts{Name: "im_load_agent_publish_403_total", Help: "Publish commands rejected with 403.", ConstLabels: labels}, func() float64 { return float64(run.publish403.Load()) }),
+		prometheus.NewCounterFunc(prometheus.CounterOpts{Name: "im_load_agent_publish_502_total", Help: "Publish commands rejected with 502.", ConstLabels: labels}, func() float64 { return float64(run.publish502.Load()) }),
+		prometheus.NewCounterFunc(prometheus.CounterOpts{Name: "im_load_agent_publish_503_total", Help: "Publish commands rejected with 503.", ConstLabels: labels}, func() float64 { return float64(run.publish503.Load()) }),
+		prometheus.NewCounterFunc(prometheus.CounterOpts{Name: "im_load_agent_publish_ctrl_timeout_total", Help: "Publish commands with control-response timeout.", ConstLabels: labels}, func() float64 { return float64(run.publishTimeout.Load()) }),
+		prometheus.NewCounterFunc(prometheus.CounterOpts{Name: "im_load_agent_publish_socket_error_total", Help: "Publish commands with socket errors.", ConstLabels: labels}, func() float64 { return float64(run.publishSocket.Load()) }),
 		prometheus.NewCounterFunc(prometheus.CounterOpts{Name: "im_load_agent_received_total", Help: "Unique matching data deliveries received.", ConstLabels: labels}, func() float64 {
 			received, _, _, _ := run.recorder.metricCounts()
 			return float64(received)

loadtest/im-pressure/collector.go

@@ -27,6 +27,7 @@ func (n *namedURLs) Set(value string) error {
 
 type metricSnapshot struct {
 	Timestamp time.Time `json:"timestamp"`
+	Phase     string    `json:"phase"`
 	Name      string    `json:"name"`
 	URL       string    `json:"url"`
 	Status    int       `json:"status"`
@@ -35,7 +36,8 @@ type metricSnapshot struct {
 	Error     string    `json:"error,omitempty"`
 }
 
-func runCollector(ctx context.Context, sources namedURLs, output string, interval, duration time.Duration) error {
+func runCollector(ctx context.Context, sources namedURLs, output string, interval, duration time.Duration,
+	coordinatorURL, coordinatorOutput string) error {
 	if len(sources) == 0 {
 		return fmt.Errorf("at least one --url name=url is required")
 	}
@@ -56,8 +58,10 @@ func runCollector(ctx context.Context, sources namedURLs, output string, interva
 	defer writer.Flush()
 	client := &http.Client{Timeout: 10 * time.Second}
 	deadline := time.Now().Add(duration)
+	phase := "before"
 	for {
-		snapshots := collectOnce(ctx, client, sources)
+		phase = detectCollectorPhase(ctx, client, phase, coordinatorURL, coordinatorOutput)
+		snapshots := collectOnce(ctx, client, sources, phase)
 		for _, snapshot := range snapshots {
 			raw, _ := json.Marshal(snapshot)
 			if _, err = writer.Write(append(raw, '\n')); err != nil {
@@ -76,7 +80,36 @@ func runCollector(ctx context.Context, sources namedURLs, output string, interva
 	}
 }
 
-func collectOnce(ctx context.Context, client *http.Client, sources namedURLs) []metricSnapshot {
+func detectCollectorPhase(ctx context.Context, client *http.Client, current, coordinatorURL, coordinatorOutput string) string {
+	if coordinatorOutput != "" {
+		if _, err := os.Stat(coordinatorOutput); err == nil {
+			return "after"
+		}
+	}
+	if coordinatorURL == "" {
+		return current
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet,
+		strings.TrimRight(coordinatorURL, "/")+"/v1/status", nil)
+	if err != nil {
+		return current
+	}
+	response, err := client.Do(req)
+	if err != nil {
+		return current
+	}
+	defer response.Body.Close()
+	var status struct {
+		Start StartInfo `json:"start"`
+	}
+	if response.StatusCode != http.StatusOK || json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&status) != nil ||
+		status.Start.StartAt.IsZero() || time.Now().Before(status.Start.StartAt) {
+		return current
+	}
+	return "during"
+}
+
+func collectOnce(ctx context.Context, client *http.Client, sources namedURLs, phase string) []metricSnapshot {
 	out := make([]metricSnapshot, len(sources))
 	var wg sync.WaitGroup
 	for i, source := range sources {
@@ -85,7 +118,7 @@ func collectOnce(ctx context.Context, client *http.Client, sources namedURLs) []
 		go func() {
 			defer wg.Done()
 			parts := strings.SplitN(source, "=", 2)
-			snapshot := metricSnapshot{Timestamp: time.Now().UTC(), Name: parts[0], URL: parts[1]}
+			snapshot := metricSnapshot{Timestamp: time.Now().UTC(), Phase: phase, Name: parts[0], URL: parts[1]}
 			started := time.Now()
 			req, err := http.NewRequestWithContext(ctx, http.MethodGet, parts[1], nil)
 			if err == nil {

loadtest/im-pressure/collector_test.go

@@ -0,0 +1,31 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+func TestDetectCollectorPhase(t *testing.T) {
+	start := StartInfo{RunID: "run1", StartAt: time.Now().Add(-time.Second)}
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		_ = json.NewEncoder(w).Encode(map[string]any{"start": start})
+	}))
+	defer server.Close()
+	client := &http.Client{Timeout: time.Second}
+	if got := detectCollectorPhase(context.Background(), client, "before", server.URL, ""); got != "during" {
+		t.Fatalf("phase=%s", got)
+	}
+	output := filepath.Join(t.TempDir(), "coordinator.json")
+	if err := os.WriteFile(output, []byte("{}"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	if got := detectCollectorPhase(context.Background(), client, "during", server.URL, output); got != "after" {
+		t.Fatalf("phase=%s", got)
+	}
+}

loadtest/im-pressure/config.example.json

@@ -14,6 +14,10 @@
   "prepare_workers": 20,
   "connect_ramp": "20s",
   "command_timeout": "20s",
+  "preflight_workers": 32,
+  "skip_p2p_preflight": false,
+  "history_workers": 48,
+  "history_timeout": "3m",
   "p2p_messages_per_user": 10,
   "p2p_interval": "1s",
   "drain_p2p": "30s",

loadtest/im-pressure/dbverify.go

@@ -0,0 +1,166 @@
+package main
+
+import (
+	"context"
+	"database/sql"
+	"encoding/json"
+	"fmt"
+	"os"
+	"regexp"
+	"sort"
+	"strings"
+
+	_ "github.com/go-sql-driver/mysql"
+)
+
+type dbVerifyResult struct {
+	RunID         string   `json:"run_id"`
+	TenantCode    string   `json:"tenant_code"`
+	TenantID      int64    `json:"tenant_id"`
+	Expected      int      `json:"expected"`
+	FoundExact    int      `json:"found_exact"`
+	Missing       int      `json:"missing"`
+	Mismatch      int      `json:"mismatch"`
+	DuplicateRows int      `json:"duplicate_rows"`
+	Unexpected    int      `json:"unexpected_rows"`
+	Tables        []string `json:"tables"`
+	MissingMIDs   []string `json:"missing_client_mids,omitempty"`
+}
+
+type persistedIdentity struct {
+	Topic string
+	Seq   int64
+}
+
+var dbIdentifierRE = regexp.MustCompile(`^[A-Za-z0-9_]+$`)
+
+func runDBVerify(ctx context.Context, coordinatorPath, output, tenantCode, dsnEnv, registryDB string) error {
+	raw, err := os.ReadFile(coordinatorPath)
+	if err != nil {
+		return err
+	}
+	var coordinator struct {
+		Start   StartInfo                    `json:"start"`
+		Ledgers map[string][]AcceptedMessage `json:"ledgers"`
+	}
+	if err = json.Unmarshal(raw, &coordinator); err != nil {
+		return fmt.Errorf("decode coordinator: %w", err)
+	}
+	if coordinator.Start.RunID == "" {
+		return fmt.Errorf("coordinator has no run id")
+	}
+	dsn := strings.TrimSpace(os.Getenv(dsnEnv))
+	if dsn == "" {
+		return fmt.Errorf("environment variable %s is required", dsnEnv)
+	}
+	if registryDB == "" {
+		registryDB = strings.TrimSpace(os.Getenv("TENANT_REGISTRY_DB"))
+	}
+	if registryDB == "" {
+		registryDB = "meeting_im"
+	}
+	if !dbIdentifierRE.MatchString(registryDB) {
+		return fmt.Errorf("invalid registry database identifier")
+	}
+	db, err := sql.Open("mysql", dsn)
+	if err != nil {
+		return err
+	}
+	defer db.Close()
+	if err = db.PingContext(ctx); err != nil {
+		return err
+	}
+	var tenantID int64
+	query := fmt.Sprintf("SELECT id FROM `%s`.im_tenant WHERE code=?", registryDB)
+	if err = db.QueryRowContext(ctx, query, tenantCode).Scan(&tenantID); err != nil {
+		return err
+	}
+
+	expected := make(map[string]AcceptedMessage)
+	months := make(map[string]struct{})
+	for _, ledger := range coordinator.Ledgers {
+		for _, message := range ledger {
+			if message.RunID != coordinator.Start.RunID {
+				continue
+			}
+			expected[message.ClientMid] = message
+			if message.Kind == "group" {
+				months[message.SentAt.UTC().Format("200601")] = struct{}{}
+			}
+		}
+	}
+	tables := []string{fmt.Sprintf("im_%03d_messages", tenantID)}
+	for month := range months {
+		tables = append(tables, fmt.Sprintf("im_%03d_group_messages_%s", tenantID, month))
+	}
+	sort.Strings(tables)
+	actual := make(map[string][]persistedIdentity)
+	for _, table := range tables {
+		if !dbIdentifierRE.MatchString(table) {
+			return fmt.Errorf("invalid generated table name %q", table)
+		}
+		rows, queryErr := db.QueryContext(ctx, fmt.Sprintf(
+			"SELECT client_msg_id,topic,seqid FROM `%s` WHERE tenant_id=? AND client_msg_id LIKE ?", table),
+			tenantID, coordinator.Start.RunID+"-%")
+		if queryErr != nil {
+			return fmt.Errorf("query %s: %w", table, queryErr)
+		}
+		for rows.Next() {
+			var mid, topic string
+			var seq int64
+			if err = rows.Scan(&mid, &topic, &seq); err != nil {
+				_ = rows.Close()
+				return err
+			}
+			actual[mid] = append(actual[mid], persistedIdentity{Topic: topic, Seq: seq})
+		}
+		if err = rows.Close(); err != nil {
+			return err
+		}
+	}
+	result := dbVerifyResult{RunID: coordinator.Start.RunID, TenantCode: tenantCode, TenantID: tenantID,
+		Expected: len(expected), Tables: tables}
+	for mid, want := range expected {
+		rows := actual[mid]
+		if len(rows) == 0 {
+			result.Missing++
+			result.MissingMIDs = append(result.MissingMIDs, mid)
+			continue
+		}
+		matched := false
+		for _, got := range rows {
+			if got.Topic == want.Topic && got.Seq == want.Seq {
+				matched = true
+			}
+		}
+		if matched {
+			result.FoundExact++
+		} else {
+			result.Mismatch++
+		}
+		if len(rows) > 1 {
+			result.DuplicateRows += len(rows) - 1
+		}
+	}
+	for mid := range actual {
+		if _, ok := expected[mid]; !ok {
+			result.Unexpected++
+		}
+	}
+	sort.Strings(result.MissingMIDs)
+	resultRaw, _ := json.MarshalIndent(result, "", "  ")
+	if output == "" || output == "-" {
+		if _, err = fmt.Fprintln(os.Stdout, string(resultRaw)); err != nil {
+			return err
+		}
+	} else {
+		if err = writeJSONFile(output, result); err != nil {
+			return err
+		}
+	}
+	if result.Missing != 0 || result.Mismatch != 0 || result.DuplicateRows != 0 || result.Unexpected != 0 {
+		return fmt.Errorf("database verification failed: missing=%d mismatch=%d duplicates=%d unexpected=%d",
+			result.Missing, result.Mismatch, result.DuplicateRows, result.Unexpected)
+	}
+	return nil
+}

loadtest/im-pressure/main.go

@@ -7,6 +7,7 @@ import (
 	"log"
 	"os"
 	"os/signal"
+	"strings"
 	"syscall"
 	"time"
 )
@@ -33,6 +34,8 @@ func main() {
 		err = mergeCommand(os.Args[2:])
 	case "collect":
 		err = collectCommand(ctx, os.Args[2:])
+	case "verify-db":
+		err = verifyDBCommand(ctx, os.Args[2:])
 	default:
 		usage()
 		os.Exit(2)
@@ -42,6 +45,22 @@ func main() {
 	}
 }
 
+func verifyDBCommand(ctx context.Context, args []string) error {
+	fs := flag.NewFlagSet("verify-db", flag.ContinueOnError)
+	coordinator := fs.String("coordinator", "loadtest/im-pressure/results/coordinator.json", "coordinator ledger path")
+	output := fs.String("output", "-", "verification JSON output path, or - for stdout")
+	tenant := fs.String("tenant", "", "tenant code")
+	dsnEnv := fs.String("dsn-env", "MYSQL_DSN", "environment variable containing the business database DSN")
+	registryDB := fs.String("registry-db", "", "tenant registry database; defaults to TENANT_REGISTRY_DB")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+	if strings.TrimSpace(*tenant) == "" {
+		return fmt.Errorf("--tenant is required")
+	}
+	return runDBVerify(ctx, *coordinator, *output, *tenant, *dsnEnv, *registryDB)
+}
+
 func prepareCommand(ctx context.Context, args []string) error {
 	fs := flag.NewFlagSet("prepare", flag.ContinueOnError)
 	configPath := fs.String("config", "loadtest/im-pressure/config.example.json", "JSON configuration path")
@@ -135,12 +154,14 @@ func collectCommand(ctx context.Context, args []string) error {
 	output := fs.String("output", "loadtest/im-pressure/results/metrics.jsonl", "snapshot output")
 	interval := fs.Duration("interval", 5*time.Second, "collection interval")
 	duration := fs.Duration("duration", 10*time.Minute, "collection duration")
+	coordinatorURL := fs.String("coordinator", "", "optional coordinator base URL used to label before/during phases")
+	coordinatorOutput := fs.String("coordinator-output", "", "coordinator JSON path used to detect the after phase")
 	if err := fs.Parse(args); err != nil {
 		return err
 	}
-	return runCollector(ctx, urls, *output, *interval, *duration)
+	return runCollector(ctx, urls, *output, *interval, *duration, *coordinatorURL, *coordinatorOutput)
 }
 
 func usage() {
-	fmt.Fprintln(os.Stderr, "usage: im-pressure <prepare|verify|coordinator|agent|merge|collect> [flags]")
+	fmt.Fprintln(os.Stderr, "usage: im-pressure <prepare|verify|coordinator|agent|merge|collect|verify-db> [flags]")
 }

loadtest/im-pressure/merge.go

@@ -18,6 +18,10 @@ type CombinedReport struct {
 	PublishAttempted   int64             `json:"publish_attempted"`
 	PublishAccepted    int64             `json:"publish_accepted"`
 	PublishRejected    int64             `json:"publish_rejected"`
+	PublishErrors      map[string]int64  `json:"publish_errors,omitempty"`
+	PublishSuccessRate float64           `json:"publish_success_rate"`
+	AckP95MS           float64           `json:"ack_p95_ms"`
+	DeliveryP95MS      float64           `json:"delivery_p95_ms"`
 	ExpectedDeliveries int64             `json:"expected_deliveries"`
 	ReceivedDeliveries int64             `json:"received_deliveries"`
 	RealtimeLost       int64             `json:"realtime_lost"`
@@ -47,12 +51,16 @@ func runMerge(aPath, bPath, output string) error {
 	if a.RunID != b.RunID || a.Scenario != b.Scenario {
 		return fmt.Errorf("agent summaries do not belong to the same run")
 	}
-	report := CombinedReport{RunID: a.RunID, Scenario: a.Scenario, GeneratedAt: time.Now().UTC(), Agents: []AgentSummary{a, b}, Passed: true}
+	report := CombinedReport{RunID: a.RunID, Scenario: a.Scenario, GeneratedAt: time.Now().UTC(), Agents: []AgentSummary{a, b}, Passed: true,
+		PublishErrors: make(map[string]int64)}
 	for _, summary := range report.Agents {
 		report.ReadyConnections += summary.ReadyConnections
 		report.PublishAttempted += summary.PublishAttempted
 		report.PublishAccepted += summary.PublishAccepted
 		report.PublishRejected += summary.PublishRejected
+		for category, count := range summary.PublishErrors {
+			report.PublishErrors[category] += count
+		}
 		report.ExpectedDeliveries += summary.ExpectedPeer + summary.ExpectedGroup
 		report.ReceivedDeliveries += summary.ReceivedPeer + summary.ReceivedGroup
 		report.RealtimeLost += summary.RealtimeLost
@@ -68,6 +76,11 @@ func runMerge(aPath, bPath, output string) error {
 			report.FailureReasons = append(report.FailureReasons, fmt.Sprintf("agent %s errors=%d", summary.Machine, len(summary.Errors)))
 		}
 	}
+	if report.PublishAttempted > 0 {
+		report.PublishSuccessRate = float64(report.PublishAccepted) / float64(report.PublishAttempted)
+	}
+	report.AckP95MS = mergedHistogramP95(report.Agents, func(summary AgentSummary) HistogramSnapshot { return summary.AckLatency })
+	report.DeliveryP95MS = mergedHistogramP95(report.Agents, func(summary AgentSummary) HistogramSnapshot { return summary.DeliveryLatency })
 	if report.ExpectedDeliveries > 0 {
 		report.ArrivalRate = float64(report.ReceivedDeliveries) / float64(report.ExpectedDeliveries)
 		report.LossRate = float64(report.RealtimeLost) / float64(report.ExpectedDeliveries)
@@ -75,6 +88,17 @@ func runMerge(aPath, bPath, output string) error {
 	if report.ReadyConnections != 2000 {
 		report.FailureReasons = append(report.FailureReasons, fmt.Sprintf("ready connections=%d expected=2000", report.ReadyConnections))
 	}
+	if report.PublishAttempted > 0 && report.PublishSuccessRate < 0.9999 {
+		report.FailureReasons = append(report.FailureReasons,
+			fmt.Sprintf("publish success rate=%.6f expected>=0.999900", report.PublishSuccessRate))
+	}
+	if report.AckP95MS > 500 {
+		report.FailureReasons = append(report.FailureReasons, fmt.Sprintf("ack p95=%.0fms expected<=500ms", report.AckP95MS))
+	}
+	if report.DeliveryP95MS > 1000 {
+		report.FailureReasons = append(report.FailureReasons,
+			fmt.Sprintf("delivery p95=%.0fms expected<=1000ms", report.DeliveryP95MS))
+	}
 	if report.PublishRejected > 0 || report.RealtimeLost > 0 || report.PersistenceLost > 0 || report.Duplicates > 0 || report.OutOfOrder > 0 || report.Disconnects > 0 {
 		report.FailureReasons = append(report.FailureReasons, "message correctness gate failed")
 	}
@@ -88,6 +112,38 @@ func runMerge(aPath, bPath, output string) error {
 	return writeJSONFile(output, report)
 }
 
+func mergedHistogramP95(agents []AgentSummary, selectHistogram func(AgentSummary) HistogramSnapshot) float64 {
+	var count int64
+	buckets := make([]int64, len(latencyBounds)+1)
+	maxMS := float64(0)
+	for _, agent := range agents {
+		histogram := selectHistogram(agent)
+		count += histogram.Count
+		if histogram.MaxMS > maxMS {
+			maxMS = histogram.MaxMS
+		}
+		for i, bound := range latencyBounds {
+			buckets[i] += histogram.Buckets[bound.String()]
+		}
+		buckets[len(latencyBounds)] += histogram.Buckets["+Inf"]
+	}
+	if count == 0 {
+		return 0
+	}
+	target := (count*95 + 99) / 100
+	var cumulative int64
+	for i, bucket := range buckets {
+		cumulative += bucket
+		if cumulative >= target {
+			if i == len(latencyBounds) {
+				return maxMS
+			}
+			return float64(latencyBounds[i]) / float64(time.Millisecond)
+		}
+	}
+	return maxMS
+}
+
 func readAgentSummary(path string) (AgentSummary, error) {
 	var summary AgentSummary
 	raw, err := os.ReadFile(path)

loadtest/im-pressure/merge_test.go

@@ -37,6 +37,16 @@ func TestRunMergeComputesArrivalAndPersistenceGate(t *testing.T) {
 	}
 }
 
+func TestMergedHistogramP95(t *testing.T) {
+	agents := []AgentSummary{
+		{AckLatency: HistogramSnapshot{Count: 10, MaxMS: 900, Buckets: map[string]int64{"200ms": 9, "1s": 1}}},
+		{AckLatency: HistogramSnapshot{Count: 10, MaxMS: 300, Buckets: map[string]int64{"200ms": 10}}},
+	}
+	if got := mergedHistogramP95(agents, func(summary AgentSummary) HistogramSnapshot { return summary.AckLatency }); got != 200 {
+		t.Fatalf("p95=%v", got)
+	}
+}
+
 func writeSummaryForTest(t *testing.T, path string, summary AgentSummary) {
 	t.Helper()
 	raw, err := json.Marshal(summary)

loadtest/im-pressure/model.go

@@ -15,22 +15,26 @@ import (
 const manifestVersion = 1
 
 type Config struct {
-	Tenant          string              `json:"tenant"`
-	HTTPBase        string              `json:"http_base"`
-	APIKeyEnv       string              `json:"api_key_env"`
-	PasswordEnv     string              `json:"password_env"`
-	UsersPerMachine int                 `json:"users_per_machine"`
-	Endpoints       map[string][]string `json:"endpoints"`
-	InsecureTLS     bool                `json:"insecure_tls"`
-	PrepareWorkers  int                 `json:"prepare_workers"`
-	ConnectRamp     string              `json:"connect_ramp"`
-	CommandTimeout  string              `json:"command_timeout"`
-	P2PMessages     int                 `json:"p2p_messages_per_user"`
-	P2PInterval     string              `json:"p2p_interval"`
-	DrainP2P        string              `json:"drain_p2p"`
-	DrainGroup      string              `json:"drain_group"`
-	PhonePrefix     string              `json:"registration_phone_prefix"`
-	VerifyCodeEnv   string              `json:"verification_code_env"`
+	Tenant           string              `json:"tenant"`
+	HTTPBase         string              `json:"http_base"`
+	APIKeyEnv        string              `json:"api_key_env"`
+	PasswordEnv      string              `json:"password_env"`
+	UsersPerMachine  int                 `json:"users_per_machine"`
+	Endpoints        map[string][]string `json:"endpoints"`
+	InsecureTLS      bool                `json:"insecure_tls"`
+	PrepareWorkers   int                 `json:"prepare_workers"`
+	ConnectRamp      string              `json:"connect_ramp"`
+	CommandTimeout   string              `json:"command_timeout"`
+	P2PMessages      int                 `json:"p2p_messages_per_user"`
+	P2PInterval      string              `json:"p2p_interval"`
+	DrainP2P         string              `json:"drain_p2p"`
+	DrainGroup       string              `json:"drain_group"`
+	PhonePrefix      string              `json:"registration_phone_prefix"`
+	VerifyCodeEnv    string              `json:"verification_code_env"`
+	HistoryWorkers   int                 `json:"history_workers"`
+	HistoryTimeout   string              `json:"history_timeout"`
+	PreflightWorkers int                 `json:"preflight_workers"`
+	SkipP2PPreflight bool                `json:"skip_p2p_preflight"`
 }
 
 func loadConfig(path string) (*Config, error) {
@@ -69,6 +73,24 @@ func loadConfig(path string) (*Config, error) {
 	if cfg.DrainGroup == "" {
 		cfg.DrainGroup = "60s"
 	}
+	if cfg.HistoryWorkers <= 0 {
+		cfg.HistoryWorkers = 48
+	}
+	if cfg.HistoryWorkers > 128 {
+		return nil, errors.New("history_workers must not exceed 128")
+	}
+	if cfg.HistoryTimeout == "" {
+		cfg.HistoryTimeout = "3m"
+	}
+	if _, err = configDuration(cfg.HistoryTimeout); err != nil {
+		return nil, fmt.Errorf("history_timeout: %w", err)
+	}
+	if cfg.PreflightWorkers <= 0 {
+		cfg.PreflightWorkers = 32
+	}
+	if cfg.PreflightWorkers > 128 {
+		return nil, errors.New("preflight_workers must not exceed 128")
+	}
 	if strings.TrimSpace(cfg.HTTPBase) == "" {
 		return nil, errors.New("http_base is required")
 	}
@@ -331,6 +353,7 @@ type AgentSummary struct {
 	PublishAttempted   int64             `json:"publish_attempted"`
 	PublishAccepted    int64             `json:"publish_accepted"`
 	PublishRejected    int64             `json:"publish_rejected"`
+	PublishErrors      map[string]int64  `json:"publish_errors,omitempty"`
 	ExpectedPeer       int64             `json:"expected_peer"`
 	ReceivedPeer       int64             `json:"received_peer"`
 	ExpectedEcho       int64             `json:"expected_echo"`

loadtest/im-pressure/model_test.go

@@ -1,6 +1,25 @@
 package main
 
-import "testing"
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestLoadConfigAppliesBoundedHistoryDefaults(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "config.json")
+	raw := []byte(`{"http_base":"https://example.test","endpoints":{"a":["wss://a"],"b":["wss://b"]}}`)
+	if err := os.WriteFile(path, raw, 0o600); err != nil {
+		t.Fatal(err)
+	}
+	cfg, err := loadConfig(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if cfg.HistoryWorkers != 48 || cfg.HistoryTimeout != "3m" || cfg.PreflightWorkers != 32 {
+		t.Fatalf("unexpected defaults: %+v", cfg)
+	}
+}
 
 func TestNewManifestDistributesUsersAndEndpoints(t *testing.T) {
 	cfg := &Config{Tenant: "006", UsersPerMachine: 3, Endpoints: map[string][]string{