fix loadtest p2p preflight replay

.gitea/workflows/deploy.yml

@@ -34,5 +34,39 @@ jobs:
           mkdir -p ~/.kube
           echo "${{ secrets.KUBECONFIG }}" > ~/.kube/config
           chmod 600 ~/.kube/config
+          kubectl set resources statefulset/im-server -n im-server-system -c im-server \
+            --requests=cpu=1,memory=512Mi --limits=cpu=2,memory=1Gi
+          kubectl set env statefulset/im-server -n im-server-system -c im-server \
+            CLUSTER_PROXY_TO_MASTER_BUFFER=4096 \
+            CLUSTER_PROXY_TO_MASTER_WORKERS=16 \
+            CLUSTER_PROXY_TO_MASTER_ENQUEUE_TIMEOUT_MS=300 \
+            CLUSTER_ROUTE_RETRIES=3 \
+            CLUSTER_ROUTE_RETRY_BACKOFF_MS=25
+          kubectl patch statefulset/im-server -n im-server-system --type=strategic -p \
+            '{"spec":{"template":{"spec":{"containers":[{"name":"im-server","readinessProbe":{"httpGet":{"path":"/healthz","port":6060},"timeoutSeconds":3,"periodSeconds":10,"failureThreshold":3},"livenessProbe":{"httpGet":{"path":"/livez","port":6060},"timeoutSeconds":3,"periodSeconds":20,"failureThreshold":3}}]}}}}'
+          if kubectl auth can-i get services -n im-server-system >/dev/null 2>&1 && \
+             kubectl auth can-i patch services -n im-server-system >/dev/null 2>&1; then
+            kubectl patch service/im-server -n im-server-system --type=merge -p \
+              '{"spec":{"sessionAffinity":"None"}}'
+          else
+            echo "::warning::ci-deployer cannot get/patch Service; skipped sessionAffinity update"
+          fi
+          if kubectl auth can-i get ingresses.networking.k8s.io -n im-server-system >/dev/null 2>&1 && \
+             kubectl auth can-i patch ingresses.networking.k8s.io -n im-server-system >/dev/null 2>&1 && \
+             kubectl get ingress/im-server -n im-server-system >/dev/null 2>&1; then
+            kubectl annotate ingress/im-server -n im-server-system --overwrite \
+              nginx.ingress.kubernetes.io/load-balance=least_conn
+          else
+            echo "::warning::ci-deployer cannot manage Ingress/im-server or it does not exist; skipped least_conn update"
+          fi
           kubectl rollout restart statefulset/im-server -n im-server-system
           kubectl rollout status statefulset/im-server -n im-server-system --timeout=300s
+
+          resources="$(kubectl get statefulset/im-server -n im-server-system -o \
+            jsonpath='{.spec.template.spec.containers[?(@.name=="im-server")].resources}')"
+          liveness="$(kubectl get statefulset/im-server -n im-server-system -o \
+            jsonpath='{.spec.template.spec.containers[?(@.name=="im-server")].livenessProbe.httpGet.path}')"
+          echo "effective resources=${resources} liveness=${liveness}"
+          if [ "${liveness}" != "/livez" ] || ! echo "${resources}" | grep -q '"cpu":"2"'; then
+            echo "::warning::StatefulSet resources/probes were reconciled away; update the ArgoCD source kustomization"
+          fi

docker/README.md

@@ -188,6 +188,11 @@ You can specify the following environment variables when issuing `docker run` co
 | `AWS_S3_ENDPOINT`[^2] | string |  | An endpoint URL (hostname only or fully qualified URI) to override the default endpoint; can be of any S3-compatible service, such as `minio-api.x.io` |
 | `AWS_SECRET_ACCESS_KEY`[^2] | string |  | AWS [Secret Access Key](https://aws.amazon.com/blogs/security/wheres-my-secret-access-key/) when using `s3` media handler. |
 | `CLUSTER_SELF` | string |  | Node name if the server is running in a Tinode cluster. |
+| `CLUSTER_PROXY_TO_MASTER_BUFFER` | number | `4096` | Total bounded proxy-to-master queue capacity for each peer. |
+| `CLUSTER_PROXY_TO_MASTER_WORKERS` | number | `16` | Ordered queue shards and synchronous RPC workers for each peer. |
+| `CLUSTER_PROXY_TO_MASTER_ENQUEUE_TIMEOUT_MS` | number | `300` | Maximum wait for a full proxy-to-master shard before returning 503. |
+| `CLUSTER_ROUTE_RETRIES` | number | `3` | Number of retries after a remote server-message route is rejected as overloaded. |
+| `CLUSTER_ROUTE_RETRY_BACKOFF_MS` | number | `25` | Initial exponential retry delay for remote server-message routes. |
 | `DEBUG_EMAIL_VERIFICATION_CODE`[^2] | string |  | Enable dummy email verification code, e.g. `123456`. Disabled by default (empty string). |
 | `DEFAULT_COUNTRY_CODE`[^2] | string | `US` | 2-letter country code to assign to sessions by default when the country isn't specified by the client explicitly and it's impossible to infer it. |
 | `EXT_CONFIG`[^1] | string |  | Path to external config file to use instead of the built-in one. If this parameter is used, most other variables are ignored[^1]. |

docker/tinode/Dockerfile

@@ -87,6 +87,9 @@ ENV KAFKA_SASL_USERNAME=
 ENV KAFKA_SASL_PASSWORD=
 ENV KAFKA_MAX_RETRY_COUNT=3
 ENV KAFKA_RETRY_BACKOFF_MS='[10000,30000,60000,300000,900000]'
+ENV KAFKA_MESSAGE_BATCH_SIZE=100
+ENV KAFKA_MESSAGE_BATCH_WAIT_MS=10
+ENV KAFKA_MESSAGE_SINGLE_RETRY_COUNT=3
 ENV KAFKA_DEAD_LETTER_RETENTION_SECONDS=7776000
 ENV KAFKA_TOPIC_ADMIN_COMMAND=im.admin.command
 ENV KAFKA_TOPIC_ADMIN_COMMAND_RESULT=im.admin.command.result

docker/tinode/Dockerfile.build

@@ -67,6 +67,9 @@ ENV KAFKA_SASL_USERNAME=
 ENV KAFKA_SASL_PASSWORD=
 ENV KAFKA_MAX_RETRY_COUNT=3
 ENV KAFKA_RETRY_BACKOFF_MS='[10000,30000,60000,300000,900000]'
+ENV KAFKA_MESSAGE_BATCH_SIZE=100
+ENV KAFKA_MESSAGE_BATCH_WAIT_MS=10
+ENV KAFKA_MESSAGE_SINGLE_RETRY_COUNT=3
 ENV KAFKA_DEAD_LETTER_RETENTION_SECONDS=7776000
 ENV KAFKA_TOPIC_ADMIN_COMMAND=im.admin.command
 ENV KAFKA_TOPIC_ADMIN_COMMAND_RESULT=im.admin.command.result

docker/tinode/config.template

@@ -140,6 +140,9 @@
 		"sasl_password": "",
 		"max_retry_count": 3,
 		"retry_backoff_ms": [10000, 30000, 60000, 300000, 900000],
+		"message_batch_size": $KAFKA_MESSAGE_BATCH_SIZE,
+		"message_batch_wait_ms": $KAFKA_MESSAGE_BATCH_WAIT_MS,
+		"message_single_retry_count": $KAFKA_MESSAGE_SINGLE_RETRY_COUNT,
 		"dead_letter_retention_seconds": 7776000,
 		"topics": {
 			"admin_command": "im.admin.command",
@@ -278,6 +281,11 @@
 	"cluster_config": {
 		"self": "",
 		"nodes": $CLUSTER_NODES_JSON,
+		"proxy_to_master_buffer": $CLUSTER_PROXY_TO_MASTER_BUFFER,
+		"proxy_to_master_workers": $CLUSTER_PROXY_TO_MASTER_WORKERS,
+		"proxy_to_master_enqueue_timeout_ms": $CLUSTER_PROXY_TO_MASTER_ENQUEUE_TIMEOUT_MS,
+		"route_retries": $CLUSTER_ROUTE_RETRIES,
+		"route_retry_backoff_ms": $CLUSTER_ROUTE_RETRY_BACKOFF_MS,
 		"failover": {
 			"enabled": true,
 			"heartbeat": 100,

docker/tinode/entrypoint.sh

@@ -13,6 +13,11 @@ else
 	CONFIG=working.config
 
 	: "${REDIS_KEY_PREFIX:=im}"
+	: "${CLUSTER_PROXY_TO_MASTER_BUFFER:=4096}"
+	: "${CLUSTER_PROXY_TO_MASTER_WORKERS:=16}"
+	: "${CLUSTER_PROXY_TO_MASTER_ENQUEUE_TIMEOUT_MS:=300}"
+	: "${CLUSTER_ROUTE_RETRIES:=3}"
+	: "${CLUSTER_ROUTE_RETRY_BACKOFF_MS:=25}"
 	if [ -z "$CLUSTER_NODES_JSON" ] ; then
 		CLUSTER_NODES_JSON='[{"name":"tinode-0","addr":"tinode-0:12000"},{"name":"tinode-1","addr":"tinode-1:12001"},{"name":"tinode-2","addr":"tinode-2:12002"}]'
 	fi

docs/IM-006商户双机消息压力测试方案.md

@@ -0,0 +1,794 @@
+# IM 006 商户双机消息压力测试方案
+
+日期：2026-09-25
+
+## 1. 测试目标
+
+> 双机实际操作以 [`loadtest/im-pressure/README.md`](../loadtest/im-pressure/README.md) 为准。当前开发机固定为机器 A，负责数据准备、协调器、agent A、集中采集和最终报告；另一台机器固定为机器 B，只运行 agent B 并回传结果。
+
+本次测试使用 006 商户的 2000 个账号，在两台压测机上各运行 1000 个在线 WebSocket 用户，验证多 Pod 场景下：
+
+1. 1000 对 P2P 会话同时收发时，消息是否被接受、实时投递并最终持久化。
+2. 2000 人大群中 1000 个成员轮流发言时，跨 Pod 群广播的丢失率、重复率、乱序率和延迟。
+3. 300 对 P2P 与 500 个群发送者同时工作时，混合负载是否相互影响。
+4. Tinode 原生 Cluster、Kafka 消息持久化、数据库分表和 WebSocket 下行是否出现容量瓶颈。
+
+本轮不在正式发压窗口内注册账号、加好友或建群。所有数据准备完成并核验后，再开始消息压力测试。
+
+## 2. 本方案对需求的精确定义
+
+### 2.1 账号与机器分片
+
+账号使用只包含小写字母和数字、且不超过 10 个字符的用户名，以满足当前注册校验：
+
+| 压测机 | 账号 | 数量 | 用户名示例 |
+| --- | --- | ---: | --- |
+| Machine A | A0001-A1000 | 1000 | `lta0001`、`lta1000` |
+| Machine B | B0001-B1000 | 1000 | `ltb0001`、`ltb1000` |
+
+固定配对关系为 `A0001 <-> B0001`，依此类推到 `A1000 <-> B1000`。“号码相同”指账号序号相同，不使用相同手机号。
+
+每个账号最终记录：`username`、`user_id`、`token`、`token_expires`、`machine`、`pair_no`、`p2p_topic`、`group_topic`。密码只从环境变量读取，不写入结果文件或日志。
+
+### 2.2 群聊需求解释
+
+“1000 个人间隔 0.1 秒发消息，2 轮合计 2000 条”按全局调度解释：
+
+- 每 100ms 只安排一个发送者发送一条消息。
+- 每轮 1000 个发送者各发送 1 条，共 1000 条，持续约 100 秒。
+- 连续执行 2 轮，共 2000 条，正式发压约 200 秒。
+- 不是每个发送者每 100ms 发一条；后一种解释会产生 10000 条/秒和 100 万条/100 秒，与“合计 2000 条”矛盾。
+
+### 2.3 P2P 场景默认强度
+
+原需求未指定每对消息数。本方案默认每对双方各发 10 条：
+
+- 1000 对同时开始。
+- 每个用户每秒发送 1 条，共 10 秒；每次加 0-200ms 确定性抖动，避免所有请求落在同一毫秒。
+- 总发布量：`1000 × 2 × 10 = 20000` 条。
+- 主要期望投递量：20000 次对端投递；发送方自身回显单独统计，不混入对端丢失率。
+
+消息数、间隔和抖动必须可通过命令行参数修改，但每轮结果必须记录实际参数。
+
+## 3. 协议与接口契约
+
+### 3.1 WebSocket 建连与账号注册
+
+WebSocket 地址为 `/v0/channels`，Upgrade 携带普通 API Key。每个连接严格执行：
+
+```text
+Upgrade
+-> hi(tenant="006")
+-> acc 或 login
+-> sub(me)
+-> sub(目标 P2P/群 Topic)
+-> READY
+```
+
+006 是企业商户，注册使用 `registration_type: 0` 和 `tenant_code: "006"`：
+
+```json
+{
+  "acc": {
+    "id": "prep-acc-lta0001",
+    "user": "new",
+    "scheme": "basic",
+    "username": "lta0001",
+    "registration_type": 0,
+    "tenant_code": "006",
+    "secret": "base64(lta0001:password)",
+    "login": true,
+    "desc": {"public": {"fn": "LT A0001"}}
+  }
+}
+```
+
+当前 006 环境必须确认未强制手机号或邀请码注册。准备程序应先尝试 basic 登录：已存在且密码正确则复用；仅在账号不存在时注册。账号存在但密码不匹配必须停止，禁止覆盖真实账号。
+
+### 3.2 HTTP 公共请求头
+
+好友、隐私和群接口使用登录返回的同一 Token：
+
+```text
+X-MeetingIM-APIKey: <ordinary api key>
+X-IM-Schema: 1
+X-Request-ID: <unique request id>
+Authorization: Bearer <token>
+Idempotency-Key: <unique key for write requests>
+```
+
+所有密钥和 Token 通过环境变量或权限受控文件传入，日志只记录账号索引和脱敏 ID。
+
+### 3.3 好友准备
+
+对每个序号执行：
+
+1. A 调用 `POST /v1/im/friend-requests`，Body 为 `{"target_user_id":"usr_b","message":"loadtest","source":"search","source_topic":""}`。
+2. 若返回 `pending`，B 调用 `POST /v1/im/friend-requests/{request_id}/accept`。
+3. 若已经是好友，读取好友列表或会话 context 并复用关系。
+4. 保存响应中的 P2P `topic`；必要时调用 `POST /v1/im/conversations/p2p`，Body 为 `{"peer_user_id":"usr_b"}`。
+5. 最终核验 1000 对关系均为 `friend`，且双方能订阅同一个 P2P Topic。
+
+好友创建会产生 `friend_added` 系统消息，因此正式统计只识别本轮 `run_id` 对应的 `x-im-client-mid`，不能把 Topic 的初始 `seq` 当成 0。
+
+### 3.4 2000 人群准备
+
+1. 使用 A0001 调用 `POST /v1/im/groups` 创建群。创建接口最多附带 99 个成员，建议只创建群主。
+2. 先确认所有账号的 `group_invite_verification=false`，避免进入当前尚未完整开放的邀请确认链路。
+3. 调用 `POST /v1/im/groups/{topic}/invitations` 分批邀请，每批不超过 100 人，每批使用独立 `Idempotency-Key`。
+4. 邀请完成后分页调用 `GET /v1/im/groups/{topic}/members`，核验去重后的有效成员恰好为 2000。
+5. 2000 个 WebSocket 全部订阅该群，并完成屏障消息验证后才允许发压。
+
+直接邀请会产生群业务事件和系统消息。准备结束后至少静默 30 秒，并记录正式测试前的群 `baseline_seq`；统计时仍以 `run_id/client_mid` 为准。
+
+## 4. 环境硬门槛
+
+任一门槛不满足，本轮标记为 `INVALID`，不能把结果当作服务端性能结论：
+
+1. `max_subscriber_count >= 2500`。仓库配置默认是 128，必须在所有 Pod 上统一调高并滚动重启。
+2. 006 商户 `max_group_members` 为 0（不限）或不小于 2000；目标群 `max_member_count` 同样满足要求。
+3. IM Server 所有 Pod 版本和配置一致，原生 Cluster 成员数与 Running Pod 数一致。
+4. Kafka 的 P2P、群消息持久化 Topic 可写，消费组无持续积压，死信在基线时为 0。
+5. 两台压测机到目标入口网络稳定，文件描述符上限、临时端口和 NAT 容量足够。
+6. 两台压测机使用 chrony/NTP，同一时刻的时钟偏差绝对值小于 5ms，否则跨机器单向延迟无效。
+7. 每台压测机至少预留 2 CPU、4GB 内存和稳定的 100Mbps 网络；测试进程自身 CPU 不应长期超过 70%。
+8. 2000 个连接全部 READY；正式阶段不允许把连接失败用户从分母中静默删除。
+
+## 5. 多 Pod 拓扑
+
+仅通过同一个负载均衡入口连接，不能证明消息确实跨 Pod。推荐提供仅供压测使用的三个直连入口，并保持业务数据路径不变：
+
+```text
+Machine A: A0001-A0500 -> Pod 0
+           A0501-A1000 -> Pod 1
+Machine B: B0001-B1000 -> Pod 2
+```
+
+这样 1000 对 P2P 的双方一定不在同一个 Pod；群成员也覆盖全部 3 个 Pod。若只能提供两个直连入口，则 Machine A 固定 Pod 0、Machine B 固定 Pod 1，并在另一轮把 Machine B 切到 Pod 2 重跑。
+
+入口必须能明确确认实际 Pod，例如握手诊断日志或反向代理访问日志能关联 `session -> pod`。不要依赖负载均衡概率分布。
+
+Topic 消息在启用 Tinode 原生 Cluster 时由 Cluster 路由；此时 Redis realtime 指标中的 `publish/skip_native_cluster` 是预期状态，不代表丢失。Redis 仍需观察用户级业务事件和 Session 控制类 fanout。
+
+## 6. 统一消息格式
+
+所有正式测试消息都使用 JSON 文本消息，并设置 `noecho:false`：
+
+```json
+{
+  "pub": {
+    "id": "s1-a0001-00001",
+    "topic": "p2p_or_grp",
+    "noecho": false,
+    "head": {
+      "mime": "application/json",
+      "x-im-schema": 1,
+      "x-im-type": "text",
+      "x-im-client-mid": "lt-20260925-s1-a0001-00001"
+    },
+    "content": {
+      "text": "loadtest",
+      "run_id": "lt-20260925-s1",
+      "scenario": "p2p",
+      "sender_index": 1,
+      "message_index": 1,
+      "sent_at_unix_nano": 0
+    }
+  }
+}
+```
+
+`x-im-client-mid` 在 `tenant + topic + sender` 范围内必须唯一，重试时必须复用原值。发送程序用 `{ctrl.id}` 关联请求，用 `ctrl.params.seq` 记录服务端分配的 Topic 序号。
+
+## 7. 执行阶段
+
+### 7.1 阶段 P0：预检与热身
+
+1. 两台机器各以 50 连接/秒建立 1000 条连接，总建连时间约 20 秒。
+2. 每个用户订阅 `me`、自己的 P2P Topic 和群 Topic。
+3. 群主发送 1 条不计入正式结果的群屏障消息；必须在 10 秒内被 2000 个连接全部收到。
+4. 每个 P2P Topic 发送 1 条不计入正式结果的屏障消息；双方必须收到，且对端投递为 1000/1000。
+5. 等待 30 秒，确认错误计数、Kafka lag、CPU、内存和连接数回到稳定状态。
+
+### 7.2 阶段 S1：1000 对 P2P 同时聊天
+
+参与者：全部 2000 人，1000 个 P2P Topic。
+
+默认调度：
+
+- A、B 双方各发送 10 条，每用户 1 条/秒，持续 10 秒。
+- 每个用户使用由账号索引计算的 0-200ms 固定抖动，确保可复现。
+- 总发布量 20000，平均发布速率约 2000 条/秒。
+- 发布完成后保留连接并等待 30 秒排空，再进行历史核对。
+
+主要期望：每条被服务端接受的消息恰好被对应的另一方实时收到一次。发送方回显也应收到一次，但作为独立指标。
+
+### 7.3 阶段 S2：2000 人群、1000 人发言
+
+参与者：2000 个在线订阅者。发送者选择 A0001-A0500 与 B0001-B0500，共 1000 人，避免所有发布压力只落在一台压测机上。
+
+默认调度：
+
+- Machine A 每 200ms 发 1 条，Machine B 在其后错开 100ms 发 1 条，合并后全局间隔为 100ms。
+- 每轮每个发送者恰好发送 1 条，共 1000 条、约 100 秒。
+- 连续 2 轮，共 2000 条、约 200 秒。
+- `noecho:false`，因此每条被接受的消息期望投递到 2000 个在线连接。
+- 理论实时投递总量：`2000 × 2000 = 4000000` 次，平均约 20000 次/秒。
+- 发布结束后等待 60 秒排空，然后由多个校验账号分页拉取历史。
+
+### 7.4 阶段 S3：P2P 与群聊混合
+
+正式窗口为 100 秒：
+
+- P2P：A0001-A0300 与 B0001-B0300，共 300 对；双方各发 5 条，总发布量 `300 × 2 × 5 = 3000`。在第 0、20、40、60、80 秒附近发送，并加入固定抖动。
+- 群聊：A0301-A0550 与 B0301-B0550，共 500 个发送者；每人发 2 条。两台机器错峰形成全局 100ms 间隔，两轮共 1000 条，持续约 100 秒。
+- 其余用户保持在线并接收群消息。
+- 群实时理论投递总量：`1000 × 2000 = 2000000` 次。
+- P2P 主要理论对端投递总量：3000 次；发送方回显另计 3000 次。
+- 发布结束后等待 60 秒排空，再做 P2P 和群历史核对。
+
+混合场景的群发送者与 P2P 发送者故意分开，便于判断瓶颈来自连接写队列、群广播还是同一连接并发写。通过后可增加一轮“发送者重叠”增强测试。
+
+## 8. 丢失率与正确性口径
+
+不能只统计压测端成功调用 `WriteJSON` 的数量。每轮必须同时输出以下四层结果。
+
+### 8.1 发送接受率
+
+```text
+publish_accept_rate = ctrl_202_with_seq / publish_attempted
+publish_reject_rate = (ctrl_4xx + ctrl_5xx + ctrl_timeout + write_error) / publish_attempted
+```
+
+只有收到 `ctrl.code=202` 且包含有效 `params.seq` 的消息，才进入后续“期望投递”分母。
+
+### 8.2 实时投递丢失率
+
+实时到达率必须由 2000 个 WebSocket 连接实际监听得到，不能使用发送 ACK、Kafka offset 或数据库行数代替。每个连接建立后启动一个常驻 read loop，全量读取服务端下行，并把 `{ctrl}`、`{data}`、`{meta}`、`{pres}`、`{info}` 分流处理。
+
+监听器要求：
+
+1. 每个 WebSocket 只能有一个 goroutine 调用 `ReadMessage/ReadJSON`，避免多个读取者争抢数据；`ctrl` 通过 `id` 分发给等待中的发布请求。
+2. 只统计 `head.x-im-client-mid` 前缀和 `content.run_id` 都匹配本轮的 `{data}`，忽略准备阶段系统消息、其他业务消息和历史噪声。
+3. 记录 `receiver_index + client_mid + seq + first_received_at`。相同接收者再次收到同一 `client_mid` 计为重复，不增加到达数。
+4. 服务端允许 `{data}` 早于对应 `{ctrl 202}` 到达，因此监听器先保存观测事实，最终再与 ACK ledger 对账，不能因为 ACK 尚未到达而丢弃该 `{data}`。
+5. P2P 以配对的另一方为主要期望接收者；发送者自身回显单独统计。
+6. 群消息以发压开始时的 2000 个 READY 成员为固定分母。因为 `noecho:false`，发送者本人也属于期望接收者。
+7. 正式窗口内连接断开时不得从分母中删除。记录断开时间、所属机器和 Pod；服务端导致的断开计入结果，压测机自身饱和导致的断开则将整轮标记为无效。
+8. P2P 发布结束后继续监听 30 秒，群聊和混合场景继续监听 60 秒。窗口内未收到计为实时丢失；窗口后到达另记 `late_delivery`，不能把它改写为正常实时到达。
+
+期望投递账本由成功 ACK 生成：
+
+```text
+P2P expected key   = run_id + client_mid + peer_receiver_index
+Group expected key = run_id + client_mid + each_ready_receiver_index
+```
+
+群场景 2000 条消息对应 400 万个期望键。实现时为每个接收者维护按消息序号索引的 bitset，不保存 400 万份完整 JSON；延迟使用 HDR Histogram 或固定桶，缺失和重复才输出逐条明细。
+
+P2P 主要口径：
+
+```text
+p2p_realtime_loss =
+  (accepted_messages - unique_peer_data_received_within_drain_window)
+  / accepted_messages
+```
+
+群聊口径：
+
+```text
+group_expected_deliveries = accepted_group_messages × ready_group_subscribers
+group_realtime_loss =
+  (group_expected_deliveries - unique_group_deliveries_within_drain_window)
+  / group_expected_deliveries
+```
+
+每个接收者按 `run_id + x-im-client-mid` 去重。重复到达不用于抵消其他消息的缺失。
+
+同时输出：
+
+```text
+arrival_rate = unique_deliveries_within_drain_window / expected_deliveries
+loss_rate = 1 - arrival_rate
+late_rate = unique_late_deliveries / expected_deliveries
+duplicate_rate = duplicate_deliveries / expected_deliveries
+```
+
+实时监听结束后继续执行历史回查，将缺失分为：
+
+- `realtime_lost_but_persisted`：WS 窗口内没收到，但历史中存在，说明实时投递链路丢失，客户端可通过重连补偿。
+- `persisted_but_late`：超过实时排空窗口才由 WS 收到，同时历史中存在。
+- `durable_lost`：WS 未收到且最长等待后历史也不存在，属于真正的持久化消息丢失。
+- `receiver_disconnected`：期望接收者在消息窗口内断线，单独统计并保留在实时丢失分母中。
+
+### 8.3 持久化丢失率
+
+Kafka 持久化是异步链路，`ctrl 202` 不等于数据库已经可查。排空后通过 `{get what:"data"}` 分页回查：
+
+```text
+persistence_loss =
+  (accepted_unique_messages - unique_messages_found_in_history)
+  / accepted_unique_messages
+```
+
+- P2P：1000 个 Topic 各由一侧账号回查。
+- 群聊：至少使用分别连接不同 Pod 的 3 个成员回查，三份结果应完全一致。
+- 最长等待 120 秒，每 5 秒重试一次；超时仍不存在才记为持久化丢失。
+
+### 8.4 重复与乱序
+
+```text
+duplicate_rate = duplicate_deliveries / expected_deliveries
+out_of_order_rate = receivers_observing_seq_regression / ready_receivers
+```
+
+同时记录：同一 `client_mid` 对应多个 `seq`、同一 `seq` 对应多个测试消息、接收端 Topic `seq` 回退、ACK `seq` 与下行 `seq` 不一致。这些都必须单独报错，不能归入普通丢失。
+
+## 9. 延迟口径
+
+记录以下时间点：
+
+- `t_write`：压测端开始写 `{pub}`。
+- `t_ack`：收到对应 `{ctrl 202}`。
+- `t_data`：接收者收到并完成 JSON 解析。
+
+输出：
+
+```text
+ack_latency = t_ack - t_write
+e2e_realtime_latency = t_data - sent_at
+```
+
+每个场景输出 p50、p95、p99、p99.9 和 max。跨机器延迟仅在 NTP 偏差达标时有效；否则仍可统计 ACK 延迟，但端到端单向延迟标记为无效。
+
+群聊 400 万次投递不应把所有延迟样本常驻内存。使用固定桶直方图或 HDR Histogram，并按接收端机器和 Pod 分组。
+
+## 10. 建议通过标准
+
+第一轮基线建议使用：
+
+| 指标 | 标准 |
+| --- | --- |
+| READY 连接 | 2000/2000 |
+| 发布接受率 | 100% |
+| 持久化丢失率 | 0 |
+| 同一 `client_mid` 多 `seq` | 0 |
+| P2P 对端实时丢失率 | <= 0.001%，且任何丢失都要输出明细 |
+| 群实时投递丢失率 | <= 0.001%，发布版本验收目标为 0 |
+| 重复投递率 | 0 |
+| Topic `seq` 回退 | 0 |
+| ACK p95 / p99 | < 100ms / < 300ms |
+| 实时投递 p95 / p99 | < 200ms / < 500ms |
+| `deliver/hub_full`、WS 写队列满 | 0 |
+| Kafka producer/commit 错误 | 0 |
+| Kafka 消费重试、单条降级、死信新增 | 0 |
+| Kafka 排空 | 发布结束后 60 秒内 lag 回到基线/0 |
+| Kafka 到 MySQL 落库 p95 / p99 | < 200ms / < 500ms |
+| MySQL 实际落库数 | 与 `ctrl 202` 唯一消息数完全一致 |
+| MySQL deadlock、lock timeout、连接池耗尽 | 0 |
+| Redis 操作错误、淘汰、拒绝连接 | 0 |
+| Redis command p95 / p99 | < 5ms / < 10ms |
+| Pod CPU throttling ratio | < 10% |
+| Pod CPU、内存峰值 | < limit 的 80% |
+| 容器/节点网络 drop、error | 0 新增或不高于基线噪声 |
+| 压测机 CPU、内存、网络 | < 70%，无 read-loop backlog |
+| Pod 重启、OOM、持续 GC/内存增长 | 0 |
+
+如果实时有缺失但历史回查完整，结论应写“实时链路丢失、可由历史恢复”，不能写成“消息未丢失”。如果历史也缺失，则属于持久化消息丢失，直接判失败。
+
+## 11. 全链路观测与资源采集
+
+### 11.1 观测目标与时间窗口
+
+监控必须能把同一轮测试拆成以下链路：
+
+```text
+压测机 WriteJSON
+-> IM Pod 收到 pub
+-> Redis 分配消息 seq / 幂等 claim
+-> Kafka producer 写入消息持久化 Topic
+-> IM Pod 返回 ctrl 202 并实时广播
+-> Kafka consumer 拉取与组批
+-> MySQL 批量提交
+-> Kafka offset commit
+-> WebSocket 历史查询可见
+```
+
+每轮采集窗口为：正式 `start_at` 前 10 分钟、正式测试全程、排空与历史核对结束后 10 分钟。Prometheus 在压测期间建议使用 5 秒抓取周期，重要 counter 以 `increase` 计算，不使用瞬时值推算总量。
+
+每轮在 Grafana 或结果文件中记录四个时间点：
+
+- `baseline_start`：基线开始。
+- `load_start`：正式发送开始。
+- `load_end`：最后一次发布结束。
+- `reconcile_end`：历史核对完成。
+
+所有图表使用相同 UTC 时间轴，并标注 `run_id`、scenario、Git commit、Pod 镜像版本和压测参数。
+
+### 11.2 采集组件
+
+推荐最小采集栈：
+
+| 对象 | 采集方式 | 主要用途 |
+| --- | --- | --- |
+| IM Server | `/metrics`、`/debug/vars` | WS、HTTP、DB、fanout、Go runtime、连接池 |
+| Kubernetes Pod/Container | kube-state-metrics + kubelet/cAdvisor | CPU、内存、磁盘 IO、网络、重启、限流 |
+| Kubernetes Node | node-exporter | 节点 CPU、内存、磁盘、网络、conntrack |
+| Kafka | kafka-exporter + Broker JMX Exporter | consumer lag、吞吐、ISR、请求队列、磁盘 |
+| Redis | redis-exporter | OPS、延迟、内存、连接、命中率、淘汰、网络 |
+| MySQL | mysqld-exporter + performance_schema | QPS、事务、锁、Buffer Pool、慢 SQL、IO |
+| 两台压测机 | node-exporter + 压测 agent `/metrics` | 生成端 CPU/网络/连接和客户端统计 |
+| 链路探测 | blackbox-exporter 或压测 agent 主动探测 | DNS、TCP、HTTP、依赖 RTT |
+
+如果环境没有完整 Prometheus 栈，压测前至少以 5 秒周期抓取上述 HTTP 指标，并同步保存 `kubectl top`、Pod 日志、Kafka consumer group、Redis INFO 和 MySQL status。手工快照只能作为兜底，不能替代连续时间序列。
+
+### 11.3 当前已有与必须补充的应用指标
+
+当前 IM Server 已经提供：
+
+| 指标 | 状态 | 用途 |
+| --- | --- | --- |
+| `ws_connections_active` | 已有 | 当前 Pod/实例的活跃 WS 数 |
+| `http_requests_total` | 已有 | HTTP 请求量和错误码 |
+| `http_request_duration_seconds` | 已有 | HTTP 延迟 |
+| `db_query_duration_seconds` | 已有 | MySQL query/exec/commit 等通用耗时 |
+| `realtime_fanout_events_total` | 已有 | Redis realtime 发布、接收和本地投递结果 |
+| `realtime_fanout_delivery_duration_seconds` | 已有 | Redis realtime 消息年龄 |
+| `/debug/vars` 中 `DbStats` | 已有 | Go SQL 连接池状态 |
+| `/debug/vars` 中 Cluster/Session/Topic 计数 | 已有 | Cluster 节点、会话和活跃 Topic |
+
+为准确测量 Kafka 落库速度，压测版本必须再补以下低基数指标。严禁把 `topic`、`user_id`、`client_mid`、`run_id` 或 tenant ID 放入 Prometheus label：
+
+```text
+message_persist_events_total{stage,kind,status}
+message_persist_duration_seconds{stage,kind,status}
+message_persist_end_to_end_seconds{kind,status}
+message_persist_batch_size{kind}
+message_persist_inflight{kind}
+kafka_consumer_commit_total{topic_kind,status}
+redis_operation_duration_seconds{operation,status}
+redis_operation_errors_total{operation}
+mysql_pool_connections{state}
+mysql_pool_wait_total
+mysql_pool_wait_duration_seconds_total
+cluster_route_events_total{operation,status}
+cluster_route_duration_seconds{operation,status}
+ws_outbound_events_total{status}
+ws_outbound_queue_depth
+```
+
+标签约定：
+
+- `kind`：仅允许 `p2p`、`group`。
+- `stage`：`producer_write`、`consumer_fetch`、`db_batch`、`consume_finish`。
+- `status`：`ok`、`error`、`duplicate`、`retry`、`dead`。
+- `operation`：Redis 使用 `seq_reserve`、`unique_claim`、`hot_cache`、`fanout_publish`、`fanout_receive`；Cluster 使用 `route`、`proxy_join`、`proxy_leave`。
+
+`message_persist_end_to_end_seconds` 从 Kafka envelope 的 `occurred_at` 计算到 MySQL 提交并标记消费成功的时刻。它是回答“消息从服务端接受到真正落库用了多久”的核心指标。
+
+另外应在自定义 Prometheus Registry 中注册 Go collector 和 process collector，提供标准 `go_*`、`process_*` 指标；在完成前从 `/debug/vars` 采集对应 runtime 数据。
+
+### 11.4 Kafka 处理速度与积压
+
+Kafka 需要同时观察应用层和 Broker 层，至少包含：
+
+1. P2P Topic `im.message.persist` 的生产/消费条数与字节数。
+2. 群 Topic `im.group.message.persist` 的生产/消费条数与字节数。
+3. 对应 consumer group 每个 partition 的 lag、总 lag 和 lag 恢复到 0 的时间。
+4. producer 写入延迟、错误、重试和 timeout。
+5. consumer fetch 速率、批量大小、DB 批次耗时、offset commit 延迟和失败。
+6. 批量落库失败后降级为单条写入的次数。
+7. 消费重试、Redis 幂等 claim 失败和死信新增数。
+8. Broker request queue、network processor idle、request handler idle、ISR shrink、under-replicated/offline partition 和磁盘使用率。
+
+不同 exporter 的 lag 指标名可能是 `kafka_consumergroup_lag` 或 `kafka_consumergroup_group_lag`，部署时必须以实际 `/metrics` 为准建立 dashboard，不允许直接照抄不存在的指标名。
+
+典型查询：
+
+```promql
+sum by (kind, stage, status) (
+  rate(message_persist_events_total[1m])
+)
+```
+
+```promql
+histogram_quantile(
+  0.99,
+  sum by (le, kind) (
+    rate(message_persist_end_to_end_seconds_bucket[5m])
+  )
+)
+```
+
+```promql
+sum by (consumergroup, topic) (
+  kafka_consumergroup_lag{
+    topic=~"im.message.persist|im.group.message.persist"
+  }
+)
+```
+
+每轮报告必须给出：峰值生产速度、峰值消费速度、峰值 lag、lag 清零耗时、平均/最大 batch size、DB batch p95/p99、端到端落库 p95/p99/max、重试数和死信数。
+
+### 11.5 数据库落库正确性与速度
+
+006 商户使用默认库 `meeting_im`，但消息表名按数据库中的数值 `tenant_id` 生成，不能直接把企业码字符串当表后缀。准备阶段先从租户注册信息解析并记录 006 的真实 `tenant_id`，再生成只读核对 SQL。
+
+当前消息存储位置：
+
+- P2P：`im_<tenant_id三位补零>_messages`，例如真实 tenant ID 为 6 时是 `im_006_messages`。
+- 群聊：`im_<tenant_id三位补零>_group_messages_YYYYMM`。
+- 群消息分片注册：`im_group_message_shards`。
+- Kafka 消费幂等状态：Redis `kafka_consume` namespace，TTL 7 天。
+- Kafka 死信：MySQL `im_kafka_dead_letters`。
+
+正式核对优先使用 WebSocket 历史接口，SQL 作为服务端审计证据。SQL 由核对程序使用白名单表名生成，不允许把外部输入直接拼入表名。
+
+P2P 落库数量示例：
+
+```sql
+SELECT
+  COUNT(*) AS rows_written,
+  COUNT(DISTINCT client_msg_id) AS unique_client_mid,
+  MIN(createdat) AS first_persisted_at,
+  MAX(createdat) AS last_persisted_at
+FROM im_006_messages
+WHERE client_msg_id LIKE 'lt-20260925-s1-%';
+```
+
+群消息落库数量示例：
+
+```sql
+SELECT
+  COUNT(*) AS rows_written,
+  COUNT(DISTINCT client_msg_id) AS unique_client_mid,
+  MIN(created_at) AS first_persisted_at,
+  MAX(created_at) AS last_persisted_at
+FROM im_006_group_messages_202609
+WHERE client_msg_id LIKE 'lt-20260925-s2-%';
+```
+
+重复检查：
+
+```sql
+SELECT client_msg_id, COUNT(*) AS cnt, MIN(seqid), MAX(seqid)
+FROM im_006_messages
+WHERE client_msg_id LIKE 'lt-20260925-s1-%'
+GROUP BY client_msg_id
+HAVING COUNT(*) <> 1;
+```
+
+群表执行同样检查。P2P 有 1000 个 Topic，不能用整张表的 `MAX(seq)-MIN(seq)+1` 判断丢失；必须以压测端保存的 `client_mid -> topic -> ack_seq` manifest 与数据库逐条比对。
+
+MySQL 性能至少采集：
+
+- `Threads_connected`、`Threads_running`、连接使用率和连接池 wait count/duration。
+- QPS、Com_insert、事务提交/回滚、行写入速度。
+- InnoDB Buffer Pool 命中率、dirty pages、checkpoint age、log waits。
+- row lock waits/time、deadlock、慢 SQL、临时磁盘表。
+- data/log fsync、读写 IOPS、吞吐、磁盘 await、util 和剩余空间。
+- P2P 批量 INSERT、群月表批量 INSERT 的 p95/p99。
+- 主从架构下的 replication lag；压测历史回查必须明确读主还是读从。
+
+数据库落库速度报告包含两个值：
+
+```text
+db_write_throughput = 测试消息成功落库数 / 首次到最后一次 DB commit 时间
+drain_time = 最后一条 ctrl 202 到 Kafka lag=0 且全部历史可见的时间
+```
+
+### 11.6 Redis 观测
+
+Redis 在当前消息链路中承担 seq 分配、`client_mid` 幂等 claim、群热消息缓存、Kafka 消费幂等状态和用户级 realtime fanout。需要采集：
+
+- `instantaneous_ops_per_sec`、命令总量与各命令耗时。
+- connected/blocked clients、连接拒绝和连接创建速度。
+- used memory、RSS、碎片率、maxmemory 使用率。
+- evicted/expired keys、keyspace hit/miss。
+- network input/output bytes、Pub/Sub channel/pattern/subscriber 数。
+- Redis 主线程 CPU、事件循环延迟、slowlog 数量。
+- Sentinel/主从环境的 role、replication offset、replication lag、failover。
+- `seq_reserve`、`unique_claim`、`hot_cache`、`kafka_consume` 操作错误和 p95/p99。
+
+压测期间禁止使用无界 `KEYS` 或全库 `SCAN` 做统计，这会污染结果。Kafka 消费状态只通过应用指标抽样统计，不逐条扫描 Redis key。
+
+常见异常判定：
+
+- evicted keys 或 rejected connections 增加：直接判失败。
+- used memory 持续增长且排空后不回落：检查 hot message、幂等 key TTL 和消费记录泄漏。
+- p99 command latency 超过 10ms：标记风险；超过 50ms 且与消息延迟尖刺相关时判失败。
+- replication lag 持续扩大或发生 failover：本轮单独标记，不与无故障基线混合。
+
+### 11.7 Pod、容器与节点资源
+
+每个 IM Pod 单独展示，不只看 namespace 汇总：
+
+```promql
+sum by (pod) (
+  rate(container_cpu_usage_seconds_total{namespace="im-server-system",container="im-server"}[1m])
+)
+```
+
+```promql
+sum by (pod) (
+  container_memory_working_set_bytes{namespace="im-server-system",container="im-server"}
+)
+```
+
+```promql
+sum by (pod) (
+  rate(container_network_receive_bytes_total{namespace="im-server-system",pod=~"im-server-.*"}[1m])
+)
+```
+
+```promql
+sum by (pod) (
+  rate(container_network_transmit_bytes_total{namespace="im-server-system",pod=~"im-server-.*"}[1m])
+)
+```
+
+必须采集：
+
+- CPU 使用核数、CPU limit 使用率、CFS throttled seconds/periods。
+- memory working set、RSS、cache、limit 使用率、OOM 和 page fault。
+- 容器文件系统 read/write bytes、IOPS、IO wait；若根文件系统不承载业务数据，也要观察日志写入压力。
+- 网络接收/发送 bytes、packets、errors、drops、TCP retransmit。
+- Pod restart、readiness、liveness、Pending/NotReady 时间。
+- Go goroutine、heap、stack、GC 次数、GC pause、进程文件描述符。
+- `LiveSessions`、`LiveTopics`、`LiveClusterNodes`、Cluster leader 和代理 Topic 数。
+- Node CPU steal/iowait、内存压力、磁盘压力、网络错误、conntrack 使用率。
+
+资源报告既要给峰值，也要给排空后是否回落。单个 Pod 明显高于其他 Pod 时，结合连接分布、Topic master 位置和网络流量判断是否发生热点。
+
+建议风险线：
+
+- Pod CPU 持续超过 limit 的 80%，或 throttling ratio 超过 10%。
+- memory working set 超过 limit 的 80%，或排空 10 分钟后仍持续增长。
+- 节点磁盘 util 持续超过 80%，await 持续超过 20ms。
+- 网络 drop/error 或 TCP retransmit 明显增长。
+- 任一 Pod restart/OOM/NotReady，直接判本轮失败。
+
+### 11.8 网络时延与压测机资源
+
+两台压测机分别持续测量：
+
+- 到各 Pod 直连入口和负载均衡入口的 DNS、TCP connect、TLS handshake、HTTP `/healthz` RTT。
+- WebSocket Ping/Pong RTT p50/p95/p99/max，每 5 秒一次，不能用业务 `note kp` 代替协议 Ping。
+- 到 Kafka、Redis、MySQL 服务地址的 TCP connect RTT；只做低频只读探测，不绕过业务链路写数据。
+- 两台压测机之间的 RTT、丢包和 NTP offset。
+- 压测进程 CPU、RSS、GC、goroutine、FD、socket、send/receive bytes、read-loop queue depth。
+
+压力生成端出现 CPU > 70%、网卡 > 70%、FD 接近上限、read-loop backlog 或 GC 尖刺时，本轮标记 `LOAD_GENERATOR_SATURATED`，不能把客户端处理不过来算成服务端丢消息。
+
+### 11.9 现有 fanout 与 Cluster 查询
+
+每轮开始前记录 counter 快照，结束后记录增量：
+
+```promql
+sum by (stage, kind, target_type, status) (
+  increase(realtime_fanout_events_total[10m])
+)
+```
+
+```promql
+histogram_quantile(
+  0.99,
+  sum by (le, kind, target_type) (
+    rate(realtime_fanout_delivery_duration_seconds_bucket[5m])
+  )
+)
+```
+
+Topic 消息使用原生 Cluster 时，`publish/skip_native_cluster` 增加属于正常现象。重点检查 Cluster 连接数、route/proxy 错误、`node for topic not found`、gob 编解码错误、Hub 队列满和 WS 写失败。
+
+### 11.10 日志、快照与关联规则
+
+压测日志统一带 `run_id`、scenario、machine、account_index 和脱敏 Topic hash。服务端日志不新增用户 ID 或 `client_mid` 的高基数 Prometheus label；需要定位时通过压测端保存的 manifest 关联。
+
+每轮保存：
+
+- Prometheus 查询结果或 Grafana dashboard snapshot。
+- IM Pod、Kafka、Redis、MySQL 在测试窗口内的错误日志。
+- Kafka consumer group 压测前、峰值、结束、排空后的 offset/lag 快照。
+- Redis INFO、SLOWLOG 摘要和 exporter 指标快照。
+- MySQL GLOBAL STATUS、InnoDB status 摘要、慢 SQL 摘要和表行数核对。
+- `kubectl top pod/node`、Pod restart/condition/event 和容器日志摘要。
+- 两台压测机 node/process 指标及 agent summary。
+
+日志采集必须过滤 Token、密码、API Key、Kafka/Redis/MySQL 凭证和完整消息正文。
+
+### 11.11 建议仪表盘布局
+
+一张压测 dashboard 分为六行：
+
+1. 业务结果：attempt、ACK、实时接收、历史可见、丢失、重复、乱序、p95/p99。
+2. IM Pod：连接数、发布/下行速率、Cluster route、CPU、内存、GC、goroutine。
+3. Kafka：producer/consumer rate、lag、batch、commit、retry、dead letter、broker health。
+4. Redis：OPS、command latency、memory、clients、network、eviction、replication。
+5. MySQL：insert/transaction rate、DB batch latency、pool wait、locks、Buffer Pool、IO。
+6. 基础设施：Pod/Node CPU、memory、disk IO、network、压测机资源和网络 RTT。
+
+所有面板至少按 Pod、Kafka Topic/consumer group、Redis instance、MySQL instance 和压测机分组；不得只展示总和。
+
+## 12. 压测程序结构
+
+现有 `loadtest/realtime_fanout_probe.go` 只支持单群和单进程，不能直接用于本方案。新工具使用 Go 实现并提供六个子命令：
+
+```text
+im-pressure prepare   注册、好友、群和 manifest
+im-pressure verify    只读核验 2000 账号、1000 对关系和 2000 群成员
+im-pressure coordinator 协调双机 READY、统一起跑时间、ACK 台账与结果上报
+im-pressure agent     在每台压测机运行连接与场景
+im-pressure collect   抓取 Prometheus、K8s 和依赖组件快照
+im-pressure merge     合并两台机器结果并生成报告
+```
+
+推荐文件：
+
+```text
+loadtest/im-pressure/config.example.yaml
+loadtest/im-pressure/cmd/im-pressure/
+loadtest/im-pressure/internal/protocol/
+loadtest/im-pressure/internal/prepare/
+loadtest/im-pressure/internal/runner/
+loadtest/im-pressure/internal/collector/
+loadtest/im-pressure/internal/report/
+```
+
+协调方式：Machine A 启动 coordinator，等待 A/B 两个 agent 都上报 1000 个 READY，然后发布一个至少晚于当前时间 30 秒的 UTC `start_at`。两台机器按相同 `run_id` 和确定性调度执行。任何 agent 失联都将本轮标记为 `INVALID`。
+
+`agent` 自身暴露 `/metrics`，提供连接状态、发布、接收、重复、乱序、断连及 Go/process 指标；ACK 和投递延迟分布写入最终 summary。`collect` 只执行只读查询，并把各数据源的原始快照与查询时间一起保存；连接凭证通过环境变量传入，不写入结果目录。
+
+## 13. 结果文件
+
+每轮保留以下文件：
+
+```text
+results/<run_id>/config.redacted.json
+results/<run_id>/manifest.sha256
+results/<run_id>/agent-a-summary.json
+results/<run_id>/agent-b-summary.json
+results/<run_id>/publish.jsonl
+results/<run_id>/missing.jsonl
+results/<run_id>/duplicate.jsonl
+results/<run_id>/history-reconcile.json
+results/<run_id>/db-reconcile-p2p.json
+results/<run_id>/db-reconcile-group.json
+results/<run_id>/timeline.json
+results/<run_id>/metrics/im-server.prom
+results/<run_id>/metrics/load-agent.prom
+results/<run_id>/metrics/kafka.json
+results/<run_id>/metrics/redis.json
+results/<run_id>/metrics/mysql.json
+results/<run_id>/metrics/kubernetes.json
+results/<run_id>/metrics/network.json
+results/<run_id>/logs/im-server-errors.log
+results/<run_id>/logs/kafka-errors.log
+results/<run_id>/logs/mysql-slow-summary.log
+results/<run_id>/grafana-snapshot.json
+results/<run_id>/report.md
+```
+
+缺失明细至少包含 `run_id`、场景、Topic 脱敏哈希、发送者索引、接收者索引、`client_mid`、ACK `seq`、发送时间、最后等待时间、发送 Pod 和接收 Pod。不得写入 Token、密码、API Key 或完整手机号。
+
+最终 `report.md` 必须分别给出：
+
+- 压测业务量和消息正确性结论。
+- 实时投递与历史恢复结论。
+- Kafka 生产、消费、lag、排空和死信结论。
+- MySQL 落库数量、吞吐、耗时、锁与 IO 结论。
+- Redis OPS、耗时、内存、幂等/seq 错误和 fanout 结论。
+- 每个 Pod/Node 以及两台压测机的资源峰值和瓶颈判断。
+- 测试是否有效、是否通过，以及所有未通过项的原始证据路径。
+
+## 14. 推荐执行顺序
+
+1. 在测试窗口前完成环境门槛检查和 2000 账号准备。
+2. 完成 1000 对好友关系与 2000 人群，运行 `verify` 直到全部通过。
+3. 先用 20 对 P2P、100 人群做冒烟测试，验证统计口径。
+4. 执行 P0 热身。
+5. 执行 S1，等待服务端和 Kafka 恢复基线至少 5 分钟。
+6. 执行 S2，等待恢复基线至少 10 分钟。
+7. 执行 S3。
+8. 合并结果，按实时链路与持久化链路分别给出结论。
+9. 全部稳定后再单独进行 Pod 重启、断网或 Kafka/Redis 故障注入，不与本轮无故障基线混跑。

docs/IM-Kafka接入方案.md

@@ -724,11 +724,19 @@ IM Server 已接入 `im.message.persist`，用于把在线发送链路和最终
 1. WS/HTTP 发送入口先完成权限校验，并由 topic 内存态分配 `seq_id`。
 2. Kafka 开启时，私聊发送入口直接生产 `im.message.persist`，群聊发送入口直接生产 `im.group.message.persist`，不查库、不落库、不写 DB outbox；客户端在 Kafka 写入成功后拿到已分配的 `seq_id`。
 3. Kafka 关闭时，继续走原同步落库路径，兼容本地和未接 Kafka 环境。
-4. 消费者按 `message_kind` 区分私聊与群聊：
+4. 私聊和群聊使用独立 Topic、独立消费组和独立物理表：私聊批量写入租户分表 `im_%03d_messages`，群聊按租户和月份批量写入 `im_%03d_group_messages_YYYYMM`。
-   - `p2p` 写入 `im_messages`。
-   - `group` 写入 `im_group_messages`。
 5. 消费侧复用 `im_kafka_consume_log` 做 `event_id + consumer_group` 幂等。
-6. 失败按统一退避重试 3 次，超过上限先投递 `im.dead.letter`，再由 dead letter consumer 写入 `im_kafka_dead_letters`。
+6. 消费者最多聚合 `message_batch_size` 条消息，在 `message_batch_wait_ms` 时间窗内形成批次并执行一条多值 `INSERT`。
+7. 批量写入失败后逐条降级；每条执行首次写入和 `message_single_retry_count` 次重试，默认共尝试 4 次。
+8. 单条仍失败时，完成 `im.dead.letter` 投递和 `im_kafka_dead_letters` 终态写入后才提交该批 offset；提交失败会持续重试，避免跳过失败消息。
+
+消息批量落库参数：
+
+| 配置项 | 默认值 | 说明 |
+| --- | ---: | --- |
+| `message_batch_size` | `100` | 单批最大消息数，上限 500。 |
+| `message_batch_wait_ms` | `10` | 从取得第一条消息开始的批次聚合等待时间。 |
+| `message_single_retry_count` | `3` | 批量失败后单条额外重试次数，不含首次执行。 |
 
 ### 11.10 IM Server 已接入的缓存失效消费
 

docs/IM-Kafka接入联调测试文档.md

@@ -545,27 +545,27 @@ Payload：
 1. 用户 A 与用户 B 建立私聊。
 2. A 发送带 `x-im-client-mid` 的私聊消息。
 3. 查询 Kafka `im.message.persist`。
-4. 查询租户库 `im_messages`。
+4. 查询租户私聊分表 `im_%03d_messages`。
 5. A 使用同一 `x-im-client-mid` 重复发送。
 
 预期：
 
 1. A 立即收到服务端 ack 和 `seq_id`。
 2. `seq_id` 为 BIGINT/int64。
-3. 消费成功后 `im_messages` 有一条记录。
+3. 消费成功后对应租户私聊分表有一条记录。
 4. 重复发送不重复落库。
 5. 消费日志 succeeded。
 
 ### TC-KAFKA-014 群聊消息异步落库
 
-目标：验证群消息写入 `im_group_messages`，不混入私聊表。
+目标：验证群消息写入租户月度群消息分表，不混入私聊表。
 
 步骤：
 
 1. 创建群并加入至少两个成员。
 2. 成员发送群消息。
-3. 查询 `im_group_messages`。
+3. 查询 `im_%03d_group_messages_YYYYMM`。
-4. 查询 `im_messages`。
+4. 查询 `im_%03d_messages`。
 5. 重连客户端按 Topic seq 拉取消息。
 
 预期：
@@ -574,24 +574,27 @@ Payload：
 2. `seq_id` 单调增长，类型为 BIGINT/int64。
 3. 客户端补偿拉取成功。
 4. 24 小时热消息可进入 Redis 热缓存。
+5. 私聊与群聊分别由 `im.message.persist`、`im.group.message.persist` 消费。
 
 ### TC-KAFKA-015 消息消费失败重试
 
-目标：验证 `im.message.persist` 消费失败可重试并最终成功。
+目标：验证私聊和群聊批量落库失败后均按“逐条降级、重试 3 次、死信后提交 offset”处理。
 
 步骤：
 
 1. 临时制造租户库写入失败，如锁表或停止库连接。
-2. 发送消息。
+2. 在批次窗口内连续发送多条私聊和群聊消息。
-3. 恢复数据库。
+3. 恢复数据库，验证逐条重试可成功落库。
-4. 等待重试。
+4. 再次持续制造单条失败直至超过重试次数。
+5. 查询 `im_kafka_consume_log`、`im_kafka_dead_letters` 和 `im.dead.letter`。
 
 预期：
 
 1. `im_kafka_consume_log.retry_count` 增加。
-2. 未超过重试上限时不进入死信。
+2. 批量失败后降级为单条写入，每条最多执行 4 次（首次加 3 次重试）。
-3. 数据库恢复后消息最终落库。
+3. 未超过重试上限时不进入死信，数据库恢复后消息最终落库并标记 succeeded。
-4. 状态变为 succeeded。
+4. 持续失败的单条消息写入死信表并投递死信 Topic。
+5. 只有批内全部消息达到 succeeded/dead 终态后才提交 offset，不能跳过失败消息。
 
 ### TC-KAFKA-016 文件上传完成事件
 

docs/IM-前后端对接文档.md

@@ -362,7 +362,7 @@ X-MeetingIM-APIKey: <预置的普通 API Key>
 | 类型 | 适用场景 | 服务端下行示例 | 客户端渲染 |
 | --- | --- | --- | --- |
 | `system_notice` | “你已添加了安妮，现在可以开始聊天了”、撤回提示、群名修改、成员进出群等纯提示 | `{"data":{"topic":"p2p...","seq":19,"from":"","head":{"x-im-schema":1,"x-im-type":"system_notice","x-im-template":"friend_added","x-im-from":"system"},"content":{"text":"你已添加了安妮，现在可以开始聊天了。","level":"normal","template":"friend_added","actors":[{"user_id":"usr_anne","name":"安妮"}],"actions":[]}}}` | 居中灰色提示；不显示发送人头像；如 `actors.click` 为空则不可点 |
-| `system_action` | 群邀请确认说明、审批入口、群设置入口、可点击人名或群名提示 | `{"data":{"topic":"grp...","seq":20,"from":"","head":{"x-im-schema":1,"x-im-type":"system_action","x-im-template":"group_invite_confirm_enabled","x-im-from":"system"},"content":{"text":"安妮邀请栗子来了加入了群聊","template":"group_invite_confirm_enabled","actors":[{"user_id":"usr_anne","name":"安妮","click":{"type":"user_profile","target_id":"usr_anne"}},{"user_id":"usr_lizi","name":"栗子来了","click":{"type":"user_profile","target_id":"usr_lizi"}}],"actions":[{"type":"open_group_invite_settings","label":"群聊邀请确认","target_id":"grp..."}]}}}` | 居中灰色或业务提示样式；可点击片段按 `actors.click` 高亮；按钮或文字入口按 `actions` 渲染 |
+| `system_action` | 审批入口、群设置入口、可点击人名或群名提示；仅在当前消息对所有接收者都可操作时下发 `actions` | `{"data":{"topic":"grp...","seq":20,"from":"","head":{"x-im-schema":1,"x-im-type":"system_action","x-im-template":"group_member_joined","x-im-from":"system"},"content":{"text":"安妮邀请栗子来了加入了群聊","template":"group_member_joined","actors":[{"user_id":"usr_anne","name":"安妮","click":{"type":"user_profile","target_id":"usr_anne"}},{"user_id":"usr_lizi","name":"栗子来了","click":{"type":"user_profile","target_id":"usr_lizi"}}],"actions":[]}}}` | 居中灰色或业务提示样式；可点击片段按 `actors.click` 高亮；按钮或文字入口按 `actions` 渲染。若 `actions` 为空，不得将普通文案渲染为可点击入口 |
 
 `content` 字段约定：
 
@@ -372,7 +372,7 @@ X-MeetingIM-APIKey: <预置的普通 API Key>
 | `template` | string | 是 | 服务端稳定模板编码，例如 `friend_added`、`group_member_joined`、`group_invite_confirm_enabled` |
 | `level` | string | 否 | `normal`、`warning`、`danger`，影响颜色和强调程度 |
 | `actors` | array | 否 | 文案中的用户、群、文件等实体；支持局部高亮和点击 |
-| `actions` | array | 否 | 当前消息可触发的操作入口；只允许客户端白名单动作 |
+| `actions` | array | 否 | 当前消息可触发的操作入口；只允许客户端白名单动作。群邀请确认开关、全员禁言开关、成员私信权限开关等设置变更公告面向全员广播，服务端不下发仅管理员可操作的 `actions`；客户端不得仅凭模板名高亮入口。若历史消息或兼容字段携带 `min_role`，客户端必须按当前成员角色过滤，角色不足时忽略该 action |
 | `extra` | object | 否 | 模板扩展参数；客户端不得依赖未知字段完成关键流程 |
 
 `actors[].click` 和 `actions[]` 的 `type` 首版白名单：
@@ -381,7 +381,7 @@ X-MeetingIM-APIKey: <预置的普通 API Key>
 | --- | --- | --- |
 | `user_profile` | `target_id` | 打开用户资料页 |
 | `group_profile` | `target_id` | 打开群资料页 |
-| `open_group_invite_settings` | `target_id` | 打开群邀请确认设置 |
+| `open_group_invite_settings` | `target_id` | 打开群邀请确认设置；仅用于明确发给管理员/群主或全员都可执行的入口，不用于全员广播的开关结果公告 |
 | `open_group_join_requests` | `target_id` | 打开群入群审批列表 |
 | `open_message` | `topic,seq` | 定位到指定消息 |
 

docs/IM-原生Cluster运维部署说明.md

@@ -175,6 +175,7 @@ metadata:
   name: im-server
   namespace: default
 spec:
+  sessionAffinity: None
   selector:
     app: im-server
   ports:
@@ -219,6 +220,33 @@ spec:
                   fieldPath: metadata.name
             - name: EXT_CONFIG
               value: /etc/tinode/tinode.conf
+            - name: CLUSTER_PROXY_TO_MASTER_BUFFER
+              value: "4096"
+            - name: CLUSTER_PROXY_TO_MASTER_WORKERS
+              value: "16"
+            - name: CLUSTER_PROXY_TO_MASTER_ENQUEUE_TIMEOUT_MS
+              value: "300"
+            - name: CLUSTER_ROUTE_RETRIES
+              value: "3"
+            - name: CLUSTER_ROUTE_RETRY_BACKOFF_MS
+              value: "25"
+          resources:
+            requests:
+              cpu: "1"
+              memory: 512Mi
+            limits:
+              cpu: "2"
+              memory: 1Gi
+          readinessProbe:
+            httpGet:
+              path: /healthz
+              port: 6060
+            timeoutSeconds: 3
+          livenessProbe:
+            httpGet:
+              path: /livez
+              port: 6060
+            timeoutSeconds: 3
           volumeMounts:
             - name: tinode-config
               mountPath: /etc/tinode

docs/IM-多开跨Pod实时投递压测方案.md

@@ -2,6 +2,9 @@
 
 日期：2026-09-13
 
+> 006 商户、2000 账号、两台压测机的可执行场景与统计口径见
+> [IM-006商户双机消息压力测试方案](./IM-006商户双机消息压力测试方案.md)。本文保留为通用阶梯与指标说明。
+
 ## 1. 目标
 
 验证 IM Server 使用 Redis realtime fanout 后，在多 Pod、多端登录、群消息和群业务事件场景下的实时投递能力。

docs/sql/IM-消息表Snowflake-ID-BIGINT修复SQL.sql

@@ -0,0 +1,69 @@
+-- IM private-message Snowflake ID compatibility migration.
+-- Scope: the shared business database (for example meeting_im).
+-- Review and back up before execution. This script does not replay dead letters.
+
+-- 1. Preflight: every listed id column must become BIGINT.
+SELECT TABLE_NAME, COLUMN_TYPE, IS_NULLABLE, EXTRA
+FROM information_schema.COLUMNS
+WHERE TABLE_SCHEMA = DATABASE()
+  AND (TABLE_NAME = 'im_messages' OR TABLE_NAME REGEXP '^im_[0-9]+_messages$')
+  AND COLUMN_NAME = 'id'
+ORDER BY TABLE_NAME;
+
+-- 2. Upgrade the template/base table used by new tenant shards.
+ALTER TABLE `im_messages`
+  MODIFY COLUMN `id` BIGINT NOT NULL AUTO_INCREMENT;
+
+-- 3. Upgrade every existing tenant private-message shard.
+DELIMITER $$
+
+DROP PROCEDURE IF EXISTS migrate_private_message_id_to_bigint$$
+CREATE PROCEDURE migrate_private_message_id_to_bigint()
+BEGIN
+    DECLARE done INT DEFAULT 0;
+    DECLARE shard_table VARCHAR(64);
+    DECLARE shard_cursor CURSOR FOR
+        SELECT TABLE_NAME
+        FROM information_schema.TABLES
+        WHERE TABLE_SCHEMA = DATABASE()
+          AND TABLE_NAME REGEXP '^im_[0-9]+_messages$'
+        ORDER BY TABLE_NAME;
+    DECLARE CONTINUE HANDLER FOR NOT FOUND SET done = 1;
+
+    OPEN shard_cursor;
+    shard_loop: LOOP
+        FETCH shard_cursor INTO shard_table;
+        IF done = 1 THEN
+            LEAVE shard_loop;
+        END IF;
+        SET @alter_message_id_sql = CONCAT(
+            'ALTER TABLE `', REPLACE(shard_table, '`', '``'),
+            '` MODIFY COLUMN `id` BIGINT NOT NULL AUTO_INCREMENT'
+        );
+        PREPARE alter_message_id_stmt FROM @alter_message_id_sql;
+        EXECUTE alter_message_id_stmt;
+        DEALLOCATE PREPARE alter_message_id_stmt;
+    END LOOP;
+    CLOSE shard_cursor;
+END$$
+
+CALL migrate_private_message_id_to_bigint()$$
+DROP PROCEDURE migrate_private_message_id_to_bigint$$
+
+DELIMITER ;
+
+-- 4. Post-check: this query must return zero rows.
+SELECT TABLE_NAME, COLUMN_TYPE, IS_NULLABLE, EXTRA
+FROM information_schema.COLUMNS
+WHERE TABLE_SCHEMA = DATABASE()
+  AND (TABLE_NAME = 'im_messages' OR TABLE_NAME REGEXP '^im_[0-9]+_messages$')
+  AND COLUMN_NAME = 'id'
+  AND DATA_TYPE <> 'bigint';
+
+-- 5. Observe pending recovery work. Replay only after the post-check passes.
+SELECT status, last_reason, COUNT(*) AS message_count,
+       MIN(created_at) AS first_created_at, MAX(created_at) AS last_created_at
+FROM im_kafka_dead_letters
+WHERE event_type = 'message.persist.p2p'
+GROUP BY status, last_reason
+ORDER BY status, last_reason;

docs/sql/mysql-multitenant-tenant-id-review.sql

@@ -249,9 +249,9 @@ ALTER TABLE im_topictags
 
 ALTER TABLE im_subscriptions
     ADD UNIQUE KEY uk_subscriptions_tenant_topic_user (tenant_id, topic, userid),
-    ADD KEY idx_subscriptions_tenant_user_topic_deleted
+    ADD KEY idx_subscriptions_tenant_user_topic_deletedat
         (tenant_id, userid, topic, deletedat),
-    ADD KEY idx_subscriptions_tenant_topic_deleted
+    ADD KEY idx_subscriptions_tenant_topic_deletedat
         (tenant_id, topic, deletedat),
     DROP INDEX subscriptions_topic_userid;
 

docs/test/IM-allrich双机压测执行记录-20260927.md

@@ -0,0 +1,91 @@
+# IM allrich 双机压测执行记录
+
+日期：2026-09-27
+
+## 1. 测试环境
+
+| 项目 | 值 |
+|---|---|
+| 企业码 | `allrich` |
+| 企业名称 | 大家一起发 |
+| HTTP API | `https://api.118im.com` |
+| WebSocket | `wss://api.118im.com/v0/channels` |
+| API Key | 通过 `IM_API_KEY` 注入，不写入 Git |
+| 测试验证码 | `123456`，仅测试环境使用 |
+| 测试密码 | 通过 `IM_LOAD_PASSWORD` 注入，不写入 Git |
+| 机器 A | 当前开发机，`192.168.1.135` |
+| 机器 B | 另一台压测机，运行 `-machine b` |
+
+`config.json`、`.env.test`、`manifest.json` 均被 Git 忽略。manifest 含 2000 个账号 Token，只能通过受控内网传输。
+
+## 2. 数据准备结果
+
+2026-09-27 已完成并通过硬校验：
+
+- 账号：2000，机器 A/B 各 1000。
+- 用户名：`lta0001-lta1000`、`ltb0001-ltb1000`。
+- 测试手机号：A 使用配置前缀加 `0001-1000`，B 使用配置前缀加 `5001-6000`。
+- 好友关系：1000 对，序号相同的 A/B 用户互为好友。
+- P2P Topic：1000 个，全部写入 manifest。
+- 测试群：1 个，成员数 2000，Topic 仅在 manifest 中保存，文档只记录哈希 `98372d54fe65`。
+
+准备过程观察到注册和好友同意接口存在少量瞬时 500/404，100 人群邀请批次存在 502、超过 2 分钟及 `IDEMPOTENCY_IN_PROGRESS`。工具使用相同业务幂等键和退避重试后全部恢复；这些属于准备链路现象，不计入正式消息到达率。
+
+## 3. 当前运行状态
+
+| 项目 | 值 |
+|---|---|
+| Run ID | `p2p-20260927-01` |
+| 场景 | `p2p` |
+| Coordinator | `http://192.168.1.135:8090` |
+| 机器 A agent | `1000/1000 READY`，意外断连 0 |
+| 机器 B agent | 等待启动 |
+| 远端 dev | `6d17154` 或更新版本 |
+
+机器 A 的 coordinator 和 agent A 已运行。机器 B 启动并达到 1000 READY 后，coordinator 自动下发统一开始时间，无需人工再次触发。
+
+## 4. 机器 B 运行文件
+
+机器 A 已生成：
+
+```text
+/tmp/allrich-machine-b-runtime-p2p-20260927-01.zip
+SHA-256: 479f119e7d01e1829c7262cace282c72a5f53ad8e7f5128abc2ade75d046db64
+```
+
+机器 B 必须先安全获取该文件，再执行：
+
+```bash
+git switch dev
+git pull --ff-only origin dev
+
+# 运行包不在 Git 中，必须从机器 A 单独复制
+scp youshunlai@192.168.1.135:/tmp/allrich-machine-b-runtime-p2p-20260927-01.zip /tmp/
+shasum -a 256 /tmp/allrich-machine-b-runtime-p2p-20260927-01.zip
+
+unzip -o /tmp/allrich-machine-b-runtime-p2p-20260927-01.zip \
+  -d loadtest/im-pressure
+chmod 600 loadtest/im-pressure/.env.test \
+  loadtest/im-pressure/config.json \
+  loadtest/im-pressure/manifest.json
+
+set -a
+source loadtest/im-pressure/.env.test
+set +a
+cd loadtest/im-pressure
+go build -o im-pressure .
+
+./im-pressure agent \
+  -config config.json \
+  -manifest manifest.json \
+  -machine b \
+  -coordinator http://192.168.1.135:8090 \
+  -metrics-listen :9105 \
+  -output results/p2p-20260927-01/agent-b-summary.json
+```
+
+完整 B 机操作和故障处理见 [`loadtest/im-pressure/MACHINE-B.md`](../../loadtest/im-pressure/MACHINE-B.md)。
+
+## 5. 多 Pod 口径说明
+
+当前 A/B 配置均使用 `wss://api.118im.com/v0/channels` 公共入口。该配置验证 LB 实际分配下的多 Pod 能力，但不能保证两台压测机固定落到不同 Pod。若验收要求强制指定不同 Pod，需要提供两个独立 WS 定向入口，分别配置到 `endpoints.a` 和 `endpoints.b`。

docs/test/IM-单聊压测Pod日志根因分析与改进方案-20260927.md

@@ -0,0 +1,299 @@
+# IM 单聊压测 Pod 日志根因分析与改进方案
+
+日期：2026-09-27
+
+## 1. 结论
+
+`p2p-20260927-01` 暴露了两个相互独立的 P0 故障：
+
+1. **Tinode 原生 Cluster 转发能力在突发流量下耗尽**，造成 8133 次 publish 拒绝或超时。
+2. **Kafka 异步落库写入 Snowflake ID 时与 MySQL 32 位 `INT` 字段不兼容**，造成客户端已 ACK 的消息没有落入 MySQL。
+
+两者都不是 Pod 崩溃、Pod 重启或集群节点断开导致。实时投递接近成功不代表持久化成功，本轮必须同时判定 Cluster 投递和 MySQL 落库不通过。
+
+直接故障链路：
+
+1. 客户端连接所在 Pod 不是 P2P Topic 的 master 时，请求进入 proxy-to-master 转发。
+2. 当前每个远端节点只有 1 个同步 RPC 发送协程，队列容量只有 64。
+3. 队列满后仅等待 20ms，随即返回 `cluster: load exceeded`。
+4. Topic proxy 将该错误转换为 WebSocket `502 cluster unreachable`。
+5. 反向投递进入 `hub.routeSrv` 时，4096 容量队列也出现满载；当前 RPC 调用方忽略 `rejected` 结果，存在静默丢弃路径。
+
+因此，本轮 `40.665%` 的发送失败率是 Cluster 内部排队与串行 RPC 的结果。Redis/Kafka 没有参与这条 Topic proxy-to-master 请求路径，不是发送失败的故障点；但 Kafka 消费到 MySQL 的异步持久化链路另有独立故障。
+
+## 2. 压测结果对照
+
+| 指标 | 结果 |
+|---|---:|
+| WS Ready | 2000/2000 |
+| Publish 尝试 | 20000 |
+| 服务端 ACK | 11867 |
+| 拒绝或超时 | 8133 |
+| ACK 成功率 | 59.335% |
+| ACK 后应到达 | 11867 |
+| ACK 后实际到达 | 11859 |
+| 实时丢失 | 8 |
+| ACK 后到达率 | 99.9326% |
+| 重复/乱序/断连 | 0 / 2 / 1 |
+| ACK 平均/最大延迟 | 3.079s / 18.715s |
+| 投递平均/最大延迟 | 4.178s / 21.741s |
+
+8 条丢失全部集中在 `B -> A` 用户序号 0959，与 A 机记录到的 1 次 WS 断连同时出现。
+
+### 2.1 MySQL 直接核验
+
+2026-09-27 通过 Pod 当前的只读数据库连接，对租户 `allrich`（`tenant_id=8`）进行了两层核验：
+
+| 核验项 | 结果 |
+|---|---:|
+| 客户端收到 ACK | 11867 |
+| `im_008_messages` 按本次 `client_msg_id` 前缀命中 | 0 |
+| `im_008_messages` 按 `content.run_id=p2p-20260927-01` 命中 | 0 |
+| `im_messages` 按 `content.run_id=p2p-20260927-01` 命中 | 0 |
+| ACK 消息精确匹配 `client_mid + topic + seq` | 0/11867 |
+| 本轮已生成 Kafka 死信（核验时点） | 27 |
+| 死信错误 | `Error 1264: Out of range value for column 'id'` |
+
+结论：修复前，本轮 ACK 消息的 **MySQL 落库率为 0%**。27 是首次核验时已完成重试并转入死信的数量，不是最终总数；当时其余消息仍位于 Kafka backlog 或处理中。
+
+### 2.2 修复执行与数据恢复结果
+
+2026-09-27 已在本地和测试环境执行 `docs/sql/IM-消息表Snowflake-ID-BIGINT修复SQL.sql` 对应迁移：
+
+- 本地 `meeting_im.im_messages.id` 已由 `INT` 改为 `BIGINT NOT NULL AUTO_INCREMENT`。
+- 测试环境 `meeting_im.im_messages` 以及 `im_001_messages` 至 `im_009_messages` 共 10 张表已全部改为 `BIGINT NOT NULL AUTO_INCREMENT`，后置校验异常数为 0。
+- 修复后 Kafka backlog 自动恢复落库 11835 条；本轮已有 32 条消息进入死信，通过系统 `kafka.dead_letter.replay` 命令完成重放。
+
+使用原始 ACK ledger 对恢复后的数据库逐条校验：
+
+| 核验项 | 最终结果 |
+|---|---:|
+| ACK ledger | 11867 |
+| 数据库本轮消息行 | 11867 |
+| `client_mid + topic + seq` 精确匹配 | 11867 |
+| 缺失 | 0 |
+| 错配 | 0 |
+| 重复 | 0 |
+| 拒绝/超时请求意外落库 | 0/8133 |
+
+因此本轮已确认 ACK 消息最终 100% 落库。这里的最终结果是执行字段迁移、等待 Kafka backlog 排空并重放死信后的恢复结果，不代表故障发生时的异步持久化链路正常。
+
+## 3. Kubernetes 现状
+
+| 项目 | 实际值 |
+|---|---|
+| Namespace | `im-server-system` |
+| Workload | `StatefulSet/im-server` |
+| 副本 | 3/3 Ready |
+| Pod | `im-server-0/1/2` |
+| 节点分布 | 3 个 Pod 在 3 个不同 K8s Node |
+| 重启 | 全部为 0 |
+| 镜像 | 3 Pod 使用相同 digest |
+| CPU request/limit | `100m / 500m` |
+| Memory request/limit | `128Mi / 256Mi` |
+| Cluster 端口 | 12000 |
+
+Pod 没有 OOM/Kill，内存 cgroup 没有 `oom` 或 `oom_kill`。未观察到集群 TCP 断开、`node not connected`、panic 或 fatal。
+
+## 4. Pod 日志证据
+
+压测发送窗口：2026-09-26 18:02:45Z 至 18:03:24Z（北京时间 02:02:45 至 02:03:24）。
+
+`im-server-0` 的可用完整窗口日志中：
+
+| 日志 | 次数 |
+|---|---:|
+| `route broadcast request from proxy to master failed - cluster: load exceeded` | 2259 |
+| `cluster Route: server busy` | 3889 |
+| `cluster: node ... not connected` | 0 |
+| cluster RPC connection error | 0 |
+| panic/fatal | 0 |
+
+`im-server-2` 当前保留的日志从 18:03:19Z 才开始，仅最后约 5 秒已有 496 次 `cluster Route: server busy`。`im-server-1` 当前日志从 18:15:47Z 开始，本次发送窗口已被轮转覆盖。
+
+因此 2259 和 4385（3889+496）都是 **可见日志下界**，不是整个集群的全量。这也是日志量与客户端 8133 次拒绝/超时不完全相等的主要原因之一。
+
+错误从发送开始后立即出现，`im-server-0` 前 10 秒每秒约有 180-226 条 Cluster 过载日志，说明它不是某个后台定时任务引起，而是与本轮 publish 突发直接相关。
+
+## 5. 代码级根因
+
+### 5.1 proxy-to-master 队列过小
+
+`server/cluster.go` 将 `clusterProxyToMasterBuffer` 固定为 64，`clusterP2MTimeout` 固定为 20ms。3 节点时 `len(c.nodes)==2`，扩容逻辑不生效，所以每个远端 peer 实际只有 64 个缓冲位。
+
+队列满后，`proxyToMasterAsync` 在 20ms 后返回 `cluster: load exceeded`。`topic_proxy.go` 随即给客户端返回 502。日志与代码路径一致。
+
+### 5.2 每个 peer 只有一个同步 RPC worker
+
+`p2mSenderLoop` 只启动一次，循环内调用同步 `endpoint.Call`。该 peer 上任何一个较慢的 TopicMaster RPC 都会阻塞后续所有 Topic。在 1000 对用户同时 publish 时，64 队列很快被填满。
+
+### 5.3 接收侧有静默丢弃路径
+
+`Cluster.Route` 在 `globals.hub.routeSrv` 满时设置 `rejected=true`并打印 `cluster Route: server busy`。但 `ClusterNode.route` 只检查 RPC error，传入的 bool 命名为 `unused`，没有检查被远端拒绝的结果。
+
+这意味着发送方可能认为 RPC 成功，但接收方已经因队列满而丢弃消息。本轮 8 条 ACK 后实时丢失与此风险方向一致，但因全量日志已部分轮转，无法将每一条 Client MID 与具体 server-busy 日志逐条对齐。
+
+### 5.4 资源限制会放大排队
+
+每个 Pod 的 CPU limit 只有 500m，同时承担 1000 量级 WS、Topic goroutine、net/rpc、MySQL 持久化和实时投递。当前 cgroup 累计存在 CPU throttling，但没有压测时间窗的 Prometheus 历史样本，所以只能将 CPU 限制定义为高概率放大因素，不将其单独认定为根因。
+
+### 5.5 Kafka 异步落库主键类型不兼容
+
+当前代码在 Kafka 入队前通过 `store.Store.GetUid()` 生成 Snowflake 消息 ID。消费者批量落库时，`TenantMessageSaveBatch` 显式把该 ID 写入 `im_008_messages.id`。
+
+线上实际字段类型为：
+
+| 表 | `id` 类型 |
+|---|---|
+| `im_messages` | `INT` |
+| `im_008_messages` | `INT` |
+
+Snowflake ID 超过有符号 32 位 `INT` 上限，批量插入立即报 1264。代码降级为逐条处理时仍调用同一个批量写入方法，因此不会绕过该错误；达到重试上限后消息进入 `im_kafka_dead_letters`。与此同时，WebSocket 202/ACK 在 Kafka publish 成功后就返回，因而客户端会观察到“ACK 成功、实时广播成功、历史和数据库没有消息”。
+
+这是数据库 DDL 与当前异步持久化代码契约不一致。仓库初始化 SQL 也仍把 `im_messages.id` 定义为 `INT`，因此新环境同样会复现。
+
+## 6. 次要问题
+
+### 6.1 403 permission denied
+
+两台 agent 保留的前 1000 条错误中，A 有 11 条、B 有 24 条 403。多个相同用户序号在 A/B 两侧同时出现，例如 0082、0175、0304、0401、0773。这表明少量 P2P Topic 的发布权限或好友/Topic 初始化状态不一致，与 Cluster 过载是独立问题。
+
+### 6.2 历史回查程序串行阻塞
+
+压测程序对 1000 个 Topic 串行调用 history GET，每次控制应答最长等 20 秒，导致收尾耗时达 13-19 分钟。原始 `history_found=0` 本身不能证明 MySQL 丢数据；本报告新增的 MySQL 直查与 Kafka 死信证据已经独立确认落库失败。
+
+### 6.3 日志保留不足
+
+压测后不到 20 分钟，`im-server-1` 和 `im-server-2` 的关键窗口已部分或全部被容器日志轮转覆盖。没有集中日志时，无法完整重建三 Pod 时间线。
+
+## 7. 改进方案
+
+### P0：修复 Cluster 转发实现
+
+1. 将 cluster buffer、enqueue timeout、worker 数改为可配置项，不再使用 64/20ms 硬编码。
+2. 每个 peer 使用有界并发 RPC worker pool。为保证同一 Topic 的顺序，应按 `tenant_id + topic` 做固定分片，同一 Topic 始终进入同一 worker。
+3. 首轮建议值：每 peer 16 个 worker、总 buffer 4096、enqueue timeout 200-500ms。该数值必须通过阶梯压测定标，不应作为无上限扩容。
+4. 修复 `ClusterNode.route`：检查远端 `rejected` 并返回明确 overload error，禁止静默成功。
+5. 对已持久化的 ServerComMessage 建立有界重试（例如 3 次指数退避）。超过重试上限要记录可检索的 tenant/topic/seq/target pod，并由客户端重连后历史同步补齐。
+6. 将 overload 与 node unreachable 分开：队列满返回 503/`cluster overloaded`，只有节点断连才返回 502/`cluster unreachable`。
+
+### P0：恢复 Kafka 消息持久化
+
+1. 暂停新的压力流量，避免继续扩大 Kafka backlog 和死信数量。
+2. 审核并执行 `docs/sql/IM-消息表Snowflake-ID-BIGINT修复SQL.sql`，把基础表及所有 `im_NNN_messages.id` 改为 `BIGINT NOT NULL AUTO_INCREMENT`。保留 `AUTO_INCREMENT` 是为了兼容 Kafka 关闭时不显式传 ID 的同步写入路径。
+3. 变更后先发 1 条带唯一 `client_mid` 的探针消息，确认 Kafka 消费、MySQL 行、Topic seq 和历史查询一致。
+4. 再通过现有死信管理接口重放本轮 `message.persist.p2p` 死信；Kafka 中尚未提交的 backlog 会由消费者继续处理。不要直接删除死信或重置 offset。
+5. 以本轮 11867 个 ACK ledger 做最终精确核对，要求 `client_mid + topic + seq` 为 11867/11867，重复为 0。
+6. 增加启动前 schema contract 检查：Kafka 持久化启用时，若私聊消息基础表或任何租户分表的 `id` 不是 BIGINT，Pod readiness 必须失败并输出明确的表名和字段类型。
+7. 增加 `kafka_message_persist_total{result=batch_ok|single_ok|retry|dead}`、消费 lag、死信累计量和最老 backlog age 告警。客户端 ACK 语义应明确为“Kafka 已接收（pending）”，服务端另行提供最终持久化状态；若产品要求 ACK 等于已落库，则必须改为同步确认或增加持久化回执。
+
+### P0：调整部署资源与入口
+
+1. 测试环境首轮将每 Pod request 调整为 `cpu: 1, memory: 512Mi`，limit 调整为 `cpu: 2, memory: 1Gi`；运行阶梯压测后再根据 P95 调整。
+2. 核对客户端 Service/Ingress：`sessionAffinity: None`，WebSocket upstream 不使用 `ip_hash`，建议 `least_conn` 或等价的长连接均衡策略。
+3. 在压测开始前记录每 Pod `ws_connections_active`，单 Pod 连接数不得超过平均值的 1.3 倍。
+4. readiness 增加 `LiveClusterNodes == TotalClusterNodes-1` 和 cluster queue 长时间高水位检查，但不应因瞬时高水位造成 Pod 反复摘除。
+
+### P1：增加可观测性
+
+新增以下 Prometheus 指标，必须带 `pod` 和 `peer` 维度，Topic 只记 hash：
+
+- `im_cluster_p2m_queue_length/capacity`
+- `im_cluster_p2m_enqueue_total{result=ok|timeout}`
+- `im_cluster_rpc_inflight`
+- `im_cluster_rpc_duration_seconds`
+- `im_cluster_route_total{result=ok|hub_full|peer_rejected|disconnected}`
+- `im_cluster_proxy_response_total{result=ok|channel_full|unknown_topic}`
+- `im_ws_connections_active{pod}`
+- `im_message_persist_duration_seconds`
+
+告警建议：
+
+- queue 使用率连续 30s 超过 70%：预警；超过 90%：严重。
+- `enqueue timeout` 或 `hub_full` 1 分钟内大于 0：严重。
+- `LiveClusterNodes` 低于期望值：严重。
+- WS 连接不均衡比超过 1.3：预警。
+
+日志应接入 Loki/ELK 等集中系统，至少保留 7 天。限速相同 overload 日志，用计数指标补全被限速的数量，避免大量重复日志反过来抢占 CPU 和 IO。
+
+### P1：修复压测程序
+
+1. history 回查改为 32-64 有界并发，总收尾超时不超过 3 分钟。
+2. 分开统计 `publish_502`、`publish_503`、`publish_403`、`publish_ctrl_timeout`、`socket_error`，不再只保留前 1000 条文本错误。
+3. 对所有 1000 对用户做发布权限预检，修复 403 后再开始正式计时。
+4. 在两台压测机启动前、发送中、收尾后自动抓取 Pod/Redis/Kafka/MySQL 指标快照。
+5. 历史落库验证增加 MySQL 直查作为独立通道，以 `tenant_id + topic + client_mid/seq` 分批校验。
+
+## 8. 复测顺序与通过门槛
+
+1. 200 对、500 对、1000 对分三档，每档至少跑 3 轮。
+2. 每档同时执行“均匀爬坡”和“同时突发”两种模式。
+3. 最终 1000 对场景通过门槛：
+   - publish ACK 成功率不低于 99.99%。
+   - ACK 后对端到达率为 100%，重复为 0。
+   - Cluster enqueue timeout、hub full、peer rejected 均为 0。
+   - ACK P95 < 500ms，对端投递 P95 < 1s。
+   - MySQL 落库校验 100%。
+   - Kafka consumer lag 在发送停止后 60 秒内归零，死信新增为 0。
+   - 单 Pod CPU P95 < 70%，内存 P95 < limit 的 70%，无持续 CPU throttling。
+
+## 9. 本次调查限制
+
+- 调试 ServiceAccount 可读 Pod/StatefulSet/log，但不可读 Service/Endpoint 和 metrics-server 历史数据。
+- `im-server-1/2` 的压测窗口日志已部分轮转，日志次数只能作为下界。
+- 未获得当时的 CPU、内存、网络、MySQL、Redis、Kafka 时序监控，不对这些组件的历史利用率做无证据推断。
+- Kafka backlog 的精确条数无法通过当前调试 ServiceAccount 直接读取；死信 27 条只代表核验时已经完成失败重试的部分。
+
+即使存在上述限制，现有证据已足以分别确认 Cluster 转发过载和 Kafka 持久化主键溢出。修复并完成死信/backlog 恢复前，本轮数据不能视为已持久化。
+
+## 10. P0/P1 实施状态
+
+2026-09-27 已完成代码与发布配置改造：
+
+- Cluster 使用按 `tenant_id + topic` 固定分片的 16 worker、每 peer 总队列 4096、入队超时 300ms；远端 `hub_full` 有 3 次指数退避重试，过载返回 503，节点断连保持 502。
+- TopicMaster 的客户端 publish 接收队列满时会通过 multiplex session 回 503，不再只记录日志后等待客户端超时；同类过载日志每秒限速，完整次数由指标保留。
+- Kafka 启用时，启动阶段校验基础私聊表及所有活动租户私聊分表的 `id` 必须为 `BIGINT NOT NULL AUTO_INCREMENT`，不满足时进程不会进入 Ready。
+- 增加 Cluster 队列/RPC/路由、WebSocket 连接、Kafka 持久化结果/lag/backlog age/死信及持久化耗时指标和 PrometheusRule 告警。
+- `/healthz` 检查全部 Cluster peer，并仅在队列超过 90% 持续 30 秒时摘除 readiness；新增 `/livez` 供 liveness 使用，避免 Cluster 短暂异常触发 Pod 重启。
+- 测试环境发布流水线应用 `request=1C/512Mi`、`limit=2C/1Gi`，设置 Cluster 参数、Service `sessionAffinity=None`，并在存在 `Ingress/im-server` 时设置 NGINX `least_conn`。
+- 压测程序增加全量 P2P 预检、48 worker 历史回查、3 分钟总超时、publish 错误分类、指标快照阶段标记、P95/ACK 成功率门禁及 MySQL 独立精确核验。
+
+当前调试 kubeconfig 对 StatefulSet/Service/Ingress 没有 patch 权限，因此现存 Pod 资源尚未被本地会话直接修改；以上部署项会在新提交触发有权限的 Gitea deploy runner 后生效。生效后必须按第 8 节执行 200/500/1000 对阶梯复测，不能用单元测试替代容量验收。
+
+## 11. 200 对阶梯复测结果
+
+Run ID：`p2p-200-20260927-01`，A/B 两台压测机各模拟 200 个用户，共 200 对、400 条 WebSocket 连接。
+
+| 指标 | 结果 |
+|---|---:|
+| WS Ready | 400/400 |
+| Publish 尝试 | 4000 |
+| 服务端 ACK | 3890 |
+| 403 拒绝 | 110 |
+| 502 / 503 / ctrl timeout / socket error | 0 / 0 / 0 / 0 |
+| ACK 后应到达 / 实际到达 | 3890 / 3890 |
+| ACK 后到达率 | 100% |
+| 实时丢失 / 重复 / 乱序 / 断连 | 0 / 0 / 0 / 0 |
+| Kafka P2P `batch_ok` | 3890 |
+| Kafka consumer lag / backlog age | 0 / 0 |
+| MySQL 精确匹配 | 3890/3890 |
+| MySQL 缺失 / 错配 / 重复 / 非 ACK 落库 | 0 / 0 / 0 / 0 |
+
+本轮没有再出现 `cluster: load exceeded`、`server busy`、502 或 503，说明 Cluster P0 改造在 200 对档位已生效。Kafka 消费指标和 MySQL 对 ACK ledger 的独立精确查询均确认 3890 条消息已全部落库。
+
+压测报告当时的 `history_found=2213`、`persistence_lost=1677` 不是真实落库丢失。旧实现首轮查询后仅等待 2 秒，再进行两次 2 秒重试，并未持续到配置的 3 分钟截止时间。已改为在 `history_timeout` 内持续重试，并按 Topic 合并缺失区间查询，避免过早判定和大量单条请求。
+
+### 11.1 403 根因与修复
+
+110 次 403 稳定集中在用户序号 `0018`、`0043`、`0065`、`0082`、`0175`、`0183` 的 6 个 P2P Topic，与 Cluster 转发无关。`EnsureP2P` 已将 MySQL 中双方订阅修复为 `JRWPA`，但已加载 Topic 的 `perUser` 仍保留旧权限，因此预检通过后 WebSocket publish 仍被内存态拒绝。
+
+已增加两层修复：
+
+1. HTTP `EnsureP2P` 成功后向 Topic master 发布内部权限刷新事件，原生 Cluster 由 `routeSrv` 路由，非原生多 Pod 模式由 Redis realtime fanout 路由。
+2. P2P publish 发现内存态无写权限时，先从 MySQL 重载该用户订阅再判定，作为刷新事件延迟或丢失时的自愈保障。数据库仍无写权限时依然返回 403。
+
+### 11.2 当前未达标项
+
+- ACK P95 和到达 P95 均落在 2 秒直方图上界，未达 500ms / 1s 门槛。
+- ArgoCD 已将 StatefulSet 资源回滚为 `request 100m/128Mi`、`limit 500m/256Mi`，且 liveness 仍使用 `/healthz`。压测期间存在 CPU throttling 累计，部署配置必须在 GitOps 声明源中固化，否则应用后仍会被回滚。
+- 403 修复发布后应重跑 200 对；只有 ACK 成功率、实时到达率、MySQL 精确落库均达标，才进入 500 对档位。

docs/对接文档最新/IM-前后端对接文档.md

@@ -323,7 +323,7 @@ X-MeetingIM-APIKey: <预置的普通 API Key>
 | 类型 | 适用场景 | 服务端下行示例 | 客户端渲染 |
 | --- | --- | --- | --- |
 | `system_notice` | “你已添加了安妮，现在可以开始聊天了”、撤回提示、群名修改、成员进出群等纯提示 | `{"data":{"topic":"p2p...","seq":19,"from":"","head":{"x-im-schema":1,"x-im-type":"system_notice","x-im-template":"friend_added","x-im-from":"system"},"content":{"text":"你已添加了安妮，现在可以开始聊天了。","level":"normal","template":"friend_added","actors":[{"user_id":"usr_anne","name":"安妮"}],"actions":[]}}}` | 居中灰色提示；不显示发送人头像；如 `actors.click` 为空则不可点 |
-| `system_action` | 群邀请确认说明、审批入口、群设置入口、可点击人名或群名提示 | `{"data":{"topic":"grp...","seq":20,"from":"","head":{"x-im-schema":1,"x-im-type":"system_action","x-im-template":"group_invite_confirm_enabled","x-im-from":"system"},"content":{"text":"安妮邀请栗子来了加入了群聊","template":"group_invite_confirm_enabled","actors":[{"user_id":"usr_anne","name":"安妮","click":{"type":"user_profile","target_id":"usr_anne"}},{"user_id":"usr_lizi","name":"栗子来了","click":{"type":"user_profile","target_id":"usr_lizi"}}],"actions":[{"type":"open_group_invite_settings","label":"群聊邀请确认","target_id":"grp..."}]}}}` | 居中灰色或业务提示样式；可点击片段按 `actors.click` 高亮；按钮或文字入口按 `actions` 渲染 |
+| `system_action` | 审批入口、群设置入口、可点击人名或群名提示；仅在当前消息对所有接收者都可操作时下发 `actions` | `{"data":{"topic":"grp...","seq":20,"from":"","head":{"x-im-schema":1,"x-im-type":"system_action","x-im-template":"group_member_joined","x-im-from":"system"},"content":{"text":"安妮邀请栗子来了加入了群聊","template":"group_member_joined","actors":[{"user_id":"usr_anne","name":"安妮","click":{"type":"user_profile","target_id":"usr_anne"}},{"user_id":"usr_lizi","name":"栗子来了","click":{"type":"user_profile","target_id":"usr_lizi"}}],"actions":[]}}}` | 居中灰色或业务提示样式；可点击片段按 `actors.click` 高亮；按钮或文字入口按 `actions` 渲染。若 `actions` 为空，不得将普通文案渲染为可点击入口 |
 
 `content` 字段约定：
 
@@ -333,7 +333,7 @@ X-MeetingIM-APIKey: <预置的普通 API Key>
 | `template` | string | 是 | 服务端稳定模板编码，例如 `friend_added`、`group_member_joined`、`group_invite_confirm_enabled` |
 | `level` | string | 否 | `normal`、`warning`、`danger`，影响颜色和强调程度 |
 | `actors` | array | 否 | 文案中的用户、群、文件等实体；支持局部高亮和点击 |
-| `actions` | array | 否 | 当前消息可触发的操作入口；只允许客户端白名单动作 |
+| `actions` | array | 否 | 当前消息可触发的操作入口；只允许客户端白名单动作。群邀请确认开关、全员禁言开关、成员私信权限开关等设置变更公告面向全员广播，服务端不下发仅管理员可操作的 `actions`；客户端不得仅凭模板名高亮入口。若历史消息或兼容字段携带 `min_role`，客户端必须按当前成员角色过滤，角色不足时忽略该 action |
 | `extra` | object | 否 | 模板扩展参数；客户端不得依赖未知字段完成关键流程 |
 
 `actors[].click` 和 `actions[]` 的 `type` 首版白名单：
@@ -342,7 +342,7 @@ X-MeetingIM-APIKey: <预置的普通 API Key>
 | --- | --- | --- |
 | `user_profile` | `target_id` | 打开用户资料页 |
 | `group_profile` | `target_id` | 打开群资料页 |
-| `open_group_invite_settings` | `target_id` | 打开群邀请确认设置 |
+| `open_group_invite_settings` | `target_id` | 打开群邀请确认设置；仅用于明确发给管理员/群主或全员都可执行的入口，不用于全员广播的开关结果公告 |
 | `open_group_join_requests` | `target_id` | 打开群入群审批列表 |
 | `open_message` | `topic,seq` | 定位到指定消息 |
 

loadtest/im-pressure/.gitignore

@@ -0,0 +1,9 @@
+/im-pressure
+/config.json
+/config.local*.json
+/.env.test
+/manifest*.json
+/results/
+/*-summary.json
+/report.json
+/metrics.jsonl

loadtest/im-pressure/MACHINE-B.md

@@ -0,0 +1,202 @@
+# 机器 B 压测执行手册
+
+## 给机器 B 上 AI 的指令
+
+你当前操作的是 **机器 B**，代码与机器 A 完全相同。你的职责只有：核验运行文件、启动 `agent -machine b`、观察运行状态、把结果留给机器 A。不要执行 `prepare`、`coordinator` 或 `merge`，不要修改账号分片。
+
+机器 B 固定模拟 `ltb0001` 至 `ltb1000` 共 1000 个用户。测试场景和统一开始时间由机器 A 的 coordinator 下发。
+
+## 当前 allrich 测试批次
+
+以下参数适用于 2026-09-27 正在执行的第一轮单聊压测：
+
+| 参数 | 值 |
+|---|---|
+| Git 分支 | `dev` |
+| 最低提交 | `6d17154` |
+| 企业码 | `allrich` |
+| HTTP | `https://api.118im.com` |
+| WebSocket | `wss://api.118im.com/v0/channels` |
+| 机器 A 内网地址 | `192.168.1.135` |
+| Coordinator | `http://192.168.1.135:8090` |
+| Run ID | `p2p-20260927-01` |
+| 场景 | `p2p` |
+| 机器 A 状态 | `1000/1000 READY`，等待机器 B |
+
+机器 A 已生成安全运行包：
+
+```text
+/tmp/allrich-machine-b-runtime-p2p-20260927-01.zip
+SHA-256: 479f119e7d01e1829c7262cace282c72a5f53ad8e7f5128abc2ade75d046db64
+```
+
+运行包内只有三个运行时文件：`.env.test`、`config.json`、`manifest.json`。其中包含 API Key、测试账号密码、固定测试验证码和账号 Token，不进入 Git，必须通过受控内网传输。测试环境验证码为 `123456`，正式环境禁止使用固定验证码。
+
+因此，机器 B 只执行 `git pull` 后看不到这三个文件是正常现象。机器 B 需要从机器 A 单独复制运行包：
+
+```bash
+scp youshunlai@192.168.1.135:/tmp/allrich-machine-b-runtime-p2p-20260927-01.zip /tmp/
+
+shasum -a 256 /tmp/allrich-machine-b-runtime-p2p-20260927-01.zip
+# 必须输出：479f119e7d01e1829c7262cace282c72a5f53ad8e7f5128abc2ade75d046db64
+```
+
+如果机器 B 无法主动连接 A，也可以由机器 A 执行 `scp` 推送；目标机器的用户名和地址由 B 机提供。禁止通过 Git、公开网盘或无访问控制的 HTTP 服务传输该运行包。
+
+本轮 A/B 都通过同一个公共 WebSocket LB 接入。它能覆盖负载均衡后的多 Pod 投递，但不能保证 A、B 被固定分配到不同 Pod；需要强制跨指定 Pod 时，应另行提供两个定向 WS 入口并更新 `config.json`。
+
+## 运行前向机器 A 获取
+
+开始前必须拿到以下信息：
+
+- `MACHINE_A_IP`：机器 A 的内网 IP。
+- `RUN_ID`：本轮唯一标识，例如 `p2p-20260927-01`。
+- `config.json`：两台机器使用同一份配置。
+- `manifest.json`：由机器 A 执行 `prepare` 生成，包含账号 Token 和 Topic。
+- `IM_API_KEY`、`IM_LOAD_PASSWORD`：通过安全方式提供，不写进代码或结果文件。
+- 机器 A 已启动 coordinator 的确认。
+
+同一份代码不代表运行时 manifest 会自动同步。如果本机没有最新的 `manifest.json`，必须让机器 A 传过来；机器 B 不得自行执行 `prepare`。
+
+## 第一步：进入仓库并构建
+
+把 `/absolute/path/to/im_server` 换成本机仓库绝对路径：
+
+```bash
+cd /absolute/path/to/im_server
+git rev-parse HEAD
+go build -o loadtest/im-pressure/im-pressure ./loadtest/im-pressure
+ulimit -n 100000
+```
+
+将 `git rev-parse HEAD` 的结果发给机器 A，确认两边代码版本一致。
+
+## 第二步：解压运行文件
+
+在机器 B 的仓库根目录执行：
+
+```bash
+unzip -o /tmp/allrich-machine-b-runtime-p2p-20260927-01.zip \
+  -d loadtest/im-pressure
+chmod 600 \
+  loadtest/im-pressure/.env.test \
+  loadtest/im-pressure/config.json \
+  loadtest/im-pressure/manifest.json
+```
+
+解压后必须存在：
+
+```text
+loadtest/im-pressure/.env.test
+loadtest/im-pressure/config.json
+loadtest/im-pressure/manifest.json
+```
+
+检查文件：
+
+```bash
+test -s loadtest/im-pressure/.env.test
+test -s loadtest/im-pressure/config.json
+test -s loadtest/im-pressure/manifest.json
+```
+
+## 第三步：设置环境变量并核验
+
+```bash
+set -a
+source loadtest/im-pressure/.env.test
+set +a
+
+export MACHINE_A_IP='192.168.1.135'
+export RUN_ID='p2p-20260927-01'
+
+./loadtest/im-pressure/im-pressure verify \
+  -config loadtest/im-pressure/config.json \
+  -manifest loadtest/im-pressure/manifest.json
+```
+
+`verify` 必须成功后才能继续。它只读核验 2000 个账号、P2P Topic 和 2000 人群，不会开始压测。
+
+## 第四步：检查机器 A 协调器
+
+```bash
+curl -fsS "http://${MACHINE_A_IP}:8090/v1/status"
+```
+
+如果连接失败，停止操作并让机器 A 检查 coordinator、内网 IP、防火墙和 `8090` 端口。不要改为本机启动 coordinator。
+
+## 第五步：启动机器 B agent
+
+```bash
+mkdir -p "loadtest/im-pressure/results/${RUN_ID}"
+
+./loadtest/im-pressure/im-pressure agent \
+  -config loadtest/im-pressure/config.json \
+  -manifest loadtest/im-pressure/manifest.json \
+  -machine b \
+  -coordinator "http://${MACHINE_A_IP}:8090" \
+  -metrics-listen :9105 \
+  -output "loadtest/im-pressure/results/${RUN_ID}/agent-b-summary.json"
+```
+
+启动后会依次发生：
+
+1. 机器 B 建立 1000 条 WebSocket 连接并订阅对应 P2P Topic 和群 Topic。
+2. agent 向机器 A coordinator 报告 READY。
+3. 如果机器 A 尚未 READY，当前命令会等待，这是正常现象。
+4. coordinator 下发 `run_id`、场景和统一 UTC 开始时间。
+5. agent 完成发送、WS 到达统计、drain 和历史持久化回查。
+6. agent 写出 summary、上报 coordinator，然后正常退出。
+
+机器 B 不需要传场景参数。`p2p`、`group` 或 `mixed` 由机器 A coordinator 决定。
+
+## 第六步：检查并回传结果
+
+```bash
+test -s "loadtest/im-pressure/results/${RUN_ID}/agent-b-summary.json"
+```
+
+将以下文件路径和运行是否成功告知机器 A：
+
+```text
+loadtest/im-pressure/results/<RUN_ID>/agent-b-summary.json
+```
+
+如果机器 A 无法直接读取机器 B 文件，由机器 A 使用 `scp` 拉取，或从机器 B 推送：
+
+```bash
+scp "loadtest/im-pressure/results/${RUN_ID}/agent-b-summary.json" \
+  user@机器A内网IP:/absolute/path/to/im_server/loadtest/im-pressure/results/${RUN_ID}/
+```
+
+## 运行中观察
+
+另开一个终端可以查看本机 agent 指标：
+
+```bash
+curl -fsS http://127.0.0.1:9105/metrics | \
+  grep -E 'im_load_agent_(ready_connections|publish|received|duplicates|out_of_order|disconnects)'
+```
+
+关键判断：
+
+- `im_load_agent_ready_connections` 应到达 `1000`。
+- `disconnects_total`、`duplicates_total`、`out_of_order_total` 期望为 `0`。
+- agent 进程退出前不要关闭终端或停止网络。
+
+## 常见问题
+
+- `coordinator connection refused`：机器 A coordinator 未启动、IP 错误或 `8090` 被拦截。
+- 一直等待开始：通常是另一台 agent 尚未达到 1000 个 READY，查看机器 A coordinator 状态。
+- 登录返回 `401`：确认本机设置了正确的 `IM_LOAD_PASSWORD`，并让机器 A 重新同步最新 manifest。
+- `listen :9105 address already in use`：换成空闲端口，例如 `-metrics-listen :9106`，并通知机器 A 修改采集地址。
+- `ready connections` 不足 1000：不要继续压测，检查 WS 地址、文件描述符限制、Ingress 和网络连接限制。
+- `verify` 失败：停止，不要绕过；让机器 A 核对数据准备结果和本机运行文件版本。
+
+## 机器 B 禁止事项
+
+- 禁止运行 `prepare`，避免重复创建或改变测试数据。
+- 禁止运行 `coordinator`，全场只能使用机器 A 的 coordinator。
+- 禁止把参数改成 `-machine a`。
+- 禁止自行改变 `RUN_ID`、config 或 manifest。
+- 禁止提交或传播包含 Token 的 manifest。

loadtest/im-pressure/README.md

@@ -0,0 +1,261 @@
+# IM 双机压力测试工具
+
+该工具按配置指定的企业租户运行：两台压测机各承载 1000 个 WebSocket 用户，共 2000 个用户、1000 对好友和一个 2000 人群。当前测试环境使用 `allrich`，本轮实时参数见 [MACHINE-B.md](./MACHINE-B.md) 和 [allrich 双机压测执行记录](../../docs/test/IM-allrich双机压测执行记录-20260927.md)。
+
+机器 B 上的 AI 或操作人请直接阅读 [MACHINE-B.md](./MACHINE-B.md)。该文件不重复讲方案，只给出机器 B 从核验到启动、观察和回传结果的完整命令。
+
+## 两台机器的固定分工
+
+当前仓库所在机器固定作为 **机器 A**。两台机器不要交换 `-machine` 参数，否则账号分片和结果归属会混乱。
+
+| 项目 | 机器 A（当前机器，主控） | 机器 B（另一台机器，发压） |
+|---|---|---|
+| 模拟账号 | `lta0001` 至 `lta1000` | `ltb0001` 至 `ltb1000` |
+| WS 连接数 | 1000 | 1000 |
+| 固定参数 | `-machine a` | `-machine b` |
+| 数据准备 | 执行 `prepare`、`verify` | 不执行 |
+| 双机协调 | 运行 `coordinator :8090` | 连接机器 A 的 `:8090` |
+| 场景发压 | 运行 agent A | 运行 agent B |
+| 指标 | agent `:9105`，运行中央 `collect` | agent `:9105`，供机器 A 抓取 |
+| 结果 | 保存 A 结果、接收 B 结果、执行 `merge` | 保存 B 结果并传回机器 A |
+
+机器 A 至少准备三个终端：协调器、agent A、指标采集。机器 B 至少准备一个终端运行 agent B；需要单独采集本机指标时再开第二个终端。
+
+### 机器 B 最短操作清单
+
+机器 B 的完整命令见 [MACHINE-B.md](./MACHINE-B.md)。机器 B 操作人只需完成以下事项：
+
+1. 检出与机器 A 相同的 Git 提交并构建 `im-pressure`。
+2. 接收机器 A 生成的 `config.json` 和 `manifest.json`，不要自行运行 `prepare`。
+3. 设置与机器 A 相同的 `IM_API_KEY`、`IM_LOAD_PASSWORD`，执行一次 `verify`。
+4. 从机器 A 获取本轮 `RUN_ID` 和机器 A 内网 IP。
+5. 等机器 A 启动 coordinator 后，固定使用 `-machine b` 启动 agent B。
+6. agent B 退出后，将 `agent-b-summary.json` 传回机器 A。
+
+机器 B 不负责建账号、加好友、建群、选择场景或合并报告。
+
+### 网络要求
+
+- 机器 B 必须能访问 `http://机器A内网IP:8090`。
+- 如果由机器 A 集中采集 agent 指标，机器 A 必须能访问 `http://机器B内网IP:9105/metrics`。
+- 两台机器都必须能访问配置中的 HTTP API 和各自的 WS 地址。
+- `8090` 是无鉴权的压测协调端口，只允许压测内网访问，不要暴露到公网。
+- 两台机器必须使用 NTP/chrony 同步时间；端到端延迟依赖两边时钟一致。
+
+## 统计口径
+
+- 发送成功以 `{ctrl code=2xx, params.seq>0}` 为准，形成两台机器共享的 ACK 台账。
+- 实时到达只统计同时匹配本轮 `run_id` 和 `x-im-client-mid` 的 `{data}`。
+- 群消息设置 `noecho=false`，每条服务端接受的群消息期望 2000 个连接各收到一次。
+- 实时 drain 结束后通过 `{get what=data}` 按 ACK 的 Topic/Seq 回查历史，区分跨 Pod 实时漏投与数据库持久化缺失。
+- 每轮建立 WS 后、正式计时前，以 32 个 worker 对本机 1000 个账号做好友关系、P2P Topic 和发布权限预检；任一异常会终止本轮。
+- 历史回查默认使用 48 个 worker、总超时 3 分钟，并对缺失项再并发重试两轮。
+- publish 错误完整分类为 `403`、`502`、`503`、`ctrl_timeout`、`socket_error` 和 `other`，不受文本错误样本上限影响。
+- `manifest.json` 含登录 token，文件权限为 `0600`，不得提交到 Git 或放入结果包。
+
+## 两台机器初始化
+
+两台机器先检出完全相同的 Git 提交，并分别构建本机二进制：
+
+```bash
+git rev-parse HEAD
+go build -o loadtest/im-pressure/im-pressure ./loadtest/im-pressure
+ulimit -n 100000
+```
+
+只在机器 A 创建配置：
+
+```bash
+cp loadtest/im-pressure/config.example.json loadtest/im-pressure/config.json
+export IM_API_KEY='...'
+export IM_LOAD_PASSWORD='...'
+```
+
+修改 `config.json`：`http_base` 指向 IM HTTP API；`endpoints.a/b` 分别填写两台机器应直连的 WS Pod/LB 地址，多个地址时用户会轮询分配。正式测试保持 `insecure_tls=false`。
+
+测试环境要求短信验证注册时，将 `registration_phone_prefix` 配置为 7 位数字。工具为机器 A 追加后缀 `0001` 起的序号，为机器 B 追加后缀 `5001` 起的序号，生成互不冲突的 11 位测试手机号；验证码从 `verification_code_env` 指定的环境变量读取。正式环境不得使用固定测试验证码。
+
+`endpoints.a` 是机器 A 的 1000 个用户使用的 WS Pod/LB 地址，`endpoints.b` 是机器 B 的 1000 个用户使用的地址。要验证跨 Pod 投递时，应让 A、B 连接到不同 Pod 或不同的定向入口；如果都填同一个普通 LB，连接落在哪个 Pod 将由 LB 决定。
+
+机器 B 同样需要设置 `IM_API_KEY` 和 `IM_LOAD_PASSWORD`。密码用于 Token 失效时自动回退 Basic 登录，不能只在机器 A 设置。
+
+## 机器 A：一次性准备
+
+以下命令只在当前机器 A 执行。命令可重复运行，会复用已有账号、好友和群：
+
+```bash
+./loadtest/im-pressure/im-pressure prepare \
+  -config loadtest/im-pressure/config.json \
+  -manifest loadtest/im-pressure/manifest.json
+
+./loadtest/im-pressure/im-pressure verify \
+  -config loadtest/im-pressure/config.json \
+  -manifest loadtest/im-pressure/manifest.json
+```
+
+准备成功后，机器 A 的 manifest 中包含 2000 个账号 Token、1000 个 P2P Topic 和群 Topic。将配置和 manifest 安全复制到机器 B：
+
+```bash
+export MACHINE_B_SSH='user@机器B内网IP'
+export MACHINE_B_REPO='/home/user/im_server'
+
+scp loadtest/im-pressure/config.json \
+  "${MACHINE_B_SSH}:${MACHINE_B_REPO}/loadtest/im-pressure/config.json"
+scp loadtest/im-pressure/manifest.json \
+  "${MACHINE_B_SSH}:${MACHINE_B_REPO}/loadtest/im-pressure/manifest.json"
+```
+
+机器 B 收到后执行：
+
+```bash
+cd /home/user/im_server
+chmod 600 loadtest/im-pressure/manifest.json
+export IM_API_KEY='...'
+export IM_LOAD_PASSWORD='...'
+./loadtest/im-pressure/im-pressure verify \
+  -config loadtest/im-pressure/config.json \
+  -manifest loadtest/im-pressure/manifest.json
+```
+
+`MACHINE_B_REPO` 必须填写机器 B 上仓库的绝对路径。每次机器 A 重新执行 `prepare` 并改变了 manifest，都必须重新同步到机器 B。
+
+## 每轮双机运行
+
+以下示例执行单聊场景。机器 A 设置本轮场景；两台机器使用同一个、每轮唯一且不超过 24 个字符的 `RUN_ID`，保证结果目录一致：
+
+```bash
+export RUN_ID='p2p-20260927-01'
+export SCENARIO='p2p'
+export MACHINE_A_IP='机器A内网IP'
+export MACHINE_B_IP='机器B内网IP'
+mkdir -p "loadtest/im-pressure/results/${RUN_ID}"
+```
+
+机器 A 应在正式起压前启动终端 A3 的指标采集，然后按 A1、A2、B1 的顺序启动协调器和两个 agent。
+
+### 第一步：机器 A 启动协调器
+
+机器 A 终端 A1：
+
+```bash
+./loadtest/im-pressure/im-pressure coordinator \
+  -listen :8090 \
+  -run-id "${RUN_ID}" \
+  -scenario "${SCENARIO}" \
+  -output "loadtest/im-pressure/results/${RUN_ID}/coordinator.json"
+```
+
+协调器会等待两个 agent 都报告 1000 条连接 READY，再统一下发至少 30 秒后的开始时间。不要跳过协调器直接启动 agent。
+
+### 第二步：机器 A 启动 agent A
+
+机器 A 终端 A2：
+
+```bash
+./loadtest/im-pressure/im-pressure agent \
+  -config loadtest/im-pressure/config.json \
+  -manifest loadtest/im-pressure/manifest.json \
+  -machine a \
+  -coordinator "http://${MACHINE_A_IP}:8090" \
+  -metrics-listen :9105 \
+  -output "loadtest/im-pressure/results/${RUN_ID}/agent-a-summary.json"
+```
+
+### 第三步：机器 B 启动 agent B
+
+机器 B 终端 B1 需要先设置与机器 A 相同的值：
+
+```bash
+cd /home/user/im_server
+export IM_API_KEY='...'
+export IM_LOAD_PASSWORD='...'
+export RUN_ID='p2p-20260927-01'
+export MACHINE_A_IP='机器A内网IP'
+mkdir -p "loadtest/im-pressure/results/${RUN_ID}"
+
+./loadtest/im-pressure/im-pressure agent \
+  -config loadtest/im-pressure/config.json \
+  -manifest loadtest/im-pressure/manifest.json \
+  -machine b \
+  -coordinator "http://${MACHINE_A_IP}:8090" \
+  -metrics-listen :9105 \
+  -output "loadtest/im-pressure/results/${RUN_ID}/agent-b-summary.json"
+```
+
+机器 B 不需要传 `SCENARIO`，场景由机器 A 的 coordinator 下发。两个 agent 可以任意先后启动，但 coordinator 必须最先启动。
+
+两个 agent 都建立 1000 条连接后，协调器下发统一的 UTC `start_at`。场景可替换为：
+
+- `p2p`：1000 对用户两侧同时发送，默认每人 10 条、间隔 1 秒。
+- `group`：每台机器 500 个发送者，每人 2 条；全局每 100ms 一条，共 2000 条。
+- `mixed`：300 对私聊用户每人 5 条，同时每台 250 个群发送者各发 2 条。
+
+## 机器 A：回收并合并报告
+
+两个 agent 完成历史回查并退出后，在机器 A 拉取机器 B 的 summary：
+
+```bash
+scp "${MACHINE_B_SSH}:${MACHINE_B_REPO}/loadtest/im-pressure/results/${RUN_ID}/agent-b-summary.json" \
+  "loadtest/im-pressure/results/${RUN_ID}/agent-b-summary.json"
+```
+
+然后仍在机器 A 合并报告：
+
+```bash
+./loadtest/im-pressure/im-pressure merge \
+  -a "loadtest/im-pressure/results/${RUN_ID}/agent-a-summary.json" \
+  -b "loadtest/im-pressure/results/${RUN_ID}/agent-b-summary.json" \
+  -output "loadtest/im-pressure/results/${RUN_ID}/report.json"
+```
+
+`report.json` 给出 ACK、实时投递、重复、乱序、断连和历史持久化核验结果。当前正确性门禁要求上述错误计数全部为 0。
+
+每个 agent 在 `:9105/metrics` 暴露连接、发送、接收、重复、乱序、断连以及 Go/process 指标；端口冲突时可通过 `-metrics-listen` 修改，传空字符串可关闭。
+
+## 机器 A：集中采集指标
+
+`collect` 可并行抓取 Prometheus、Redis exporter、Kafka exporter、mysqld exporter、kube-state-metrics/cAdvisor 和压测机 node-exporter，原始响应按时间写入 JSONL：
+
+```bash
+./loadtest/im-pressure/im-pressure collect \
+  -duration 20m -interval 5s \
+  -coordinator "http://${MACHINE_A_IP}:8090" \
+  -coordinator-output "loadtest/im-pressure/results/${RUN_ID}/coordinator.json" \
+  -url agent-a=http://127.0.0.1:9105/metrics \
+  -url agent-b=http://${MACHINE_B_IP}:9105/metrics \
+  -url im=http://PROMETHEUS_HOST/api/v1/query?query=up \
+  -url redis=http://REDIS_EXPORTER/metrics \
+  -url kafka=http://KAFKA_EXPORTER/metrics \
+  -url mysql=http://MYSQLD_EXPORTER/metrics \
+  -url node-a=http://NODE_A_EXPORTER/metrics \
+  -url node-b=http://NODE_B_EXPORTER/metrics \
+  -output "loadtest/im-pressure/results/${RUN_ID}/metrics.jsonl"
+```
+
+机器 A 终端 A3 运行采集器。每条 JSONL 快照会自动标记 `phase=before|during|after`：coordinator 下发开始时间后进入 `during`，coordinator 结果文件生成后进入 `after`。URL 含 `&` 时需用单引号包住整个 `name=url` 参数。建议起压前 2 分钟开始采集，结束后继续 5 分钟，以观察 Kafka lag、数据库落库追平时间及资源恢复。如果机器 A 无法访问机器 B 的 `9105`，就在机器 B 单独运行一份 `collect`，结束后把 JSONL 一并传回机器 A。
+
+## MySQL 独立落库核验
+
+合并报告后在机器 A 使用只读 DSN 对 ACK ledger 做独立校验。工具会根据租户和消息时间自动查询 `im_NNN_messages` 及涉及的群消息月表，精确比较 `client_mid + topic + seq`，并统计缺失、错配、重复和非 ACK 消息：
+
+```bash
+export MYSQL_DSN='readonly:password@tcp(mysql-host:3306)/meeting_im?parseTime=true'
+./loadtest/im-pressure/im-pressure verify-db \
+  -coordinator "loadtest/im-pressure/results/${RUN_ID}/coordinator.json" \
+  -tenant allrich \
+  -dsn-env MYSQL_DSN \
+  -registry-db hotgo \
+  -output "loadtest/im-pressure/results/${RUN_ID}/db-verification.json"
+```
+
+DSN 只能通过环境变量传入，不要写入配置、命令脚本或结果包。命令在 missing、mismatch、duplicate 或 unexpected 任一计数非零时返回失败。
+
+## 运行前检查
+
+```bash
+ulimit -n 100000
+curl -fsS "http://${MACHINE_A_IP}:8090/v1/status"
+curl -fsS http://127.0.0.1:9105/metrics | head
+```
+
+两台压测机应完成时钟同步，并确认压测出口、Ingress、Pod 与 Redis/MySQL/Kafka 连接数限制可以容纳本轮负载。协调器没有鉴权，只能监听在受控压测网络中。

loadtest/im-pressure/agent.go

@@ -0,0 +1,624 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+)
+
+var runIDPattern = regexp.MustCompile(`^[a-zA-Z0-9_-]{1,24}$`)
+
+type agentRun struct {
+	cfg            *Config
+	machine        string
+	runID          string
+	scenario       string
+	clients        map[int]*WSClient
+	recorder       *deliveryRecorder
+	ack            *DurationHistogram
+	attempted      atomic.Int64
+	rejected       atomic.Int64
+	ready          atomic.Int64
+	publish403     atomic.Int64
+	publish502     atomic.Int64
+	publish503     atomic.Int64
+	publishTimeout atomic.Int64
+	publishSocket  atomic.Int64
+	publishOther   atomic.Int64
+
+	mu       sync.Mutex
+	accepted []AcceptedMessage
+	errors   []string
+}
+
+func validScenario(scenario string) bool {
+	return scenario == "p2p" || scenario == "group" || scenario == "mixed"
+}
+
+func preflightP2P(ctx context.Context, cfg *Config, manifest *Manifest, local []AccountRecord, apiKey string) error {
+	accounts := manifestAccountMap(manifest)
+	api := newAPIClient(cfg, apiKey)
+	jobs := make(chan AccountRecord)
+	errCh := make(chan error, len(local))
+	var wg sync.WaitGroup
+	workers := cfg.PreflightWorkers
+	if workers > len(local) {
+		workers = len(local)
+	}
+	for i := 0; i < workers; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for source := range jobs {
+				peerMachine := "a"
+				if source.Machine == "a" {
+					peerMachine = "b"
+				}
+				peer := accounts[accountKey(peerMachine, source.Index)]
+				if peer == nil {
+					errCh <- fmt.Errorf("p2p preflight %s-%04d: peer is missing", source.Machine, source.Index)
+					continue
+				}
+				conversation, err := api.ensureP2P(ctx, source, *peer)
+				if err == nil && (conversation.Relationship != "friend" || conversation.Topic != source.P2PTopic) {
+					err = fmt.Errorf("relationship=%s topic_match=%t", conversation.Relationship, conversation.Topic == source.P2PTopic)
+				}
+				if err != nil {
+					errCh <- fmt.Errorf("p2p preflight %s-%04d: %w", source.Machine, source.Index, err)
+				}
+			}
+		}()
+	}
+	for _, account := range local {
+		jobs <- account
+	}
+	close(jobs)
+	wg.Wait()
+	close(errCh)
+	failures := make([]string, 0, len(errCh))
+	for err := range errCh {
+		if len(failures) < 20 {
+			failures = append(failures, err.Error())
+		}
+	}
+	if len(failures) > 0 {
+		return fmt.Errorf("p2p permission preflight failed: %s", strings.Join(failures, "; "))
+	}
+	return nil
+}
+
+func runAgent(ctx context.Context, cfg *Config, manifest *Manifest, machine, coordinatorURL, output, metricsListen string) error {
+	if machine != "a" && machine != "b" {
+		return fmt.Errorf("machine must be a or b")
+	}
+	apiKey, err := cfg.apiKey()
+	if err != nil {
+		return err
+	}
+	password := ""
+	if name := cfg.PasswordEnv; name != "" {
+		password = os.Getenv(name)
+	} else {
+		password = os.Getenv("IM_LOAD_PASSWORD")
+	}
+	accounts := accountsForMachine(manifest, machine)
+	if len(accounts) != cfg.UsersPerMachine {
+		return fmt.Errorf("machine %s accounts=%d expected=%d", machine, len(accounts), cfg.UsersPerMachine)
+	}
+	for i := range accounts {
+		accounts[i].Endpoint = cfg.Endpoints[machine][(accounts[i].Index-1)%len(cfg.Endpoints[machine])]
+	}
+	if !cfg.SkipP2PPreflight {
+		if err = preflightP2P(ctx, cfg, manifest, accounts, apiKey); err != nil {
+			return err
+		}
+	}
+	coord := newCoordinatorClient(coordinatorURL)
+	placeholderRunID := "pending"
+	run := &agentRun{cfg: cfg, machine: machine, clients: make(map[int]*WSClient), ack: NewDurationHistogram()}
+	run.recorder = newDeliveryRecorder(placeholderRunID, machine, len(accounts))
+	stopMetrics, err := startAgentMetrics(metricsListen, run)
+	if err != nil {
+		return err
+	}
+	defer stopMetrics()
+	if err = run.connect(ctx, accounts, manifest.GroupTopic, apiKey, password); err != nil {
+		run.close()
+		return err
+	}
+	defer run.close()
+	start, err := coord.ready(ctx, AgentReady{Machine: machine, ReadyConnections: len(run.clients)})
+	if err != nil {
+		return err
+	}
+	if !runIDPattern.MatchString(start.RunID) || !validScenario(start.Scenario) {
+		return fmt.Errorf("invalid coordinator start: %+v", start)
+	}
+	run.runID = start.RunID
+	run.scenario = start.Scenario
+	run.recorder.setRunID(start.RunID)
+	if err = waitUntil(ctx, start.StartAt); err != nil {
+		return err
+	}
+	startedAt := time.Now().UTC()
+	log.Printf("agent=%s scenario=%s started", machine, start.Scenario)
+	if err = run.execute(ctx, start.StartAt, manifest.GroupTopic); err != nil {
+		return err
+	}
+	loadEndedAt := time.Now().UTC()
+	local := run.acceptedCopy()
+	if err = coord.submitLedger(ctx, machine, local); err != nil {
+		return err
+	}
+	ledger, err := coord.ledger(ctx)
+	if err != nil {
+		return err
+	}
+	drain, err := run.drainDuration()
+	if err != nil {
+		return err
+	}
+	if err = waitContext(ctx, drain); err != nil {
+		return err
+	}
+	base := AgentSummary{
+		RunID: start.RunID, Scenario: start.Scenario, Machine: machine, StartedAt: startedAt,
+		LoadEndedAt: loadEndedAt, FinishedAt: time.Now().UTC(), ExpectedConnections: cfg.UsersPerMachine,
+		ReadyConnections: len(run.clients),
+		PublishAttempted: run.attempted.Load(), PublishAccepted: int64(len(local)), PublishRejected: run.rejected.Load(),
+		PublishErrors: run.publishErrorCounts(),
+		AckLatency:    run.ack.Snapshot(), AcceptedLocal: local,
+	}
+	run.mu.Lock()
+	base.Errors = append(base.Errors, run.errors...)
+	run.mu.Unlock()
+	summary := run.recorder.summarize(base, ledger)
+	checked, found, persistenceMissing := run.reconcileHistory(ctx, ledger)
+	summary.HistoryChecked = int64(checked)
+	summary.HistoryFound = int64(found)
+	summary.PersistenceLost = int64(len(persistenceMissing))
+	summary.RealtimeLostStored = run.recorder.countRealtimeLostStored(ledger)
+	if len(persistenceMissing) > 10000 {
+		summary.PersistenceMissing = append([]string(nil), persistenceMissing[:10000]...)
+		summary.Errors = append(summary.Errors, "persistence missing details truncated at 10000")
+	} else {
+		summary.PersistenceMissing = persistenceMissing
+	}
+	summary.FinishedAt = time.Now().UTC()
+	if err = writeAgentSummary(output, summary); err != nil {
+		return err
+	}
+	if err = coord.submitSummary(ctx, summary); err != nil {
+		return err
+	}
+	log.Printf("agent=%s accepted=%d expected_peer=%d received_peer=%d expected_group=%d received_group=%d lost=%d duplicates=%d",
+		machine, summary.PublishAccepted, summary.ExpectedPeer, summary.ReceivedPeer,
+		summary.ExpectedGroup, summary.ReceivedGroup, summary.RealtimeLost, summary.Duplicates)
+	return nil
+}
+
+type historyQuery struct {
+	client *WSClient
+	topic  string
+	since  int64
+	before int64
+	limit  int
+}
+
+func (r *agentRun) reconcileHistory(ctx context.Context, ledger []AcceptedMessage) (int, int, []string) {
+	expected := make(map[string]AcceptedMessage)
+	byTopic := make(map[string][]AcceptedMessage)
+	for _, message := range ledger {
+		if message.RunID != r.runID {
+			continue
+		}
+		if message.Kind == "p2p" && message.PeerMachine == r.machine {
+			expected[message.ClientMid] = message
+			byTopic[message.Topic] = append(byTopic[message.Topic], message)
+		} else if message.Kind == "group" && r.machine == "a" {
+			expected[message.ClientMid] = message
+			byTopic[message.Topic] = append(byTopic[message.Topic], message)
+		}
+	}
+	if len(expected) == 0 {
+		return 0, 0, nil
+	}
+	r.recorder.beginHistory()
+	queries := make([]historyQuery, 0, len(byTopic))
+	for topic, messages := range byTopic {
+		sort.Slice(messages, func(i, j int) bool { return messages[i].Seq < messages[j].Seq })
+		client := r.clients[messages[0].PeerIndex]
+		if messages[0].Kind == "group" {
+			client = r.clients[1]
+		}
+		if client == nil {
+			r.addError(fmt.Sprintf("history topic=%s has no verifier client", topicHash(topic)))
+			continue
+		}
+		for start := 0; start < len(messages); start += 200 {
+			end := start + 200
+			if end > len(messages) {
+				end = len(messages)
+			}
+			queries = append(queries, historyQuery{client: client, topic: topic,
+				since: messages[start].Seq, before: messages[end-1].Seq + 1, limit: 256})
+		}
+	}
+	historyTimeout, _ := configDuration(r.cfg.HistoryTimeout)
+	historyCtx, cancel := context.WithTimeout(ctx, historyTimeout)
+	defer cancel()
+	r.issueHistoryQueries(historyCtx, queries)
+	_ = waitContext(historyCtx, 2*time.Second)
+	for {
+		missing := missingHistory(r.recorder, expected)
+		if len(missing) == 0 || historyCtx.Err() != nil {
+			break
+		}
+		retryQueries := r.historyQueriesForMessages(missing)
+		r.issueHistoryQueries(historyCtx, retryQueries)
+		_ = waitContext(historyCtx, 5*time.Second)
+	}
+	missing := missingHistory(r.recorder, expected)
+	missingIDs := make([]string, 0, len(missing))
+	for _, message := range missing {
+		missingIDs = append(missingIDs, message.ClientMid)
+	}
+	sort.Strings(missingIDs)
+	return len(expected), len(expected) - len(missing), missingIDs
+}
+
+func (r *agentRun) historyQueriesForMessages(messages []AcceptedMessage) []historyQuery {
+	byTopic := make(map[string][]AcceptedMessage)
+	for _, message := range messages {
+		byTopic[message.Topic] = append(byTopic[message.Topic], message)
+	}
+	queries := make([]historyQuery, 0, len(byTopic))
+	for topic, topicMessages := range byTopic {
+		sort.Slice(topicMessages, func(i, j int) bool { return topicMessages[i].Seq < topicMessages[j].Seq })
+		client := r.clients[topicMessages[0].PeerIndex]
+		if topicMessages[0].Kind == "group" {
+			client = r.clients[1]
+		}
+		if client == nil {
+			continue
+		}
+		for start := 0; start < len(topicMessages); start += 200 {
+			end := start + 200
+			if end > len(topicMessages) {
+				end = len(topicMessages)
+			}
+			queries = append(queries, historyQuery{client: client, topic: topic,
+				since: topicMessages[start].Seq, before: topicMessages[end-1].Seq + 1, limit: 256})
+		}
+	}
+	return queries
+}
+
+func (r *agentRun) issueHistoryQueries(ctx context.Context, queries []historyQuery) {
+	workers := r.cfg.HistoryWorkers
+	if workers > len(queries) {
+		workers = len(queries)
+	}
+	if workers == 0 {
+		return
+	}
+	jobs := make(chan historyQuery)
+	var wg sync.WaitGroup
+	for i := 0; i < workers; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for query := range jobs {
+				if err := query.client.GetData(ctx, query.topic, query.since, query.before, query.limit); err != nil && ctx.Err() == nil {
+					r.addError(fmt.Sprintf("history get topic=%s range=%d-%d: %v", topicHash(query.topic), query.since, query.before, err))
+				}
+			}
+		}()
+	}
+	for _, query := range queries {
+		select {
+		case jobs <- query:
+		case <-ctx.Done():
+			close(jobs)
+			wg.Wait()
+			return
+		}
+	}
+	close(jobs)
+	wg.Wait()
+}
+
+func missingHistory(recorder *deliveryRecorder, expected map[string]AcceptedMessage) []AcceptedMessage {
+	missing := make([]AcceptedMessage, 0)
+	for mid, message := range expected {
+		if !recorder.historyFound(mid) {
+			missing = append(missing, message)
+		}
+	}
+	return missing
+}
+
+func (r *agentRun) addError(message string) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	if len(r.errors) < 1000 {
+		r.errors = append(r.errors, message)
+	}
+}
+
+func (r *agentRun) connect(ctx context.Context, accounts []AccountRecord, groupTopic, apiKey, password string) error {
+	ramp, err := configDuration(r.cfg.ConnectRamp)
+	if err != nil {
+		return err
+	}
+	interval := ramp / time.Duration(len(accounts))
+	var wg sync.WaitGroup
+	errCh := make(chan error, len(accounts))
+	var mu sync.Mutex
+	for _, account := range accounts {
+		account := account
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			client, connectErr := r.connectAccountWithRetry(ctx, account, groupTopic, apiKey, password)
+			if connectErr != nil {
+				errCh <- connectErr
+				return
+			}
+			mu.Lock()
+			r.clients[account.Index] = client
+			mu.Unlock()
+			r.ready.Add(1)
+		}()
+		if interval > 0 {
+			if err = waitContext(ctx, interval); err != nil {
+				break
+			}
+		}
+	}
+	wg.Wait()
+	close(errCh)
+	var failures []string
+	for connectErr := range errCh {
+		if len(failures) < 20 {
+			failures = append(failures, connectErr.Error())
+		}
+	}
+	if len(r.clients) != len(accounts) {
+		return fmt.Errorf("ready connections=%d expected=%d failures=%v", len(r.clients), len(accounts), failures)
+	}
+	return nil
+}
+
+func (r *agentRun) connectAccountWithRetry(ctx context.Context, account AccountRecord, groupTopic, apiKey, password string) (*WSClient, error) {
+	var lastErr error
+	for attempt := 1; attempt <= 6; attempt++ {
+		client, err := connectWSClient(ctx, r.cfg, account, apiKey, password, r.recorder.onData, r.recorder.onDisconnect)
+		if err == nil {
+			if err = client.Subscribe(ctx, account.P2PTopic); err == nil {
+				err = client.Subscribe(ctx, groupTopic)
+			}
+		}
+		if err == nil {
+			return client, nil
+		}
+		if client != nil {
+			client.Close()
+		}
+		lastErr = err
+		if ctx.Err() != nil {
+			return nil, ctx.Err()
+		}
+		delay := time.Duration(attempt*2) * time.Second
+		log.Printf("account=%s connect attempt=%d failed; retry_in=%s: %v", account.Username, attempt, delay, err)
+		if err = waitContext(ctx, delay); err != nil {
+			return nil, err
+		}
+	}
+	return nil, fmt.Errorf("connect %s failed after retries: %w", account.Username, lastErr)
+}
+
+func (r *agentRun) execute(ctx context.Context, start time.Time, groupTopic string) error {
+	if groupTopic == "" {
+		return fmt.Errorf("group topic is empty")
+	}
+	switch r.scenario {
+	case "p2p":
+		return r.scheduleP2P(ctx, start, r.cfg.UsersPerMachine, r.cfg.P2PMessages, mustDuration(r.cfg.P2PInterval))
+	case "group":
+		return r.scheduleGroup(ctx, start, groupTopic, 1, 500, 2)
+	case "mixed":
+		var wg sync.WaitGroup
+		errCh := make(chan error, 2)
+		wg.Add(2)
+		go func() { defer wg.Done(); errCh <- r.scheduleP2P(ctx, start, 300, 5, 20*time.Second) }()
+		go func() { defer wg.Done(); errCh <- r.scheduleGroup(ctx, start, groupTopic, 301, 550, 2) }()
+		wg.Wait()
+		close(errCh)
+		for err := range errCh {
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	default:
+		return fmt.Errorf("unknown scenario %s", r.scenario)
+	}
+}
+
+func (r *agentRun) scheduleP2P(ctx context.Context, start time.Time, users, messages int, interval time.Duration) error {
+	var wg sync.WaitGroup
+	for index := 1; index <= users; index++ {
+		client := r.clients[index]
+		if client == nil {
+			return fmt.Errorf("missing client %d", index)
+		}
+		wg.Add(1)
+		go func(index int, client *WSClient) {
+			defer wg.Done()
+			jitter := time.Duration((index*37)%200) * time.Millisecond
+			for messageIndex := 1; messageIndex <= messages; messageIndex++ {
+				target := start.Add(time.Duration(messageIndex-1)*interval + jitter)
+				if waitUntil(ctx, target) != nil {
+					return
+				}
+				mid := fmt.Sprintf("%s-p-%s-%04d-%02d", r.runID, r.machine, index, messageIndex)
+				accepted, err := r.publish(ctx, client, client.account.P2PTopic, mid, messageIndex)
+				if err == nil {
+					accepted.PeerIndex = index
+					if r.machine == "a" {
+						accepted.PeerMachine = "b"
+					} else {
+						accepted.PeerMachine = "a"
+					}
+					r.addAccepted(accepted)
+				}
+			}
+		}(index, client)
+	}
+	wg.Wait()
+	return ctx.Err()
+}
+
+func (r *agentRun) scheduleGroup(ctx context.Context, start time.Time, topic string, first, last, rounds int) error {
+	if first < 1 || last > len(r.clients) || first > last {
+		return fmt.Errorf("invalid group publisher range %d-%d", first, last)
+	}
+	count := last - first + 1
+	roundDuration := time.Duration(count) * 200 * time.Millisecond
+	offset := time.Duration(0)
+	if r.machine == "b" {
+		offset = 100 * time.Millisecond
+	}
+	var wg sync.WaitGroup
+	for index := first; index <= last; index++ {
+		client := r.clients[index]
+		position := index - first
+		wg.Add(1)
+		go func(index, position int, client *WSClient) {
+			defer wg.Done()
+			for round := 1; round <= rounds; round++ {
+				target := start.Add(time.Duration(round-1)*roundDuration + time.Duration(position)*200*time.Millisecond + offset)
+				if waitUntil(ctx, target) != nil {
+					return
+				}
+				mid := fmt.Sprintf("%s-g-%s-%04d-%02d", r.runID, r.machine, index, round)
+				accepted, err := r.publish(ctx, client, topic, mid, round)
+				if err == nil {
+					r.addAccepted(accepted)
+				}
+			}
+		}(index, position, client)
+	}
+	wg.Wait()
+	return ctx.Err()
+}
+
+func (r *agentRun) publish(ctx context.Context, client *WSClient, topic, mid string, messageIndex int) (AcceptedMessage, error) {
+	r.attempted.Add(1)
+	accepted, err := client.Publish(ctx, topic, mid, r.runID, r.scenario, messageIndex)
+	if err != nil {
+		r.rejected.Add(1)
+		r.classifyPublishError(err)
+		r.mu.Lock()
+		if len(r.errors) < 1000 {
+			r.errors = append(r.errors, fmt.Sprintf("publish %s: %v", mid, err))
+		}
+		r.mu.Unlock()
+		return accepted, err
+	}
+	r.ack.Observe(accepted.AckAt.Sub(accepted.SentAt))
+	return accepted, nil
+}
+
+func (r *agentRun) classifyPublishError(err error) {
+	var ctrlErr *ctrlError
+	switch {
+	case errors.As(err, &ctrlErr) && ctrlErr.Code == 403:
+		r.publish403.Add(1)
+	case errors.As(err, &ctrlErr) && ctrlErr.Code == 502:
+		r.publish502.Add(1)
+	case errors.As(err, &ctrlErr) && ctrlErr.Code == 503:
+		r.publish503.Add(1)
+	case strings.Contains(err.Error(), "ctrl timeout") || errors.Is(err, context.DeadlineExceeded):
+		r.publishTimeout.Add(1)
+	case strings.Contains(err.Error(), "websocket") || strings.Contains(err.Error(), "broken pipe") ||
+		strings.Contains(err.Error(), "connection reset"):
+		r.publishSocket.Add(1)
+	default:
+		r.publishOther.Add(1)
+	}
+}
+
+func (r *agentRun) publishErrorCounts() map[string]int64 {
+	return map[string]int64{
+		"403": r.publish403.Load(), "502": r.publish502.Load(), "503": r.publish503.Load(),
+		"ctrl_timeout": r.publishTimeout.Load(), "socket": r.publishSocket.Load(), "other": r.publishOther.Load(),
+	}
+}
+
+func (r *agentRun) addAccepted(message AcceptedMessage) {
+	r.mu.Lock()
+	r.accepted = append(r.accepted, message)
+	r.mu.Unlock()
+}
+
+func (r *agentRun) acceptedCopy() []AcceptedMessage {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	out := append([]AcceptedMessage(nil), r.accepted...)
+	sort.Slice(out, func(i, j int) bool { return out[i].ClientMid < out[j].ClientMid })
+	return out
+}
+
+func (r *agentRun) drainDuration() (time.Duration, error) {
+	if r.scenario == "p2p" {
+		return configDuration(r.cfg.DrainP2P)
+	}
+	return configDuration(r.cfg.DrainGroup)
+}
+
+func (r *agentRun) close() {
+	for _, client := range r.clients {
+		client.Close()
+	}
+}
+
+func waitUntil(ctx context.Context, target time.Time) error {
+	delay := time.Until(target)
+	if delay <= 0 {
+		return nil
+	}
+	return waitContext(ctx, delay)
+}
+
+func mustDuration(value string) time.Duration {
+	d, err := time.ParseDuration(value)
+	if err != nil || d <= 0 {
+		return time.Second
+	}
+	return d
+}
+
+func writeAgentSummary(path string, summary AgentSummary) error {
+	raw, err := json.MarshalIndent(summary, "", "  ")
+	if err != nil {
+		return err
+	}
+	if dir := filepath.Dir(path); dir != "." {
+		if err = os.MkdirAll(dir, 0o750); err != nil {
+			return err
+		}
+	}
+	return os.WriteFile(path, append(raw, '\n'), 0o600)
+}

loadtest/im-pressure/agent_history_test.go

@@ -0,0 +1,41 @@
+package main
+
+import "testing"
+
+func TestHistoryQueriesForMessagesGroupsRangesByTopic(t *testing.T) {
+	p2pClient := &WSClient{}
+	groupClient := &WSClient{}
+	run := &agentRun{clients: map[int]*WSClient{1: groupClient, 7: p2pClient}}
+	messages := make([]AcceptedMessage, 0, 207)
+	for seq := int64(205); seq >= 1; seq-- {
+		messages = append(messages, AcceptedMessage{Kind: "p2p", Topic: "p2p-a", PeerIndex: 7, Seq: seq})
+	}
+	messages = append(messages,
+		AcceptedMessage{Kind: "group", Topic: "grp-a", PeerIndex: 99, Seq: 11},
+		AcceptedMessage{Kind: "group", Topic: "grp-a", PeerIndex: 99, Seq: 12},
+	)
+
+	queries := run.historyQueriesForMessages(messages)
+	if len(queries) != 3 {
+		t.Fatalf("queries=%d want=3", len(queries))
+	}
+	p2pQueries := 0
+	groupQueries := 0
+	for _, query := range queries {
+		switch query.topic {
+		case "p2p-a":
+			p2pQueries++
+			if query.client != p2pClient || query.limit != 256 {
+				t.Fatalf("unexpected p2p query: %#v", query)
+			}
+		case "grp-a":
+			groupQueries++
+			if query.client != groupClient || query.since != 11 || query.before != 13 {
+				t.Fatalf("unexpected group query: %#v", query)
+			}
+		}
+	}
+	if p2pQueries != 2 || groupQueries != 1 {
+		t.Fatalf("p2p queries=%d group queries=%d", p2pQueries, groupQueries)
+	}
+}

loadtest/im-pressure/agent_metrics.go

@@ -0,0 +1,73 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"net/http"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/collectors"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+)
+
+func startAgentMetrics(address string, run *agentRun) (func(), error) {
+	if address == "" {
+		return func() {}, nil
+	}
+	listener, err := net.Listen("tcp", address)
+	if err != nil {
+		return nil, fmt.Errorf("listen for agent metrics on %s: %w", address, err)
+	}
+	labels := prometheus.Labels{"machine": run.machine}
+	registry := prometheus.NewRegistry()
+	registry.MustRegister(
+		collectors.NewGoCollector(),
+		collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),
+		prometheus.NewGaugeFunc(prometheus.GaugeOpts{Name: "im_load_agent_ready_connections", Help: "Current ready WebSocket connections.", ConstLabels: labels}, func() float64 {
+			return float64(run.ready.Load())
+		}),
+		prometheus.NewCounterFunc(prometheus.CounterOpts{Name: "im_load_agent_publish_attempted_total", Help: "Publish commands attempted.", ConstLabels: labels}, func() float64 {
+			return float64(run.attempted.Load())
+		}),
+		prometheus.NewCounterFunc(prometheus.CounterOpts{Name: "im_load_agent_publish_accepted_total", Help: "Publish commands acknowledged by the server.", ConstLabels: labels}, func() float64 {
+			run.mu.Lock()
+			defer run.mu.Unlock()
+			return float64(len(run.accepted))
+		}),
+		prometheus.NewCounterFunc(prometheus.CounterOpts{Name: "im_load_agent_publish_rejected_total", Help: "Publish commands rejected or timed out.", ConstLabels: labels}, func() float64 {
+			return float64(run.rejected.Load())
+		}),
+		prometheus.NewCounterFunc(prometheus.CounterOpts{Name: "im_load_agent_publish_403_total", Help: "Publish commands rejected with 403.", ConstLabels: labels}, func() float64 { return float64(run.publish403.Load()) }),
+		prometheus.NewCounterFunc(prometheus.CounterOpts{Name: "im_load_agent_publish_502_total", Help: "Publish commands rejected with 502.", ConstLabels: labels}, func() float64 { return float64(run.publish502.Load()) }),
+		prometheus.NewCounterFunc(prometheus.CounterOpts{Name: "im_load_agent_publish_503_total", Help: "Publish commands rejected with 503.", ConstLabels: labels}, func() float64 { return float64(run.publish503.Load()) }),
+		prometheus.NewCounterFunc(prometheus.CounterOpts{Name: "im_load_agent_publish_ctrl_timeout_total", Help: "Publish commands with control-response timeout.", ConstLabels: labels}, func() float64 { return float64(run.publishTimeout.Load()) }),
+		prometheus.NewCounterFunc(prometheus.CounterOpts{Name: "im_load_agent_publish_socket_error_total", Help: "Publish commands with socket errors.", ConstLabels: labels}, func() float64 { return float64(run.publishSocket.Load()) }),
+		prometheus.NewCounterFunc(prometheus.CounterOpts{Name: "im_load_agent_received_total", Help: "Unique matching data deliveries received.", ConstLabels: labels}, func() float64 {
+			received, _, _, _ := run.recorder.metricCounts()
+			return float64(received)
+		}),
+		prometheus.NewCounterFunc(prometheus.CounterOpts{Name: "im_load_agent_duplicates_total", Help: "Duplicate matching deliveries received.", ConstLabels: labels}, func() float64 {
+			_, duplicates, _, _ := run.recorder.metricCounts()
+			return float64(duplicates)
+		}),
+		prometheus.NewCounterFunc(prometheus.CounterOpts{Name: "im_load_agent_out_of_order_total", Help: "Per-connection topic sequence regressions.", ConstLabels: labels}, func() float64 {
+			_, _, outOfOrder, _ := run.recorder.metricCounts()
+			return float64(outOfOrder)
+		}),
+		prometheus.NewCounterFunc(prometheus.CounterOpts{Name: "im_load_agent_disconnects_total", Help: "Unexpected WebSocket disconnects.", ConstLabels: labels}, func() float64 {
+			_, _, _, disconnects := run.recorder.metricCounts()
+			return float64(disconnects)
+		}),
+	)
+	mux := http.NewServeMux()
+	mux.Handle("/metrics", promhttp.HandlerFor(registry, promhttp.HandlerOpts{}))
+	server := &http.Server{Handler: mux, ReadHeaderTimeout: 5 * time.Second}
+	go func() { _ = server.Serve(listener) }()
+	return func() {
+		ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+		defer cancel()
+		_ = server.Shutdown(ctx)
+	}, nil
+}

loadtest/im-pressure/collector.go

@@ -0,0 +1,147 @@
+package main
+
+import (
+	"bufio"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+)
+
+type namedURLs []string
+
+func (n *namedURLs) String() string { return strings.Join(*n, ",") }
+func (n *namedURLs) Set(value string) error {
+	if !strings.Contains(value, "=") {
+		return fmt.Errorf("URL must use name=https://host/path")
+	}
+	*n = append(*n, value)
+	return nil
+}
+
+type metricSnapshot struct {
+	Timestamp time.Time `json:"timestamp"`
+	Phase     string    `json:"phase"`
+	Name      string    `json:"name"`
+	URL       string    `json:"url"`
+	Status    int       `json:"status"`
+	Duration  int64     `json:"duration_ms"`
+	Body      string    `json:"body,omitempty"`
+	Error     string    `json:"error,omitempty"`
+}
+
+func runCollector(ctx context.Context, sources namedURLs, output string, interval, duration time.Duration,
+	coordinatorURL, coordinatorOutput string) error {
+	if len(sources) == 0 {
+		return fmt.Errorf("at least one --url name=url is required")
+	}
+	if interval <= 0 || duration <= 0 {
+		return fmt.Errorf("interval and duration must be positive")
+	}
+	if dir := filepath.Dir(output); dir != "." {
+		if err := os.MkdirAll(dir, 0o750); err != nil {
+			return err
+		}
+	}
+	file, err := os.OpenFile(output, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+	writer := bufio.NewWriterSize(file, 256<<10)
+	defer writer.Flush()
+	client := &http.Client{Timeout: 10 * time.Second}
+	deadline := time.Now().Add(duration)
+	phase := "before"
+	for {
+		phase = detectCollectorPhase(ctx, client, phase, coordinatorURL, coordinatorOutput)
+		snapshots := collectOnce(ctx, client, sources, phase)
+		for _, snapshot := range snapshots {
+			raw, _ := json.Marshal(snapshot)
+			if _, err = writer.Write(append(raw, '\n')); err != nil {
+				return err
+			}
+		}
+		if err = writer.Flush(); err != nil {
+			return err
+		}
+		if time.Now().Add(interval).After(deadline) {
+			return nil
+		}
+		if err = waitContext(ctx, interval); err != nil {
+			return err
+		}
+	}
+}
+
+func detectCollectorPhase(ctx context.Context, client *http.Client, current, coordinatorURL, coordinatorOutput string) string {
+	if coordinatorOutput != "" {
+		if _, err := os.Stat(coordinatorOutput); err == nil {
+			return "after"
+		}
+	}
+	if coordinatorURL == "" {
+		return current
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet,
+		strings.TrimRight(coordinatorURL, "/")+"/v1/status", nil)
+	if err != nil {
+		return current
+	}
+	response, err := client.Do(req)
+	if err != nil {
+		return current
+	}
+	defer response.Body.Close()
+	var status struct {
+		Start StartInfo `json:"start"`
+	}
+	if response.StatusCode != http.StatusOK || json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&status) != nil ||
+		status.Start.StartAt.IsZero() || time.Now().Before(status.Start.StartAt) {
+		return current
+	}
+	return "during"
+}
+
+func collectOnce(ctx context.Context, client *http.Client, sources namedURLs, phase string) []metricSnapshot {
+	out := make([]metricSnapshot, len(sources))
+	var wg sync.WaitGroup
+	for i, source := range sources {
+		i, source := i, source
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			parts := strings.SplitN(source, "=", 2)
+			snapshot := metricSnapshot{Timestamp: time.Now().UTC(), Phase: phase, Name: parts[0], URL: parts[1]}
+			started := time.Now()
+			req, err := http.NewRequestWithContext(ctx, http.MethodGet, parts[1], nil)
+			if err == nil {
+				var response *http.Response
+				response, err = client.Do(req)
+				if err == nil {
+					snapshot.Status = response.StatusCode
+					raw, readErr := io.ReadAll(io.LimitReader(response.Body, 16<<20))
+					_ = response.Body.Close()
+					if readErr != nil {
+						err = readErr
+					} else {
+						snapshot.Body = string(raw)
+					}
+				}
+			}
+			snapshot.Duration = time.Since(started).Milliseconds()
+			if err != nil {
+				snapshot.Error = err.Error()
+			}
+			out[i] = snapshot
+		}()
+	}
+	wg.Wait()
+	return out
+}

loadtest/im-pressure/collector_test.go

@@ -0,0 +1,31 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+func TestDetectCollectorPhase(t *testing.T) {
+	start := StartInfo{RunID: "run1", StartAt: time.Now().Add(-time.Second)}
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		_ = json.NewEncoder(w).Encode(map[string]any{"start": start})
+	}))
+	defer server.Close()
+	client := &http.Client{Timeout: time.Second}
+	if got := detectCollectorPhase(context.Background(), client, "before", server.URL, ""); got != "during" {
+		t.Fatalf("phase=%s", got)
+	}
+	output := filepath.Join(t.TempDir(), "coordinator.json")
+	if err := os.WriteFile(output, []byte("{}"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	if got := detectCollectorPhase(context.Background(), client, "during", server.URL, output); got != "after" {
+		t.Fatalf("phase=%s", got)
+	}
+}

loadtest/im-pressure/config.example.json

@@ -0,0 +1,25 @@
+{
+  "tenant": "006",
+  "http_base": "https://im-api.example.com",
+  "api_key_env": "IM_API_KEY",
+  "password_env": "IM_LOAD_PASSWORD",
+  "verification_code_env": "IM_VERIFICATION_CODE",
+  "registration_phone_prefix": "",
+  "users_per_machine": 1000,
+  "endpoints": {
+    "a": ["wss://im-ws-pod-a.example.com/v0/channels"],
+    "b": ["wss://im-ws-pod-b.example.com/v0/channels"]
+  },
+  "insecure_tls": false,
+  "prepare_workers": 20,
+  "connect_ramp": "20s",
+  "command_timeout": "20s",
+  "preflight_workers": 32,
+  "skip_p2p_preflight": false,
+  "history_workers": 48,
+  "history_timeout": "3m",
+  "p2p_messages_per_user": 10,
+  "p2p_interval": "1s",
+  "drain_p2p": "30s",
+  "drain_group": "60s"
+}

loadtest/im-pressure/coordinator.go

@@ -0,0 +1,323 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+)
+
+type StartInfo struct {
+	RunID    string    `json:"run_id"`
+	Scenario string    `json:"scenario"`
+	StartAt  time.Time `json:"start_at"`
+}
+
+type AgentReady struct {
+	Machine          string `json:"machine"`
+	ReadyConnections int    `json:"ready_connections"`
+}
+
+type coordinatorState struct {
+	mu             sync.Mutex
+	start          StartInfo
+	startDelay     time.Duration
+	expectedAgents int
+	ready          map[string]AgentReady
+	ledgers        map[string][]AcceptedMessage
+	summaries      map[string]AgentSummary
+	complete       chan struct{}
+	completeOnce   sync.Once
+}
+
+func runCoordinator(ctx context.Context, listen, runID, scenario, output string, expectedAgents int, startDelay time.Duration) error {
+	if !runIDPattern.MatchString(runID) {
+		return fmt.Errorf("run-id must match %s", runIDPattern.String())
+	}
+	if !validScenario(scenario) {
+		return fmt.Errorf("unsupported scenario %q", scenario)
+	}
+	if expectedAgents <= 0 {
+		expectedAgents = 2
+	}
+	state := &coordinatorState{
+		start: StartInfo{RunID: runID, Scenario: scenario}, startDelay: startDelay, expectedAgents: expectedAgents,
+		ready: make(map[string]AgentReady), ledgers: make(map[string][]AcceptedMessage),
+		summaries: make(map[string]AgentSummary), complete: make(chan struct{}),
+	}
+	mux := http.NewServeMux()
+	mux.HandleFunc("/v1/ready", state.handleReady)
+	mux.HandleFunc("/v1/start", state.handleStart)
+	mux.HandleFunc("/v1/ledger", state.handleLedger)
+	mux.HandleFunc("/v1/summary", state.handleSummary)
+	mux.HandleFunc("/v1/status", state.handleStatus)
+	server := &http.Server{Addr: listen, Handler: mux, ReadHeaderTimeout: 5 * time.Second}
+	errCh := make(chan error, 1)
+	go func() {
+		log.Printf("coordinator listening=%s run_id=%s scenario=%s expected_agents=%d", listen, runID, scenario, expectedAgents)
+		if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
+			errCh <- err
+		}
+	}()
+	select {
+	case <-ctx.Done():
+	case err := <-errCh:
+		return err
+	case <-state.complete:
+		log.Printf("all agent summaries received")
+	}
+	shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	_ = server.Shutdown(shutdownCtx)
+	if output != "" {
+		if err := state.writeOutput(output); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (s *coordinatorState) handleReady(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodPost {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	var ready AgentReady
+	if !decodeLimitedJSON(w, r, &ready) || (ready.Machine != "a" && ready.Machine != "b") || ready.ReadyConnections <= 0 {
+		if ready.Machine != "a" && ready.Machine != "b" {
+			http.Error(w, "invalid machine", http.StatusBadRequest)
+		}
+		return
+	}
+	s.mu.Lock()
+	s.ready[ready.Machine] = ready
+	if len(s.ready) == s.expectedAgents && s.start.StartAt.IsZero() {
+		s.start.StartAt = time.Now().UTC().Add(s.startDelay)
+		log.Printf("all agents ready; start_at=%s", s.start.StartAt.Format(time.RFC3339Nano))
+	}
+	start := s.start
+	s.mu.Unlock()
+	writeJSON(w, http.StatusOK, start)
+}
+
+func (s *coordinatorState) handleStart(w http.ResponseWriter, r *http.Request) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if s.start.StartAt.IsZero() {
+		writeJSON(w, http.StatusAccepted, map[string]any{"ready_agents": len(s.ready), "expected_agents": s.expectedAgents})
+		return
+	}
+	writeJSON(w, http.StatusOK, s.start)
+}
+
+func (s *coordinatorState) handleLedger(w http.ResponseWriter, r *http.Request) {
+	machine := strings.ToLower(strings.TrimSpace(r.URL.Query().Get("machine")))
+	switch r.Method {
+	case http.MethodPost:
+		if machine != "a" && machine != "b" {
+			http.Error(w, "invalid machine", http.StatusBadRequest)
+			return
+		}
+		var messages []AcceptedMessage
+		if !decodeLimitedJSON(w, r, &messages) {
+			return
+		}
+		s.mu.Lock()
+		s.ledgers[machine] = messages
+		count := len(s.ledgers)
+		s.mu.Unlock()
+		writeJSON(w, http.StatusOK, map[string]int{"submitted_agents": count})
+	case http.MethodGet:
+		s.mu.Lock()
+		defer s.mu.Unlock()
+		if len(s.ledgers) < s.expectedAgents {
+			writeJSON(w, http.StatusAccepted, map[string]int{"submitted_agents": len(s.ledgers), "expected_agents": s.expectedAgents})
+			return
+		}
+		var merged []AcceptedMessage
+		for _, messages := range s.ledgers {
+			merged = append(merged, messages...)
+		}
+		sort.Slice(merged, func(i, j int) bool { return merged[i].ClientMid < merged[j].ClientMid })
+		writeJSON(w, http.StatusOK, merged)
+	default:
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+	}
+}
+
+func (s *coordinatorState) handleSummary(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodPost {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	var summary AgentSummary
+	if !decodeLimitedJSON(w, r, &summary) {
+		return
+	}
+	if summary.Machine != "a" && summary.Machine != "b" {
+		http.Error(w, "invalid machine", http.StatusBadRequest)
+		return
+	}
+	s.mu.Lock()
+	s.summaries[summary.Machine] = summary
+	complete := len(s.summaries) == s.expectedAgents
+	s.mu.Unlock()
+	writeJSON(w, http.StatusOK, map[string]any{"complete": complete})
+	if complete {
+		s.completeOnce.Do(func() {
+			go func() {
+				time.Sleep(500 * time.Millisecond)
+				close(s.complete)
+			}()
+		})
+	}
+}
+
+func (s *coordinatorState) handleStatus(w http.ResponseWriter, _ *http.Request) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	writeJSON(w, http.StatusOK, map[string]any{
+		"start": s.start, "ready": s.ready, "ledger_agents": len(s.ledgers), "summary_agents": len(s.summaries),
+	})
+}
+
+func (s *coordinatorState) writeOutput(path string) error {
+	s.mu.Lock()
+	data := struct {
+		Start     StartInfo                    `json:"start"`
+		Ready     map[string]AgentReady        `json:"ready"`
+		Ledgers   map[string][]AcceptedMessage `json:"ledgers"`
+		Summaries map[string]AgentSummary      `json:"summaries"`
+	}{s.start, s.ready, s.ledgers, s.summaries}
+	s.mu.Unlock()
+	raw, err := json.MarshalIndent(data, "", "  ")
+	if err != nil {
+		return err
+	}
+	if err = os.MkdirAll(filepath.Dir(path), 0o750); err != nil && filepath.Dir(path) != "." {
+		return err
+	}
+	return os.WriteFile(path, append(raw, '\n'), 0o600)
+}
+
+func decodeLimitedJSON(w http.ResponseWriter, r *http.Request, output any) bool {
+	dec := json.NewDecoder(io.LimitReader(r.Body, 32<<20))
+	if err := dec.Decode(output); err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return false
+	}
+	return true
+}
+
+func writeJSON(w http.ResponseWriter, status int, value any) {
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(status)
+	_ = json.NewEncoder(w).Encode(value)
+}
+
+type coordinatorClient struct {
+	base   string
+	client *http.Client
+}
+
+func newCoordinatorClient(base string) *coordinatorClient {
+	return &coordinatorClient{base: strings.TrimRight(base, "/"), client: &http.Client{Timeout: 35 * time.Second}}
+}
+
+func (c *coordinatorClient) ready(ctx context.Context, ready AgentReady) (StartInfo, error) {
+	var start StartInfo
+	if _, err := c.request(ctx, http.MethodPost, "/v1/ready", ready, &start); err != nil {
+		return start, err
+	}
+	for start.StartAt.IsZero() {
+		if err := waitContext(ctx, time.Second); err != nil {
+			return start, err
+		}
+		status, err := c.request(ctx, http.MethodGet, "/v1/start", nil, &start)
+		if err != nil {
+			return start, err
+		}
+		if status != http.StatusOK {
+			start = StartInfo{}
+		}
+	}
+	return start, nil
+}
+
+func (c *coordinatorClient) submitLedger(ctx context.Context, machine string, messages []AcceptedMessage) error {
+	_, err := c.request(ctx, http.MethodPost, "/v1/ledger?machine="+machine, messages, nil)
+	return err
+}
+
+func (c *coordinatorClient) ledger(ctx context.Context) ([]AcceptedMessage, error) {
+	for {
+		var messages []AcceptedMessage
+		status, err := c.request(ctx, http.MethodGet, "/v1/ledger", nil, &messages)
+		if err != nil {
+			return nil, err
+		}
+		if status == http.StatusOK {
+			return messages, nil
+		}
+		if err = waitContext(ctx, time.Second); err != nil {
+			return nil, err
+		}
+	}
+}
+
+func (c *coordinatorClient) submitSummary(ctx context.Context, summary AgentSummary) error {
+	_, err := c.request(ctx, http.MethodPost, "/v1/summary", summary, nil)
+	return err
+}
+
+func (c *coordinatorClient) request(ctx context.Context, method, path string, input, output any) (int, error) {
+	var body io.Reader
+	if input != nil {
+		raw, err := json.Marshal(input)
+		if err != nil {
+			return 0, err
+		}
+		body = strings.NewReader(string(raw))
+	}
+	req, err := http.NewRequestWithContext(ctx, method, c.base+path, body)
+	if err != nil {
+		return 0, err
+	}
+	if input != nil {
+		req.Header.Set("Content-Type", "application/json")
+	}
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return 0, err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode >= 400 {
+		raw, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
+		return resp.StatusCode, fmt.Errorf("coordinator HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(raw)))
+	}
+	if output != nil && resp.StatusCode == http.StatusOK {
+		if err = json.NewDecoder(resp.Body).Decode(output); err != nil {
+			return resp.StatusCode, err
+		}
+	}
+	return resp.StatusCode, nil
+}
+
+func waitContext(ctx context.Context, delay time.Duration) error {
+	timer := time.NewTimer(delay)
+	defer timer.Stop()
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case <-timer.C:
+		return nil
+	}
+}

loadtest/im-pressure/coordinator_test.go

@@ -0,0 +1,60 @@
+package main
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+)
+
+func TestCoordinatorClientWaitsForBothAgentsAndMergesLedger(t *testing.T) {
+	state := &coordinatorState{
+		start: StartInfo{RunID: "run1", Scenario: "p2p"}, startDelay: time.Second, expectedAgents: 2,
+		ready: make(map[string]AgentReady), ledgers: make(map[string][]AcceptedMessage),
+		summaries: make(map[string]AgentSummary), complete: make(chan struct{}),
+	}
+	server := httptest.NewServer(testCoordinatorMux(state))
+	defer server.Close()
+	client := newCoordinatorClient(server.URL)
+
+	aDone := make(chan StartInfo, 1)
+	go func() {
+		start, _ := client.ready(context.Background(), AgentReady{Machine: "a", ReadyConnections: 1000})
+		aDone <- start
+	}()
+	time.Sleep(50 * time.Millisecond)
+	startB, err := client.ready(context.Background(), AgentReady{Machine: "b", ReadyConnections: 1000})
+	if err != nil || startB.StartAt.IsZero() {
+		t.Fatalf("machine B ready: start=%+v err=%v", startB, err)
+	}
+	select {
+	case startA := <-aDone:
+		if !startA.StartAt.Equal(startB.StartAt) {
+			t.Fatalf("different start times A=%s B=%s", startA.StartAt, startB.StartAt)
+		}
+	case <-time.After(2 * time.Second):
+		t.Fatal("machine A did not receive start barrier")
+	}
+
+	if err = client.submitLedger(context.Background(), "a", []AcceptedMessage{{ClientMid: "b"}}); err != nil {
+		t.Fatal(err)
+	}
+	if err = client.submitLedger(context.Background(), "b", []AcceptedMessage{{ClientMid: "a"}}); err != nil {
+		t.Fatal(err)
+	}
+	ledger, err := client.ledger(context.Background())
+	if err != nil || len(ledger) != 2 || ledger[0].ClientMid != "a" || ledger[1].ClientMid != "b" {
+		t.Fatalf("ledger=%+v err=%v", ledger, err)
+	}
+}
+
+func testCoordinatorMux(state *coordinatorState) *http.ServeMux {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/v1/ready", state.handleReady)
+	mux.HandleFunc("/v1/start", state.handleStart)
+	mux.HandleFunc("/v1/ledger", state.handleLedger)
+	mux.HandleFunc("/v1/summary", state.handleSummary)
+	mux.HandleFunc("/v1/status", state.handleStatus)
+	return mux
+}

loadtest/im-pressure/dbverify.go

@@ -0,0 +1,166 @@
+package main
+
+import (
+	"context"
+	"database/sql"
+	"encoding/json"
+	"fmt"
+	"os"
+	"regexp"
+	"sort"
+	"strings"
+
+	_ "github.com/go-sql-driver/mysql"
+)
+
+type dbVerifyResult struct {
+	RunID         string   `json:"run_id"`
+	TenantCode    string   `json:"tenant_code"`
+	TenantID      int64    `json:"tenant_id"`
+	Expected      int      `json:"expected"`
+	FoundExact    int      `json:"found_exact"`
+	Missing       int      `json:"missing"`
+	Mismatch      int      `json:"mismatch"`
+	DuplicateRows int      `json:"duplicate_rows"`
+	Unexpected    int      `json:"unexpected_rows"`
+	Tables        []string `json:"tables"`
+	MissingMIDs   []string `json:"missing_client_mids,omitempty"`
+}
+
+type persistedIdentity struct {
+	Topic string
+	Seq   int64
+}
+
+var dbIdentifierRE = regexp.MustCompile(`^[A-Za-z0-9_]+$`)
+
+func runDBVerify(ctx context.Context, coordinatorPath, output, tenantCode, dsnEnv, registryDB string) error {
+	raw, err := os.ReadFile(coordinatorPath)
+	if err != nil {
+		return err
+	}
+	var coordinator struct {
+		Start   StartInfo                    `json:"start"`
+		Ledgers map[string][]AcceptedMessage `json:"ledgers"`
+	}
+	if err = json.Unmarshal(raw, &coordinator); err != nil {
+		return fmt.Errorf("decode coordinator: %w", err)
+	}
+	if coordinator.Start.RunID == "" {
+		return fmt.Errorf("coordinator has no run id")
+	}
+	dsn := strings.TrimSpace(os.Getenv(dsnEnv))
+	if dsn == "" {
+		return fmt.Errorf("environment variable %s is required", dsnEnv)
+	}
+	if registryDB == "" {
+		registryDB = strings.TrimSpace(os.Getenv("TENANT_REGISTRY_DB"))
+	}
+	if registryDB == "" {
+		registryDB = "meeting_im"
+	}
+	if !dbIdentifierRE.MatchString(registryDB) {
+		return fmt.Errorf("invalid registry database identifier")
+	}
+	db, err := sql.Open("mysql", dsn)
+	if err != nil {
+		return err
+	}
+	defer db.Close()
+	if err = db.PingContext(ctx); err != nil {
+		return err
+	}
+	var tenantID int64
+	query := fmt.Sprintf("SELECT id FROM `%s`.im_tenant WHERE code=?", registryDB)
+	if err = db.QueryRowContext(ctx, query, tenantCode).Scan(&tenantID); err != nil {
+		return err
+	}
+
+	expected := make(map[string]AcceptedMessage)
+	months := make(map[string]struct{})
+	for _, ledger := range coordinator.Ledgers {
+		for _, message := range ledger {
+			if message.RunID != coordinator.Start.RunID {
+				continue
+			}
+			expected[message.ClientMid] = message
+			if message.Kind == "group" {
+				months[message.SentAt.UTC().Format("200601")] = struct{}{}
+			}
+		}
+	}
+	tables := []string{fmt.Sprintf("im_%03d_messages", tenantID)}
+	for month := range months {
+		tables = append(tables, fmt.Sprintf("im_%03d_group_messages_%s", tenantID, month))
+	}
+	sort.Strings(tables)
+	actual := make(map[string][]persistedIdentity)
+	for _, table := range tables {
+		if !dbIdentifierRE.MatchString(table) {
+			return fmt.Errorf("invalid generated table name %q", table)
+		}
+		rows, queryErr := db.QueryContext(ctx, fmt.Sprintf(
+			"SELECT client_msg_id,topic,seqid FROM `%s` WHERE tenant_id=? AND client_msg_id LIKE ?", table),
+			tenantID, coordinator.Start.RunID+"-%")
+		if queryErr != nil {
+			return fmt.Errorf("query %s: %w", table, queryErr)
+		}
+		for rows.Next() {
+			var mid, topic string
+			var seq int64
+			if err = rows.Scan(&mid, &topic, &seq); err != nil {
+				_ = rows.Close()
+				return err
+			}
+			actual[mid] = append(actual[mid], persistedIdentity{Topic: topic, Seq: seq})
+		}
+		if err = rows.Close(); err != nil {
+			return err
+		}
+	}
+	result := dbVerifyResult{RunID: coordinator.Start.RunID, TenantCode: tenantCode, TenantID: tenantID,
+		Expected: len(expected), Tables: tables}
+	for mid, want := range expected {
+		rows := actual[mid]
+		if len(rows) == 0 {
+			result.Missing++
+			result.MissingMIDs = append(result.MissingMIDs, mid)
+			continue
+		}
+		matched := false
+		for _, got := range rows {
+			if got.Topic == want.Topic && got.Seq == want.Seq {
+				matched = true
+			}
+		}
+		if matched {
+			result.FoundExact++
+		} else {
+			result.Mismatch++
+		}
+		if len(rows) > 1 {
+			result.DuplicateRows += len(rows) - 1
+		}
+	}
+	for mid := range actual {
+		if _, ok := expected[mid]; !ok {
+			result.Unexpected++
+		}
+	}
+	sort.Strings(result.MissingMIDs)
+	resultRaw, _ := json.MarshalIndent(result, "", "  ")
+	if output == "" || output == "-" {
+		if _, err = fmt.Fprintln(os.Stdout, string(resultRaw)); err != nil {
+			return err
+		}
+	} else {
+		if err = writeJSONFile(output, result); err != nil {
+			return err
+		}
+	}
+	if result.Missing != 0 || result.Mismatch != 0 || result.DuplicateRows != 0 || result.Unexpected != 0 {
+		return fmt.Errorf("database verification failed: missing=%d mismatch=%d duplicates=%d unexpected=%d",
+			result.Missing, result.Mismatch, result.DuplicateRows, result.Unexpected)
+	}
+	return nil
+}

loadtest/im-pressure/histogram.go

@@ -0,0 +1,64 @@
+package main
+
+import (
+	"sort"
+	"sync"
+	"time"
+)
+
+var latencyBounds = []time.Duration{
+	time.Millisecond, 3 * time.Millisecond, 5 * time.Millisecond, 10 * time.Millisecond,
+	25 * time.Millisecond, 50 * time.Millisecond, 100 * time.Millisecond, 200 * time.Millisecond,
+	500 * time.Millisecond, time.Second, 2 * time.Second, 5 * time.Second, 10 * time.Second,
+}
+
+type DurationHistogram struct {
+	mu     sync.Mutex
+	count  int64
+	sum    time.Duration
+	max    time.Duration
+	bucket []int64
+}
+
+type HistogramSnapshot struct {
+	Count   int64            `json:"count"`
+	SumMS   float64          `json:"sum_ms"`
+	MaxMS   float64          `json:"max_ms"`
+	Buckets map[string]int64 `json:"buckets"`
+}
+
+func NewDurationHistogram() *DurationHistogram {
+	return &DurationHistogram{bucket: make([]int64, len(latencyBounds)+1)}
+}
+
+func (h *DurationHistogram) Observe(value time.Duration) {
+	if value < 0 {
+		return
+	}
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	h.count++
+	h.sum += value
+	if value > h.max {
+		h.max = value
+	}
+	idx := sort.Search(len(latencyBounds), func(i int) bool { return value <= latencyBounds[i] })
+	h.bucket[idx]++
+}
+
+func (h *DurationHistogram) Snapshot() HistogramSnapshot {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	buckets := make(map[string]int64, len(h.bucket))
+	for i, count := range h.bucket {
+		label := "+Inf"
+		if i < len(latencyBounds) {
+			label = latencyBounds[i].String()
+		}
+		buckets[label] = count
+	}
+	return HistogramSnapshot{
+		Count: h.count, SumMS: float64(h.sum) / float64(time.Millisecond),
+		MaxMS: float64(h.max) / float64(time.Millisecond), Buckets: buckets,
+	}
+}

loadtest/im-pressure/httpapi.go

@@ -0,0 +1,300 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"crypto/tls"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"sync/atomic"
+	"time"
+)
+
+type apiClient struct {
+	baseURL string
+	apiKey  string
+	tenant  string
+	client  *http.Client
+	seq     atomic.Uint64
+}
+
+type apiEnvelope struct {
+	Code      int             `json:"code"`
+	Data      json.RawMessage `json:"data"`
+	Reason    string          `json:"reason"`
+	Message   string          `json:"message"`
+	RequestID string          `json:"request_id"`
+}
+
+type apiError struct {
+	Status  int
+	Code    int
+	Reason  string
+	Message string
+}
+
+func (e *apiError) Error() string {
+	return fmt.Sprintf("HTTP %d code=%d reason=%s message=%s", e.Status, e.Code, e.Reason, e.Message)
+}
+
+func newAPIClient(cfg *Config, apiKey string) *apiClient {
+	transport := http.DefaultTransport.(*http.Transport).Clone()
+	if cfg.InsecureTLS {
+		transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} // #nosec G402: explicit load-test option.
+	}
+	return &apiClient{
+		baseURL: strings.TrimRight(cfg.HTTPBase, "/"), apiKey: apiKey, tenant: cfg.Tenant,
+		client: &http.Client{Timeout: 2 * time.Minute, Transport: transport},
+	}
+}
+
+func (a *apiClient) do(ctx context.Context, method, path, token, idempotencyKey string, body any, output any) error {
+	var reader io.Reader
+	if body != nil {
+		raw, err := json.Marshal(body)
+		if err != nil {
+			return err
+		}
+		reader = bytes.NewReader(raw)
+	}
+	req, err := http.NewRequestWithContext(ctx, method, a.baseURL+path, reader)
+	if err != nil {
+		return err
+	}
+	req.Header.Set("X-MeetingIM-APIKey", a.apiKey)
+	req.Header.Set("X-Tenant-Code", a.tenant)
+	req.Header.Set("X-IM-Schema", "1")
+	req.Header.Set("X-Request-ID", fmt.Sprintf("loadtest-%d-%d", time.Now().UnixNano(), a.seq.Add(1)))
+	if token != "" {
+		req.Header.Set("Authorization", "Bearer "+token)
+	}
+	if idempotencyKey != "" {
+		req.Header.Set("Idempotency-Key", idempotencyKey)
+	}
+	if body != nil {
+		req.Header.Set("Content-Type", "application/json")
+	}
+	resp, err := a.client.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	raw, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
+	if err != nil {
+		return err
+	}
+	var envelope apiEnvelope
+	if err = json.Unmarshal(raw, &envelope); err != nil {
+		return fmt.Errorf("decode HTTP %d response: %w", resp.StatusCode, err)
+	}
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 || envelope.Code != 0 {
+		return &apiError{Status: resp.StatusCode, Code: envelope.Code, Reason: envelope.Reason, Message: envelope.Message}
+	}
+	if output != nil && len(envelope.Data) > 0 {
+		if err = json.Unmarshal(envelope.Data, output); err != nil {
+			return fmt.Errorf("decode response data: %w", err)
+		}
+	}
+	return nil
+}
+
+type privacyResponse struct {
+	FriendRequestVerification bool `json:"friend_request_verification"`
+	GroupInviteVerification   bool `json:"group_invite_verification"`
+	Version                   int  `json:"version"`
+}
+
+type registrationCredential struct {
+	ChallengeID       string
+	VerificationToken string
+}
+
+func (a *apiClient) createRegistrationCredential(ctx context.Context, account AccountRecord, code string) (*registrationCredential, error) {
+	if account.Phone == "" {
+		return nil, nil
+	}
+	var challenge struct {
+		ChallengeID string `json:"challenge_id"`
+	}
+	key := fmt.Sprintf("loadtest-verification-%s-%d", account.Username, time.Now().UnixNano())
+	body := map[string]any{"scene": "register", "registration_type": 0, "tenant_code": a.tenant, "phone": account.Phone}
+	if err := a.do(ctx, http.MethodPost, "/v1/im/auth/verification-codes", "", key, body, &challenge); err != nil {
+		return nil, fmt.Errorf("create verification for %s: %w", account.Username, err)
+	}
+	if challenge.ChallengeID == "" {
+		return nil, fmt.Errorf("create verification for %s: missing challenge_id", account.Username)
+	}
+	var verified struct {
+		VerificationToken string `json:"verification_token"`
+	}
+	key = fmt.Sprintf("loadtest-verify-%s-%d", account.Username, time.Now().UnixNano())
+	path := "/v1/im/auth/verification-codes/" + url.PathEscape(challenge.ChallengeID) + "/verify"
+	if err := a.do(ctx, http.MethodPost, path, "", key, map[string]any{"code": code}, &verified); err != nil {
+		return nil, fmt.Errorf("verify code for %s: %w", account.Username, err)
+	}
+	if verified.VerificationToken == "" {
+		return nil, fmt.Errorf("verify code for %s: missing verification_token", account.Username)
+	}
+	return &registrationCredential{ChallengeID: challenge.ChallengeID, VerificationToken: verified.VerificationToken}, nil
+}
+
+func (a *apiClient) disableGroupInviteVerification(ctx context.Context, account AccountRecord) error {
+	var privacy privacyResponse
+	if err := a.do(ctx, http.MethodGet, "/v1/im/me/privacy", account.Token, "", nil, &privacy); err != nil {
+		return err
+	}
+	if !privacy.GroupInviteVerification {
+		return nil
+	}
+	body := map[string]any{"group_invite_verification": false, "version": privacy.Version}
+	return a.do(ctx, http.MethodPatch, "/v1/im/me/privacy", account.Token,
+		"loadtest-privacy-"+account.Username, body, &privacy)
+}
+
+type friendRequestResponse struct {
+	RequestID string `json:"request_id"`
+	Status    string `json:"status"`
+	Topic     string `json:"topic"`
+}
+
+type friendRequestListResponse struct {
+	Items []struct {
+		RequestID   string `json:"request_id"`
+		ApplicantID string `json:"applicant_id"`
+		Status      string `json:"status"`
+	} `json:"items"`
+	NextCursor string `json:"next_cursor"`
+}
+
+func (a *apiClient) findPendingFriendRequest(ctx context.Context, target AccountRecord, applicantID string) (string, error) {
+	cursor := ""
+	for {
+		path := "/v1/im/friend-requests?direction=inbound&status=pending&limit=100"
+		if cursor != "" {
+			path += "&cursor=" + url.QueryEscape(cursor)
+		}
+		var page friendRequestListResponse
+		if err := a.do(ctx, http.MethodGet, path, target.Token, "", nil, &page); err != nil {
+			return "", err
+		}
+		for _, item := range page.Items {
+			if item.ApplicantID == applicantID && item.Status == "pending" {
+				return item.RequestID, nil
+			}
+		}
+		if page.NextCursor == "" || page.NextCursor == cursor {
+			return "", nil
+		}
+		cursor = page.NextCursor
+	}
+}
+
+func (a *apiClient) createFriendRequest(ctx context.Context, source, target AccountRecord) (*friendRequestResponse, error) {
+	var result friendRequestResponse
+	body := map[string]any{"target_user_id": target.UserID, "message": "loadtest", "source": "search", "source_topic": ""}
+	err := a.do(ctx, http.MethodPost, "/v1/im/friend-requests", source.Token,
+		fmt.Sprintf("loadtest-friend-%04d", source.Index), body, &result)
+	return &result, err
+}
+
+func (a *apiClient) acceptFriendRequest(ctx context.Context, target AccountRecord, requestID string) (*friendRequestResponse, error) {
+	var result friendRequestResponse
+	err := a.do(ctx, http.MethodPost, "/v1/im/friend-requests/"+url.PathEscape(requestID)+"/accept", target.Token,
+		"loadtest-friend-accept-"+requestID, map[string]any{}, &result)
+	return &result, err
+}
+
+type p2pResponse struct {
+	Topic        string `json:"topic"`
+	Relationship string `json:"relationship"`
+}
+
+func (a *apiClient) ensureP2P(ctx context.Context, source, target AccountRecord) (*p2pResponse, error) {
+	var result p2pResponse
+	// EnsureP2P is itself idempotent and must execute on every preflight so it
+	// can repair subscriptions and refresh a loaded topic. A stable HTTP
+	// idempotency key would replay an old response and skip that repair.
+	err := a.do(ctx, http.MethodPost, "/v1/im/conversations/p2p", source.Token,
+		"",
+		map[string]any{"peer_user_id": target.UserID}, &result)
+	return &result, err
+}
+
+type groupCreateResponse struct {
+	Topic       string `json:"topic"`
+	MemberCount int    `json:"member_count"`
+}
+
+func (a *apiClient) createGroup(ctx context.Context, owner AccountRecord) (*groupCreateResponse, error) {
+	var result groupCreateResponse
+	body := map[string]any{"name": "IM 006 Load Test", "description": "im-pressure", "member_user_ids": []string{}}
+	err := a.do(ctx, http.MethodPost, "/v1/im/groups", owner.Token, "loadtest-group-create", body, &result)
+	return &result, err
+}
+
+type groupInvitationItem struct {
+	UserID string `json:"user_id"`
+	Status string `json:"status"`
+	Reason string `json:"reason"`
+}
+
+type groupInvitationResponse struct {
+	Topic string                `json:"topic"`
+	Items []groupInvitationItem `json:"items"`
+}
+
+func (a *apiClient) inviteGroup(ctx context.Context, owner AccountRecord, topic string, batch int, userIDs []string) (*groupInvitationResponse, error) {
+	var result groupInvitationResponse
+	membersHash := topicHash(strings.Join(userIDs, ","))
+	err := a.do(ctx, http.MethodPost, "/v1/im/groups/"+url.PathEscape(topic)+"/invitations", owner.Token,
+		fmt.Sprintf("loadtest-group-%s-invite-%03d-%s", topicHash(topic), batch, membersHash),
+		map[string]any{"user_ids": userIDs, "message": "loadtest"}, &result)
+	return &result, err
+}
+
+type groupMembersResponse struct {
+	Items []struct {
+		UserID string `json:"user_id"`
+	} `json:"items"`
+	NextCursor string `json:"next_cursor"`
+	HasMore    bool   `json:"has_more"`
+}
+
+func (a *apiClient) countGroupMembers(ctx context.Context, account AccountRecord, topic string) (int, error) {
+	cursor := ""
+	seen := make(map[string]bool)
+	for {
+		path := "/v1/im/groups/" + url.PathEscape(topic) + "/members?limit=100"
+		if cursor != "" {
+			path += "&cursor=" + url.QueryEscape(cursor)
+		}
+		var page groupMembersResponse
+		if err := a.do(ctx, http.MethodGet, path, account.Token, "", nil, &page); err != nil {
+			return 0, err
+		}
+		for _, item := range page.Items {
+			seen[item.UserID] = true
+		}
+		if page.NextCursor == "" || (!page.HasMore && len(page.Items) < 100) {
+			break
+		}
+		if page.NextCursor == cursor {
+			return 0, fmt.Errorf("group member cursor did not advance: %s", cursor)
+		}
+		cursor = page.NextCursor
+	}
+	return len(seen), nil
+}
+
+func queryInt(value string, fallback int) int {
+	n, err := strconv.Atoi(value)
+	if err != nil {
+		return fallback
+	}
+	return n
+}

loadtest/im-pressure/main.go

@@ -0,0 +1,167 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"os/signal"
+	"strings"
+	"syscall"
+	"time"
+)
+
+func main() {
+	log.SetFlags(log.LstdFlags | log.Lmicroseconds | log.LUTC)
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stop()
+	if len(os.Args) < 2 {
+		usage()
+		os.Exit(2)
+	}
+	var err error
+	switch os.Args[1] {
+	case "prepare":
+		err = prepareCommand(ctx, os.Args[2:])
+	case "verify":
+		err = verifyCommand(ctx, os.Args[2:])
+	case "coordinator":
+		err = coordinatorCommand(ctx, os.Args[2:])
+	case "agent":
+		err = agentCommand(ctx, os.Args[2:])
+	case "merge":
+		err = mergeCommand(os.Args[2:])
+	case "collect":
+		err = collectCommand(ctx, os.Args[2:])
+	case "verify-db":
+		err = verifyDBCommand(ctx, os.Args[2:])
+	default:
+		usage()
+		os.Exit(2)
+	}
+	if err != nil {
+		log.Fatal(err)
+	}
+}
+
+func verifyDBCommand(ctx context.Context, args []string) error {
+	fs := flag.NewFlagSet("verify-db", flag.ContinueOnError)
+	coordinator := fs.String("coordinator", "loadtest/im-pressure/results/coordinator.json", "coordinator ledger path")
+	output := fs.String("output", "-", "verification JSON output path, or - for stdout")
+	tenant := fs.String("tenant", "", "tenant code")
+	dsnEnv := fs.String("dsn-env", "MYSQL_DSN", "environment variable containing the business database DSN")
+	registryDB := fs.String("registry-db", "", "tenant registry database; defaults to TENANT_REGISTRY_DB")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+	if strings.TrimSpace(*tenant) == "" {
+		return fmt.Errorf("--tenant is required")
+	}
+	return runDBVerify(ctx, *coordinator, *output, *tenant, *dsnEnv, *registryDB)
+}
+
+func prepareCommand(ctx context.Context, args []string) error {
+	fs := flag.NewFlagSet("prepare", flag.ContinueOnError)
+	configPath := fs.String("config", "loadtest/im-pressure/config.example.json", "JSON configuration path")
+	manifestPath := fs.String("manifest", "loadtest/im-pressure/manifest.json", "manifest output path")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+	cfg, err := loadConfig(*configPath)
+	if err != nil {
+		return err
+	}
+	return runPrepare(ctx, cfg, *manifestPath)
+}
+
+func verifyCommand(ctx context.Context, args []string) error {
+	fs := flag.NewFlagSet("verify", flag.ContinueOnError)
+	configPath := fs.String("config", "loadtest/im-pressure/config.example.json", "JSON configuration path")
+	manifestPath := fs.String("manifest", "loadtest/im-pressure/manifest.json", "manifest path")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+	cfg, err := loadConfig(*configPath)
+	if err != nil {
+		return err
+	}
+	manifest, err := loadManifest(*manifestPath)
+	if err != nil {
+		return err
+	}
+	apiKey, err := cfg.apiKey()
+	if err != nil {
+		return err
+	}
+	return runVerify(ctx, cfg, manifest, newAPIClient(cfg, apiKey))
+}
+
+func coordinatorCommand(ctx context.Context, args []string) error {
+	fs := flag.NewFlagSet("coordinator", flag.ContinueOnError)
+	listen := fs.String("listen", ":8090", "coordinator listen address")
+	runID := fs.String("run-id", "", "unique run ID, max 24 characters")
+	scenario := fs.String("scenario", "", "p2p, group, or mixed")
+	output := fs.String("output", "loadtest/im-pressure/results/coordinator.json", "coordinator result path")
+	expected := fs.Int("expected-agents", 2, "number of agents")
+	delay := fs.Duration("start-delay", 30*time.Second, "delay after all agents are ready")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+	return runCoordinator(ctx, *listen, *runID, *scenario, *output, *expected, *delay)
+}
+
+func agentCommand(ctx context.Context, args []string) error {
+	fs := flag.NewFlagSet("agent", flag.ContinueOnError)
+	configPath := fs.String("config", "loadtest/im-pressure/config.example.json", "JSON configuration path")
+	manifestPath := fs.String("manifest", "loadtest/im-pressure/manifest.json", "manifest path")
+	machine := fs.String("machine", "", "a or b")
+	coordinatorURL := fs.String("coordinator", "", "coordinator base URL")
+	output := fs.String("output", "loadtest/im-pressure/results/agent-summary.json", "agent result path")
+	metricsListen := fs.String("metrics-listen", ":9105", "Prometheus listen address; empty disables it")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+	if *coordinatorURL == "" {
+		return fmt.Errorf("--coordinator is required")
+	}
+	cfg, err := loadConfig(*configPath)
+	if err != nil {
+		return err
+	}
+	manifest, err := loadManifest(*manifestPath)
+	if err != nil {
+		return err
+	}
+	return runAgent(ctx, cfg, manifest, *machine, *coordinatorURL, *output, *metricsListen)
+}
+
+func mergeCommand(args []string) error {
+	fs := flag.NewFlagSet("merge", flag.ContinueOnError)
+	a := fs.String("a", "loadtest/im-pressure/results/agent-a-summary.json", "machine A summary")
+	b := fs.String("b", "loadtest/im-pressure/results/agent-b-summary.json", "machine B summary")
+	output := fs.String("output", "loadtest/im-pressure/results/report.json", "combined report")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+	return runMerge(*a, *b, *output)
+}
+
+func collectCommand(ctx context.Context, args []string) error {
+	fs := flag.NewFlagSet("collect", flag.ContinueOnError)
+	var urls namedURLs
+	fs.Var(&urls, "url", "repeatable name=https://host/metrics source")
+	output := fs.String("output", "loadtest/im-pressure/results/metrics.jsonl", "snapshot output")
+	interval := fs.Duration("interval", 5*time.Second, "collection interval")
+	duration := fs.Duration("duration", 10*time.Minute, "collection duration")
+	coordinatorURL := fs.String("coordinator", "", "optional coordinator base URL used to label before/during phases")
+	coordinatorOutput := fs.String("coordinator-output", "", "coordinator JSON path used to detect the after phase")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+	return runCollector(ctx, urls, *output, *interval, *duration, *coordinatorURL, *coordinatorOutput)
+}
+
+func usage() {
+	fmt.Fprintln(os.Stderr, "usage: im-pressure <prepare|verify|coordinator|agent|merge|collect|verify-db> [flags]")
+}

loadtest/im-pressure/merge.go

@@ -0,0 +1,177 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"time"
+)
+
+type CombinedReport struct {
+	RunID               string            `json:"run_id"`
+	Scenario            string            `json:"scenario"`
+	GeneratedAt         time.Time         `json:"generated_at"`
+	Agents              []AgentSummary    `json:"agents"`
+	ExpectedConnections int               `json:"expected_connections"`
+	ReadyConnections    int               `json:"ready_connections"`
+	PublishAttempted    int64             `json:"publish_attempted"`
+	PublishAccepted     int64             `json:"publish_accepted"`
+	PublishRejected     int64             `json:"publish_rejected"`
+	PublishErrors       map[string]int64  `json:"publish_errors,omitempty"`
+	PublishSuccessRate  float64           `json:"publish_success_rate"`
+	AckP95MS            float64           `json:"ack_p95_ms"`
+	DeliveryP95MS       float64           `json:"delivery_p95_ms"`
+	ExpectedDeliveries  int64             `json:"expected_deliveries"`
+	ReceivedDeliveries  int64             `json:"received_deliveries"`
+	RealtimeLost        int64             `json:"realtime_lost"`
+	RealtimeLostStored  int64             `json:"realtime_lost_but_persisted"`
+	ArrivalRate         float64           `json:"arrival_rate"`
+	LossRate            float64           `json:"loss_rate"`
+	Duplicates          int64             `json:"duplicates"`
+	OutOfOrder          int64             `json:"out_of_order"`
+	Disconnects         int64             `json:"disconnects"`
+	HistoryChecked      int64             `json:"history_checked"`
+	HistoryFound        int64             `json:"history_found"`
+	PersistenceLost     int64             `json:"persistence_lost"`
+	Missing             []MissingDelivery `json:"missing,omitempty"`
+	Passed              bool              `json:"passed"`
+	FailureReasons      []string          `json:"failure_reasons,omitempty"`
+}
+
+func runMerge(aPath, bPath, output string) error {
+	a, err := readAgentSummary(aPath)
+	if err != nil {
+		return err
+	}
+	b, err := readAgentSummary(bPath)
+	if err != nil {
+		return err
+	}
+	if a.RunID != b.RunID || a.Scenario != b.Scenario {
+		return fmt.Errorf("agent summaries do not belong to the same run")
+	}
+	report := CombinedReport{RunID: a.RunID, Scenario: a.Scenario, GeneratedAt: time.Now().UTC(), Agents: []AgentSummary{a, b}, Passed: true,
+		PublishErrors: make(map[string]int64)}
+	for _, summary := range report.Agents {
+		expectedConnections := summary.ExpectedConnections
+		if expectedConnections == 0 {
+			expectedConnections = 1000
+		}
+		report.ExpectedConnections += expectedConnections
+		report.ReadyConnections += summary.ReadyConnections
+		report.PublishAttempted += summary.PublishAttempted
+		report.PublishAccepted += summary.PublishAccepted
+		report.PublishRejected += summary.PublishRejected
+		for category, count := range summary.PublishErrors {
+			report.PublishErrors[category] += count
+		}
+		report.ExpectedDeliveries += summary.ExpectedPeer + summary.ExpectedGroup
+		report.ReceivedDeliveries += summary.ReceivedPeer + summary.ReceivedGroup
+		report.RealtimeLost += summary.RealtimeLost
+		report.RealtimeLostStored += summary.RealtimeLostStored
+		report.Duplicates += summary.Duplicates
+		report.OutOfOrder += summary.OutOfOrder
+		report.Disconnects += summary.Disconnects
+		report.HistoryChecked += summary.HistoryChecked
+		report.HistoryFound += summary.HistoryFound
+		report.PersistenceLost += summary.PersistenceLost
+		report.Missing = append(report.Missing, summary.Missing...)
+		if len(summary.Errors) > 0 {
+			report.FailureReasons = append(report.FailureReasons, fmt.Sprintf("agent %s errors=%d", summary.Machine, len(summary.Errors)))
+		}
+	}
+	if report.PublishAttempted > 0 {
+		report.PublishSuccessRate = float64(report.PublishAccepted) / float64(report.PublishAttempted)
+	}
+	report.AckP95MS = mergedHistogramP95(report.Agents, func(summary AgentSummary) HistogramSnapshot { return summary.AckLatency })
+	report.DeliveryP95MS = mergedHistogramP95(report.Agents, func(summary AgentSummary) HistogramSnapshot { return summary.DeliveryLatency })
+	if report.ExpectedDeliveries > 0 {
+		report.ArrivalRate = float64(report.ReceivedDeliveries) / float64(report.ExpectedDeliveries)
+		report.LossRate = float64(report.RealtimeLost) / float64(report.ExpectedDeliveries)
+	}
+	if report.ReadyConnections != report.ExpectedConnections {
+		report.FailureReasons = append(report.FailureReasons, fmt.Sprintf("ready connections=%d expected=%d",
+			report.ReadyConnections, report.ExpectedConnections))
+	}
+	if report.PublishAttempted > 0 && report.PublishSuccessRate < 0.9999 {
+		report.FailureReasons = append(report.FailureReasons,
+			fmt.Sprintf("publish success rate=%.6f expected>=0.999900", report.PublishSuccessRate))
+	}
+	if report.AckP95MS > 500 {
+		report.FailureReasons = append(report.FailureReasons, fmt.Sprintf("ack p95=%.0fms expected<=500ms", report.AckP95MS))
+	}
+	if report.DeliveryP95MS > 1000 {
+		report.FailureReasons = append(report.FailureReasons,
+			fmt.Sprintf("delivery p95=%.0fms expected<=1000ms", report.DeliveryP95MS))
+	}
+	if report.PublishRejected > 0 || report.RealtimeLost > 0 || report.PersistenceLost > 0 || report.Duplicates > 0 || report.OutOfOrder > 0 || report.Disconnects > 0 {
+		report.FailureReasons = append(report.FailureReasons, "message correctness gate failed")
+	}
+	report.Passed = len(report.FailureReasons) == 0
+	sort.Slice(report.Missing, func(i, j int) bool {
+		if report.Missing[i].ClientMid == report.Missing[j].ClientMid {
+			return report.Missing[i].ReceiverIndex < report.Missing[j].ReceiverIndex
+		}
+		return report.Missing[i].ClientMid < report.Missing[j].ClientMid
+	})
+	return writeJSONFile(output, report)
+}
+
+func mergedHistogramP95(agents []AgentSummary, selectHistogram func(AgentSummary) HistogramSnapshot) float64 {
+	var count int64
+	buckets := make([]int64, len(latencyBounds)+1)
+	maxMS := float64(0)
+	for _, agent := range agents {
+		histogram := selectHistogram(agent)
+		count += histogram.Count
+		if histogram.MaxMS > maxMS {
+			maxMS = histogram.MaxMS
+		}
+		for i, bound := range latencyBounds {
+			buckets[i] += histogram.Buckets[bound.String()]
+		}
+		buckets[len(latencyBounds)] += histogram.Buckets["+Inf"]
+	}
+	if count == 0 {
+		return 0
+	}
+	target := (count*95 + 99) / 100
+	var cumulative int64
+	for i, bucket := range buckets {
+		cumulative += bucket
+		if cumulative >= target {
+			if i == len(latencyBounds) {
+				return maxMS
+			}
+			return float64(latencyBounds[i]) / float64(time.Millisecond)
+		}
+	}
+	return maxMS
+}
+
+func readAgentSummary(path string) (AgentSummary, error) {
+	var summary AgentSummary
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return summary, err
+	}
+	if err = json.Unmarshal(raw, &summary); err != nil {
+		return summary, err
+	}
+	return summary, nil
+}
+
+func writeJSONFile(path string, value any) error {
+	raw, err := json.MarshalIndent(value, "", "  ")
+	if err != nil {
+		return err
+	}
+	if dir := filepath.Dir(path); dir != "." {
+		if err = os.MkdirAll(dir, 0o750); err != nil {
+			return err
+		}
+	}
+	return os.WriteFile(path, append(raw, '\n'), 0o600)
+}

loadtest/im-pressure/merge_test.go

@@ -0,0 +1,61 @@
+package main
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestRunMergeComputesArrivalAndPersistenceGate(t *testing.T) {
+	dir := t.TempDir()
+	a := AgentSummary{RunID: "run1", Scenario: "p2p", Machine: "a", ReadyConnections: 1000,
+		ExpectedConnections: 1000,
+		PublishAttempted:    10, PublishAccepted: 10, ExpectedPeer: 10, ReceivedPeer: 9,
+		RealtimeLost: 1, HistoryChecked: 10, HistoryFound: 10}
+	b := AgentSummary{RunID: "run1", Scenario: "p2p", Machine: "b", ReadyConnections: 1000,
+		ExpectedConnections: 1000,
+		PublishAttempted:    10, PublishAccepted: 10, ExpectedPeer: 10, ReceivedPeer: 10,
+		HistoryChecked: 10, HistoryFound: 9, PersistenceLost: 1}
+	aPath, bPath, reportPath := filepath.Join(dir, "a.json"), filepath.Join(dir, "b.json"), filepath.Join(dir, "report.json")
+	writeSummaryForTest(t, aPath, a)
+	writeSummaryForTest(t, bPath, b)
+	if err := runMerge(aPath, bPath, reportPath); err != nil {
+		t.Fatal(err)
+	}
+	var report CombinedReport
+	raw, err := os.ReadFile(reportPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err = json.Unmarshal(raw, &report); err != nil {
+		t.Fatal(err)
+	}
+	if report.ExpectedDeliveries != 20 || report.ReceivedDeliveries != 19 || report.ArrivalRate != 0.95 || report.LossRate != 0.05 {
+		t.Fatalf("unexpected delivery totals: %+v", report)
+	}
+	if report.Passed || report.PersistenceLost != 1 {
+		t.Fatalf("correctness gate did not fail: %+v", report)
+	}
+}
+
+func TestMergedHistogramP95(t *testing.T) {
+	agents := []AgentSummary{
+		{AckLatency: HistogramSnapshot{Count: 10, MaxMS: 900, Buckets: map[string]int64{"200ms": 9, "1s": 1}}},
+		{AckLatency: HistogramSnapshot{Count: 10, MaxMS: 300, Buckets: map[string]int64{"200ms": 10}}},
+	}
+	if got := mergedHistogramP95(agents, func(summary AgentSummary) HistogramSnapshot { return summary.AckLatency }); got != 200 {
+		t.Fatalf("p95=%v", got)
+	}
+}
+
+func writeSummaryForTest(t *testing.T, path string, summary AgentSummary) {
+	t.Helper()
+	raw, err := json.Marshal(summary)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err = os.WriteFile(path, raw, 0o600); err != nil {
+		t.Fatal(err)
+	}
+}

loadtest/im-pressure/model.go

@@ -0,0 +1,387 @@
+package main
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+	"sort"
+	"strings"
+	"time"
+)
+
+const manifestVersion = 1
+
+type Config struct {
+	Tenant           string              `json:"tenant"`
+	HTTPBase         string              `json:"http_base"`
+	APIKeyEnv        string              `json:"api_key_env"`
+	PasswordEnv      string              `json:"password_env"`
+	UsersPerMachine  int                 `json:"users_per_machine"`
+	Endpoints        map[string][]string `json:"endpoints"`
+	InsecureTLS      bool                `json:"insecure_tls"`
+	PrepareWorkers   int                 `json:"prepare_workers"`
+	ConnectRamp      string              `json:"connect_ramp"`
+	CommandTimeout   string              `json:"command_timeout"`
+	P2PMessages      int                 `json:"p2p_messages_per_user"`
+	P2PInterval      string              `json:"p2p_interval"`
+	DrainP2P         string              `json:"drain_p2p"`
+	DrainGroup       string              `json:"drain_group"`
+	PhonePrefix      string              `json:"registration_phone_prefix"`
+	VerifyCodeEnv    string              `json:"verification_code_env"`
+	HistoryWorkers   int                 `json:"history_workers"`
+	HistoryTimeout   string              `json:"history_timeout"`
+	PreflightWorkers int                 `json:"preflight_workers"`
+	SkipP2PPreflight bool                `json:"skip_p2p_preflight"`
+}
+
+func loadConfig(path string) (*Config, error) {
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	var cfg Config
+	if err = json.Unmarshal(raw, &cfg); err != nil {
+		return nil, fmt.Errorf("decode config: %w", err)
+	}
+	if cfg.Tenant == "" {
+		cfg.Tenant = "006"
+	}
+	if cfg.UsersPerMachine <= 0 {
+		cfg.UsersPerMachine = 1000
+	}
+	if cfg.PrepareWorkers <= 0 {
+		cfg.PrepareWorkers = 20
+	}
+	if cfg.ConnectRamp == "" {
+		cfg.ConnectRamp = "20s"
+	}
+	if cfg.CommandTimeout == "" {
+		cfg.CommandTimeout = "20s"
+	}
+	if cfg.P2PMessages <= 0 {
+		cfg.P2PMessages = 10
+	}
+	if cfg.P2PInterval == "" {
+		cfg.P2PInterval = "1s"
+	}
+	if cfg.DrainP2P == "" {
+		cfg.DrainP2P = "30s"
+	}
+	if cfg.DrainGroup == "" {
+		cfg.DrainGroup = "60s"
+	}
+	if cfg.HistoryWorkers <= 0 {
+		cfg.HistoryWorkers = 48
+	}
+	if cfg.HistoryWorkers > 128 {
+		return nil, errors.New("history_workers must not exceed 128")
+	}
+	if cfg.HistoryTimeout == "" {
+		cfg.HistoryTimeout = "3m"
+	}
+	if _, err = configDuration(cfg.HistoryTimeout); err != nil {
+		return nil, fmt.Errorf("history_timeout: %w", err)
+	}
+	if cfg.PreflightWorkers <= 0 {
+		cfg.PreflightWorkers = 32
+	}
+	if cfg.PreflightWorkers > 128 {
+		return nil, errors.New("preflight_workers must not exceed 128")
+	}
+	if strings.TrimSpace(cfg.HTTPBase) == "" {
+		return nil, errors.New("http_base is required")
+	}
+	for _, machine := range []string{"a", "b"} {
+		if len(cfg.Endpoints[machine]) == 0 {
+			return nil, fmt.Errorf("at least one endpoint is required for machine %s", machine)
+		}
+	}
+	if cfg.PhonePrefix != "" {
+		if len(cfg.PhonePrefix) != 7 || strings.Trim(cfg.PhonePrefix, "0123456789") != "" {
+			return nil, errors.New("registration_phone_prefix must contain exactly 7 digits")
+		}
+		if cfg.UsersPerMachine > 4999 {
+			return nil, errors.New("registration phone suffix supports at most 4999 users per machine")
+		}
+	}
+	return &cfg, nil
+}
+
+func (c *Config) apiKey() (string, error) {
+	name := c.APIKeyEnv
+	if name == "" {
+		name = "IM_API_KEY"
+	}
+	value := strings.TrimSpace(os.Getenv(name))
+	if value == "" {
+		return "", fmt.Errorf("environment variable %s is required", name)
+	}
+	return value, nil
+}
+
+func (c *Config) password() (string, error) {
+	name := c.PasswordEnv
+	if name == "" {
+		name = "IM_LOAD_PASSWORD"
+	}
+	value := os.Getenv(name)
+	if value == "" {
+		return "", fmt.Errorf("environment variable %s is required", name)
+	}
+	return value, nil
+}
+
+func (c *Config) verificationCode() (string, error) {
+	name := c.VerifyCodeEnv
+	if name == "" {
+		name = "IM_VERIFICATION_CODE"
+	}
+	value := strings.TrimSpace(os.Getenv(name))
+	if c.PhonePrefix != "" && value == "" {
+		return "", fmt.Errorf("environment variable %s is required", name)
+	}
+	return value, nil
+}
+
+func configDuration(value string) (time.Duration, error) {
+	d, err := time.ParseDuration(value)
+	if err != nil || d <= 0 {
+		return 0, fmt.Errorf("invalid positive duration %q", value)
+	}
+	return d, nil
+}
+
+type Manifest struct {
+	Version    int             `json:"version"`
+	Tenant     string          `json:"tenant"`
+	GroupTopic string          `json:"group_topic,omitempty"`
+	CreatedAt  time.Time       `json:"created_at"`
+	Accounts   []AccountRecord `json:"accounts"`
+}
+
+type AccountRecord struct {
+	Machine      string    `json:"machine"`
+	Index        int       `json:"index"`
+	Username     string    `json:"username"`
+	Phone        string    `json:"phone,omitempty"`
+	UserID       string    `json:"user_id"`
+	Token        string    `json:"token"`
+	TokenExpires time.Time `json:"token_expires,omitempty"`
+	P2PTopic     string    `json:"p2p_topic,omitempty"`
+	Endpoint     string    `json:"endpoint,omitempty"`
+}
+
+func newManifest(cfg *Config) *Manifest {
+	m := &Manifest{Version: manifestVersion, Tenant: cfg.Tenant, CreatedAt: time.Now().UTC()}
+	for _, machine := range []string{"a", "b"} {
+		for i := 1; i <= cfg.UsersPerMachine; i++ {
+			phone, _ := registrationPhone(cfg, machine, i)
+			m.Accounts = append(m.Accounts, AccountRecord{
+				Machine:  machine,
+				Index:    i,
+				Username: fmt.Sprintf("lt%s%04d", machine, i),
+				Phone:    phone,
+				Endpoint: cfg.Endpoints[machine][(i-1)%len(cfg.Endpoints[machine])],
+			})
+		}
+	}
+	return m
+}
+
+func applyConfigEndpoints(cfg *Config, manifest *Manifest) error {
+	if manifest.Tenant != cfg.Tenant {
+		return fmt.Errorf("manifest tenant=%s config tenant=%s", manifest.Tenant, cfg.Tenant)
+	}
+	if len(manifest.Accounts) != cfg.UsersPerMachine*2 {
+		return fmt.Errorf("manifest accounts=%d expected=%d", len(manifest.Accounts), cfg.UsersPerMachine*2)
+	}
+	for i := range manifest.Accounts {
+		account := &manifest.Accounts[i]
+		endpoints := cfg.Endpoints[account.Machine]
+		if account.Index <= 0 || len(endpoints) == 0 {
+			return fmt.Errorf("invalid endpoint assignment for %s", accountKey(account.Machine, account.Index))
+		}
+		account.Endpoint = endpoints[(account.Index-1)%len(endpoints)]
+		phone, err := registrationPhone(cfg, account.Machine, account.Index)
+		if err != nil {
+			return err
+		}
+		account.Phone = phone
+	}
+	return nil
+}
+
+func reconcileManifest(cfg *Config, existing *Manifest) (*Manifest, error) {
+	if existing.Tenant != cfg.Tenant {
+		return nil, fmt.Errorf("manifest tenant=%s config tenant=%s", existing.Tenant, cfg.Tenant)
+	}
+	desired := newManifest(cfg)
+	if len(existing.Accounts) > len(desired.Accounts) {
+		return nil, fmt.Errorf("refusing to shrink manifest accounts from %d to %d", len(existing.Accounts), len(desired.Accounts))
+	}
+	desired.GroupTopic = existing.GroupTopic
+	desired.CreatedAt = existing.CreatedAt
+	old := manifestAccountMap(existing)
+	for i := range desired.Accounts {
+		account := &desired.Accounts[i]
+		if previous := old[accountKey(account.Machine, account.Index)]; previous != nil {
+			account.UserID = previous.UserID
+			account.Token = previous.Token
+			account.TokenExpires = previous.TokenExpires
+			account.P2PTopic = previous.P2PTopic
+		}
+	}
+	return desired, nil
+}
+
+func registrationPhone(cfg *Config, machine string, index int) (string, error) {
+	if cfg.PhonePrefix == "" {
+		return "", nil
+	}
+	ordinal := index
+	if machine == "b" {
+		ordinal += 5000
+	} else if machine != "a" {
+		return "", fmt.Errorf("invalid machine %q", machine)
+	}
+	if index <= 0 || ordinal > 9999 {
+		return "", fmt.Errorf("invalid phone ordinal %d", ordinal)
+	}
+	return fmt.Sprintf("%s%04d", cfg.PhonePrefix, ordinal), nil
+}
+
+func loadManifest(path string) (*Manifest, error) {
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	var manifest Manifest
+	if err = json.Unmarshal(raw, &manifest); err != nil {
+		return nil, err
+	}
+	if manifest.Version != manifestVersion {
+		return nil, fmt.Errorf("unsupported manifest version %d", manifest.Version)
+	}
+	return &manifest, manifest.validate()
+}
+
+func (m *Manifest) validate() error {
+	seen := make(map[string]bool, len(m.Accounts))
+	for _, account := range m.Accounts {
+		key := accountKey(account.Machine, account.Index)
+		if account.Machine != "a" && account.Machine != "b" {
+			return fmt.Errorf("invalid machine %q", account.Machine)
+		}
+		if account.Index <= 0 || account.Username == "" || seen[key] {
+			return fmt.Errorf("invalid or duplicate account %s", key)
+		}
+		seen[key] = true
+	}
+	return nil
+}
+
+func saveManifest(path string, manifest *Manifest) error {
+	if err := manifest.validate(); err != nil {
+		return err
+	}
+	raw, err := json.MarshalIndent(manifest, "", "  ")
+	if err != nil {
+		return err
+	}
+	tmp := path + ".tmp"
+	if err = os.WriteFile(tmp, append(raw, '\n'), 0o600); err != nil {
+		return err
+	}
+	return os.Rename(tmp, path)
+}
+
+func accountKey(machine string, index int) string {
+	return strings.ToLower(machine) + fmt.Sprintf("-%04d", index)
+}
+
+func accountsForMachine(manifest *Manifest, machine string) []AccountRecord {
+	var out []AccountRecord
+	for _, account := range manifest.Accounts {
+		if account.Machine == machine {
+			out = append(out, account)
+		}
+	}
+	sort.Slice(out, func(i, j int) bool { return out[i].Index < out[j].Index })
+	return out
+}
+
+func manifestAccountMap(manifest *Manifest) map[string]*AccountRecord {
+	out := make(map[string]*AccountRecord, len(manifest.Accounts))
+	for i := range manifest.Accounts {
+		account := &manifest.Accounts[i]
+		out[accountKey(account.Machine, account.Index)] = account
+	}
+	return out
+}
+
+func topicHash(topic string) string {
+	sum := sha256.Sum256([]byte(topic))
+	return hex.EncodeToString(sum[:6])
+}
+
+type AcceptedMessage struct {
+	RunID         string    `json:"run_id"`
+	Scenario      string    `json:"scenario"`
+	Kind          string    `json:"kind"`
+	ClientMid     string    `json:"client_mid"`
+	Topic         string    `json:"topic"`
+	Seq           int64     `json:"seq"`
+	SenderMachine string    `json:"sender_machine"`
+	SenderIndex   int       `json:"sender_index"`
+	PeerMachine   string    `json:"peer_machine,omitempty"`
+	PeerIndex     int       `json:"peer_index,omitempty"`
+	SentAt        time.Time `json:"sent_at"`
+	AckAt         time.Time `json:"ack_at"`
+}
+
+type AgentSummary struct {
+	RunID               string            `json:"run_id"`
+	Scenario            string            `json:"scenario"`
+	Machine             string            `json:"machine"`
+	StartedAt           time.Time         `json:"started_at"`
+	LoadEndedAt         time.Time         `json:"load_ended_at"`
+	FinishedAt          time.Time         `json:"finished_at"`
+	ExpectedConnections int               `json:"expected_connections"`
+	ReadyConnections    int               `json:"ready_connections"`
+	PublishAttempted    int64             `json:"publish_attempted"`
+	PublishAccepted     int64             `json:"publish_accepted"`
+	PublishRejected     int64             `json:"publish_rejected"`
+	PublishErrors       map[string]int64  `json:"publish_errors,omitempty"`
+	ExpectedPeer        int64             `json:"expected_peer"`
+	ReceivedPeer        int64             `json:"received_peer"`
+	ExpectedEcho        int64             `json:"expected_echo"`
+	ReceivedEcho        int64             `json:"received_echo"`
+	ExpectedGroup       int64             `json:"expected_group"`
+	ReceivedGroup       int64             `json:"received_group"`
+	RealtimeLost        int64             `json:"realtime_lost"`
+	RealtimeLostStored  int64             `json:"realtime_lost_but_persisted"`
+	Duplicates          int64             `json:"duplicates"`
+	OutOfOrder          int64             `json:"out_of_order"`
+	Disconnects         int64             `json:"disconnects"`
+	UnexpectedMessages  int64             `json:"unexpected_messages"`
+	HistoryChecked      int64             `json:"history_checked"`
+	HistoryFound        int64             `json:"history_found"`
+	PersistenceLost     int64             `json:"persistence_lost"`
+	AckLatency          HistogramSnapshot `json:"ack_latency"`
+	DeliveryLatency     HistogramSnapshot `json:"delivery_latency"`
+	Errors              []string          `json:"errors,omitempty"`
+	Missing             []MissingDelivery `json:"missing,omitempty"`
+	PersistenceMissing  []string          `json:"persistence_missing,omitempty"`
+	AcceptedLocal       []AcceptedMessage `json:"accepted_local,omitempty"`
+}
+
+type MissingDelivery struct {
+	ClientMid     string `json:"client_mid"`
+	Kind          string `json:"kind"`
+	TopicHash     string `json:"topic_hash"`
+	Seq           int64  `json:"seq"`
+	ReceiverIndex int    `json:"receiver_index"`
+}

loadtest/im-pressure/model_test.go

@@ -0,0 +1,100 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestLoadConfigAppliesBoundedHistoryDefaults(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "config.json")
+	raw := []byte(`{"http_base":"https://example.test","endpoints":{"a":["wss://a"],"b":["wss://b"]}}`)
+	if err := os.WriteFile(path, raw, 0o600); err != nil {
+		t.Fatal(err)
+	}
+	cfg, err := loadConfig(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if cfg.HistoryWorkers != 48 || cfg.HistoryTimeout != "3m" || cfg.PreflightWorkers != 32 {
+		t.Fatalf("unexpected defaults: %+v", cfg)
+	}
+}
+
+func TestNewManifestDistributesUsersAndEndpoints(t *testing.T) {
+	cfg := &Config{Tenant: "006", UsersPerMachine: 3, Endpoints: map[string][]string{
+		"a": {"ws://a1", "ws://a2"}, "b": {"ws://b1"},
+	}}
+	manifest := newManifest(cfg)
+	if len(manifest.Accounts) != 6 {
+		t.Fatalf("accounts=%d", len(manifest.Accounts))
+	}
+	if got := accountsForMachine(manifest, "a"); got[0].Username != "lta0001" || got[1].Endpoint != "ws://a2" || got[2].Endpoint != "ws://a1" {
+		t.Fatalf("unexpected A accounts: %+v", got)
+	}
+	if err := manifest.validate(); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestManifestRejectsDuplicateAccount(t *testing.T) {
+	manifest := &Manifest{Version: manifestVersion, Accounts: []AccountRecord{
+		{Machine: "a", Index: 1, Username: "lta0001"},
+		{Machine: "a", Index: 1, Username: "duplicate"},
+	}}
+	if err := manifest.validate(); err == nil {
+		t.Fatal("expected duplicate account validation error")
+	}
+}
+
+func TestApplyConfigEndpointsRefreshesManifest(t *testing.T) {
+	cfg := &Config{Tenant: "006", UsersPerMachine: 1, Endpoints: map[string][]string{
+		"a": {"ws://new-a"}, "b": {"ws://new-b"},
+	}}
+	manifest := &Manifest{Tenant: "006", Accounts: []AccountRecord{
+		{Machine: "a", Index: 1, Username: "lta0001", Endpoint: "ws://old"},
+		{Machine: "b", Index: 1, Username: "ltb0001", Endpoint: "ws://old"},
+	}}
+	if err := applyConfigEndpoints(cfg, manifest); err != nil {
+		t.Fatal(err)
+	}
+	if manifest.Accounts[0].Endpoint != "ws://new-a" || manifest.Accounts[1].Endpoint != "ws://new-b" {
+		t.Fatalf("endpoints were not refreshed: %+v", manifest.Accounts)
+	}
+}
+
+func TestRegistrationPhoneUsesDistinctMachineRanges(t *testing.T) {
+	cfg := &Config{UsersPerMachine: 1000, PhonePrefix: "1990927"}
+	a, err := registrationPhone(cfg, "a", 1)
+	if err != nil || a != "19909270001" {
+		t.Fatalf("machine A phone=%q err=%v", a, err)
+	}
+	b, err := registrationPhone(cfg, "b", 1)
+	if err != nil || b != "19909275001" {
+		t.Fatalf("machine B phone=%q err=%v", b, err)
+	}
+	last, err := registrationPhone(cfg, "b", 1000)
+	if err != nil || last != "19909276000" {
+		t.Fatalf("last phone=%q err=%v", last, err)
+	}
+}
+
+func TestReconcileManifestExpandsSmokeRun(t *testing.T) {
+	existing := &Manifest{Version: manifestVersion, Tenant: "allrich", GroupTopic: "grp-one", Accounts: []AccountRecord{
+		{Machine: "a", Index: 1, Username: "lta0001", UserID: "usr-a", Token: "token-a", P2PTopic: "p2p-one"},
+		{Machine: "b", Index: 1, Username: "ltb0001", UserID: "usr-b", Token: "token-b", P2PTopic: "p2p-one"},
+	}}
+	cfg := &Config{Tenant: "allrich", UsersPerMachine: 2, PhonePrefix: "1990927", Endpoints: map[string][]string{
+		"a": {"wss://a"}, "b": {"wss://b"},
+	}}
+	expanded, err := reconcileManifest(cfg, existing)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(expanded.Accounts) != 4 || expanded.GroupTopic != "grp-one" || expanded.Accounts[0].Token != "token-a" {
+		t.Fatalf("unexpected expanded manifest: %+v", expanded)
+	}
+	if expanded.Accounts[3].Phone != "19909275002" {
+		t.Fatalf("machine B phone=%q", expanded.Accounts[3].Phone)
+	}
+}

loadtest/im-pressure/prepare.go

@@ -0,0 +1,318 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log"
+	"net/http"
+	"sort"
+	"sync"
+	"sync/atomic"
+	"time"
+)
+
+func runPrepare(ctx context.Context, cfg *Config, manifestPath string) error {
+	apiKey, err := cfg.apiKey()
+	if err != nil {
+		return err
+	}
+	password, err := cfg.password()
+	if err != nil {
+		return err
+	}
+	verificationCode, err := cfg.verificationCode()
+	if err != nil {
+		return err
+	}
+	manifest := newManifest(cfg)
+	if existing, loadErr := loadManifest(manifestPath); loadErr == nil {
+		manifest, err = reconcileManifest(cfg, existing)
+		if err != nil {
+			return err
+		}
+		log.Printf("resuming manifest with %d existing accounts, target=%d", len(existing.Accounts), len(manifest.Accounts))
+	}
+	if err = applyConfigEndpoints(cfg, manifest); err != nil {
+		return err
+	}
+	api := newAPIClient(cfg, apiKey)
+	if err = prepareAccounts(ctx, cfg, manifest, api, password, verificationCode); err != nil {
+		return err
+	}
+	if err = saveManifest(manifestPath, manifest); err != nil {
+		return err
+	}
+	if err = prepareFriendships(ctx, cfg, manifest, api); err != nil {
+		return err
+	}
+	if err = saveManifest(manifestPath, manifest); err != nil {
+		return err
+	}
+	if err = prepareGroup(ctx, cfg, manifest, api); err != nil {
+		return err
+	}
+	if err = saveManifest(manifestPath, manifest); err != nil {
+		return err
+	}
+	return runVerify(ctx, cfg, manifest, api)
+}
+
+func prepareAccounts(ctx context.Context, cfg *Config, manifest *Manifest, api *apiClient, password, verificationCode string) error {
+	var created atomic.Int64
+	var existing atomic.Int64
+	var mu sync.Mutex
+	err := parallelAccounts(ctx, manifest.Accounts, cfg.PrepareWorkers, func(account AccountRecord) error {
+		if account.UserID != "" && account.Token != "" && account.TokenExpires.After(time.Now().Add(5*time.Minute)) {
+			existing.Add(1)
+			return nil
+		}
+		client, wasCreated, err := connectForPrepareWithRetry(ctx, cfg, account, api, password, verificationCode)
+		if err != nil {
+			return fmt.Errorf("account %s: %w", account.Username, err)
+		}
+		updated := client.Account()
+		client.Close()
+		mu.Lock()
+		defer mu.Unlock()
+		for i := range manifest.Accounts {
+			if manifest.Accounts[i].Machine == account.Machine && manifest.Accounts[i].Index == account.Index {
+				manifest.Accounts[i].UserID = updated.UserID
+				manifest.Accounts[i].Token = updated.Token
+				manifest.Accounts[i].TokenExpires = updated.TokenExpires
+				break
+			}
+		}
+		if wasCreated {
+			created.Add(1)
+		} else {
+			existing.Add(1)
+		}
+		return nil
+	})
+	if err == nil {
+		log.Printf("accounts ready=%d created=%d existing=%d", len(manifest.Accounts), created.Load(), existing.Load())
+	}
+	return err
+}
+
+func connectForPrepareWithRetry(ctx context.Context, cfg *Config, account AccountRecord, api *apiClient,
+	password, verificationCode string) (*WSClient, bool, error) {
+	var lastErr error
+	for attempt := 1; attempt <= 6; attempt++ {
+		client, created, err := connectForPrepare(ctx, cfg, account, api, password, verificationCode)
+		if err == nil {
+			return client, created, nil
+		}
+		lastErr = err
+		if ctx.Err() != nil {
+			return nil, false, ctx.Err()
+		}
+		delay := time.Duration(attempt*5) * time.Second
+		log.Printf("account=%s prepare attempt=%d failed; retry_in=%s: %v", account.Username, attempt, delay, err)
+		if err = waitContext(ctx, delay); err != nil {
+			return nil, false, err
+		}
+	}
+	return nil, false, fmt.Errorf("account %s prepare failed after retries: %w", account.Username, lastErr)
+}
+
+func prepareFriendships(ctx context.Context, cfg *Config, manifest *Manifest, api *apiClient) error {
+	accounts := manifestAccountMap(manifest)
+	pairs := make([]AccountRecord, 0, cfg.UsersPerMachine)
+	for i := 1; i <= cfg.UsersPerMachine; i++ {
+		pairs = append(pairs, *accounts[accountKey("a", i)])
+	}
+	var mu sync.Mutex
+	err := parallelAccounts(ctx, pairs, cfg.PrepareWorkers, func(a AccountRecord) error {
+		return retryPrepareOperation(ctx, fmt.Sprintf("friend-pair-%04d", a.Index), func() error {
+			b := *accounts[accountKey("b", a.Index)]
+			if a.P2PTopic != "" && b.P2PTopic == a.P2PTopic {
+				return nil
+			}
+			if a.UserID == "" || b.UserID == "" {
+				return fmt.Errorf("pair %d has missing user ID", a.Index)
+			}
+			request, requestErr := api.createFriendRequest(ctx, a, b)
+			if requestErr == nil && request.Status == "pending" {
+				if _, requestErr = api.acceptFriendRequest(ctx, b, request.RequestID); requestErr != nil {
+					return fmt.Errorf("accept pair %d: %w", a.Index, requestErr)
+				}
+			} else if requestErr != nil {
+				var apiErr *apiError
+				if !errors.As(requestErr, &apiErr) || apiErr.Status != http.StatusConflict {
+					return fmt.Errorf("friend pair %d: %w", a.Index, requestErr)
+				}
+				pendingID, findErr := api.findPendingFriendRequest(ctx, b, a.UserID)
+				if findErr != nil {
+					return fmt.Errorf("find pending friend pair %d: %w", a.Index, findErr)
+				}
+				if pendingID != "" {
+					if _, acceptErr := api.acceptFriendRequest(ctx, b, pendingID); acceptErr != nil {
+						return fmt.Errorf("resume friend pair %d: %w", a.Index, acceptErr)
+					}
+				}
+			}
+			conversation, err := api.ensureP2P(ctx, a, b)
+			if err != nil {
+				return fmt.Errorf("ensure P2P pair %d: %w", a.Index, err)
+			}
+			if conversation.Relationship != "friend" || conversation.Topic == "" {
+				return fmt.Errorf("pair %d relationship=%s topic=%q", a.Index, conversation.Relationship, conversation.Topic)
+			}
+			mu.Lock()
+			accounts[accountKey("a", a.Index)].P2PTopic = conversation.Topic
+			accounts[accountKey("b", a.Index)].P2PTopic = conversation.Topic
+			mu.Unlock()
+			return nil
+		})
+	})
+	if err == nil {
+		log.Printf("friendships ready=%d", cfg.UsersPerMachine)
+	}
+	return err
+}
+
+func prepareGroup(ctx context.Context, cfg *Config, manifest *Manifest, api *apiClient) error {
+	if err := parallelAccounts(ctx, manifest.Accounts, cfg.PrepareWorkers, func(account AccountRecord) error {
+		return retryPrepareOperation(ctx, "privacy-"+account.Username, func() error {
+			return api.disableGroupInviteVerification(ctx, account)
+		})
+	}); err != nil {
+		return fmt.Errorf("prepare group invite privacy: %w", err)
+	}
+	accounts := manifestAccountMap(manifest)
+	owner := *accounts[accountKey("a", 1)]
+	if manifest.GroupTopic == "" {
+		group, err := api.createGroup(ctx, owner)
+		if err != nil {
+			return err
+		}
+		manifest.GroupTopic = group.Topic
+		log.Printf("created load-test group topic_hash=%s", topicHash(group.Topic))
+	}
+	memberIDs := make([]string, 0, len(manifest.Accounts)-1)
+	for _, account := range manifest.Accounts {
+		if account.UserID != owner.UserID {
+			memberIDs = append(memberIDs, account.UserID)
+		}
+	}
+	sort.Strings(memberIDs)
+	for offset, batch := 0, 1; offset < len(memberIDs); offset, batch = offset+100, batch+1 {
+		end := offset + 100
+		if end > len(memberIDs) {
+			end = len(memberIDs)
+		}
+		var response *groupInvitationResponse
+		err := retryPrepareOperation(ctx, fmt.Sprintf("group-invite-%03d", batch), func() error {
+			var inviteErr error
+			response, inviteErr = api.inviteGroup(ctx, owner, manifest.GroupTopic, batch, memberIDs[offset:end])
+			return inviteErr
+		})
+		if err != nil {
+			return fmt.Errorf("invite group batch %d: %w", batch, err)
+		}
+		for _, item := range response.Items {
+			if item.Status != "joined" && item.Status != "already_member" {
+				return fmt.Errorf("invite user %s status=%s reason=%s", item.UserID, item.Status, item.Reason)
+			}
+		}
+	}
+	count, err := api.countGroupMembers(ctx, owner, manifest.GroupTopic)
+	if err != nil {
+		return err
+	}
+	if count != len(manifest.Accounts) {
+		return fmt.Errorf("group has %d members, expected %d", count, len(manifest.Accounts))
+	}
+	log.Printf("group ready members=%d topic_hash=%s", count, topicHash(manifest.GroupTopic))
+	return nil
+}
+
+func retryPrepareOperation(ctx context.Context, label string, operation func() error) error {
+	var lastErr error
+	for attempt := 1; attempt <= 6; attempt++ {
+		if err := operation(); err == nil {
+			return nil
+		} else {
+			lastErr = err
+		}
+		if ctx.Err() != nil {
+			return ctx.Err()
+		}
+		delay := time.Duration(attempt*5) * time.Second
+		log.Printf("operation=%s attempt=%d failed; retry_in=%s: %v", label, attempt, delay, lastErr)
+		if err := waitContext(ctx, delay); err != nil {
+			return err
+		}
+	}
+	return fmt.Errorf("operation %s failed after retries: %w", label, lastErr)
+}
+
+func runVerify(ctx context.Context, cfg *Config, manifest *Manifest, api *apiClient) error {
+	if len(manifest.Accounts) != cfg.UsersPerMachine*2 {
+		return fmt.Errorf("manifest accounts=%d expected=%d", len(manifest.Accounts), cfg.UsersPerMachine*2)
+	}
+	for _, account := range manifest.Accounts {
+		if account.UserID == "" || account.Token == "" || account.P2PTopic == "" {
+			return fmt.Errorf("account %s is incomplete", account.Username)
+		}
+	}
+	if manifest.GroupTopic == "" {
+		return errors.New("manifest group topic is empty")
+	}
+	owner := manifestAccountMap(manifest)[accountKey("a", 1)]
+	count, err := api.countGroupMembers(ctx, *owner, manifest.GroupTopic)
+	if err != nil {
+		return err
+	}
+	if count != len(manifest.Accounts) {
+		return fmt.Errorf("group member count=%d expected=%d", count, len(manifest.Accounts))
+	}
+	log.Printf("verify passed accounts=%d pairs=%d group_members=%d", len(manifest.Accounts), cfg.UsersPerMachine, count)
+	return nil
+}
+
+func parallelAccounts(ctx context.Context, accounts []AccountRecord, workers int, fn func(AccountRecord) error) error {
+	if workers <= 0 {
+		workers = 1
+	}
+	workCtx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	jobs := make(chan AccountRecord)
+	errCh := make(chan error, 1)
+	var wg sync.WaitGroup
+	for i := 0; i < workers; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for account := range jobs {
+				if err := fn(account); err != nil {
+					select {
+					case errCh <- err:
+					default:
+					}
+					cancel()
+					return
+				}
+			}
+		}()
+	}
+	go func() {
+		defer close(jobs)
+		for _, account := range accounts {
+			select {
+			case jobs <- account:
+			case <-workCtx.Done():
+				return
+			}
+		}
+	}()
+	wg.Wait()
+	select {
+	case err := <-errCh:
+		return err
+	default:
+		return ctx.Err()
+	}
+}

loadtest/im-pressure/recorder.go

@@ -0,0 +1,229 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"math/bits"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+)
+
+type observation struct {
+	topic string
+	seq   int64
+	bits  []uint64
+}
+
+type deliveryRecorder struct {
+	machine     string
+	localUsers  int
+	latency     *DurationHistogram
+	disconnects atomic.Int64
+	received    atomic.Int64
+	runID       atomic.Value
+	historyMode atomic.Bool
+
+	mu         sync.Mutex
+	observed   map[string]*observation
+	lastSeq    []map[string]int64
+	duplicates int64
+	outOfOrder int64
+	malformed  int64
+	history    map[string]bool
+}
+
+func newDeliveryRecorder(runID, machine string, localUsers int) *deliveryRecorder {
+	lastSeq := make([]map[string]int64, localUsers)
+	for i := range lastSeq {
+		lastSeq[i] = make(map[string]int64)
+	}
+	recorder := &deliveryRecorder{
+		machine: machine, localUsers: localUsers, latency: NewDurationHistogram(),
+		observed: make(map[string]*observation), lastSeq: lastSeq, history: make(map[string]bool),
+	}
+	recorder.runID.Store(runID)
+	return recorder
+}
+
+func (r *deliveryRecorder) setRunID(runID string) { r.runID.Store(runID) }
+
+func (r *deliveryRecorder) onData(account AccountRecord, data dataPacket) {
+	runID, _ := r.runID.Load().(string)
+	mid, _ := data.Head["x-im-client-mid"].(string)
+	if mid == "" || !strings.HasPrefix(mid, runID+"-") {
+		return
+	}
+	var content struct {
+		RunID  string `json:"run_id"`
+		SentAt int64  `json:"sent_at_unix_nano"`
+	}
+	if json.Unmarshal(data.Content, &content) != nil || content.RunID != runID {
+		r.mu.Lock()
+		r.malformed++
+		r.mu.Unlock()
+		return
+	}
+	if r.historyMode.Load() {
+		r.mu.Lock()
+		r.history[mid] = true
+		r.mu.Unlock()
+		return
+	}
+	index := account.Index - 1
+	if index < 0 || index >= r.localUsers {
+		return
+	}
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	obs := r.observed[mid]
+	if obs == nil {
+		obs = &observation{topic: data.Topic, seq: data.Seq, bits: make([]uint64, (r.localUsers+63)/64)}
+		r.observed[mid] = obs
+	}
+	word, bit := index/64, uint(index%64)
+	mask := uint64(1) << bit
+	if obs.bits[word]&mask != 0 {
+		r.duplicates++
+		return
+	}
+	obs.bits[word] |= mask
+	r.received.Add(1)
+	if previous := r.lastSeq[index][data.Topic]; previous > 0 && data.Seq < previous {
+		r.outOfOrder++
+	}
+	if data.Seq > r.lastSeq[index][data.Topic] {
+		r.lastSeq[index][data.Topic] = data.Seq
+	}
+	if content.SentAt > 0 {
+		r.latency.Observe(time.Since(time.Unix(0, content.SentAt)))
+	}
+}
+
+func (r *deliveryRecorder) metricCounts() (received, duplicates, outOfOrder, disconnects int64) {
+	r.mu.Lock()
+	duplicates = r.duplicates
+	outOfOrder = r.outOfOrder
+	r.mu.Unlock()
+	return r.received.Load(), duplicates, outOfOrder, r.disconnects.Load()
+}
+
+func (r *deliveryRecorder) beginHistory() { r.historyMode.Store(true) }
+
+func (r *deliveryRecorder) historyFound(mid string) bool {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	return r.history[mid]
+}
+
+func (r *deliveryRecorder) onDisconnect(_ AccountRecord, _ error) {
+	r.disconnects.Add(1)
+}
+
+func (r *deliveryRecorder) summarize(base AgentSummary, ledger []AcceptedMessage) AgentSummary {
+	base.DeliveryLatency = r.latency.Snapshot()
+	base.Disconnects = r.disconnects.Load()
+	runID, _ := r.runID.Load().(string)
+	accepted := make(map[string]AcceptedMessage, len(ledger))
+	const missingLimit = 100000
+
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	base.Duplicates = r.duplicates
+	base.OutOfOrder = r.outOfOrder
+	if r.malformed > 0 {
+		base.Errors = append(base.Errors, fmt.Sprintf("malformed run messages=%d", r.malformed))
+	}
+	for _, message := range ledger {
+		if message.RunID != runID {
+			continue
+		}
+		accepted[message.ClientMid] = message
+		obs := r.observed[message.ClientMid]
+		switch message.Kind {
+		case "group":
+			base.ExpectedGroup += int64(r.localUsers)
+			for index := 1; index <= r.localUsers; index++ {
+				if observationHas(obs, index) {
+					base.ReceivedGroup++
+				} else if len(base.Missing) < missingLimit {
+					base.Missing = append(base.Missing, missingFor(message, index))
+				}
+			}
+		case "p2p":
+			if message.PeerMachine == r.machine && message.PeerIndex > 0 {
+				base.ExpectedPeer++
+				if observationHas(obs, message.PeerIndex) {
+					base.ReceivedPeer++
+				} else if len(base.Missing) < missingLimit {
+					base.Missing = append(base.Missing, missingFor(message, message.PeerIndex))
+				}
+			}
+			if message.SenderMachine == r.machine && message.SenderIndex > 0 {
+				base.ExpectedEcho++
+				if observationHas(obs, message.SenderIndex) {
+					base.ReceivedEcho++
+				}
+			}
+		}
+	}
+	for mid, obs := range r.observed {
+		if _, ok := accepted[mid]; !ok {
+			base.UnexpectedMessages += bitsetCount(obs.bits)
+		}
+	}
+	base.RealtimeLost = (base.ExpectedPeer - base.ReceivedPeer) + (base.ExpectedGroup - base.ReceivedGroup)
+	if len(base.Missing) == missingLimit {
+		base.Errors = append(base.Errors, "missing delivery details truncated at 100000")
+	}
+	return base
+}
+
+func (r *deliveryRecorder) countRealtimeLostStored(ledger []AcceptedMessage) int64 {
+	runID, _ := r.runID.Load().(string)
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	var count int64
+	for _, message := range ledger {
+		if message.RunID != runID || !r.history[message.ClientMid] {
+			continue
+		}
+		obs := r.observed[message.ClientMid]
+		switch message.Kind {
+		case "group":
+			for index := 1; index <= r.localUsers; index++ {
+				if !observationHas(obs, index) {
+					count++
+				}
+			}
+		case "p2p":
+			if message.PeerMachine == r.machine && message.PeerIndex > 0 && !observationHas(obs, message.PeerIndex) {
+				count++
+			}
+		}
+	}
+	return count
+}
+
+func observationHas(obs *observation, receiverIndex int) bool {
+	if obs == nil || receiverIndex <= 0 {
+		return false
+	}
+	index := receiverIndex - 1
+	word, bit := index/64, uint(index%64)
+	return word < len(obs.bits) && obs.bits[word]&(uint64(1)<<bit) != 0
+}
+
+func bitsetCount(words []uint64) int64 {
+	var count int64
+	for _, word := range words {
+		count += int64(bits.OnesCount64(word))
+	}
+	return count
+}
+
+func missingFor(message AcceptedMessage, receiver int) MissingDelivery {
+	return MissingDelivery{ClientMid: message.ClientMid, Kind: message.Kind, TopicHash: topicHash(message.Topic),
+		Seq: message.Seq, ReceiverIndex: receiver}
+}

loadtest/im-pressure/recorder_test.go

@@ -0,0 +1,55 @@
+package main
+
+import (
+	"encoding/json"
+	"testing"
+	"time"
+)
+
+func TestDeliveryRecorderSummarizesPeerGroupAndDuplicates(t *testing.T) {
+	recorder := newDeliveryRecorder("run1", "a", 2)
+	peer := AcceptedMessage{RunID: "run1", Kind: "p2p", ClientMid: "run1-p-b-0001-01", Topic: "p2p-one", Seq: 10,
+		SenderMachine: "b", SenderIndex: 1, PeerMachine: "a", PeerIndex: 1}
+	group := AcceptedMessage{RunID: "run1", Kind: "group", ClientMid: "run1-g-b-0001-01", Topic: "grp-one", Seq: 20,
+		SenderMachine: "b", SenderIndex: 1}
+
+	recorder.onData(AccountRecord{Index: 1}, testData(peer, time.Now()))
+	recorder.onData(AccountRecord{Index: 1}, testData(peer, time.Now()))
+	recorder.onData(AccountRecord{Index: 1}, testData(group, time.Now()))
+
+	summary := recorder.summarize(AgentSummary{}, []AcceptedMessage{peer, group})
+	if summary.ExpectedPeer != 1 || summary.ReceivedPeer != 1 {
+		t.Fatalf("peer expected/received = %d/%d", summary.ExpectedPeer, summary.ReceivedPeer)
+	}
+	if summary.ExpectedGroup != 2 || summary.ReceivedGroup != 1 || summary.RealtimeLost != 1 {
+		t.Fatalf("group expected/received/lost = %d/%d/%d", summary.ExpectedGroup, summary.ReceivedGroup, summary.RealtimeLost)
+	}
+	if summary.Duplicates != 1 || len(summary.Missing) != 1 || summary.Missing[0].ReceiverIndex != 2 {
+		t.Fatalf("duplicates=%d missing=%+v", summary.Duplicates, summary.Missing)
+	}
+}
+
+func TestDeliveryRecorderSeparatesHistoryFromRealtime(t *testing.T) {
+	recorder := newDeliveryRecorder("run1", "a", 1)
+	message := AcceptedMessage{RunID: "run1", Kind: "p2p", ClientMid: "run1-p-b-0001-01", Topic: "p2p-one", Seq: 10,
+		SenderMachine: "b", SenderIndex: 1, PeerMachine: "a", PeerIndex: 1}
+	recorder.beginHistory()
+	recorder.onData(AccountRecord{Index: 1}, testData(message, time.Now()))
+
+	summary := recorder.summarize(AgentSummary{}, []AcceptedMessage{message})
+	if summary.ReceivedPeer != 0 || summary.RealtimeLost != 1 {
+		t.Fatalf("history response polluted realtime result: %+v", summary)
+	}
+	if !recorder.historyFound(message.ClientMid) {
+		t.Fatal("history message was not recorded")
+	}
+	if got := recorder.countRealtimeLostStored([]AcceptedMessage{message}); got != 1 {
+		t.Fatalf("realtime lost but persisted=%d", got)
+	}
+}
+
+func testData(message AcceptedMessage, sentAt time.Time) dataPacket {
+	content, _ := json.Marshal(map[string]any{"run_id": message.RunID, "sent_at_unix_nano": sentAt.UnixNano()})
+	return dataPacket{Topic: message.Topic, Seq: message.Seq,
+		Head: map[string]any{"x-im-client-mid": message.ClientMid}, Content: content}
+}

loadtest/im-pressure/wsclient.go

@@ -0,0 +1,376 @@
+package main
+
+import (
+	"context"
+	"crypto/tls"
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/gorilla/websocket"
+)
+
+type ctrlPacket struct {
+	ID     string                     `json:"id"`
+	Topic  string                     `json:"topic"`
+	Code   int                        `json:"code"`
+	Text   string                     `json:"text"`
+	Params map[string]json.RawMessage `json:"params"`
+}
+
+type dataPacket struct {
+	Topic     string          `json:"topic"`
+	From      string          `json:"from"`
+	Timestamp time.Time       `json:"ts"`
+	Seq       int64           `json:"seq"`
+	Head      map[string]any  `json:"head"`
+	Content   json.RawMessage `json:"content"`
+}
+
+type ctrlError struct {
+	Code int
+	Text string
+}
+
+func (e *ctrlError) Error() string {
+	return fmt.Sprintf("ctrl code=%d text=%s", e.Code, e.Text)
+}
+
+type WSClient struct {
+	account        AccountRecord
+	conn           *websocket.Conn
+	commandTimeout time.Duration
+	onData         func(AccountRecord, dataPacket)
+	onDisconnect   func(AccountRecord, error)
+
+	sendMu sync.Mutex
+	mu     sync.Mutex
+	seq    uint64
+	waits  map[string]chan ctrlPacket
+	done   chan struct{}
+	closed atomic.Bool
+}
+
+func connectWSClient(ctx context.Context, cfg *Config, account AccountRecord, apiKey, password string,
+	onData func(AccountRecord, dataPacket), onDisconnect func(AccountRecord, error)) (*WSClient, error) {
+	timeout, err := configDuration(cfg.CommandTimeout)
+	if err != nil {
+		return nil, err
+	}
+	endpoint, err := websocketEndpoint(account.Endpoint, apiKey)
+	if err != nil {
+		return nil, err
+	}
+	dialer := *websocket.DefaultDialer
+	if cfg.InsecureTLS {
+		dialer.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} // #nosec G402: explicit load-test option.
+	}
+	conn, response, err := dialer.DialContext(ctx, endpoint, http.Header{})
+	if err != nil {
+		if response != nil {
+			return nil, fmt.Errorf("websocket dial status=%d: %w", response.StatusCode, err)
+		}
+		return nil, err
+	}
+	client := &WSClient{
+		account: account, conn: conn, commandTimeout: timeout, onData: onData, onDisconnect: onDisconnect,
+		waits: make(map[string]chan ctrlPacket), done: make(chan struct{}),
+	}
+	go client.readLoop()
+	if _, err = client.command(ctx, "hi", map[string]any{"hi": map[string]any{
+		"id": client.nextID("hi"), "ver": "0.25", "ua": "im-pressure/1.0",
+		"platf": "loadtest", "tenant": cfg.Tenant,
+	}}); err != nil {
+		client.Close()
+		return nil, fmt.Errorf("hi %s: %w", account.Username, err)
+	}
+	login := map[string]any{"id": client.nextID("login")}
+	if account.Token != "" {
+		login["scheme"] = "token"
+		login["secret"] = account.Token
+	} else {
+		login["scheme"] = "basic"
+		login["secret"] = base64.StdEncoding.EncodeToString([]byte(account.Username + ":" + password))
+		login["login_type"] = 0
+		login["tenant_code"] = cfg.Tenant
+	}
+	ctrl, err := client.command(ctx, "login", map[string]any{"login": login})
+	if err != nil && account.Token != "" && password != "" {
+		login["id"] = client.nextID("login-basic")
+		login["scheme"] = "basic"
+		login["secret"] = base64.StdEncoding.EncodeToString([]byte(account.Username + ":" + password))
+		login["login_type"] = 0
+		login["tenant_code"] = cfg.Tenant
+		ctrl, err = client.command(ctx, "login", map[string]any{"login": login})
+	}
+	if err != nil {
+		client.Close()
+		return nil, fmt.Errorf("login %s: %w", account.Username, err)
+	}
+	client.applyLogin(ctrl)
+	return client, nil
+}
+
+func connectForPrepare(ctx context.Context, cfg *Config, account AccountRecord, api *apiClient, password, verificationCode string) (*WSClient, bool, error) {
+	timeout, err := configDuration(cfg.CommandTimeout)
+	if err != nil {
+		return nil, false, err
+	}
+	endpoint, err := websocketEndpoint(account.Endpoint, api.apiKey)
+	if err != nil {
+		return nil, false, err
+	}
+	dialer := *websocket.DefaultDialer
+	if cfg.InsecureTLS {
+		dialer.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} // #nosec G402: explicit load-test option.
+	}
+	conn, _, err := dialer.DialContext(ctx, endpoint, http.Header{})
+	if err != nil {
+		return nil, false, err
+	}
+	client := &WSClient{account: account, conn: conn, commandTimeout: timeout, waits: make(map[string]chan ctrlPacket), done: make(chan struct{})}
+	go client.readLoop()
+	if _, err = client.command(ctx, "hi", map[string]any{"hi": map[string]any{
+		"id": client.nextID("hi"), "ver": "0.25", "ua": "im-pressure-prepare/1.0", "tenant": cfg.Tenant,
+	}}); err != nil {
+		client.Close()
+		return nil, false, err
+	}
+	secret := base64.StdEncoding.EncodeToString([]byte(account.Username + ":" + password))
+	ctrl, loginErr := client.command(ctx, "login", map[string]any{"login": map[string]any{
+		"id": client.nextID("login"), "scheme": "basic", "secret": secret,
+		"login_type": 0, "tenant_code": cfg.Tenant,
+	}})
+	created := false
+	if loginErr != nil {
+		var ce *ctrlError
+		if !errors.As(loginErr, &ce) || ce.Code != http.StatusUnauthorized {
+			client.Close()
+			return nil, false, loginErr
+		}
+		credential, credentialErr := api.createRegistrationCredential(ctx, account, verificationCode)
+		if credentialErr != nil {
+			client.Close()
+			return nil, false, credentialErr
+		}
+		acc := map[string]any{
+			"id": client.nextID("acc"), "user": "new", "scheme": "basic", "username": account.Username,
+			"registration_type": 0, "tenant_code": cfg.Tenant, "secret": secret, "login": true,
+			"desc": map[string]any{"public": map[string]any{"fn": strings.ToUpper(account.Username)}},
+		}
+		if credential != nil {
+			acc["phone"] = account.Phone
+			acc["cred"] = []map[string]any{{"meth": "tel", "val": account.Phone,
+				"resp": credential.VerificationToken, "params": map[string]any{"challenge_id": credential.ChallengeID}}}
+		}
+		ctrl, err = client.command(ctx, "acc", map[string]any{"acc": acc})
+		if err != nil {
+			client.Close()
+			return nil, false, err
+		}
+		created = true
+	}
+	client.applyLogin(ctrl)
+	return client, created, nil
+}
+
+func websocketEndpoint(raw, apiKey string) (string, error) {
+	u, err := url.Parse(strings.TrimSpace(raw))
+	if err != nil {
+		return "", err
+	}
+	if u.Scheme != "ws" && u.Scheme != "wss" {
+		return "", fmt.Errorf("unsupported websocket scheme %q", u.Scheme)
+	}
+	q := u.Query()
+	if q.Get("apikey") == "" {
+		q.Set("apikey", apiKey)
+	}
+	u.RawQuery = q.Encode()
+	return u.String(), nil
+}
+
+func (c *WSClient) applyLogin(ctrl ctrlPacket) {
+	if raw := ctrl.Params["user"]; len(raw) > 0 {
+		_ = json.Unmarshal(raw, &c.account.UserID)
+	}
+	if raw := ctrl.Params["token"]; len(raw) > 0 {
+		_ = json.Unmarshal(raw, &c.account.Token)
+	}
+	if raw := ctrl.Params["expires"]; len(raw) > 0 {
+		var value string
+		if json.Unmarshal(raw, &value) == nil {
+			c.account.TokenExpires, _ = time.Parse(time.RFC3339Nano, value)
+		}
+	}
+}
+
+func (c *WSClient) Account() AccountRecord { return c.account }
+
+func (c *WSClient) Subscribe(ctx context.Context, topic string) error {
+	if topic == "" {
+		return errors.New("empty topic")
+	}
+	id := c.nextID("sub")
+	_, err := c.command(ctx, "sub", map[string]any{"sub": map[string]any{
+		"id": id, "topic": topic, "get": map[string]any{"what": "desc"},
+	}})
+	return err
+}
+
+func (c *WSClient) GetData(ctx context.Context, topic string, since, before int64, limit int) error {
+	if topic == "" || since <= 0 || before <= since || limit <= 0 {
+		return errors.New("invalid get data range")
+	}
+	id := c.nextID("get-data")
+	_, err := c.command(ctx, "get", map[string]any{"get": map[string]any{
+		"id": id, "topic": topic, "what": "data",
+		"data": map[string]any{"since": since, "before": before, "limit": limit},
+	}})
+	return err
+}
+
+func (c *WSClient) Publish(ctx context.Context, topic, clientMid, runID, scenario string, messageIndex int) (AcceptedMessage, error) {
+	id := c.nextID("pub")
+	sentAt := time.Now().UTC()
+	ctrl, err := c.command(ctx, "pub", map[string]any{"pub": map[string]any{
+		"id": id, "topic": topic, "noecho": false,
+		"head": map[string]any{"mime": "application/json", "x-im-schema": 1, "x-im-type": "text", "x-im-client-mid": clientMid},
+		"content": map[string]any{"text": "loadtest", "run_id": runID, "scenario": scenario,
+			"sender_index": c.account.Index, "message_index": messageIndex, "sent_at_unix_nano": sentAt.UnixNano()},
+	}})
+	if err != nil {
+		return AcceptedMessage{}, err
+	}
+	seq, err := ctrlInt64(ctrl.Params["seq"])
+	if err != nil || seq <= 0 {
+		return AcceptedMessage{}, fmt.Errorf("publish ACK has invalid seq: %w", err)
+	}
+	kind := "p2p"
+	if strings.HasPrefix(topic, "grp") {
+		kind = "group"
+	}
+	return AcceptedMessage{RunID: runID, Scenario: scenario, Kind: kind, ClientMid: clientMid,
+		Topic: topic, Seq: seq, SenderMachine: c.account.Machine, SenderIndex: c.account.Index,
+		SentAt: sentAt, AckAt: time.Now().UTC()}, nil
+}
+
+func ctrlInt64(raw json.RawMessage) (int64, error) {
+	if len(raw) == 0 {
+		return 0, errors.New("missing value")
+	}
+	var number json.Number
+	if err := json.Unmarshal(raw, &number); err == nil {
+		return strconv.ParseInt(number.String(), 10, 64)
+	}
+	var text string
+	if err := json.Unmarshal(raw, &text); err != nil {
+		return 0, err
+	}
+	return strconv.ParseInt(text, 10, 64)
+}
+
+func (c *WSClient) command(ctx context.Context, _ string, message map[string]any) (ctrlPacket, error) {
+	id := nestedMessageID(message)
+	if id == "" {
+		return ctrlPacket{}, errors.New("command id is required")
+	}
+	wait := make(chan ctrlPacket, 1)
+	c.mu.Lock()
+	c.waits[id] = wait
+	c.mu.Unlock()
+	c.sendMu.Lock()
+	err := c.conn.WriteJSON(message)
+	c.sendMu.Unlock()
+	if err != nil {
+		c.removeWait(id)
+		return ctrlPacket{}, err
+	}
+	timer := time.NewTimer(c.commandTimeout)
+	defer timer.Stop()
+	select {
+	case ctrl := <-wait:
+		if ctrl.Code < 200 || ctrl.Code >= 300 {
+			return ctrl, &ctrlError{Code: ctrl.Code, Text: ctrl.Text}
+		}
+		return ctrl, nil
+	case <-timer.C:
+		c.removeWait(id)
+		return ctrlPacket{}, fmt.Errorf("ctrl timeout id=%s", id)
+	case <-ctx.Done():
+		c.removeWait(id)
+		return ctrlPacket{}, ctx.Err()
+	case <-c.done:
+		c.removeWait(id)
+		return ctrlPacket{}, errors.New("websocket closed")
+	}
+}
+
+func nestedMessageID(message map[string]any) string {
+	for _, value := range message {
+		if body, ok := value.(map[string]any); ok {
+			if id, ok := body["id"].(string); ok {
+				return id
+			}
+		}
+	}
+	return ""
+}
+
+func (c *WSClient) nextID(prefix string) string {
+	n := atomic.AddUint64(&c.seq, 1)
+	return fmt.Sprintf("%s-%s-%04d-%d", prefix, c.account.Machine, c.account.Index, n)
+}
+
+func (c *WSClient) removeWait(id string) {
+	c.mu.Lock()
+	delete(c.waits, id)
+	c.mu.Unlock()
+}
+
+func (c *WSClient) readLoop() {
+	defer close(c.done)
+	for {
+		var envelope map[string]json.RawMessage
+		if err := c.conn.ReadJSON(&envelope); err != nil {
+			if !c.closed.Load() && c.onDisconnect != nil {
+				c.onDisconnect(c.account, err)
+			}
+			return
+		}
+		if raw := envelope["ctrl"]; len(raw) > 0 {
+			var ctrl ctrlPacket
+			if json.Unmarshal(raw, &ctrl) == nil && ctrl.ID != "" {
+				c.mu.Lock()
+				wait := c.waits[ctrl.ID]
+				delete(c.waits, ctrl.ID)
+				c.mu.Unlock()
+				if wait != nil {
+					wait <- ctrl
+				}
+			}
+			continue
+		}
+		if raw := envelope["data"]; len(raw) > 0 && c.onData != nil {
+			var data dataPacket
+			if json.Unmarshal(raw, &data) == nil {
+				c.onData(c.account, data)
+			}
+		}
+	}
+}
+
+func (c *WSClient) Close() {
+	if c == nil || !c.closed.CompareAndSwap(false, true) {
+		return
+	}
+	_ = c.conn.Close()
+}

monitoring/README.md

@@ -5,3 +5,5 @@ This directory contains code related to monitoring Tinode server. Supported moni
 * [InfluxDB](https://www.influxdata.com/)
 
 See [exporter/README](./exporter/README.md) for more details.
+
+`im-server-alerts.yaml` contains the P0/P1 cluster routing, WebSocket balance, Kafka lag, backlog age, and dead-letter alert rules for a Prometheus Operator deployment. Apply it only after the cluster has the `PrometheusRule` CRD installed.

monitoring/im-server-alerts.yaml

@@ -0,0 +1,71 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: im-server-p0-p1
+  namespace: im-server-system
+spec:
+  groups:
+    - name: im-server-cluster
+      rules:
+        - alert: IMClusterP2MQueueHigh
+          expr: im_cluster_p2m_queue_length / clamp_min(im_cluster_p2m_queue_capacity, 1) > 0.70
+          for: 30s
+          labels:
+            severity: warning
+          annotations:
+            summary: "IM cluster proxy queue is above 70%"
+        - alert: IMClusterP2MQueueCritical
+          expr: im_cluster_p2m_queue_length / clamp_min(im_cluster_p2m_queue_capacity, 1) > 0.90
+          for: 30s
+          labels:
+            severity: critical
+          annotations:
+            summary: "IM cluster proxy queue is above 90%"
+        - alert: IMClusterEnqueueTimeout
+          expr: sum by (pod, peer) (increase(im_cluster_p2m_enqueue_total{result="timeout"}[1m])) > 0
+          labels:
+            severity: critical
+          annotations:
+            summary: "IM cluster rejected proxy-to-master enqueue"
+        - alert: IMClusterRouteRejected
+          expr: sum by (pod, peer, result) (increase(im_cluster_route_total{result=~"hub_full|peer_rejected"}[1m])) > 0
+          labels:
+            severity: critical
+          annotations:
+            summary: "IM cluster route was rejected"
+        - alert: IMClusterPeerUnavailable
+          expr: im_cluster_live_peers < im_cluster_expected_peers
+          for: 15s
+          labels:
+            severity: critical
+          annotations:
+            summary: "IM server has unavailable cluster peers"
+        - alert: IMWebSocketConnectionsImbalanced
+          expr: max(im_ws_connections_active) / clamp_min(avg(im_ws_connections_active), 1) > 1.30
+          for: 2m
+          labels:
+            severity: warning
+          annotations:
+            summary: "IM WebSocket connections are imbalanced across pods"
+    - name: im-server-persistence
+      rules:
+        - alert: IMKafkaPersistDeadLetter
+          expr: sum by (kind, reason) (increase(im_kafka_dead_letters_total[5m])) > 0
+          labels:
+            severity: critical
+          annotations:
+            summary: "IM Kafka persistence produced dead letters"
+        - alert: IMKafkaConsumerLag
+          expr: max by (pod, topic, group) (im_kafka_consumer_lag) > 1000
+          for: 2m
+          labels:
+            severity: warning
+          annotations:
+            summary: "IM Kafka consumer lag is above 1000"
+        - alert: IMKafkaBacklogOld
+          expr: max by (pod, topic, group) (im_kafka_backlog_oldest_age_seconds) > 60
+          for: 2m
+          labels:
+            severity: critical
+          annotations:
+            summary: "IM Kafka persistence backlog is older than 60 seconds"

scripts/IM-租户业务库初始化.sql

@@ -1104,7 +1104,7 @@ CREATE TABLE IF NOT EXISTS `im_message_revisions` (
     ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
 
 CREATE TABLE IF NOT EXISTS `im_messages` (
-                                             `id` int NOT NULL AUTO_INCREMENT,
+                                             `id` bigint NOT NULL AUTO_INCREMENT,
                                              `createdat` datetime(3) NOT NULL,
     `updatedat` datetime(3) NOT NULL,
     `deletedat` datetime(3) DEFAULT NULL,

server/auth_audit_test.go

@@ -89,6 +89,19 @@ func TestGroupMessageQuerySupportsOperationalFilters(t *testing.T) {
 	}
 }
 
+func TestLegacyGroupMessagesUseShardDirectory(t *testing.T) {
+	query := strings.ToLower(internalLegacyGroupMessagesSelect(types.TenantID(9)))
+	if strings.Contains(query, "im_group_message_topic_shards") {
+		t.Fatalf("legacy group message query must not use removed topic shard routes: %s", query)
+	}
+	if !strings.Contains(query, "im_group_message_shards") {
+		t.Fatalf("legacy group message query must use group message shard directory: %s", query)
+	}
+	if !strings.Contains(query, "legacy.created_at>=route.shard_start") || !strings.Contains(query, "legacy.created_at<route.shard_end") {
+		t.Fatalf("legacy group message query must exclude rows covered by shard windows: %s", query)
+	}
+}
+
 func TestLoginAuditQuerySpecsExposeUserIDAsString(t *testing.T) {
 	spec := internalTenantResourceSpecs["user-login-logs"]
 	if spec == nil {

server/cluster.go

@@ -4,6 +4,7 @@ import (
 	"encoding/gob"
 	"encoding/json"
 	"errors"
+	"hash/fnv"
 	"net"
 	"net/rpc"
 	"sort"
@@ -14,6 +15,7 @@ import (
 	"github.com/tinode/chat/server/auth"
 	"github.com/tinode/chat/server/concurrency"
 	"github.com/tinode/chat/server/logs"
+	immetrics "github.com/tinode/chat/server/metrics"
 	"github.com/tinode/chat/server/push"
 	rh "github.com/tinode/chat/server/ringhash"
 	"github.com/tinode/chat/server/store/types"
@@ -26,17 +28,20 @@ const (
 	clusterDefaultReconnectTime = 200 * time.Millisecond
 	// Number of replicas in ringhash.
 	clusterHashReplicas             = 20
-	// Buffer size for sending requests from proxy to master.
+	clusterDefaultP2MBuffer         = 4096
-	clusterProxyToMasterBuffer = 64
+	clusterDefaultP2MWorkers        = 16
-	// Expand buffer size by this value for nodes over the basic 3-node setup.
+	clusterDefaultP2MTimeout        = 300 * time.Millisecond
-	clusterProxyToMasterBufferPerNode = 16
+	clusterDefaultRouteRetries      = 3
-	// Timeout for attempting to enqueue a proxy-to-master request when the buffer is full.
+	clusterDefaultRouteRetryBackoff = 25 * time.Millisecond
-	clusterP2MTimeout = 20 * time.Millisecond
 	// Buffer size for receiving responses from other nodes, per node.
 	clusterRpcCompletionBuffer = 64
 )
 
-var errClusterTopicOwnerUnavailable = errors.New("cluster topic owner unavailable")
+var (
+	errClusterTopicOwnerUnavailable = errors.New("cluster topic owner unavailable")
+	errClusterOverloaded            = errors.New("cluster overloaded")
+	errClusterLoadExceeded          = errors.New("cluster load exceeded")
+)
 
 // ProxyReqType is the type of proxy requests.
 type ProxyReqType int
@@ -65,6 +70,11 @@ type clusterConfig struct {
 	ThisName string `json:"self"`
 	// Deprecated: this field is no longer used.
 	NumProxyEventGoRoutines       int `json:"-"`
+	ProxyToMasterBuffer           int `json:"proxy_to_master_buffer"`
+	ProxyToMasterWorkers          int `json:"proxy_to_master_workers"`
+	ProxyToMasterEnqueueTimeoutMS int `json:"proxy_to_master_enqueue_timeout_ms"`
+	RouteRetries                  int `json:"route_retries"`
+	RouteRetryBackoffMS           int `json:"route_retry_backoff_ms"`
 	// Failover configuration
 	Failover *clusterFailoverConfig
 }
@@ -99,8 +109,13 @@ type ClusterNode struct {
 	// Buffered, clusterRpcCompletionBuffer * number_of_nodes.
 	rpcDone chan *rpc.Call
 
-	// Channel for sending proxy to master requests; buffered, clusterProxyToMasterBuffer.
+	// Sharded queues preserve ordering for the same tenant and topic while allowing parallel RPCs.
-	p2mSender chan *ClusterReq
+	p2mSenders         []chan *ClusterReq
+	p2mEnqueueTimeout  time.Duration
+	routeRetries       int
+	routeRetryBackoff  time.Duration
+	overloadLogAt      atomic.Int64
+	overloadSuppressed atomic.Int64
 }
 
 func (n *ClusterNode) asyncRpcLoop() {
@@ -109,17 +124,49 @@ func (n *ClusterNode) asyncRpcLoop() {
 	}
 }
 
-func (n *ClusterNode) p2mSenderLoop() {
+func (n *ClusterNode) p2mSenderLoop(sender <-chan *ClusterReq) {
-	for req := range n.p2mSender {
+	for req := range sender {
 		if req == nil {
 			// Stop
 			return
 		}
 
 		if err := n.proxyToMaster(req); err != nil {
+			if errors.Is(err, errClusterOverloaded) {
+				n.logOverload("proxy_to_master", req.TenantID, req.RcptTo, 0)
+			} else {
 				logs.Warn.Println("p2mSenderLoop: call failed", n.name, err)
 			}
 		}
+		n.updateP2MQueueMetric()
+	}
+}
+
+func (n *ClusterNode) logOverload(path string, tenantID types.TenantID, topic string, seq int64) {
+	now := time.Now().Unix()
+	previous := n.overloadLogAt.Load()
+	if previous == now || !n.overloadLogAt.CompareAndSwap(previous, now) {
+		n.overloadSuppressed.Add(1)
+		return
+	}
+	suppressed := n.overloadSuppressed.Swap(0)
+	logs.Warn.Printf("cluster overload: path=%s tenant=%d topic=%s seq=%d peer=%s suppressed=%d",
+		path, tenantID, topic, seq, n.name, suppressed)
+}
+
+func (n *ClusterNode) updateP2MQueueMetric() {
+	depth := 0
+	for _, sender := range n.p2mSenders {
+		depth += len(sender)
+	}
+	immetrics.ClusterP2MQueueLength.WithLabelValues(clusterPodName(), n.name).Set(float64(depth))
+}
+
+func clusterPodName() string {
+	if globals.cluster != nil && globals.cluster.thisNodeName != "" {
+		return globals.cluster.thisNodeName
+	}
+	return "standalone"
 }
 
 // ClusterSess is a basic info on a remote session where the message was created.
@@ -318,7 +365,12 @@ func (n *ClusterNode) call(proc string, req, resp any) error {
 		return errors.New("cluster: node '" + n.name + "' not connected")
 	}
 
+	method := proc
+	started := time.Now()
+	immetrics.ClusterRPCInflight.WithLabelValues(clusterPodName(), n.name, method).Inc()
+	defer immetrics.ClusterRPCInflight.WithLabelValues(clusterPodName(), n.name, method).Dec()
 	if err := n.endpoint.Call(proc, req, resp); err != nil {
+		immetrics.ClusterRPCDuration.WithLabelValues(clusterPodName(), n.name, method, "error").Observe(time.Since(started).Seconds())
 		logs.Warn.Println("cluster: call failed", n.name, err)
 
 		n.lock.Lock()
@@ -331,6 +383,7 @@ func (n *ClusterNode) call(proc string, req, resp any) error {
 		n.lock.Unlock()
 		return err
 	}
+	immetrics.ClusterRPCDuration.WithLabelValues(clusterPodName(), n.name, method, "ok").Observe(time.Since(started).Seconds())
 
 	return nil
 }
@@ -395,29 +448,47 @@ func (n *ClusterNode) proxyToMaster(msg *ClusterReq) error {
 	var rejected bool
 	err := n.call("Cluster.TopicMaster", msg, &rejected)
 	if err == nil && rejected {
-		err = errors.New("cluster: topic master node out of sync")
+		err = errClusterOverloaded
 	}
 	return err
 }
 
 // proxyToMaster forwards request from topic proxy to topic master.
 func (n *ClusterNode) proxyToMasterAsync(msg *ClusterReq) error {
+	if len(n.p2mSenders) == 0 {
+		return errClusterTopicOwnerUnavailable
+	}
+	sender := n.p2mSenders[clusterReqShard(msg, len(n.p2mSenders))]
 	select {
-	case n.p2mSender <- msg:
+	case sender <- msg:
+		n.updateP2MQueueMetric()
+		immetrics.ClusterP2MEnqueueTotal.WithLabelValues(clusterPodName(), n.name, "ok").Inc()
 		return nil
 	default:
 	}
 	// Buffer is full. Wait briefly before giving up.
-	timer := time.NewTimer(clusterP2MTimeout)
+	timer := time.NewTimer(n.p2mEnqueueTimeout)
 	defer timer.Stop()
 	select {
-	case n.p2mSender <- msg:
+	case sender <- msg:
+		n.updateP2MQueueMetric()
+		immetrics.ClusterP2MEnqueueTotal.WithLabelValues(clusterPodName(), n.name, "ok").Inc()
 		return nil
 	case <-timer.C:
-		return errors.New("cluster: load exceeded")
+		immetrics.ClusterP2MEnqueueTotal.WithLabelValues(clusterPodName(), n.name, "timeout").Inc()
+		return errClusterLoadExceeded
 	}
 }
 
+func clusterReqShard(msg *ClusterReq, count int) int {
+	if count <= 1 || msg == nil {
+		return 0
+	}
+	h := fnv.New32a()
+	_, _ = h.Write([]byte(types.TopicKey{TenantID: msg.TenantID, Topic: msg.RcptTo}.RoutingKey()))
+	return int(h.Sum32() % uint32(count))
+}
+
 // masterToProxyAsync forwards response from topic master to topic proxy
 // in a fire-and-forget manner.
 func (n *ClusterNode) masterToProxyAsync(msg *ClusterResp) error {
@@ -430,8 +501,32 @@ func (n *ClusterNode) masterToProxyAsync(msg *ClusterResp) error {
 
 // route routes server message within the cluster.
 func (n *ClusterNode) route(msg *ClusterRoute) error {
-	var unused bool
+	for attempt := 0; attempt <= n.routeRetries; attempt++ {
-	return n.call("Cluster.Route", msg, &unused)
+		var rejected bool
+		err := n.call("Cluster.Route", msg, &rejected)
+		if err != nil {
+			immetrics.ClusterRouteTotal.WithLabelValues(clusterPodName(), n.name, "disconnected").Inc()
+			return err
+		}
+		if !rejected {
+			immetrics.ClusterRouteTotal.WithLabelValues(clusterPodName(), n.name, "ok").Inc()
+			return nil
+		}
+		immetrics.ClusterRouteTotal.WithLabelValues(clusterPodName(), n.name, "peer_rejected").Inc()
+		if attempt < n.routeRetries {
+			time.Sleep(n.routeRetryBackoff * time.Duration(1<<attempt))
+		}
+	}
+	seq := int64(0)
+	topic := ""
+	if msg != nil && msg.SrvMsg != nil && msg.SrvMsg.Data != nil {
+		seq = msg.SrvMsg.Data.SeqId
+	}
+	if msg != nil && msg.SrvMsg != nil {
+		topic = msg.SrvMsg.RcptTo
+	}
+	n.logOverload("route", msg.TenantID, topic, seq)
+	return errClusterOverloaded
 }
 
 // Cluster is the representation of the cluster.
@@ -460,6 +555,8 @@ type Cluster struct {
 	// running a separate event processing goroutine for each proxy session
 	// leads to a rather large memory usage and excessive scheduling overhead.
 	proxyEventQueue *concurrency.GoRoutinePool
+	p2mBuffer       int
+	p2mWorkers      int
 }
 
 func (n *ClusterNode) stopMultiplexingSession(msess *Session) {
@@ -600,7 +697,13 @@ func (c *Cluster) TopicMaster(msg *ClusterReq, rejected *bool) error {
 		select {
 		case globals.hub.routeCli <- msg.CliMsg:
 		default:
-			logs.Err.Println("cluster: route req failed - hub.route queue full")
+			*rejected = true
+			if sess != nil {
+				reply := ErrServiceUnavailableReply(msg.CliMsg, msg.CliMsg.Timestamp)
+				reply.Ctrl.Text = "cluster overloaded"
+				sess.queueOut(reply)
+			}
+			node.logOverload("topic_master", msg.TenantID, msg.RcptTo, 0)
 		}
 
 	case ProxyReqBgSession, ProxyReqMeUserAgent:
@@ -681,7 +784,9 @@ func (c *Cluster) Route(msg *ClusterRoute, rejected *bool) error {
 
 	select {
 	case globals.hub.routeSrv <- msg.SrvMsg:
+		immetrics.ClusterRouteTotal.WithLabelValues(clusterPodName(), msg.Node, "ok").Inc()
 	default:
+		immetrics.ClusterRouteTotal.WithLabelValues(clusterPodName(), msg.Node, "hub_full").Inc()
 		logError("cluster Route: server busy")
 	}
 	return nil
@@ -1016,6 +1121,26 @@ func clusterInit(configString json.RawMessage, self *string) int {
 	if err := json.Unmarshal(configString, &config); err != nil {
 		logs.Err.Fatal(err)
 	}
+	if config.ProxyToMasterBuffer <= 0 {
+		config.ProxyToMasterBuffer = clusterDefaultP2MBuffer
+	}
+	if config.ProxyToMasterWorkers <= 0 {
+		config.ProxyToMasterWorkers = clusterDefaultP2MWorkers
+	}
+	if config.ProxyToMasterWorkers > config.ProxyToMasterBuffer {
+		config.ProxyToMasterWorkers = config.ProxyToMasterBuffer
+	}
+	if config.ProxyToMasterEnqueueTimeoutMS <= 0 {
+		config.ProxyToMasterEnqueueTimeoutMS = int(clusterDefaultP2MTimeout / time.Millisecond)
+	}
+	if config.RouteRetries < 0 {
+		config.RouteRetries = 0
+	} else if config.RouteRetries == 0 {
+		config.RouteRetries = clusterDefaultRouteRetries
+	}
+	if config.RouteRetryBackoffMS <= 0 {
+		config.RouteRetryBackoffMS = int(clusterDefaultRouteRetryBackoff / time.Millisecond)
+	}
 
 	thisName := *self
 	if thisName == "" {
@@ -1039,6 +1164,8 @@ func clusterInit(configString json.RawMessage, self *string) int {
 		fingerprint:     time.Now().Unix(),
 		nodes:           make(map[string]*ClusterNode),
 		proxyEventQueue: concurrency.NewGoRoutinePool(len(config.Nodes) * 5),
+		p2mBuffer:       config.ProxyToMasterBuffer,
+		p2mWorkers:      config.ProxyToMasterWorkers,
 	}
 
 	var nodeNames []string
@@ -1056,6 +1183,9 @@ func clusterInit(configString json.RawMessage, self *string) int {
 			name:              host.Name,
 			done:              make(chan bool, 1),
 			msess:             make(map[string]struct{}),
+			p2mEnqueueTimeout: time.Duration(config.ProxyToMasterEnqueueTimeoutMS) * time.Millisecond,
+			routeRetries:      config.RouteRetries,
+			routeRetryBackoff: time.Duration(config.RouteRetryBackoffMS) * time.Millisecond,
 		}
 	}
 
@@ -1113,17 +1243,18 @@ func (c *Cluster) start() {
 		logs.Err.Fatal(err)
 	}
 
-	var bufferSize = clusterProxyToMasterBuffer
-	if len(c.nodes) > 2 {
-		// Expand the buffer for larger (>3 node) clusters.
-		bufferSize += clusterProxyToMasterBufferPerNode * (len(c.nodes) - 2)
-	}
 	for _, n := range c.nodes {
 		go n.reconnect()
 		n.rpcDone = make(chan *rpc.Call, len(c.nodes)*clusterRpcCompletionBuffer)
-		n.p2mSender = make(chan *ClusterReq, bufferSize)
+		workers := c.p2mWorkers
+		perWorker := (c.p2mBuffer + workers - 1) / workers
+		n.p2mSenders = make([]chan *ClusterReq, workers)
 		go n.asyncRpcLoop()
-		go n.p2mSenderLoop()
+		for i := range n.p2mSenders {
+			n.p2mSenders[i] = make(chan *ClusterReq, perWorker)
+			go n.p2mSenderLoop(n.p2mSenders[i])
+		}
+		immetrics.ClusterP2MQueueCapacity.WithLabelValues(clusterPodName(), n.name).Set(float64(perWorker * workers))
 	}
 
 	if c.fo != nil {
@@ -1147,7 +1278,9 @@ func (c *Cluster) shutdown() {
 	}
 	for _, n := range c.nodes {
 		close(n.rpcDone)
-		close(n.p2mSender)
+		for _, sender := range n.p2mSenders {
+			close(sender)
+		}
 	}
 
 	globals.cluster.proxyEventQueue.Stop()

server/cluster_overload_test.go

@@ -0,0 +1,35 @@
+package main
+
+import (
+	"errors"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/tinode/chat/server/store/types"
+)
+
+func TestClusterReqShardIsStablePerTenantTopic(t *testing.T) {
+	req := &ClusterReq{TenantID: types.TenantID(6), RcptTo: "p2p-topic"}
+	want := clusterReqShard(req, 16)
+	for i := 0; i < 100; i++ {
+		if got := clusterReqShard(req, 16); got != want {
+			t.Fatalf("unstable shard: got=%d want=%d", got, want)
+		}
+	}
+	if got := clusterReqShard(req, 1); got != 0 {
+		t.Fatalf("single worker shard=%d", got)
+	}
+}
+
+func TestClusterRouteErrorReplySeparatesOverloadFromDisconnect(t *testing.T) {
+	msg := &ClientComMessage{Id: "test", Timestamp: time.Now()}
+	overload := clusterRouteErrorReply(msg, errClusterLoadExceeded)
+	if overload.Ctrl == nil || overload.Ctrl.Code != http.StatusServiceUnavailable || overload.Ctrl.Text != "cluster overloaded" {
+		t.Fatalf("unexpected overload reply: %+v", overload.Ctrl)
+	}
+	disconnected := clusterRouteErrorReply(msg, errors.New("disconnected"))
+	if disconnected.Ctrl == nil || disconnected.Ctrl.Code != http.StatusBadGateway {
+		t.Fatalf("unexpected disconnected reply: %+v", disconnected.Ctrl)
+	}
+}

server/db/adapter.go

@@ -260,6 +260,16 @@ type TenantRoutedAdapter interface {
 	TenantAdapterFor(t.TenantID) (TenantBusinessAdapter, TenantAuthAdapter, error)
 }
 
+type TenantMessageBatchAdapter interface {
+	TenantMessageSaveBatch(t.TenantID, []*t.Message) error
+	TenantGroupMessageSaveBatch(t.TenantID, []*t.Message) error
+}
+
+// TenantMessageSchemaAdapter validates storage contracts required by asynchronous persistence.
+type TenantMessageSchemaAdapter interface {
+	TenantMessageSchemaValidate([]t.TenantID) error
+}
+
 // TenantHTTPInfrastructureAdapter is the optional durable infrastructure used
 // by Kafka dead-letter handling.
 type TenantHTTPInfrastructureAdapter interface {

server/db/mysql/adapter.go

@@ -4615,6 +4615,10 @@ func (a *adapter) FileUserHasAccess(fid string, uid t.Uid) (bool, error) {
 		SELECT COUNT(*) FROM im_fileuploads AS fu
 		WHERE (fu.id=? OR fu.location=? OR fu.location LIKE ?) AND fu.status=? AND (
 			fu.userid=?
+			OR EXISTS (
+				SELECT 1 FROM im_filemsglinks AS fml
+				WHERE fml.fileid=fu.id AND fml.topic LIKE 'cvp%'
+			)
 			OR EXISTS (
 				SELECT 1 FROM im_filemsglinks AS fml
 				WHERE fml.fileid=fu.id AND fml.userid=?

server/db/mysql/group_message_shards.go

@@ -138,6 +138,17 @@ func groupMessageTablesForSeq(q sqlx.QueryerContext, ctx context.Context, tenant
 			return nil, err
 		}
 	}
+	tables := groupMessageShardTablesForSeq(tenantID, routes, seq)
+	if len(tables) == 0 && cached {
+		if routes, err = groupMessageAllShardRoutes(q, ctx, tenantID); err != nil {
+			return nil, err
+		}
+		tables = groupMessageShardTablesForSeq(tenantID, routes, seq)
+	}
+	return tables, nil
+}
+
+func groupMessageShardTablesForSeq(tenantID t.TenantID, routes []store.GroupMessageShardRoute, seq int64) []string {
 	tables := make([]string, 0, 2)
 	for _, route := range routes {
 		if route.MinSeq <= seq && route.MaxSeq >= seq {
@@ -146,7 +157,7 @@ func groupMessageTablesForSeq(q sqlx.QueryerContext, ctx context.Context, tenant
 			}
 		}
 	}
-	return tables, nil
+	return tables
 }
 
 func groupMessageAllShardRoutes(q sqlx.QueryerContext, ctx context.Context, tenantID t.TenantID) ([]store.GroupMessageShardRoute, error) {

server/db/mysql/group_schema_test.go

@@ -7,6 +7,7 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/tinode/chat/server/store"
 	types "github.com/tinode/chat/server/store/types"
 )
 
@@ -75,6 +76,21 @@ func TestGroupMessageShardTableRejectsUnsafeMonth(t *testing.T) {
 	}
 }
 
+func TestGroupMessageShardTablesForSeqFiltersRoutes(t *testing.T) {
+	routes := []store.GroupMessageShardRoute{
+		{Month: "202608", MinSeq: 1, MaxSeq: 20},
+		{Month: "202609", MinSeq: 21, MaxSeq: 100},
+		{Month: "202613", MinSeq: 1, MaxSeq: 1000},
+	}
+	tables := groupMessageShardTablesForSeq(7, routes, 23)
+	if len(tables) != 1 || tables[0] != "im_007_group_messages_202609" {
+		t.Fatalf("unexpected shard tables for seq: %#v", tables)
+	}
+	if tables = groupMessageShardTablesForSeq(7, routes, 101); len(tables) != 0 {
+		t.Fatalf("unexpected shard tables for uncovered seq: %#v", tables)
+	}
+}
+
 func TestTenantPartitionTableNames(t *testing.T) {
 	tests := []struct {
 		tenantID types.TenantID

server/db/mysql/tenant_adapter.go

@@ -8,7 +8,9 @@ import (
 	"crypto/sha256"
 	"database/sql"
 	"encoding/json"
+	"errors"
 	"fmt"
+	"regexp"
 	"sort"
 	"strconv"
 	"strings"
@@ -2249,6 +2251,96 @@ func (a *adapter) TenantMessageSave(id t.TenantID, msg *t.Message) error {
 	return err
 }
 
+func (a *adapter) TenantMessageSchemaValidate(tenantIDs []t.TenantID) error {
+	if len(tenantIDs) == 0 {
+		return errors.New("mysql: no active tenants available for message schema validation")
+	}
+	validated := make(map[*adapter]map[string]struct{})
+	for _, tenantID := range tenantIDs {
+		tenantAdp, err := a.routedTenantAdapter(tenantID)
+		if err != nil {
+			return fmt.Errorf("mysql: resolve tenant %d for message schema validation: %w", tenantID, err)
+		}
+		table := tenantMessagesTable(tenantID)
+		if tenantAdp.tenantScopedDB {
+			table = "im_messages"
+		}
+		if validated[tenantAdp] == nil {
+			validated[tenantAdp] = make(map[string]struct{})
+		}
+		if _, ok := validated[tenantAdp][table]; ok {
+			continue
+		}
+		if err = tenantAdp.validateMessageIDColumn(table); err != nil {
+			return err
+		}
+		validated[tenantAdp][table] = struct{}{}
+	}
+	if !a.tenantScopedDB {
+		if err := a.validateMessageIDColumn("im_messages"); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (a *adapter) validateMessageIDColumn(table string) error {
+	if !regexp.MustCompile(`^im_(?:[0-9]+_)?messages$`).MatchString(table) {
+		return fmt.Errorf("mysql: unsafe message table name %q", table)
+	}
+	var dataType, nullable, extra string
+	err := a.db.QueryRow(`SELECT DATA_TYPE,IS_NULLABLE,EXTRA FROM information_schema.COLUMNS
+		WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? AND COLUMN_NAME='id'`, table).
+		Scan(&dataType, &nullable, &extra)
+	if err == sql.ErrNoRows {
+		return fmt.Errorf("mysql: required message table %s is missing", table)
+	}
+	if err != nil {
+		return fmt.Errorf("mysql: inspect %s.id: %w", table, err)
+	}
+	if !strings.EqualFold(dataType, "bigint") || !strings.EqualFold(nullable, "NO") ||
+		!strings.Contains(strings.ToLower(extra), "auto_increment") {
+		return fmt.Errorf("mysql: incompatible %s.id schema: type=%s nullable=%s extra=%s; want BIGINT NOT NULL AUTO_INCREMENT",
+			table, dataType, nullable, extra)
+	}
+	return nil
+}
+
+func (a *adapter) TenantMessageSaveBatch(id t.TenantID, messages []*t.Message) error {
+	tenantAdp, err := a.routedTenantAdapter(id)
+	if err != nil {
+		return err
+	}
+	if len(messages) == 0 {
+		return t.ErrMalformed
+	}
+	if tenantAdp != a {
+		return tenantAdp.TenantMessageSaveBatch(id, messages)
+	}
+	ctx, cancel := a.getContext()
+	if cancel != nil {
+		defer cancel()
+	}
+	values := make([]string, 0, len(messages))
+	args := make([]any, 0, len(messages)*13)
+	for _, msg := range messages {
+		if msg == nil || msg.TenantID != id || msg.Topic == "" || msg.SeqId <= 0 || msg.Uid().IsZero() {
+			return t.ErrMalformed
+		}
+		values = append(values, "(?,?,?,?,?,?,?,?,?,?,?,?,?)")
+		args = append(args, id, store.DecodeUid(msg.Uid()), msg.CreatedAt, msg.UpdatedAt, msg.SeqId, msg.Topic,
+			decodeUID(t.ParseUid(msg.From)), sql.NullString{String: msg.ClientMsgID, Valid: msg.ClientMsgID != ""},
+			messageVersion(msg.Version), msg.Head, common.ToJSON(msg.Content), msg.ContentNonce, msg.KeyVersion)
+	}
+	query := fmt.Sprintf("INSERT INTO %s(tenant_id,id,createdat,updatedat,seqid,topic,`from`,client_msg_id,version,head,content,content_nonce,key_version) VALUES %s",
+		tenantMessagesTable(id), strings.Join(values, ","))
+	_, err = a.db.ExecContext(ctx, query, args...)
+	if isDupe(err) {
+		return t.ErrDuplicate
+	}
+	return err
+}
+
 func (a *adapter) TenantGroupMessageSave(id t.TenantID, msg *t.Message) error {
 	tenantAdp, err := a.routedTenantAdapter(id)
 	if err != nil || msg == nil || msg.TenantID != id || msg.Topic == "" || msg.SeqId <= 0 {
@@ -2315,6 +2407,69 @@ func (a *adapter) TenantGroupMessageSave(id t.TenantID, msg *t.Message) error {
 	return tx.Commit()
 }
 
+func (a *adapter) TenantGroupMessageSaveBatch(id t.TenantID, messages []*t.Message) error {
+	tenantAdp, err := a.routedTenantAdapter(id)
+	if err != nil {
+		return err
+	}
+	if len(messages) == 0 {
+		return t.ErrMalformed
+	}
+	if tenantAdp != a {
+		return tenantAdp.TenantGroupMessageSaveBatch(id, messages)
+	}
+	if err = a.ensureCurrentGroupMessageShardWindow(); err != nil {
+		return err
+	}
+	ctx, cancel := a.getContext()
+	if cancel != nil {
+		defer cancel()
+	}
+	month := messages[0].ShardMonth
+	table, ok := groupMessageShardTable(id, month)
+	if !ok {
+		return t.ErrMalformed
+	}
+	groupIDs := make(map[string]int64)
+	for _, msg := range messages {
+		if msg == nil || msg.TenantID != id || msg.Topic == "" || msg.SeqId <= 0 || msg.Uid().IsZero() || msg.ShardMonth != month {
+			return t.ErrMalformed
+		}
+		if _, found := groupIDs[msg.Topic]; !found {
+			groupIDs[msg.Topic], err = a.groupIDByTopic(ctx, id, msg.Topic)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	var shardStatus string
+	if err = a.db.QueryRowContext(ctx, "SELECT status FROM im_group_message_shards WHERE tenant_id=? AND shard_month=? AND table_name=?", id, month, table).Scan(&shardStatus); err != nil {
+		return err
+	}
+	if shardStatus != "writable" && shardStatus != "sealing" {
+		return fmt.Errorf("mysql: group message shard %s is %s", month, shardStatus)
+	}
+	values := make([]string, 0, len(messages))
+	args := make([]any, 0, len(messages)*20)
+	for _, msg := range messages {
+		contentJSON := common.ToJSON(msg.Content)
+		values = append(values, "(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)")
+		args = append(args, id, store.DecodeUid(msg.Uid()), msg.Topic, groupIDs[msg.Topic], msg.SeqId,
+			decodeUID(t.ParseUid(msg.From)), sql.NullString{String: msg.ClientMsgID, Valid: msg.ClientMsgID != ""},
+			messageHeadType(msg.Head), "normal", contentJSON, msg.Head, messageHeadJSON(msg.Head, "mentioned_user_ids"),
+			messageHeadBool(msg.Head, "mention_all"), 0, len(contentJSON), messageVersion(msg.Version), msg.CreatedAt,
+			msg.UpdatedAt, msg.ContentNonce, msg.KeyVersion)
+	}
+	query := fmt.Sprintf(`INSERT INTO %s
+		(tenant_id,message_id,topic,group_id,seqid,from_user,client_msg_id,msg_type,status,content,head,mentioned_user_ids,mention_all,
+		 file_count,content_size,version,created_at,updated_at,content_nonce,key_version) VALUES %s`, table, strings.Join(values, ","))
+	_, err = a.db.ExecContext(ctx, query, args...)
+	if isDupe(err) {
+		return t.ErrDuplicate
+	}
+	return err
+}
+
 func (a *adapter) TenantMessageGetByClientID(id t.TenantID, from t.Uid, clientMsgID string) (*t.Message, error) {
 	if !a.tenantScopedDB {
 		tenantAdp, err := a.routedTenantAdapter(id)
@@ -5303,6 +5458,10 @@ func (a *adapter) TenantFileUserHasAccess(id t.TenantID, fid string, uid t.Uid)
 		SELECT COUNT(*) FROM im_fileuploads AS fu
 		WHERE fu.tenant_id=? AND (fu.id=? OR fu.location=? OR fu.location LIKE ?) AND fu.status=? AND (
 			fu.userid=?
+			OR EXISTS (
+				SELECT 1 FROM im_filemsglinks AS fml
+				WHERE fml.tenant_id=fu.tenant_id AND fml.fileid=fu.id AND fml.topic LIKE 'cvp%%'
+			)
 			OR EXISTS (
 				SELECT 1 FROM im_filemsglinks AS fml
 				WHERE fml.tenant_id=fu.tenant_id AND fml.fileid=fu.id AND fml.userid=?

server/hub.go

@@ -255,14 +255,17 @@ func (h *Hub) run() {
 				}
 			} else if msg.Note == nil {
 				// Topic is unknown or offline.
-				// Note is silently ignored, all other messages are reported as accepted to prevent
+				// Note is silently ignored. Other client messages cannot be accepted here
-				// clients from guessing valid topic names.
+				// because they are not routed to a live topic and therefore cannot reserve a
+				// message seq or be persisted.
 
 				// TODO(gene): validate topic name, discarding invalid topics.
 
 				logs.Info.Printf("Hub. Topic[%s] is unknown or offline", msg.RcptTo)
 
-				msg.sess.queueOut(NoErrAcceptedExplicitTs(msg.Id, msg.RcptTo, types.TimeNow(), msg.Timestamp))
+				if msg.sess != nil {
+					msg.sess.queueOut(ErrAttachFirst(msg, types.TimeNow()))
+				}
 			}
 		case msg := <-h.routeSrv:
 			// This is a server message from a connection not subscribed to topic

server/hub_tenant_test.go

@@ -1,8 +1,10 @@
 package main
 
 import (
+	"net/http"
 	"sync"
 	"testing"
+	"time"
 
 	"github.com/tinode/chat/server/store/types"
 )
@@ -53,3 +55,50 @@ func TestTopicAssignsTenantToServerMessage(t *testing.T) {
 		t.Fatalf("message tenant = %d, want %d", msg.TenantID, topic.tenantID)
 	}
 }
+
+func TestHubRejectsUnroutedClientMessageInsteadOfAcceptingWithoutSeq(t *testing.T) {
+	h := &Hub{
+		topics:     &sync.Map{},
+		routeCli:   make(chan *ClientComMessage, 1),
+		shutdown:   make(chan chan<- bool),
+		routeSrv:   make(chan *ServerComMessage),
+		join:       make(chan *ClientComMessage),
+		meta:       make(chan *ClientComMessage),
+		unreg:      make(chan *topicUnreg),
+		userStatus: make(chan *userStatusReq),
+		rehash:     make(chan bool),
+	}
+	go h.run()
+	done := make(chan bool)
+	defer func() {
+		h.shutdown <- done
+		<-done
+	}()
+
+	sess := &Session{tenantID: 7, send: make(chan any, 1)}
+	now := time.Now().UTC()
+	h.routeCli <- &ClientComMessage{
+		Id:        "pub-1",
+		TenantID:  7,
+		RcptTo:    "p2pMissing",
+		Original:  "p2pMissing",
+		Timestamp: now,
+		sess:      sess,
+	}
+
+	select {
+	case raw := <-sess.send:
+		msg, ok := raw.(*ServerComMessage)
+		if !ok || msg.Ctrl == nil {
+			t.Fatalf("unexpected response: %#v", raw)
+		}
+		if msg.Ctrl.Code != http.StatusConflict || msg.Ctrl.Text != "must attach first" {
+			t.Fatalf("unexpected ctrl: %#v", msg.Ctrl)
+		}
+		if msg.Ctrl.Params != nil {
+			t.Fatalf("unrouted message must not expose seq params: %#v", msg.Ctrl.Params)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("timeout waiting for unrouted message response")
+	}
+}

server/im_http.go

@@ -2367,6 +2367,7 @@ func serveIMP2PConversation(wrt http.ResponseWriter, req *http.Request) {
 		writeIMRelationError(wrt, req, err)
 		return
 	}
+	publishIMP2PPermissionsRefresh(principal.TenantID, topic, principal.Uid, peer)
 	ctx, err := store.Relations.GetRelationshipContext(principal.TenantID, principal.Uid, peer, topic)
 	if err != nil {
 		writeIMRelationError(wrt, req, err)
@@ -5451,11 +5452,11 @@ func serveIMGroupGovernance(wrt http.ResponseWriter, req *http.Request, topic st
 				text = "群聊邀请确认已启用，群成员邀请好友进群需要群主或管理员确认。"
 			}
 			publishIMConversationSystemNotice(principal.TenantID, topic,
-				"system_action", template, text,
+				"system_notice", template, text,
 				[]imSystemMessageActor{
 					{UserID: principal.Uid.UserId(), Click: map[string]any{"type": "user_profile", "target_id": principal.Uid.UserId()}},
 				},
-				[]imSystemMessageAction{{Type: "open_group_invite_settings", Label: "群聊邀请确认", TargetID: topic, MinRole: "admin"}},
+				nil,
 				map[string]any{"join_approval": *body.JoinApproval, "version": settings.Version})
 		}
 		if body.MemberPrivateChat != nil && imReliableHTTPEventsEnabled(req) {

server/im_p2p_events.go

@@ -0,0 +1,56 @@
+package main
+
+import (
+	"encoding/json"
+	"strings"
+	"time"
+
+	"github.com/tinode/chat/server/logs"
+	"github.com/tinode/chat/server/store/types"
+)
+
+const p2pPermissionsRefreshedEvent = "p2p.permissions.refreshed"
+
+type p2pPermissionsPayload struct {
+	UserIDs []string `json:"user_ids"`
+}
+
+// publishIMP2PPermissionsRefresh asks the topic master to reload permissions
+// after the HTTP API has repaired or created the two P2P subscriptions.
+func publishIMP2PPermissionsRefresh(tenantID types.TenantID, topic string, uids ...types.Uid) {
+	topic = strings.TrimSpace(topic)
+	if !tenantID.IsValid() || topic == "" {
+		return
+	}
+	payload := p2pPermissionsPayload{UserIDs: make([]string, 0, len(uids))}
+	for _, uid := range uids {
+		if !uid.IsZero() {
+			payload.UserIDs = append(payload.UserIDs, uid.UserId())
+		}
+	}
+	if len(payload.UserIDs) == 0 {
+		return
+	}
+	encoded, err := json.Marshal(payload)
+	if err != nil {
+		logs.Warn.Printf("p2p permissions refresh: encode topic=%s: %v", topic, err)
+		return
+	}
+	message := &ServerComMessage{
+		TenantID:  tenantID,
+		RcptTo:    topic,
+		Timestamp: time.Now().UTC().Round(time.Millisecond),
+		Info: &MsgServerInfo{
+			Topic: topic, What: "p2p", Event: p2pPermissionsRefreshedEvent,
+			Payload: json.RawMessage(encoded),
+		},
+	}
+	if globals.hub != nil {
+		select {
+		case globals.hub.routeSrv <- message:
+		default:
+			logs.Warn.Printf("p2p permissions refresh: hub queue full topic=%s", topic)
+		}
+	}
+	publishRealtimeTopicMessage(topic, message)
+}

server/im_p2p_events_test.go

@@ -0,0 +1,68 @@
+package main
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/golang/mock/gomock"
+	"github.com/tinode/chat/server/store/types"
+)
+
+func TestHandleP2PPermissionsRefreshReloadsLoadedTopic(t *testing.T) {
+	helper := TopicTestHelper{}
+	helper.setUp(t, 2, types.TopicCatP2P, "p2p-test", true)
+	defer helper.tearDown()
+
+	uid := helper.uids[0]
+	stale := types.ModeCP2P &^ types.ModeWrite
+	pud := helper.topic.perUser[uid]
+	pud.modeWant = stale
+	pud.modeGiven = stale
+	helper.topic.perUser[uid] = pud
+	helper.ss.EXPECT().Get(types.TenantID(1), "p2p-test", uid, true).Return(&types.Subscription{
+		TenantID: 1, User: uid.String(), Topic: "p2p-test",
+		ModeWant: types.ModeCP2P, ModeGiven: types.ModeCP2P,
+	}, nil)
+
+	payload, err := json.Marshal(p2pPermissionsPayload{UserIDs: []string{uid.UserId()}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	helper.topic.handleServerMsg(&ServerComMessage{
+		TenantID: 1,
+		Info:     &MsgServerInfo{What: "p2p", Event: p2pPermissionsRefreshedEvent, Payload: payload},
+	})
+	helper.finish()
+
+	got := helper.topic.perUser[uid]
+	if !got.modeWant.IsWriter() || !got.modeGiven.IsWriter() {
+		t.Fatalf("permissions were not refreshed: want=%s given=%s", got.modeWant, got.modeGiven)
+	}
+}
+
+func TestP2PPublishReloadsStalePermissions(t *testing.T) {
+	helper := TopicTestHelper{}
+	helper.setUp(t, 2, types.TopicCatP2P, "p2p-test", true)
+	defer helper.tearDown()
+
+	uid := helper.uids[0]
+	stale := types.ModeCP2P &^ types.ModeWrite
+	pud := helper.topic.perUser[uid]
+	pud.modeWant = stale
+	pud.modeGiven = stale
+	helper.topic.perUser[uid] = pud
+	helper.ss.EXPECT().Get(types.TenantID(1), "p2p-test", uid, true).Return(&types.Subscription{
+		TenantID: 1, User: uid.String(), Topic: "p2p-test",
+		ModeWant: types.ModeCP2P, ModeGiven: types.ModeCP2P,
+	}, nil)
+	helper.mm.EXPECT().Save(types.TenantID(1), gomock.Any(), gomock.Any(), gomock.Any()).
+		Return(&types.MessageSaveResult{SeqId: 1, MarkedReadBySender: true}, nil)
+
+	err := helper.topic.saveAndBroadcastMessage(&ClientComMessage{
+		Id: "pub-1", Original: "p2p-test", Timestamp: types.TimeNow(), sess: helper.sessions[0],
+	}, uid, true, nil, nil, "test")
+	helper.finish()
+	if err != nil {
+		t.Fatalf("publish failed after permission reload: %v", err)
+	}
+}

server/internal_admin_query_resources.go

@@ -461,14 +461,19 @@ func internalGroupMessageSource(tenantID types.TenantID) (string, error) {
 		}
 		selects = append(selects, fmt.Sprintf(`SELECT message_id AS id,tenant_id,topic,seqid,from_user,msg_type,status,content,head,reply_seqid,mentioned_user_ids,created_at,updated_at FROM %s`, table))
 	}
+	selects = append(selects, internalLegacyGroupMessagesSelect(tenantID))
+	return "(" + strings.Join(selects, " UNION ALL ") + ")", nil
+}
+
+func internalLegacyGroupMessagesSelect(tenantID types.TenantID) string {
 	// legacy 行内联租户条件（tenantID 来自租户注册表数值 ID，非用户输入）
-	selects = append(selects, fmt.Sprintf(`SELECT id,tenant_id,topic COLLATE utf8mb4_0900_ai_ci AS topic,seqid,from_user,
+	return fmt.Sprintf(`SELECT id,tenant_id,topic COLLATE utf8mb4_0900_ai_ci AS topic,seqid,from_user,
 		msg_type COLLATE utf8mb4_0900_ai_ci AS msg_type,status COLLATE utf8mb4_0900_ai_ci AS status,
 		content,head,reply_seqid,mentioned_user_ids,created_at,updated_at
 		FROM im_group_messages AS legacy WHERE legacy.tenant_id=%d AND NOT EXISTS (
-			SELECT 1 FROM im_group_message_topic_shards AS route
+			SELECT 1 FROM im_group_message_shards AS route
-			WHERE route.topic=legacy.topic COLLATE utf8mb4_0900_ai_ci AND legacy.seqid BETWEEN route.min_seq AND route.max_seq)`, int64(tenantID)))
+			WHERE route.tenant_id=legacy.tenant_id AND route.status<>'archived'
-	return "(" + strings.Join(selects, " UNION ALL ") + ")", nil
+				AND legacy.created_at>=route.shard_start AND legacy.created_at<route.shard_end)`, int64(tenantID))
 }
 
 func validInternalGroupMessageTable(table string) bool {

server/internal_cultivate.go

@@ -24,6 +24,7 @@ type cultivateRelationsRequest struct {
 
 type cultivateFilesRequest struct {
 	UserID      int64    `json:"user_id"`
+	ReferenceID int64    `json:"reference_id"`
 	FileIDs     []string `json:"file_ids"`
 }
 
@@ -162,6 +163,7 @@ func serveCultivateFiles(w http.ResponseWriter, r *http.Request, tenantID types.
 	}
 	uid := store.EncodeUid(body.UserID)
 	items := make([]map[string]any, 0, len(body.FileIDs))
+	fileIDs := make([]string, 0, len(body.FileIDs))
 	for _, supplied := range body.FileIDs {
 		fileID := normalizeIMUploadedFileID(supplied)
 		file, err := store.Files.Get(tenantID, fileID)
@@ -173,11 +175,24 @@ func serveCultivateFiles(w http.ResponseWriter, r *http.Request, tenantID types.
 			writeIMError(w, r, http.StatusUnprocessableEntity, 30000, imReasonMalformedRequest, "invalid file", map[string]any{"file_id": supplied})
 			return
 		}
+		fileIDs = append(fileIDs, fileID)
 		items = append(items, map[string]any{"file_id": fileID, "mime_type": file.MimeType, "size": file.Size, "review_status": cultivateFileReviewStatus(tenantID, fileID)})
 	}
+	if body.ReferenceID > 0 {
+		for index, fileID := range fileIDs {
+			if err := store.Files.LinkAttachments(tenantID, cultivateFileReferenceTopic(body.ReferenceID, index), types.ZeroUid, []string{fileID}); err != nil {
+				writeIMError(w, r, http.StatusInternalServerError, 90000, imReasonInternal, "failed to retain file", nil)
+				return
+			}
+		}
+	}
 	writeIMSuccess(w, r, map[string]any{"items": items})
 }
 
+func cultivateFileReferenceTopic(referenceID int64, index int) string {
+	return "cvp" + strconv.FormatInt(referenceID, 36) + "x" + strconv.FormatInt(int64(index), 36)
+}
+
 func fileOwner(file *types.FileDef) string {
 	if file == nil {
 		return ""

server/internal_cultivate_test.go

@@ -46,3 +46,17 @@ func TestInternalCultivateUsesSharedInternalCredential(t *testing.T) {
 		t.Fatalf("shared internal credential was rejected: status=%d body=%s", rec.Code, rec.Body.String())
 	}
 }
+
+func TestCultivateFileReferenceTopic(t *testing.T) {
+	first := cultivateFileReferenceTopic(1577072518975639552, 0)
+	second := cultivateFileReferenceTopic(1577072518975639552, 1)
+	if first == second {
+		t.Fatalf("file references must be unique per media item: %q", first)
+	}
+	if len(first) > 25 || len(second) > 25 {
+		t.Fatalf("file reference exceeds im_filemsglinks.topic capacity: %q %q", first, second)
+	}
+	if first != cultivateFileReferenceTopic(1577072518975639552, 0) {
+		t.Fatalf("file reference is not deterministic: %q", first)
+	}
+}

server/kafka_admin_commands.go

@@ -141,7 +141,6 @@ func handleKafkaAdminCommand(ctx context.Context, envelope kafkaEnvelope, _ kafk
 			return nil
 		}
 		if kafkaCommandReasonRetryable(reason) {
-			sleepKafkaRetryBackoff(failedRecord.RetryCount)
 			return execErr
 		}
 		if err := publishKafkaAdminCommandResult(ctx, envelope, payload.CommandID, "failed", reason, execErr.Error(), result); err != nil {
@@ -608,7 +607,7 @@ func markKafkaDeadLetter(ctx context.Context, tenantID types.TenantID, envelope
 	if err != nil {
 		return err
 	}
-	return publishKafkaDeadLetter(ctx, "dead_letter", envelope.EventID, kafkaEnvelope{
+	if err = publishKafkaDeadLetter(ctx, "dead_letter", envelope.EventID, kafkaEnvelope{
 		EventID:       deadID,
 		TraceID:       envelope.TraceID,
 		TenantCode:    envelope.TenantCode,
@@ -621,7 +620,12 @@ func markKafkaDeadLetter(ctx context.Context, tenantID types.TenantID, envelope
 		OccurredAt:    now,
 		ParentEventID: envelope.EventID,
 		Payload:       deadPayload,
-	})
+	}); err != nil {
+		return err
+	}
+	// Persist the terminal state before the source consumer commits its offset.
+	// The dead-letter consumer repeats this idempotent upsert for recovery and auditing.
+	return store.HTTPInfrastructure.MarkKafkaConsumeDead(tenantID, dead, now)
 }
 
 func shouldDeadLetterKafkaCommand(reason string, retryCount int) bool {
@@ -643,21 +647,6 @@ func kafkaCommandReasonRetryable(reason string) bool {
 	}
 }
 
-func sleepKafkaRetryBackoff(retryCount int) {
-	if kafkaBus == nil || retryCount <= 0 {
-		return
-	}
-	backoffs := kafkaBus.config.RetryBackoffMS
-	index := retryCount - 1
-	if index >= len(backoffs) {
-		index = len(backoffs) - 1
-	}
-	delay := time.Duration(backoffs[index]) * time.Millisecond
-	if delay > 0 {
-		time.Sleep(delay)
-	}
-}
-
 func parseKafkaCommandUser(raw string) types.Uid {
 	raw = strings.TrimSpace(raw)
 	if raw == "" {

server/kafka_cache_invalidate.go

@@ -70,7 +70,6 @@ func handleKafkaCacheInvalidateEvent(ctx context.Context, envelope kafkaEnvelope
 		if failed.RetryCount >= kafkaBus.config.MaxRetryCount {
 			return markKafkaDeadLetter(ctx, tenantID, envelope, failed, reason, err.Error())
 		}
-		sleepKafkaRetryBackoff(failed.RetryCount)
 		return err
 	}
 	return store.HTTPInfrastructure.MarkKafkaConsumeSucceeded(tenantID, envelope.EventID, consumerGroup, time.Now().UTC().Round(time.Millisecond))

server/kafka_config_events.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"strings"
 	"time"
 
@@ -12,6 +13,8 @@ import (
 	"github.com/tinode/chat/server/store/types"
 )
 
+var errKafkaConsumeInProgress = errors.New("kafka event is already being processed")
+
 const (
 	configEventTenantUpdated         = "tenant.updated"
 	configEventTenantStatusChanged   = "tenant.status.changed"
@@ -66,7 +69,6 @@ func handleKafkaConfigEvent(ctx context.Context, envelope kafkaEnvelope, _ kafka
 		if failed.RetryCount >= kafkaBus.config.MaxRetryCount {
 			return markKafkaDeadLetter(ctx, tenantID, envelope, failed, reason, err.Error())
 		}
-		sleepKafkaRetryBackoff(failed.RetryCount)
 		return err
 	}
 	return store.HTTPInfrastructure.MarkKafkaConsumeSucceeded(tenantID, envelope.EventID, consumerGroup, time.Now().UTC().Round(time.Millisecond))
@@ -155,7 +157,7 @@ func beginKafkaConsumeForTopicWithGroup(tenantID types.TenantID, envelope kafkaE
 		consumerGroup = kafkaBus.config.ConsumerGroup
 	}
 	deadline := time.Now().UTC().Add(5 * time.Minute).Round(time.Millisecond)
-	return store.HTTPInfrastructure.BeginKafkaConsume(tenantID, &types.KafkaConsumeRecord{
+	record, claimed, err := store.HTTPInfrastructure.BeginKafkaConsume(tenantID, &types.KafkaConsumeRecord{
 		EventID:       envelope.EventID,
 		ConsumerGroup: consumerGroup,
 		SourceTopic:   kafkaTopic(topicName),
@@ -163,4 +165,11 @@ func beginKafkaConsumeForTopicWithGroup(tenantID types.TenantID, envelope kafkaE
 		BizType:       envelope.BizType,
 		BizID:         envelope.BizID,
 	}, deadline)
+	if err != nil || claimed || record == nil {
+		return record, claimed, err
+	}
+	if record.Status != types.KafkaConsumeSucceeded && record.Status != types.KafkaConsumeDead {
+		return record, false, errKafkaConsumeInProgress
+	}
+	return record, false, nil
 }

server/kafka_message_persist.go

@@ -5,10 +5,12 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"strings"
 	"time"
 
 	"github.com/segmentio/kafka-go"
 	"github.com/tinode/chat/server/logs"
+	immetrics "github.com/tinode/chat/server/metrics"
 	"github.com/tinode/chat/server/store"
 	"github.com/tinode/chat/server/store/types"
 )
@@ -24,8 +26,12 @@ type kafkaMessagePersistPayload struct {
 var publishKafkaMessagePersist = publishKafkaEnvelope
 
 func startKafkaMessagePersistConsumer() {
-	startKafkaConsumer("message_persist", handleKafkaMessagePersist)
+	startKafkaBatchConsumer("message_persist", func(ctx context.Context, messages []kafka.Message) error {
-	startKafkaConsumer("group_message_persist", handleKafkaGroupMessagePersist)
+		return handleKafkaMessagePersistBatch(ctx, messages, "message_persist")
+	})
+	startKafkaBatchConsumer("group_message_persist", func(ctx context.Context, messages []kafka.Message) error {
+		return handleKafkaMessagePersistBatch(ctx, messages, "group_message_persist")
+	})
 }
 
 func saveMessageMaybeKafka(ctx context.Context, tenantID types.TenantID, msg *types.Message, attachmentURLs []string, readBySender bool) (*types.MessageSaveResult, error) {
@@ -132,70 +138,179 @@ func enqueueKafkaMessagePersist(ctx context.Context, tenantID types.TenantID, ms
 	return publishKafkaMessagePersist(ctx, kafkaMessagePersistTopicName(msg.Topic), msg.Topic, envelope)
 }
 
-func handleKafkaMessagePersist(ctx context.Context, envelope kafkaEnvelope, _ kafka.Message) error {
+type kafkaMessagePersistItem struct {
-	return handleKafkaMessagePersistFromTopic(ctx, envelope, "message_persist")
+	envelope kafkaEnvelope
+	tenantID types.TenantID
+	payload  kafkaMessagePersistPayload
 }
 
-func handleKafkaGroupMessagePersist(ctx context.Context, envelope kafkaEnvelope, _ kafka.Message) error {
+func handleKafkaMessagePersistBatch(ctx context.Context, messages []kafka.Message, sourceTopicName string) error {
-	return handleKafkaMessagePersistFromTopic(ctx, envelope, "group_message_persist")
+	groups := make(map[string][]*kafkaMessagePersistItem)
+	for _, message := range messages {
+		item, terminal, err := prepareKafkaMessagePersistItem(ctx, message, sourceTopicName)
+		if err != nil {
+			return err
+		}
+		if terminal {
+			continue
+		}
+		key := fmt.Sprintf("%d", item.tenantID)
+		if item.payload.MessageKind == "group" {
+			key += ":" + item.payload.Message.ShardMonth
+		}
+		groups[key] = append(groups[key], item)
+	}
+	for _, items := range groups {
+		kind := items[0].payload.MessageKind
+		if kind == "" {
+			kind = "p2p"
+		}
+		started := time.Now()
+		batch := make([]store.PreparedMessageBatchItem, 0, len(items))
+		for _, item := range items {
+			batch = append(batch, store.PreparedMessageBatchItem{Message: &item.payload.Message, ReadBySender: item.payload.ReadBySender})
+		}
+		if err := store.PersistPreparedMessageBatch(items[0].tenantID, batch); err == nil {
+			immetrics.KafkaMessagePersistTotal.WithLabelValues(kind, "batch_ok").Add(float64(len(items)))
+			immetrics.MessagePersistDuration.WithLabelValues(kind, "batch_ok").Observe(time.Since(started).Seconds())
+			for _, item := range items {
+				if err = finishKafkaMessagePersist(ctx, item); err != nil {
+					if err = persistKafkaMessageIndividually(ctx, item); err != nil {
+						return err
+					}
+				}
+			}
+			continue
+		} else {
+			immetrics.KafkaMessagePersistTotal.WithLabelValues(kind, "batch_failed").Add(float64(len(items)))
+			immetrics.MessagePersistDuration.WithLabelValues(kind, "batch_failed").Observe(time.Since(started).Seconds())
+			logs.Warn.Printf("kafka message batch persist failed, degrading to singles: tenant=%d kind=%s shard=%s size=%d err=%v",
+				items[0].tenantID, items[0].payload.MessageKind, items[0].payload.Message.ShardMonth, len(items), err)
+		}
+		for _, item := range items {
+			if err := persistKafkaMessageIndividually(ctx, item); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
 }
 
-func handleKafkaMessagePersistFromTopic(ctx context.Context, envelope kafkaEnvelope, sourceTopicName string) error {
+func prepareKafkaMessagePersistItem(ctx context.Context, message kafka.Message, sourceTopicName string) (*kafkaMessagePersistItem, bool, error) {
+	var envelope kafkaEnvelope
+	if err := json.Unmarshal(message.Value, &envelope); err != nil {
+		logs.Err.Printf("kafka message malformed envelope skipped: partition=%d offset=%d err=%v", message.Partition, message.Offset, err)
+		return nil, true, nil
+	}
 	tenantID := envelope.TenantID
 	var err error
 	if tenantID.IsZero() {
 		tenantID, err = kafkaTenantID(envelope.TenantCode)
 		if err != nil {
-			return err
+			return nil, false, err
 		}
 	}
 	consume, claimed, err := beginKafkaConsumeForTopic(tenantID, envelope, sourceTopicName)
-	if err != nil || !claimed {
+	if err != nil {
+		return nil, false, err
+	}
+	if !claimed {
+		return nil, true, nil
+	}
+	item := &kafkaMessagePersistItem{envelope: envelope, tenantID: tenantID}
+	if err = json.Unmarshal(envelope.Payload, &item.payload); err != nil {
+		return nil, true, markKafkaDeadLetter(ctx, tenantID, envelope, consume, "MALFORMED_PAYLOAD", err.Error())
+	}
+	if item.payload.Message.TenantID == types.ZeroTenantID {
+		item.payload.Message.TenantID = tenantID
+	}
+	if item.payload.Message.ShardMonth == "" {
+		item.payload.Message.ShardMonth = item.payload.ShardMonth
+	}
+	if item.payload.Message.TenantID != tenantID {
+		return nil, true, markKafkaDeadLetter(ctx, tenantID, envelope, consume, "TENANT_MISMATCH", "message tenant does not match envelope tenant")
+	}
+	if err = validateKafkaMessageKind(item.payload); err != nil {
+		return nil, true, markKafkaDeadLetter(ctx, tenantID, envelope, consume, "MALFORMED_PAYLOAD", err.Error())
+	}
+	return item, false, nil
+}
+
+func persistKafkaMessageIndividually(ctx context.Context, item *kafkaMessagePersistItem) error {
+	kind := item.payload.MessageKind
+	if kind == "" {
+		kind = "p2p"
+	}
+	started := time.Now()
+	operation := func() error {
+		err := store.PersistPreparedMessageBatch(item.tenantID, []store.PreparedMessageBatchItem{{
+			Message: &item.payload.Message, ReadBySender: item.payload.ReadBySender,
+		}})
+		if errors.Is(err, types.ErrUnsupported) {
+			_, err = store.PersistPreparedMessage(item.tenantID, &item.payload.Message, nil, item.payload.ReadBySender)
+		}
+		if err != nil && !errors.Is(err, types.ErrDuplicate) {
 			return err
 		}
-	var payload kafkaMessagePersistPayload
+		if err = finishKafkaMessagePersist(ctx, item); err != nil {
-	if err = json.Unmarshal(envelope.Payload, &payload); err != nil {
-		return markKafkaDeadLetter(ctx, tenantID, envelope, consume, "MALFORMED_PAYLOAD", err.Error())
-	}
-	if payload.Message.TenantID == types.ZeroTenantID {
-		payload.Message.TenantID = tenantID
-	}
-	if payload.Message.ShardMonth == "" {
-		payload.Message.ShardMonth = payload.ShardMonth
-	}
-	if payload.Message.TenantID != tenantID {
-		return markKafkaDeadLetter(ctx, tenantID, envelope, consume, "TENANT_MISMATCH", "message tenant does not match envelope tenant")
-	}
-	if err = validateKafkaMessageKind(payload); err != nil {
-		return markKafkaDeadLetter(ctx, tenantID, envelope, consume, "MALFORMED_PAYLOAD", err.Error())
-	}
-	if _, err = store.PersistPreparedMessage(tenantID, &payload.Message, nil, payload.ReadBySender); err != nil && !errors.Is(err, types.ErrDuplicate) {
-		failed, markErr := store.HTTPInfrastructure.MarkKafkaConsumeFailed(tenantID, envelope.EventID, kafkaBus.config.ConsumerGroup,
-			kafkaCommandErrorReason(err), err.Error(), time.Now().UTC().Round(time.Millisecond))
-		if markErr != nil {
-			return markErr
-		}
-		if failed != nil && failed.RetryCount >= kafkaBus.config.MaxRetryCount {
-			return markKafkaDeadLetter(ctx, tenantID, envelope, failed, kafkaCommandErrorReason(err), err.Error())
-		}
-		sleepKafkaRetryBackoff(failed.RetryCount)
 			return err
 		}
-	if len(payload.AttachmentURLs) > 0 {
+		immetrics.KafkaMessagePersistTotal.WithLabelValues(kind, "single_ok").Inc()
-		if err = enqueueKafkaFileAttachmentsBind(ctx, tenantID, &payload.Message, payload.AttachmentURLs); err != nil {
+		immetrics.MessagePersistDuration.WithLabelValues(kind, "single_ok").Observe(time.Since(started).Seconds())
-			failed, markErr := store.HTTPInfrastructure.MarkKafkaConsumeFailed(tenantID, envelope.EventID, kafkaBus.config.ConsumerGroup,
+		return nil
-				"KAFKA_FILE_LIFECYCLE_FAILED", err.Error(), time.Now().UTC().Round(time.Millisecond))
-			if markErr != nil {
-				return markErr
 	}
-			if failed != nil && failed.RetryCount >= kafkaBus.config.MaxRetryCount {
+	markFailed := func(err error) (*types.KafkaConsumeRecord, error) {
-				return markKafkaDeadLetter(ctx, tenantID, envelope, failed, "KAFKA_FILE_LIFECYCLE_FAILED", err.Error())
+		immetrics.KafkaMessagePersistTotal.WithLabelValues(kind, "retry").Inc()
+		return store.HTTPInfrastructure.MarkKafkaConsumeFailed(item.tenantID, item.envelope.EventID, kafkaBus.config.ConsumerGroup,
+			kafkaMessagePersistFailureReason(err), err.Error(), time.Now().UTC().Round(time.Millisecond))
 	}
-			sleepKafkaRetryBackoff(failed.RetryCount)
+	markDead := func(failed *types.KafkaConsumeRecord, err error) error {
+		immetrics.KafkaMessagePersistTotal.WithLabelValues(kind, "dead").Inc()
+		immetrics.MessagePersistDuration.WithLabelValues(kind, "dead").Observe(time.Since(started).Seconds())
+		reason := kafkaMessagePersistFailureReason(err)
+		immetrics.KafkaDeadLettersTotal.WithLabelValues(kind, reason).Inc()
+		return markKafkaDeadLetter(ctx, item.tenantID, item.envelope, failed, reason, err.Error())
+	}
+	return runKafkaMessageSingleRetries(ctx, kafkaBus.config.MessageRetries, operation, markFailed, markDead, waitKafkaRetry)
+}
+
+func runKafkaMessageSingleRetries(ctx context.Context, retryCount int, operation func() error,
+	markFailed func(error) (*types.KafkaConsumeRecord, error), markDead func(*types.KafkaConsumeRecord, error) error,
+	wait kafkaRetryWaiter) error {
+	var lastErr error
+	var failed *types.KafkaConsumeRecord
+	for attempt := 1; attempt <= retryCount+1; attempt++ {
+		if lastErr = operation(); lastErr == nil {
+			return nil
+		}
+		var err error
+		if failed, err = markFailed(lastErr); err != nil {
+			return err
+		}
+		if attempt <= retryCount {
+			if err = wait(ctx, kafkaRetryBackoff(attempt)); err != nil {
 				return err
 			}
 		}
-	return store.HTTPInfrastructure.MarkKafkaConsumeSucceeded(tenantID, envelope.EventID, kafkaBus.config.ConsumerGroup, time.Now().UTC().Round(time.Millisecond))
+	}
+	return markDead(failed, lastErr)
+}
+
+func finishKafkaMessagePersist(ctx context.Context, item *kafkaMessagePersistItem) error {
+	if len(item.payload.AttachmentURLs) > 0 {
+		if err := enqueueKafkaFileAttachmentsBind(ctx, item.tenantID, &item.payload.Message, item.payload.AttachmentURLs); err != nil {
+			return fmt.Errorf("file lifecycle enqueue failed: %w", err)
+		}
+	}
+	return store.HTTPInfrastructure.MarkKafkaConsumeSucceeded(item.tenantID, item.envelope.EventID,
+		kafkaBus.config.ConsumerGroup, time.Now().UTC().Round(time.Millisecond))
+}
+
+func kafkaMessagePersistFailureReason(err error) string {
+	if err != nil && strings.Contains(err.Error(), "file lifecycle") {
+		return "KAFKA_FILE_LIFECYCLE_FAILED"
+	}
+	return kafkaCommandErrorReason(err)
 }
 
 func kafkaMessagePersistTopicName(topic string) string {

server/kafka_runtime.go

@@ -8,12 +8,14 @@ import (
 	"regexp"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/segmentio/kafka-go"
 	"github.com/segmentio/kafka-go/sasl"
 	"github.com/segmentio/kafka-go/sasl/scram"
 	"github.com/tinode/chat/server/logs"
+	immetrics "github.com/tinode/chat/server/metrics"
 	"github.com/tinode/chat/server/store/types"
 )
 
@@ -26,6 +28,7 @@ const (
 	kafkaTopicGroupMessagePersist = "im.group.message.persist"
 	kafkaTopicMessagePush         = "im.message.push"
 	kafkaTopicFileSecurity        = "im.file.security"
+	kafkaTopicWorkbenchEvent      = "im.workbench.event"
 	kafkaTopicCacheInvalidate     = "im.cache.invalidate"
 	kafkaTopicAuditEvent          = "im.audit.event"
 	kafkaTopicDeadLetter          = "im.dead.letter"
@@ -50,6 +53,9 @@ type kafkaConfig struct {
 	SASLPassword     string            `json:"sasl_password"`
 	MaxRetryCount    int               `json:"max_retry_count"`
 	RetryBackoffMS   []int             `json:"retry_backoff_ms"`
+	MessageBatchSize int               `json:"message_batch_size"`
+	MessageBatchWait int               `json:"message_batch_wait_ms"`
+	MessageRetries   int               `json:"message_single_retry_count"`
 	DeadRetentionSec int               `json:"dead_letter_retention_seconds"`
 }
 
@@ -151,6 +157,11 @@ func initKafkaRuntime(raw json.RawMessage) error {
 	return nil
 }
 
+func kafkaConfigured(raw json.RawMessage) bool {
+	var cfg kafkaConfig
+	return len(raw) > 0 && json.Unmarshal(raw, &cfg) == nil && cfg.Enabled
+}
+
 func normalizeKafkaConfig(cfg *kafkaConfig) {
 	if cfg.ClientID == "" {
 		cfg.ClientID = "im-server"
@@ -179,6 +190,17 @@ func normalizeKafkaConfig(cfg *kafkaConfig) {
 	if len(cfg.RetryBackoffMS) == 0 {
 		cfg.RetryBackoffMS = []int{10000, 30000, 60000, 300000, 900000}
 	}
+	if cfg.MessageBatchSize <= 0 {
+		cfg.MessageBatchSize = 100
+	} else if cfg.MessageBatchSize > 500 {
+		cfg.MessageBatchSize = 500
+	}
+	if cfg.MessageBatchWait <= 0 {
+		cfg.MessageBatchWait = 10
+	}
+	if cfg.MessageRetries <= 0 {
+		cfg.MessageRetries = 3
+	}
 	if cfg.DeadRetentionSec <= 0 {
 		cfg.DeadRetentionSec = 90 * 24 * 3600
 	}
@@ -214,6 +236,7 @@ func kafkaTopics(overrides map[string]string) map[string]string {
 		"group_message_persist": kafkaTopicGroupMessagePersist,
 		"message_push":          kafkaTopicMessagePush,
 		"file_security":         kafkaTopicFileSecurity,
+		"workbench_external":    kafkaTopicWorkbenchEvent,
 		"cache_invalidate":      kafkaTopicCacheInvalidate,
 		"audit_event":           kafkaTopicAuditEvent,
 		"dead_letter":           kafkaTopicDeadLetter,
@@ -274,6 +297,11 @@ func publishKafkaEnvelope(ctx context.Context, topicName string, key string, env
 
 type kafkaMessageHandler func(context.Context, kafkaEnvelope, kafka.Message) error
 
+type kafkaMessageCommitter func(context.Context, kafka.Message) error
+type kafkaRetryWaiter func(context.Context, time.Duration) error
+type kafkaBatchHandler func(context.Context, []kafka.Message) error
+type kafkaBatchCommitter func(context.Context, ...kafka.Message) error
+
 func startKafkaConsumer(topicName string, handler kafkaMessageHandler) {
 	startKafkaConsumerWithGroup(topicName, "", handler)
 }
@@ -312,6 +340,7 @@ func startKafkaConsumerWithGroup(topicName, groupID string, handler kafkaMessage
 		MaxWait:     time.Second,
 		Dialer:      kafkaBus.dialer(),
 	})
+	readerMetrics := startKafkaReaderMetrics(kafkaBus.ctx, reader, topic, groupID)
 	kafkaBus.wg.Add(1)
 	go func() {
 		defer kafkaBus.wg.Done()
@@ -325,24 +354,218 @@ func startKafkaConsumerWithGroup(topicName, groupID string, handler kafkaMessage
 				logs.Err.Println("Kafka fetch failed:", err)
 				continue
 			}
-			var envelope kafkaEnvelope
+			readerMetrics.observe(msg.Time)
-			if err = json.Unmarshal(msg.Value, &envelope); err != nil {
+			commit := func(ctx context.Context, message kafka.Message) error {
-				logs.Err.Println("Kafka envelope decode failed:", err)
+				return reader.CommitMessages(ctx, message)
-				_ = reader.CommitMessages(context.Background(), msg)
-				continue
 			}
-			if err = handler(kafkaBus.ctx, envelope, msg); err != nil {
+			if err = consumeKafkaMessage(kafkaBus.ctx, msg, handler, commit, waitKafkaRetry); err != nil {
-				logs.Err.Println("Kafka handler failed:", envelope.EventID, envelope.EventType, err)
+				if errors.Is(err, context.Canceled) || errors.Is(kafkaBus.ctx.Err(), context.Canceled) {
-				continue
+					return
 				}
-			if err = reader.CommitMessages(context.Background(), msg); err != nil {
+				logs.Err.Printf("Kafka message stopped without commit: topic=%s partition=%d offset=%d err=%v", msg.Topic, msg.Partition, msg.Offset, err)
-				logs.Err.Println("Kafka commit failed:", err)
 			}
+			readerMetrics.clear()
 		}
 	}()
 	logs.Info.Printf("Kafka consumer started: topic=%s group=%s", topic, groupID)
 }
 
+func startKafkaBatchConsumer(topicName string, handler kafkaBatchHandler) {
+	if kafkaBus == nil || handler == nil {
+		return
+	}
+	topic := kafkaBus.topics[topicName]
+	if topic == "" {
+		topic = topicName
+	}
+	groupID := kafkaBus.config.ConsumerGroup + "." + topicName
+	reader := kafka.NewReader(kafka.ReaderConfig{
+		Brokers: kafkaBus.config.Brokers, GroupID: groupID, Topic: topic, StartOffset: kafka.FirstOffset,
+		MinBytes: 1, MaxBytes: 10e6, MaxWait: time.Second, Dialer: kafkaBus.dialer(),
+	})
+	readerMetrics := startKafkaReaderMetrics(kafkaBus.ctx, reader, topic, groupID)
+	kafkaBus.wg.Add(1)
+	go func() {
+		defer kafkaBus.wg.Done()
+		defer reader.Close()
+		for {
+			messages, err := fetchKafkaMessageBatch(kafkaBus.ctx, reader, kafkaBus.config.MessageBatchSize,
+				time.Duration(kafkaBus.config.MessageBatchWait)*time.Millisecond)
+			if err != nil {
+				if errors.Is(err, context.Canceled) || errors.Is(kafkaBus.ctx.Err(), context.Canceled) {
+					return
+				}
+				logs.Err.Println("Kafka batch fetch failed:", err)
+				continue
+			}
+			if len(messages) > 0 {
+				readerMetrics.observe(messages[0].Time)
+			}
+			if err = consumeKafkaMessageBatch(kafkaBus.ctx, messages, handler, reader.CommitMessages, waitKafkaRetry); err != nil {
+				if errors.Is(err, context.Canceled) || errors.Is(kafkaBus.ctx.Err(), context.Canceled) {
+					return
+				}
+				logs.Err.Printf("Kafka batch stopped without commit: topic=%s size=%d err=%v", topic, len(messages), err)
+			}
+			readerMetrics.clear()
+		}
+	}()
+	logs.Info.Printf("Kafka batch consumer started: topic=%s group=%s size=%d wait_ms=%d", topic, groupID, kafkaBus.config.MessageBatchSize, kafkaBus.config.MessageBatchWait)
+}
+
+type kafkaReaderMetrics struct {
+	reader     *kafka.Reader
+	topic      string
+	group      string
+	oldestNano atomic.Int64
+}
+
+func startKafkaReaderMetrics(ctx context.Context, reader *kafka.Reader, topic, group string) *kafkaReaderMetrics {
+	metrics := &kafkaReaderMetrics{reader: reader, topic: topic, group: group}
+	kafkaBus.wg.Add(1)
+	go func() {
+		defer kafkaBus.wg.Done()
+		ticker := time.NewTicker(5 * time.Second)
+		defer ticker.Stop()
+		metrics.update()
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-ticker.C:
+				metrics.update()
+			}
+		}
+	}()
+	return metrics
+}
+
+func (m *kafkaReaderMetrics) observe(messageTime time.Time) {
+	if !messageTime.IsZero() {
+		m.oldestNano.Store(messageTime.UnixNano())
+	}
+	m.update()
+}
+
+func (m *kafkaReaderMetrics) clear() {
+	m.oldestNano.Store(0)
+	m.update()
+}
+
+func (m *kafkaReaderMetrics) update() {
+	stats := m.reader.Stats()
+	if stats.Lag >= 0 {
+		immetrics.KafkaConsumerLag.WithLabelValues(clusterPodName(), m.topic, m.group).Set(float64(stats.Lag))
+	}
+	age := float64(0)
+	if oldest := m.oldestNano.Load(); oldest > 0 {
+		age = time.Since(time.Unix(0, oldest)).Seconds()
+		if age < 0 {
+			age = 0
+		}
+	}
+	immetrics.KafkaBacklogOldestAge.WithLabelValues(clusterPodName(), m.topic, m.group).Set(age)
+}
+
+func fetchKafkaMessageBatch(ctx context.Context, reader *kafka.Reader, maxSize int, maxWait time.Duration) ([]kafka.Message, error) {
+	first, err := reader.FetchMessage(ctx)
+	if err != nil {
+		return nil, err
+	}
+	messages := []kafka.Message{first}
+	deadline := time.Now().Add(maxWait)
+	for len(messages) < maxSize {
+		remaining := time.Until(deadline)
+		if remaining <= 0 {
+			break
+		}
+		fetchCtx, cancel := context.WithTimeout(ctx, remaining)
+		message, fetchErr := reader.FetchMessage(fetchCtx)
+		cancel()
+		if errors.Is(fetchErr, context.DeadlineExceeded) && ctx.Err() == nil {
+			break
+		}
+		if fetchErr != nil {
+			return nil, fetchErr
+		}
+		messages = append(messages, message)
+	}
+	return messages, nil
+}
+
+func consumeKafkaMessageBatch(ctx context.Context, messages []kafka.Message, handler kafkaBatchHandler, commit kafkaBatchCommitter, wait kafkaRetryWaiter) error {
+	for attempt := 1; ; attempt++ {
+		if err := handler(ctx, messages); err == nil {
+			return commitKafkaMessageBatch(ctx, messages, commit, wait)
+		}
+		if err := wait(ctx, kafkaRetryBackoff(attempt)); err != nil {
+			return err
+		}
+	}
+}
+
+func consumeKafkaMessage(ctx context.Context, msg kafka.Message, handler kafkaMessageHandler, commit kafkaMessageCommitter, wait kafkaRetryWaiter) error {
+	var envelope kafkaEnvelope
+	if err := json.Unmarshal(msg.Value, &envelope); err != nil {
+		return commitKafkaMessage(ctx, msg, commit, wait)
+	}
+	for attempt := 1; ; attempt++ {
+		if err := handler(ctx, envelope, msg); err == nil {
+			return commitKafkaMessage(ctx, msg, commit, wait)
+		}
+		if err := wait(ctx, kafkaRetryBackoff(attempt)); err != nil {
+			return err
+		}
+	}
+}
+
+func commitKafkaMessage(ctx context.Context, msg kafka.Message, commit kafkaMessageCommitter, wait kafkaRetryWaiter) error {
+	for attempt := 1; ; attempt++ {
+		if err := commit(ctx, msg); err == nil {
+			return nil
+		}
+		if err := wait(ctx, kafkaRetryBackoff(attempt)); err != nil {
+			return err
+		}
+	}
+}
+
+func commitKafkaMessageBatch(ctx context.Context, messages []kafka.Message, commit kafkaBatchCommitter, wait kafkaRetryWaiter) error {
+	for attempt := 1; ; attempt++ {
+		if err := commit(ctx, messages...); err == nil {
+			return nil
+		}
+		if err := wait(ctx, kafkaRetryBackoff(attempt)); err != nil {
+			return err
+		}
+	}
+}
+
+func kafkaRetryBackoff(attempt int) time.Duration {
+	if kafkaBus == nil || len(kafkaBus.config.RetryBackoffMS) == 0 {
+		return time.Second
+	}
+	if attempt < 1 {
+		attempt = 1
+	}
+	index := attempt - 1
+	if index >= len(kafkaBus.config.RetryBackoffMS) {
+		index = len(kafkaBus.config.RetryBackoffMS) - 1
+	}
+	return time.Duration(kafkaBus.config.RetryBackoffMS[index]) * time.Millisecond
+}
+
+func waitKafkaRetry(ctx context.Context, delay time.Duration) error {
+	timer := time.NewTimer(delay)
+	defer timer.Stop()
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case <-timer.C:
+		return nil
+	}
+}
+
 func kafkaBroadcastConsumerGroup(topicName string) string {
 	if kafkaBus == nil {
 		return ""

server/kafka_runtime_batch_test.go

@@ -0,0 +1,143 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/segmentio/kafka-go"
+	"github.com/tinode/chat/server/store/types"
+)
+
+func TestConsumeKafkaMessageBatchRetriesHandlerAndCommit(t *testing.T) {
+	handlerCalls := 0
+	commitCalls := 0
+	waitCalls := 0
+	messages := []kafka.Message{{Offset: 1}, {Offset: 2}}
+
+	err := consumeKafkaMessageBatch(context.Background(), messages,
+		func(context.Context, []kafka.Message) error {
+			handlerCalls++
+			if handlerCalls == 1 {
+				return errors.New("batch failed")
+			}
+			return nil
+		},
+		func(_ context.Context, committed ...kafka.Message) error {
+			commitCalls++
+			if len(committed) != len(messages) {
+				t.Fatalf("committed %d messages, want %d", len(committed), len(messages))
+			}
+			if commitCalls == 1 {
+				return errors.New("commit failed")
+			}
+			return nil
+		},
+		func(context.Context, time.Duration) error {
+			waitCalls++
+			return nil
+		})
+
+	if err != nil {
+		t.Fatalf("consumeKafkaMessageBatch() error = %v", err)
+	}
+	if handlerCalls != 2 || commitCalls != 2 || waitCalls != 2 {
+		t.Fatalf("calls handler=%d commit=%d wait=%d, want 2/2/2", handlerCalls, commitCalls, waitCalls)
+	}
+}
+
+func TestConsumeKafkaMalformedMessageRetriesCommit(t *testing.T) {
+	commitCalls := 0
+	waitCalls := 0
+	handlerCalls := 0
+	err := consumeKafkaMessage(context.Background(), kafka.Message{Value: []byte("{")},
+		func(context.Context, kafkaEnvelope, kafka.Message) error {
+			handlerCalls++
+			return nil
+		},
+		func(context.Context, kafka.Message) error {
+			commitCalls++
+			if commitCalls == 1 {
+				return errors.New("commit failed")
+			}
+			return nil
+		},
+		func(context.Context, time.Duration) error {
+			waitCalls++
+			return nil
+		})
+
+	if err != nil {
+		t.Fatalf("consumeKafkaMessage() error = %v", err)
+	}
+	if handlerCalls != 0 || commitCalls != 2 || waitCalls != 1 {
+		t.Fatalf("calls handler=%d commit=%d wait=%d, want 0/2/1", handlerCalls, commitCalls, waitCalls)
+	}
+}
+
+func TestRunKafkaMessageSingleRetriesThenDeadLetters(t *testing.T) {
+	operationCalls := 0
+	markFailedCalls := 0
+	markDeadCalls := 0
+	waitCalls := 0
+
+	err := runKafkaMessageSingleRetries(context.Background(), 3,
+		func() error {
+			operationCalls++
+			return errors.New("write failed")
+		},
+		func(error) (*types.KafkaConsumeRecord, error) {
+			markFailedCalls++
+			return &types.KafkaConsumeRecord{RetryCount: markFailedCalls}, nil
+		},
+		func(record *types.KafkaConsumeRecord, err error) error {
+			markDeadCalls++
+			if record == nil || record.RetryCount != 4 || err == nil {
+				t.Fatalf("unexpected dead-letter input: record=%+v err=%v", record, err)
+			}
+			return nil
+		},
+		func(context.Context, time.Duration) error {
+			waitCalls++
+			return nil
+		})
+
+	if err != nil {
+		t.Fatalf("runKafkaMessageSingleRetries() error = %v", err)
+	}
+	if operationCalls != 4 || markFailedCalls != 4 || waitCalls != 3 || markDeadCalls != 1 {
+		t.Fatalf("calls operation=%d failed=%d wait=%d dead=%d, want 4/4/3/1",
+			operationCalls, markFailedCalls, waitCalls, markDeadCalls)
+	}
+}
+
+func TestRunKafkaMessageSingleRetriesStopsOnSuccess(t *testing.T) {
+	operationCalls := 0
+	markFailedCalls := 0
+	markDeadCalls := 0
+	err := runKafkaMessageSingleRetries(context.Background(), 3,
+		func() error {
+			operationCalls++
+			if operationCalls < 3 {
+				return errors.New("temporary failure")
+			}
+			return nil
+		},
+		func(error) (*types.KafkaConsumeRecord, error) {
+			markFailedCalls++
+			return &types.KafkaConsumeRecord{RetryCount: markFailedCalls}, nil
+		},
+		func(*types.KafkaConsumeRecord, error) error {
+			markDeadCalls++
+			return nil
+		},
+		func(context.Context, time.Duration) error { return nil })
+
+	if err != nil {
+		t.Fatalf("runKafkaMessageSingleRetries() error = %v", err)
+	}
+	if operationCalls != 3 || markFailedCalls != 2 || markDeadCalls != 0 {
+		t.Fatalf("calls operation=%d failed=%d dead=%d, want 3/2/0", operationCalls, markFailedCalls, markDeadCalls)
+	}
+}

server/kafka_workbench_events.go

@@ -0,0 +1,161 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"strings"
+	"time"
+
+	"github.com/segmentio/kafka-go"
+	"github.com/tinode/chat/server/push"
+	"github.com/tinode/chat/server/store"
+	"github.com/tinode/chat/server/store/types"
+)
+
+type kafkaWorkbenchEvent struct {
+	EventID       string         `json:"event_id"`
+	TraceID       string         `json:"trace_id,omitempty"`
+	TenantID      types.TenantID `json:"tenant_id"`
+	TenantCode    string         `json:"tenant_code,omitempty"`
+	ActorUserID   int64          `json:"actor_user_id"`
+	Module        string         `json:"module"`
+	Action        string         `json:"action"`
+	ResourceID    string         `json:"resource_id"`
+	TargetUserIDs []int64        `json:"target_user_ids,omitempty"`
+	Title         string         `json:"title,omitempty"`
+	Summary       string         `json:"summary,omitempty"`
+	OccurredAt    time.Time      `json:"occurred_at"`
+	Data          map[string]any `json:"data,omitempty"`
+}
+
+func startKafkaWorkbenchEventConsumer() {
+	startKafkaConsumer("workbench_external", handleKafkaWorkbenchEvent)
+}
+
+func handleKafkaWorkbenchEvent(ctx context.Context, envelope kafkaEnvelope, msg kafka.Message) error {
+	var event kafkaWorkbenchEvent
+	if err := json.Unmarshal(msg.Value, &event); err != nil {
+		return err
+	}
+	event.normalize()
+	if event.EventID == "" || event.TenantID.IsZero() || event.Action == "" || event.ResourceID == "" {
+		return types.ErrMalformed
+	}
+	envelope = event.kafkaEnvelope(envelope)
+	_, claimed, err := beginKafkaConsumeForTopic(event.TenantID, envelope, "workbench_external")
+	if err != nil || !claimed {
+		return err
+	}
+	if err = executeKafkaWorkbenchEvent(event); err != nil {
+		reason := kafkaCommandErrorReason(err)
+		failed, markErr := store.HTTPInfrastructure.MarkKafkaConsumeFailed(event.TenantID, event.EventID, kafkaBus.config.ConsumerGroup,
+			reason, err.Error(), time.Now().UTC().Round(time.Millisecond))
+		if markErr != nil {
+			return markErr
+		}
+		if failed != nil && failed.RetryCount >= kafkaBus.config.MaxRetryCount {
+			return markKafkaDeadLetter(ctx, event.TenantID, envelope, failed, reason, err.Error())
+		}
+		return err
+	}
+	return store.HTTPInfrastructure.MarkKafkaConsumeSucceeded(event.TenantID, event.EventID, kafkaBus.config.ConsumerGroup, time.Now().UTC().Round(time.Millisecond))
+}
+
+func executeKafkaWorkbenchEvent(event kafkaWorkbenchEvent) error {
+	actor := store.EncodeUid(event.ActorUserID)
+	payload, err := event.realtimePayload()
+	if err != nil {
+		return err
+	}
+	pushTargets := make(map[types.Uid]push.Recipient)
+	for _, targetID := range event.TargetUserIDs {
+		target := store.EncodeUid(targetID)
+		if target.IsZero() {
+			continue
+		}
+		if target != actor && event.Title != "" {
+			action := event.notificationAction()
+			if err = createIMSystemNotification(event.TenantID, target, stableIMEventID(event.EventID, "notification", target.UserId()),
+				event.Action, event.Title, event.Summary, action); err != nil && err != types.ErrDuplicate {
+				return err
+			}
+			pushTargets[target] = push.Recipient{}
+		}
+		if err = enqueueIMBusinessEvent(event.TenantID, event.EventID, "user", target.UserId(), actor, event.Action, payload); err != nil {
+			return err
+		}
+	}
+	if len(pushTargets) > 0 {
+		sendPush(&push.Receipt{TenantID: event.TenantID, To: pushTargets, Payload: push.Payload{
+			What: push.ActMsg, Topic: "sys", Timestamp: time.Now().UTC(), ContentType: "application/json",
+			Content: map[string]any{"event_id": event.EventID, "event_type": event.Action, "title": event.Title, "summary": event.Summary, "action": event.notificationAction()},
+		}})
+	}
+	return nil
+}
+
+func (e *kafkaWorkbenchEvent) normalize() {
+	e.EventID = strings.TrimSpace(e.EventID)
+	e.TenantCode = strings.TrimSpace(e.TenantCode)
+	e.Module = strings.TrimSpace(e.Module)
+	e.Action = strings.TrimSpace(e.Action)
+	e.ResourceID = strings.TrimSpace(e.ResourceID)
+	e.Title = strings.TrimSpace(e.Title)
+	e.Summary = strings.TrimSpace(e.Summary)
+	if e.OccurredAt.IsZero() {
+		e.OccurredAt = time.Now().UTC().Round(time.Millisecond)
+	}
+	e.TargetUserIDs = compactInt64IDs(e.TargetUserIDs)
+}
+
+func (e kafkaWorkbenchEvent) kafkaEnvelope(base kafkaEnvelope) kafkaEnvelope {
+	base.EventID = e.EventID
+	base.TraceID = firstNonEmpty(base.TraceID, e.TraceID)
+	base.TenantCode = firstNonEmpty(base.TenantCode, e.TenantCode)
+	base.TenantID = e.TenantID
+	base.EventType = e.Action
+	base.BizType = e.Module
+	base.BizID = e.ResourceID
+	base.OccurredAt = e.OccurredAt
+	return base
+}
+
+func (e kafkaWorkbenchEvent) realtimePayload() (map[string]any, error) {
+	payload := make(map[string]any, len(e.Data)+3)
+	for key, value := range e.Data {
+		if strings.TrimSpace(key) != "" {
+			payload[key] = value
+		}
+	}
+	payload["module"] = e.Module
+	payload["resource_id"] = e.ResourceID
+	payload["event_id"] = e.EventID
+	return payload, nil
+}
+
+func (e kafkaWorkbenchEvent) notificationAction() types.KVMap {
+	action := types.KVMap{
+		"module":      e.Module,
+		"resource_id": e.ResourceID,
+		"event_id":    e.EventID,
+	}
+	for key, value := range e.Data {
+		if strings.TrimSpace(key) != "" {
+			action[key] = value
+		}
+	}
+	return action
+}
+
+func compactInt64IDs(ids []int64) []int64 {
+	out := make([]int64, 0, len(ids))
+	seen := make(map[int64]bool, len(ids))
+	for _, id := range ids {
+		if id <= 0 || seen[id] {
+			continue
+		}
+		seen[id] = true
+		out = append(out, id)
+	}
+	return out
+}

server/main.go

@@ -34,6 +34,7 @@ import (
 	_ "github.com/tinode/chat/server/auth/code"
 	_ "github.com/tinode/chat/server/auth/rest"
 	_ "github.com/tinode/chat/server/auth/token"
+	adapter "github.com/tinode/chat/server/db"
 	"github.com/tinode/chat/server/store/types"
 
 	// Database backends
@@ -516,6 +517,23 @@ func main() {
 		logs.Info.Println("All done, good bye")
 	}()
 	statsRegisterDbStats()
+	if kafkaConfigured(config.Kafka) {
+		tenants, tenantErr := store.Tenants.ListActive(10000)
+		if tenantErr != nil {
+			logs.Err.Fatal("Failed to list tenants for Kafka message schema validation:", tenantErr)
+		}
+		tenantIDs := make([]types.TenantID, 0, len(tenants))
+		for i := range tenants {
+			tenantIDs = append(tenantIDs, tenants[i].ID)
+		}
+		validator, ok := store.Store.GetAdapter().(adapter.TenantMessageSchemaAdapter)
+		if !ok {
+			logs.Err.Fatal("Kafka message persistence requires a message schema validator")
+		}
+		if err = validator.TenantMessageSchemaValidate(tenantIDs); err != nil {
+			logs.Err.Fatal("Kafka message persistence schema validation failed:", err)
+		}
+	}
 
 	if err = initKafkaRuntime(config.Kafka); err != nil {
 		logs.Err.Fatal("Failed to initialize Kafka:", err)
@@ -757,6 +775,7 @@ func main() {
 	startKafkaAdminCommandConsumer()
 	startKafkaConfigEventConsumer()
 	startKafkaCacheInvalidateConsumer()
+	startKafkaWorkbenchEventConsumer()
 	startKafkaMessagePersistConsumer()
 	startKafkaDeadLetterConsumer()
 
@@ -859,6 +878,7 @@ func main() {
 		mux.HandleFunc(sspath, serveStatus)
 	}
 	mux.HandleFunc("/healthz", serveHealthz)
+	mux.HandleFunc("/livez", serveLivez)
 	mux.HandleFunc("/metrics", serveMetrics)
 
 	// Public legal pages are opened directly by clients before login.

server/meetingim.conf

@@ -128,6 +128,9 @@
     "sasl_password": "gr58tNnlZOVGW3AmMbL2",
     "max_retry_count": 3,
     "retry_backoff_ms": [10000, 30000, 60000, 300000, 900000],
+    "message_batch_size": 100,
+    "message_batch_wait_ms": 10,
+    "message_single_retry_count": 3,
     "dead_letter_retention_seconds": 7776000,
     "topics": {
       "admin_command": "im.admin.command",
@@ -235,6 +238,11 @@
   "cluster_config": {
     "self": "",
     "nodes": [],
+    "proxy_to_master_buffer": 4096,
+    "proxy_to_master_workers": 16,
+    "proxy_to_master_enqueue_timeout_ms": 300,
+    "route_retries": 3,
+    "route_retry_backoff_ms": 25,
     "failover": {
       "enabled": false
     }

server/metrics/metrics.go

@@ -2,7 +2,9 @@ package metrics
 
 import (
 	"expvar"
+	"os"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
@@ -45,11 +47,98 @@ var (
 	}, func() float64 {
 		return float64(expvarIntValue("LiveSessions"))
 	})
+	IMWSConnectionsActive = prometheus.NewGaugeFunc(prometheus.GaugeOpts{
+		Name:        "im_ws_connections_active",
+		Help:        "Current active WebSocket connections, using the IM metric namespace.",
+		ConstLabels: prometheus.Labels{"pod": metricPodName()},
+	}, func() float64 {
+		return float64(expvarIntValue("LiveSessions"))
+	})
+	IMClusterLivePeers = prometheus.NewGaugeFunc(prometheus.GaugeOpts{
+		Name:        "im_cluster_live_peers",
+		Help:        "Current connected cluster peers.",
+		ConstLabels: prometheus.Labels{"pod": metricPodName()},
+	}, func() float64 {
+		return float64(expvarIntValue("LiveClusterNodes"))
+	})
+	IMClusterExpectedPeers = prometheus.NewGaugeFunc(prometheus.GaugeOpts{
+		Name:        "im_cluster_expected_peers",
+		Help:        "Expected cluster peers derived from configured total nodes.",
+		ConstLabels: prometheus.Labels{"pod": metricPodName()},
+	}, func() float64 {
+		total := expvarIntValue("TotalClusterNodes")
+		if total > 0 {
+			total--
+		}
+		return float64(total)
+	})
+
+	ClusterP2MQueueLength = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "im_cluster_p2m_queue_length",
+		Help: "Current proxy-to-master queue depth by pod and peer.",
+	}, []string{"pod", "peer"})
+	ClusterP2MQueueCapacity = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "im_cluster_p2m_queue_capacity",
+		Help: "Configured proxy-to-master queue capacity by pod and peer.",
+	}, []string{"pod", "peer"})
+	ClusterP2MEnqueueTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name: "im_cluster_p2m_enqueue_total",
+		Help: "Proxy-to-master enqueue attempts by pod, peer, and result.",
+	}, []string{"pod", "peer", "result"})
+	ClusterRPCInflight = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "im_cluster_rpc_inflight",
+		Help: "Current synchronous cluster RPC calls by pod, peer, and method.",
+	}, []string{"pod", "peer", "method"})
+	ClusterRPCDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Name:    "im_cluster_rpc_duration_seconds",
+		Help:    "Cluster RPC duration by pod, peer, method, and result.",
+		Buckets: []float64{0.001, 0.003, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5},
+	}, []string{"pod", "peer", "method", "result"})
+	ClusterRouteTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name: "im_cluster_route_total",
+		Help: "Cluster route attempts by pod, peer, and result.",
+	}, []string{"pod", "peer", "result"})
+	ClusterProxyResponseTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name: "im_cluster_proxy_response_total",
+		Help: "Cluster proxy responses by pod and result.",
+	}, []string{"pod", "result"})
+	MessagePersistDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Name:    "im_message_persist_duration_seconds",
+		Help:    "Kafka message persistence duration by message kind and result.",
+		Buckets: []float64{0.001, 0.003, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10},
+	}, []string{"kind", "result"})
+	KafkaMessagePersistTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name: "im_kafka_message_persist_total",
+		Help: "Kafka message persistence outcomes by message kind and result.",
+	}, []string{"kind", "result"})
+	KafkaConsumerLag = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "im_kafka_consumer_lag",
+		Help: "Kafka reader lag reported by kafka-go by pod, topic, and consumer group.",
+	}, []string{"pod", "topic", "group"})
+	KafkaBacklogOldestAge = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "im_kafka_backlog_oldest_age_seconds",
+		Help: "Age of the latest fetched Kafka message by pod, topic, and consumer group.",
+	}, []string{"pod", "topic", "group"})
+	KafkaDeadLettersTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name: "im_kafka_dead_letters_total",
+		Help: "Kafka messages moved to dead letter by message kind and reason.",
+	}, []string{"kind", "reason"})
 )
 
 func init() {
-	Registry.MustRegister(WSConnectionsActive, HTTPRequestsTotal, HTTPRequestDuration, DBQueryDuration,
+	Registry.MustRegister(WSConnectionsActive, IMWSConnectionsActive, IMClusterLivePeers, IMClusterExpectedPeers,
-		RealtimeFanoutEventsTotal, RealtimeFanoutDeliveryDuration)
+		HTTPRequestsTotal, HTTPRequestDuration, DBQueryDuration,
+		RealtimeFanoutEventsTotal, RealtimeFanoutDeliveryDuration, ClusterP2MQueueLength,
+		ClusterP2MQueueCapacity, ClusterP2MEnqueueTotal, ClusterRPCInflight, ClusterRPCDuration,
+		ClusterRouteTotal, ClusterProxyResponseTotal, MessagePersistDuration, KafkaMessagePersistTotal)
+	Registry.MustRegister(KafkaConsumerLag, KafkaBacklogOldestAge, KafkaDeadLettersTotal)
+}
+
+func metricPodName() string {
+	if value := strings.TrimSpace(os.Getenv("HOSTNAME")); value != "" {
+		return value
+	}
+	return "standalone"
 }
 
 func ObserveDB(driverName, operation string, started time.Time, err error) {

server/monitoring_http.go

@@ -4,12 +4,14 @@ import (
 	"bufio"
 	"context"
 	"encoding/json"
+	"fmt"
 	"io"
 	"net"
 	"net/http"
 	"regexp"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus/promhttp"
@@ -20,6 +22,8 @@ import (
 var (
 	metricsHandler        = promhttp.HandlerFor(immetrics.Registry, promhttp.HandlerOpts{})
 	uuidLikeMetricSegment = regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F-]{13,}$`)
+	clusterQueueHighSince = make(map[string]time.Time)
+	clusterQueueHealthMu  sync.Mutex
 )
 
 type healthzResponse struct {
@@ -45,6 +49,28 @@ func serveHealthz(wrt http.ResponseWriter, req *http.Request) {
 	} else {
 		checks["mysql"] = "ok"
 	}
+	if globals.cluster != nil {
+		expected := len(globals.cluster.nodes)
+		live := 0
+		for _, node := range globals.cluster.nodes {
+			node.lock.Lock()
+			connected := node.connected
+			node.lock.Unlock()
+			if connected {
+				live++
+			}
+		}
+		checks["cluster"] = fmt.Sprintf("%d/%d peers connected", live, expected)
+		if live != expected {
+			status = http.StatusServiceUnavailable
+		}
+		if healthy, detail := clusterQueueHealthy(globals.cluster, time.Now()); !healthy {
+			checks["cluster_queue"] = detail
+			status = http.StatusServiceUnavailable
+		} else {
+			checks["cluster_queue"] = detail
+		}
+	}
 
 	body := healthzResponse{Status: "ok", Checks: checks}
 	if status != http.StatusOK {
@@ -56,6 +82,47 @@ func serveHealthz(wrt http.ResponseWriter, req *http.Request) {
 	}
 }
 
+func clusterQueueHealthy(cluster *Cluster, now time.Time) (bool, string) {
+	clusterQueueHealthMu.Lock()
+	defer clusterQueueHealthMu.Unlock()
+	healthy := true
+	detail := "ok"
+	for name, node := range cluster.nodes {
+		depth, capacity := 0, 0
+		for _, sender := range node.p2mSenders {
+			depth += len(sender)
+			capacity += cap(sender)
+		}
+		if capacity == 0 || depth*100 < capacity*90 {
+			delete(clusterQueueHighSince, name)
+			continue
+		}
+		since, exists := clusterQueueHighSince[name]
+		if !exists {
+			clusterQueueHighSince[name] = now
+			continue
+		}
+		if now.Sub(since) >= 30*time.Second {
+			healthy = false
+			detail = fmt.Sprintf("peer %s queue %d/%d above 90%% for %s", name, depth, capacity,
+				now.Sub(since).Round(time.Second))
+		}
+	}
+	return healthy, detail
+}
+
+func serveLivez(wrt http.ResponseWriter, req *http.Request) {
+	if req.Method != http.MethodGet && req.Method != http.MethodHead {
+		wrt.Header().Set("Allow", http.MethodGet+", "+http.MethodHead)
+		http.Error(wrt, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	wrt.WriteHeader(http.StatusOK)
+	if req.Method != http.MethodHead {
+		_, _ = io.WriteString(wrt, "ok\n")
+	}
+}
+
 func serveMetrics(wrt http.ResponseWriter, req *http.Request) {
 	if req.Method != http.MethodGet && req.Method != http.MethodHead {
 		wrt.Header().Set("Allow", http.MethodGet+", "+http.MethodHead)

server/monitoring_http_test.go

@@ -52,7 +52,44 @@ func TestServeHealthzError(t *testing.T) {
 	}
 }
 
+func TestServeLivez(t *testing.T) {
+	wrt := httptest.NewRecorder()
+	serveLivez(wrt, httptest.NewRequest(http.MethodGet, "/livez", nil))
+	if wrt.Code != http.StatusOK || wrt.Body.String() != "ok\n" {
+		t.Fatalf("unexpected livez response: code=%d body=%q", wrt.Code, wrt.Body.String())
+	}
+}
+
+func TestClusterQueueHealthRequiresSustainedHighWatermark(t *testing.T) {
+	clusterQueueHealthMu.Lock()
+	clusterQueueHighSince = make(map[string]time.Time)
+	clusterQueueHealthMu.Unlock()
+	queue := make(chan *ClusterReq, 10)
+	for i := 0; i < 9; i++ {
+		queue <- &ClusterReq{}
+	}
+	cluster := &Cluster{nodes: map[string]*ClusterNode{"peer": {p2mSenders: []chan *ClusterReq{queue}}}}
+	now := time.Now()
+	if healthy, _ := clusterQueueHealthy(cluster, now); !healthy {
+		t.Fatal("instant high watermark must not fail readiness")
+	}
+	if healthy, _ := clusterQueueHealthy(cluster, now.Add(31*time.Second)); healthy {
+		t.Fatal("sustained high watermark must fail readiness")
+	}
+	<-queue
+	if healthy, _ := clusterQueueHealthy(cluster, now.Add(32*time.Second)); !healthy {
+		t.Fatal("queue recovery must restore readiness")
+	}
+}
+
 func TestServeMetricsIncludesWSConnections(t *testing.T) {
+	immetrics.ClusterP2MQueueLength.WithLabelValues("test", "peer").Set(0)
+	immetrics.ClusterP2MEnqueueTotal.WithLabelValues("test", "peer", "ok").Add(0)
+	immetrics.ClusterRPCDuration.WithLabelValues("test", "peer", "Cluster.Route", "ok").Observe(0)
+	immetrics.ClusterRouteTotal.WithLabelValues("test", "peer", "ok").Add(0)
+	immetrics.KafkaMessagePersistTotal.WithLabelValues("p2p", "batch_ok").Add(0)
+	immetrics.KafkaConsumerLag.WithLabelValues("test", "topic", "group").Set(0)
+	immetrics.KafkaDeadLettersTotal.WithLabelValues("p2p", "test").Add(0)
 	wrt := httptest.NewRecorder()
 	serveMetrics(wrt, httptest.NewRequest(http.MethodGet, "/metrics", nil))
 
@@ -63,6 +100,13 @@ func TestServeMetricsIncludesWSConnections(t *testing.T) {
 	if !strings.Contains(body, "# TYPE ws_connections_active gauge") || !strings.Contains(body, "ws_connections_active ") {
 		t.Fatalf("unexpected body %s", body)
 	}
+	for _, metric := range []string{"im_ws_connections_active", "im_cluster_p2m_queue_length",
+		"im_cluster_p2m_enqueue_total", "im_cluster_rpc_duration_seconds", "im_cluster_route_total",
+		"im_kafka_message_persist_total", "im_kafka_consumer_lag", "im_kafka_dead_letters_total"} {
+		if !strings.Contains(body, "# HELP "+metric+" ") {
+			t.Fatalf("metric %s was not exported", metric)
+		}
+	}
 }
 
 func TestMonitorHTTPRecordsRequestMetrics(t *testing.T) {

server/store/group_message_runtime.go

@@ -1497,7 +1497,9 @@ func (r *redisGroupMessageRuntime) loginSessionKey(tokenHash string) string {
 
 func (r *redisGroupMessageRuntime) cacheTTL(namespace string) time.Duration {
 	ttl := r.cfg.CacheTTL
-	if namespace == "verification" || namespace == "action_token" || namespace == "login_lockout" {
+	if namespace == cacheNamespaceTopic || namespace == cacheNamespaceSub {
+		ttl = 60
+	} else if namespace == "verification" || namespace == "action_token" || namespace == "login_lockout" {
 		ttl = r.cfg.ShortTTL
 	}
 	return time.Duration(ttl) * time.Second

server/store/group_message_runtime_test.go

@@ -409,3 +409,28 @@ func TestInvalidateGroupSettingsUsesTopicCacheKey(t *testing.T) {
 		t.Fatalf("unexpected deleted objects: %#v", runtime.deletedObjects)
 	}
 }
+
+func TestTopicAndSubscriptionCacheTTLIsShort(t *testing.T) {
+	runtime := &redisGroupMessageRuntime{cfg: redisConfig{CacheTTL: 300, ShortTTL: 600}}
+	for _, namespace := range []string{cacheNamespaceTopic, cacheNamespaceSub} {
+		if got := runtime.cacheTTL(namespace); got != time.Minute {
+			t.Fatalf("cache TTL for %s = %v, want 1m", namespace, got)
+		}
+	}
+	if got := runtime.cacheTTL(cacheNamespaceGroupSettings); got != 5*time.Minute {
+		t.Fatalf("cache TTL for group settings = %v, want 5m", got)
+	}
+}
+
+func TestInvalidateSubscriptionCacheWithZeroUserDeletesTopicPrefix(t *testing.T) {
+	oldRuntime := groupMsgRuntime
+	runtime := newTestGroupMessageRuntime()
+	groupMsgRuntime = runtime
+	t.Cleanup(func() { groupMsgRuntime = oldRuntime })
+
+	invalidateSubscriptionCache(7, "grpAlpha", types.ZeroUid)
+	want := []string{"grpAlpha:", "chnAlpha:"}
+	if len(runtime.deletedPrefixes) != len(want) || runtime.deletedPrefixes[0] != want[0] || runtime.deletedPrefixes[1] != want[1] {
+		t.Fatalf("unexpected deleted prefixes: %#v", runtime.deletedPrefixes)
+	}
+}

server/store/store.go

@@ -1395,7 +1395,11 @@ func (topicsMapper) UpdateSubCnt(tenantID types.TenantID, topic string) error {
 	if err != nil {
 		return err
 	}
-	return tenantAdp.TenantTopicUpdateSubCnt(tenantID, topic)
+	if err = tenantAdp.TenantTopicUpdateSubCnt(tenantID, topic); err != nil {
+		return err
+	}
+	invalidateTopicCache(tenantID, topic)
+	return nil
 }
 
 // Update is a generic topic update.
@@ -1410,7 +1414,7 @@ func (topicsMapper) Update(tenantID types.TenantID, topic string, update map[str
 	if err = tenantAdp.TenantTopicUpdate(tenantID, topic, update); err != nil {
 		return err
 	}
-	_ = DeleteCachedObject(tenantID, cacheNamespaceTopic, topic)
+	invalidateTopicCache(tenantID, topic)
 	return nil
 }
 
@@ -1421,7 +1425,7 @@ func (topicsMapper) BumpMutationVersion(tenantID types.TenantID, topic string, a
 	}
 	version, err := tenantAdp.TenantTopicBumpMutationVersion(tenantID, topic, at)
 	if err == nil {
-		_ = DeleteCachedObject(tenantID, cacheNamespaceTopic, topic)
+		invalidateTopicCache(tenantID, topic)
 	}
 	return version, err
 }
@@ -1435,7 +1439,7 @@ func (topicsMapper) OwnerChange(tenantID types.TenantID, topic string, newOwner
 	if err = tenantAdp.TenantTopicOwnerChange(tenantID, topic, newOwner); err != nil {
 		return err
 	}
-	_ = DeleteCachedObject(tenantID, cacheNamespaceTopic, topic)
+	invalidateTopicCache(tenantID, topic)
 	return nil
 }
 
@@ -1448,7 +1452,8 @@ func (topicsMapper) Delete(tenantID types.TenantID, topic string, isChan, hard b
 	if err = tenantAdp.TenantTopicDelete(tenantID, topic, isChan, hard); err != nil {
 		return err
 	}
-	_ = DeleteCachedObject(tenantID, cacheNamespaceTopic, topic)
+	invalidateTopicCache(tenantID, topic)
+	invalidateTopicSubscriptions(tenantID, topic)
 	if hard && types.GetTopicCat(topic) == types.TopicCatGrp {
 		_ = DeleteGroupMessageShardRoutes(tenantID, topic)
 	}
@@ -1533,6 +1538,10 @@ func (subsMapper) Get(tenantID types.TenantID, topic string, user types.Uid, kee
 
 // Update values of topic's subscriptions.
 func (subsMapper) Update(tenantID types.TenantID, topic string, user types.Uid, update map[string]any) error {
+	if update == nil {
+		update = map[string]any{}
+	}
+	invalidateUnread := subscriptionUpdateAffectsUnread(update)
 	update["UpdatedAt"] = types.TimeNow()
 	tenantAdp, err := tenantBusinessAdapter(tenantID)
 	if err != nil {
@@ -1541,8 +1550,10 @@ func (subsMapper) Update(tenantID types.TenantID, topic string, user types.Uid,
 	if err = tenantAdp.TenantSubsUpdate(tenantID, topic, user, update); err != nil {
 		return err
 	}
-	_ = DeleteCachedObject(tenantID, cacheNamespaceSub, subscriptionCacheID(topic, user, false))
+	invalidateSubscriptionCache(tenantID, topic, user)
-	_ = DeleteCachedObject(tenantID, cacheNamespaceSub, subscriptionCacheID(topic, user, true))
+	if invalidateUnread {
+		DeleteCachedUserUnreadCount(tenantID, user)
+	}
 	return nil
 }
 
@@ -1556,11 +1567,21 @@ func (subsMapper) Delete(tenantID types.TenantID, topic string, user types.Uid)
 	if err = tenantAdp.TenantSubsDelete(tenantID, topic, user); err != nil {
 		return err
 	}
-	_ = DeleteCachedObject(tenantID, cacheNamespaceSub, subscriptionCacheID(topic, user, false))
+	invalidateSubscriptionCache(tenantID, topic, user)
-	_ = DeleteCachedObject(tenantID, cacheNamespaceSub, subscriptionCacheID(topic, user, true))
+	DeleteCachedUserUnreadCount(tenantID, user)
 	return nil
 }
 
+func subscriptionUpdateAffectsUnread(update map[string]any) bool {
+	for key := range update {
+		switch key {
+		case "ReadSeqId", "DelId", "ModeWant", "ModeGiven", "DeletedAt":
+			return true
+		}
+	}
+	return false
+}
+
 func subscriptionCacheID(topic string, user types.Uid, keepDeleted bool) string {
 	if keepDeleted {
 		return topic + ":" + user.String() + ":any"
@@ -1568,6 +1589,32 @@ func subscriptionCacheID(topic string, user types.Uid, keepDeleted bool) string
 	return topic + ":" + user.String() + ":active"
 }
 
+func invalidateTopicCache(tenantID types.TenantID, topic string) {
+	topic = strings.TrimSpace(topic)
+	if topic == "" {
+		return
+	}
+	if err := DeleteCachedObject(tenantID, cacheNamespaceTopic, topic); err != nil {
+		logs.Warn.Printf("cache topic[%s]: delete failed: %v", topic, err)
+	}
+}
+
+func invalidateSubscriptionCache(tenantID types.TenantID, topic string, user types.Uid) {
+	topic = strings.TrimSpace(topic)
+	if topic == "" {
+		return
+	}
+	if user.IsZero() {
+		invalidateTopicSubscriptions(tenantID, topic)
+		return
+	}
+	for _, keepDeleted := range []bool{false, true} {
+		if err := DeleteCachedObject(tenantID, cacheNamespaceSub, subscriptionCacheID(topic, user, keepDeleted)); err != nil {
+			logs.Warn.Printf("cache sub[%s]: delete failed: %v", subscriptionCacheID(topic, user, keepDeleted), err)
+		}
+	}
+}
+
 // MessagesPersistenceInterface is an interface which defines methods for persistent storage of messages.
 type MessagesPersistenceInterface interface {
 	Save(tenantID types.TenantID, msg *types.Message, attachmentURLs []string, readBySender bool) (*types.MessageSaveResult, error)
@@ -3459,6 +3506,7 @@ func (messagesMapper) PreparePersist(tenantID types.TenantID, msg *types.Message
 	if err != nil {
 		return nil, err
 	}
+	invalidateTopicCache(tenantID, msg.Topic)
 	return &types.MessageSaveResult{SeqId: msg.SeqId}, nil
 }
 
@@ -3536,6 +3584,7 @@ func (messagesMapper) PersistPrepared(tenantID types.TenantID, msg *types.Messag
 	if err = tenantAdp.TenantTopicUpdateOnMessage(tenantID, msg.Topic, msg); err != nil {
 		return nil, err
 	}
+	invalidateTopicCache(tenantID, msg.Topic)
 	if isGroupMessage && groupMsgRuntime != nil && clientMsgID != "" {
 		if err := groupMsgRuntime.MarkDone(tenantID, msg.Topic, fromUid, clientMsgID, msg.SeqId, msg.Uid().String()); err != nil {
 			logs.Warn.Printf("topic[%s]: failed to mark group message unique key done: %v", msg.Topic, err)
@@ -3564,6 +3613,8 @@ func (messagesMapper) PersistPrepared(tenantID types.TenantID, msg *types.Messag
 					"ReadSeqId": msg.SeqId}); subErr != nil {
 				logs.Warn.Printf("topic[%s]: failed to mark message (seq: %d) read by sender - err: %+v", msg.Topic, msg.SeqId, subErr)
 			} else {
+				invalidateSubscriptionCache(tenantID, msg.Topic, fromUid)
+				DeleteCachedUserUnreadCount(tenantID, fromUid)
 				markedReadBySender = true
 			}
 		}
@@ -3626,6 +3677,82 @@ func PersistPreparedMessage(tenantID types.TenantID, msg *types.Message, attachm
 	return Messages.PersistPrepared(tenantID, msg, attachmentURLs, readBySender)
 }
 
+type PreparedMessageBatchItem struct {
+	Message      *types.Message
+	ReadBySender bool
+}
+
+func PersistPreparedMessageBatch(tenantID types.TenantID, items []PreparedMessageBatchItem) error {
+	if len(items) == 0 {
+		return nil
+	}
+	tenantAdp, err := tenantBusinessAdapter(tenantID)
+	if err != nil {
+		return err
+	}
+	batchAdp, ok := tenantAdp.(adapter.TenantMessageBatchAdapter)
+	if !ok {
+		return types.ErrUnsupported
+	}
+	if items[0].Message == nil {
+		return types.ErrMalformed
+	}
+	isGroup := types.GetTopicCat(items[0].Message.Topic) == types.TopicCatGrp
+	month := items[0].Message.ShardMonth
+	stored := make([]*types.Message, 0, len(items))
+	for _, item := range items {
+		msg := item.Message
+		if msg == nil || msg.TenantID != tenantID || msg.Topic == "" || msg.SeqId <= 0 || msg.Uid().IsZero() ||
+			(types.GetTopicCat(msg.Topic) == types.TopicCatGrp) != isGroup || (isGroup && msg.ShardMonth != month) {
+			return types.ErrMalformed
+		}
+		clientMsgID, normalizeErr := normalizeClientMsgID(msg.Head)
+		if normalizeErr != nil {
+			return normalizeErr
+		}
+		msg.ClientMsgID = clientMsgID
+		storedMsg, encryptErr := encryptMessageContent(msg)
+		if encryptErr != nil {
+			return encryptErr
+		}
+		stored = append(stored, storedMsg)
+	}
+	if isGroup {
+		err = batchAdp.TenantGroupMessageSaveBatch(tenantID, stored)
+	} else {
+		err = batchAdp.TenantMessageSaveBatch(tenantID, stored)
+	}
+	if err != nil {
+		return err
+	}
+	for _, item := range items {
+		msg := item.Message
+		from := types.ParseUid(msg.From)
+		if err = tenantAdp.TenantTopicUpdateOnMessage(tenantID, msg.Topic, msg); err != nil {
+			return err
+		}
+		invalidateTopicCache(tenantID, msg.Topic)
+		if isGroup && groupMsgRuntime != nil {
+			if msg.ClientMsgID != "" {
+				_ = groupMsgRuntime.MarkDone(tenantID, msg.Topic, from, msg.ClientMsgID, msg.SeqId, msg.Uid().String())
+			}
+			_ = groupMsgRuntime.PutShardRoute(tenantID, msg.Topic, GroupMessageShardRoute{
+				Month: msg.ShardMonth, MinSeq: msg.SeqId, MaxSeq: msg.SeqId, Count: 1,
+			})
+			_ = groupMsgRuntime.CacheHotMessage(tenantID, msg)
+		}
+		if item.ReadBySender && !from.IsZero() {
+			if updateErr := tenantAdp.TenantSubsUpdate(tenantID, msg.Topic, from, map[string]any{
+				"RecvSeqId": msg.SeqId, "ReadSeqId": msg.SeqId,
+			}); updateErr == nil {
+				invalidateSubscriptionCache(tenantID, msg.Topic, from)
+				DeleteCachedUserUnreadCount(tenantID, from)
+			}
+		}
+	}
+	return nil
+}
+
 func NormalizeClientMsgID(head types.KVMap) (string, error) {
 	return normalizeClientMsgID(head)
 }
@@ -3659,6 +3786,7 @@ func retryTenantMessageSaveAfterSeqConflict(tenantAdp adapter.TenantBusinessAdap
 	if err = tenantAdp.TenantTopicUpdateOnMessage(tenantID, msg.Topic, msg); err != nil {
 		return err
 	}
+	invalidateTopicCache(tenantID, msg.Topic)
 	if isGroupMessage {
 		err = tenantAdp.TenantGroupMessageSave(tenantID, msg)
 	} else {
@@ -3959,6 +4087,7 @@ func (messagesMapper) DeleteList(tenantID types.TenantID, topic string, delID in
 		if err != nil {
 			return err
 		}
+		invalidateTopicCache(tenantID, topic)
 
 		// Soft-deleting will update one subscription, hard-deleting will ipdate all.
 		// Soft- or hard- is defined by the forUser being defined.
@@ -3966,6 +4095,7 @@ func (messagesMapper) DeleteList(tenantID types.TenantID, topic string, delID in
 		if err != nil {
 			return err
 		}
+		invalidateSubscriptionCache(tenantID, topic, forUser)
 	}
 
 	return err

server/store/subscription_unread_test.go

@@ -0,0 +1,28 @@
+package store
+
+import "testing"
+
+func TestSubscriptionUpdateAffectsUnread(t *testing.T) {
+	tests := []struct {
+		name   string
+		update map[string]any
+		want   bool
+	}{
+		{name: "read seq", update: map[string]any{"ReadSeqId": int64(12)}, want: true},
+		{name: "delete marker", update: map[string]any{"DelId": 4}, want: true},
+		{name: "want mode", update: map[string]any{"ModeWant": 1}, want: true},
+		{name: "given mode", update: map[string]any{"ModeGiven": 1}, want: true},
+		{name: "deleted", update: map[string]any{"DeletedAt": "now"}, want: true},
+		{name: "recv only", update: map[string]any{"RecvSeqId": int64(12)}, want: false},
+		{name: "private only", update: map[string]any{"Private": map[string]any{"mute": true}}, want: false},
+		{name: "empty", update: map[string]any{}, want: false},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			if got := subscriptionUpdateAffectsUnread(tc.update); got != tc.want {
+				t.Fatalf("subscriptionUpdateAffectsUnread() = %v, want %v", got, tc.want)
+			}
+		})
+	}
+}

server/tinode.conf

@@ -128,6 +128,9 @@
     "sasl_password": "gr58tNnlZOVGW3AmMbL2",
     "max_retry_count": 3,
     "retry_backoff_ms": [10000, 30000, 60000, 300000, 900000],
+    "message_batch_size": 100,
+    "message_batch_wait_ms": 10,
+    "message_single_retry_count": 3,
     "dead_letter_retention_seconds": 7776000,
     "topics": {
       "admin_command": "im.admin.command",
@@ -235,6 +238,11 @@
   "cluster_config": {
     "self": "",
     "nodes": [],
+    "proxy_to_master_buffer": 4096,
+    "proxy_to_master_workers": 16,
+    "proxy_to_master_enqueue_timeout_ms": 300,
+    "route_retries": 3,
+    "route_retry_backoff_ms": 25,
     "failover": {
       "enabled": false
     }

server/topic.go

@@ -678,6 +678,10 @@ func (t *Topic) handleServerMsg(msg *ServerComMessage) {
 		// Ignore message - the topic is paused or being deleted.
 		return
 	}
+	if msg.Info != nil && msg.Info.What == "p2p" && msg.Info.Event == p2pPermissionsRefreshedEvent {
+		t.handleP2PPermissionsRefresh(msg.Info)
+		return
+	}
 	if msg.Pres != nil {
 		t.handlePresence(msg)
 	} else if msg.Info != nil || msg.Data != nil {
@@ -700,6 +704,24 @@ func (t *Topic) handleServerMsg(msg *ServerComMessage) {
 	}
 }
 
+func (t *Topic) handleP2PPermissionsRefresh(info *MsgServerInfo) {
+	if t.cat != types.TopicCatP2P || t.isProxy {
+		return
+	}
+	var payload p2pPermissionsPayload
+	if err := json.Unmarshal(info.Payload, &payload); err != nil {
+		logs.Warn.Printf("topic[%s]: failed to decode p2p permissions refresh: %v", t.name, err)
+		return
+	}
+	uids := make([]types.Uid, 0, len(payload.UserIDs))
+	for _, raw := range payload.UserIDs {
+		if uid := types.ParseUserId(raw); !uid.IsZero() {
+			uids = append(uids, uid)
+		}
+	}
+	t.refreshPerUserFromDB(types.ZeroUid, uids...)
+}
+
 // flexUID accepts a JSON string or number representation of a user id: the
 // HTTP interface publishes base32 user id strings, while Kafka admin commands
 // publish numeric database ids.
@@ -1282,8 +1304,15 @@ func (t *Topic) saveAndBroadcastMessage(msg *ClientComMessage, asUid types.Uid,
 			return types.ErrPermissionDenied
 		}
 	} else {
+		// EnsureP2P may have repaired the subscription in the database while this
+		// topic was already loaded. Reload once on denied P2P publishes so a lost
+		// or delayed refresh event cannot leave the conversation permanently stale.
+		if t.cat == types.TopicCatP2P && userFound && !(pud.modeWant & pud.modeGiven).IsWriter() && store.Subs != nil {
+			t.refreshPerUserFromDB(asUid, asUid)
+			pud, userFound = t.perUser[asUid]
+		}
 		// If it's not 'sys' check write permission.
-		if !(pud.modeWant & pud.modeGiven).IsWriter() {
+		if !userFound || !(pud.modeWant & pud.modeGiven).IsWriter() {
 			msg.sess.queueOut(ErrPermissionDenied(msg.Id, t.original(asUid), msg.Timestamp))
 			return types.ErrPermissionDenied
 		}

server/topic_proxy.go

@@ -9,12 +9,34 @@ package main
 import (
 	"errors"
 	"net/http"
+	"sync/atomic"
 	"time"
 
 	"github.com/tinode/chat/server/logs"
+	immetrics "github.com/tinode/chat/server/metrics"
 	"github.com/tinode/chat/server/store/types"
 )
 
+var (
+	clusterProxyOverloadLogAt      atomic.Int64
+	clusterProxyOverloadSuppressed atomic.Int64
+)
+
+func logClusterProxyRouteFailure(topic, operation string, err error) {
+	if !errors.Is(err, errClusterLoadExceeded) && !errors.Is(err, errClusterOverloaded) {
+		logs.Warn.Printf("topic proxy[%s]: route %s failed - %s", topic, operation, err)
+		return
+	}
+	now := time.Now().Unix()
+	previous := clusterProxyOverloadLogAt.Load()
+	if previous == now || !clusterProxyOverloadLogAt.CompareAndSwap(previous, now) {
+		clusterProxyOverloadSuppressed.Add(1)
+		return
+	}
+	logs.Warn.Printf("topic proxy overload: topic=%s operation=%s suppressed=%d", topic, operation,
+		clusterProxyOverloadSuppressed.Swap(0))
+}
+
 func (t *Topic) runProxy(hub *Hub) {
 	killTimer := time.NewTimer(time.Hour)
 	killTimer.Stop()
@@ -27,8 +49,8 @@ func (t *Topic) runProxy(hub *Hub) {
 				msg.sess.queueOut(ErrLockedReply(msg, types.TimeNow()))
 			} else if err := globals.cluster.routeToTopicMaster(ProxyReqJoin, msg, t.name, msg.sess); err != nil {
 				// Response (ctrl message) will be handled when it's received via the proxy channel.
-				logs.Warn.Printf("proxy topic[%s]: route join request from proxy to master failed - %s", t.name, err)
+				logClusterProxyRouteFailure(t.name, "join", err)
-				msg.sess.queueOut(ErrClusterUnreachableReply(msg, types.TimeNow()))
+				msg.sess.queueOut(clusterRouteErrorReply(msg, err))
 			}
 			if msg.sess.inflightReqs != nil {
 				msg.sess.inflightReqs.Done()
@@ -51,13 +73,16 @@ func (t *Topic) runProxy(hub *Hub) {
 		case msg := <-t.clientMsg:
 			// Content message intended for broadcasting to recipients
 			if err := globals.cluster.routeToTopicMaster(ProxyReqBroadcast, msg, t.name, msg.sess); err != nil {
-				logs.Warn.Printf("topic proxy[%s]: route broadcast request from proxy to master failed - %s", t.name, err)
+				logClusterProxyRouteFailure(t.name, "broadcast", err)
-				msg.sess.queueOut(ErrClusterUnreachableReply(msg, types.TimeNow()))
+				msg.sess.queueOut(clusterRouteErrorReply(msg, err))
 			}
 
 		case msg := <-t.serverMsg:
 			if msg.Data != nil {
-				t.broadcastToSessions(msg)
+				if err := globals.cluster.routeToTopicIntraCluster(t.name, msg, msg.sess); err != nil {
+					logs.Warn.Printf("topic proxy[%s]: route server data to master failed - %s", t.name, err)
+					t.handleProxyBroadcast(msg)
+				}
 			} else if msg.Info != nil || msg.Pres != nil {
 				globals.cluster.routeToTopicIntraCluster(t.name, msg, msg.sess)
 			} else {
@@ -68,8 +93,8 @@ func (t *Topic) runProxy(hub *Hub) {
 		case msg := <-t.meta:
 			// Request to get/set topic metadata
 			if err := globals.cluster.routeToTopicMaster(ProxyReqMeta, msg, t.name, msg.sess); err != nil {
-				logs.Warn.Printf("proxy topic[%s]: route meta request from proxy to master failed - %s", t.name, err)
+				logClusterProxyRouteFailure(t.name, "meta", err)
-				msg.sess.queueOut(ErrClusterUnreachableReply(msg, types.TimeNow()))
+				msg.sess.queueOut(clusterRouteErrorReply(msg, err))
 			}
 
 		case upd := <-t.supd:
@@ -119,6 +144,17 @@ func (t *Topic) runProxy(hub *Hub) {
 	}
 }
 
+func clusterRouteErrorReply(msg *ClientComMessage, err error) *ServerComMessage {
+	if errors.Is(err, errClusterLoadExceeded) || errors.Is(err, errClusterOverloaded) {
+		immetrics.ClusterProxyResponseTotal.WithLabelValues(clusterPodName(), "overloaded").Inc()
+		reply := ErrServiceUnavailableReply(msg, types.TimeNow())
+		reply.Ctrl.Text = "cluster overloaded"
+		return reply
+	}
+	immetrics.ClusterProxyResponseTotal.WithLabelValues(clusterPodName(), "unreachable").Inc()
+	return ErrClusterUnreachableReply(msg, types.TimeNow())
+}
+
 // Takes a session leave request, forwards it to the topic master and
 // modifies the local state accordingly.
 // Returns whether the operation was successful.

server/topic_test.go

@@ -818,6 +818,10 @@ func TestHandleBroadcastDataMissingWritePermission(t *testing.T) {
 	pud := helper.topic.perUser[uid1]
 	pud.modeGiven = types.ModeRead | types.ModeJoin
 	helper.topic.perUser[uid1] = pud
+	helper.ss.EXPECT().Get(types.TenantID(1), topicName, uid1, true).Return(&types.Subscription{
+		TenantID: 1, User: uid1.String(), Topic: topicName,
+		ModeWant: pud.modeWant, ModeGiven: pud.modeGiven,
+	}, nil)
 
 	// Make test message.
 	from := helper.uids[0].UserId()
@@ -1284,7 +1288,7 @@ func TestFilterHotMessagesForUserSkipsDeleted(t *testing.T) {
 	}
 }
 
-func TestMarkKafkaDeadLetterPublishesBeforeDBWrite(t *testing.T) {
+func TestMarkKafkaDeadLetterPublishesAndWritesTerminalState(t *testing.T) {
 	oldKafkaBus := kafkaBus
 	oldPublisher := publishKafkaDeadLetter
 	oldInfrastructure := store.HTTPInfrastructure
@@ -1322,8 +1326,8 @@ func TestMarkKafkaDeadLetterPublishesBeforeDBWrite(t *testing.T) {
 	if err := markKafkaDeadLetter(context.Background(), 1, original, consume, "DB_FAILED", "boom"); err != nil {
 		t.Fatalf("markKafkaDeadLetter returned error: %v", err)
 	}
-	if len(infrastructure.deadLetters) != 0 {
+	if len(infrastructure.deadLetters) != 1 || infrastructure.deadLetters[0].DeadID == "" {
-		t.Fatalf("dead letter was written before dead_letter topic consumption: %#v", infrastructure.deadLetters)
+		t.Fatalf("dead letter terminal state was not written: %#v", infrastructure.deadLetters)
 	}
 	if publishedEnvelope.EventID == "" || publishedEnvelope.TenantID != 1 {
 		t.Fatalf("dead letter event was not published correctly: %#v", publishedEnvelope)

server/user.go

@@ -56,8 +56,7 @@ func replyCreateUser(s *Session, msg *ClientComMessage, rec *auth.Rec) {
 	}
 	if err != nil || !secretMatches {
 		logs.Warn.Println("create user: invalid registration account fields, sid=", s.sid)
-		s.queueOut(decodeStoreError(types.ErrPolicy, msg.Id, msg.Timestamp,
+		s.queueOut(decodeStoreError(types.ErrPolicy, msg.Id, msg.Timestamp, registrationAccountErrorParams(err)))
-			map[string]any{"what": "account", "reason": "REGISTRATION_ACCOUNT_INVALID"}))
 		return
 	}
 	authSecret := msg.Acc.Secret
@@ -407,6 +406,7 @@ var (
 	errRegistrationTenantForbidden = errors.New("registration tenant forbidden")
 	errRegistrationTenantRequired  = errors.New("registration tenant required")
 	errRegistrationTenantMismatch  = errors.New("registration tenant mismatch")
+	errRegistrationUsernameInvalid = errors.New("registration username invalid")
 	errLoginTypeRequired           = errors.New("login_type required")
 	errLoginTypeInvalid            = errors.New("login_type invalid")
 )
@@ -579,13 +579,22 @@ func loginTenantErrorParams(err error) map[string]any {
 	return params
 }
 
+func registrationAccountErrorParams(err error) map[string]any {
+	params := map[string]any{"what": "account", "reason": "REGISTRATION_ACCOUNT_INVALID"}
+	if errors.Is(err, errRegistrationUsernameInvalid) {
+		params["what"] = "username"
+		params["reason"] = "USERNAME_INVALID_FORMAT"
+	}
+	return params
+}
+
 func validateRegistrationUsername(username string) error {
 	if username == "" || len(username) > 10 {
-		return types.ErrPolicy
+		return errRegistrationUsernameInvalid
 	}
 	for i := range username {
 		if (username[i] < 'a' || username[i] > 'z') && (username[i] < '0' || username[i] > '9') {
-			return types.ErrPolicy
+			return errRegistrationUsernameInvalid
 		}
 	}
 	return nil

server/user_registration_test.go

@@ -71,6 +71,18 @@ func TestResolveRegistrationAccount(t *testing.T) {
 	}
 }
 
+func TestRegistrationAccountErrorParams(t *testing.T) {
+	params := registrationAccountErrorParams(errRegistrationUsernameInvalid)
+	if params["what"] != "username" || params["reason"] != "USERNAME_INVALID_FORMAT" {
+		t.Fatalf("got params=%v", params)
+	}
+
+	params = registrationAccountErrorParams(types.ErrPolicy)
+	if params["what"] != "account" || params["reason"] != "REGISTRATION_ACCOUNT_INVALID" {
+		t.Fatalf("got fallback params=%v", params)
+	}
+}
+
 func TestRegistrationSecretMatchesSelectedAccount(t *testing.T) {
 	if !registrationSecretMatches([]byte("alice:password:with-colon"), "alice") {
 		t.Fatal("matching account was rejected")