Compare commits
25
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Austin
|
7c550aaf9d | ||
|
Austin
|
0718da784f | ||
|
Austin
|
2f8ded104d | ||
|
Austin
|
1ff0894205 | ||
|
richjohnrambo
|
a974572f72 | ||
|
richjohnrambo
|
d3c679878d | ||
|
richjohnrambo
|
cb2210aa3e | ||
|
richjohnrambo
|
622fc50418 | ||
|
richjohnrambo
|
6b8ed4a2cc | ||
|
richjohnrambo
|
fc875cc84e | ||
|
richjohnrambo
|
e90e92a59e | ||
|
richjohnrambo
|
72e6c7b2c7 | ||
|
richjohnrambo
|
ea8732e2b1 | ||
|
richjohnrambo
|
bde02e8eac | ||
|
richjohnrambo
|
8e778726e0 | ||
|
richjohnrambo
|
36b52e120c | ||
|
richjohnrambo
|
3bb6774786 | ||
|
richjohnrambo
|
36eb76a5e3 | ||
|
richjohnrambo
|
0cf88e6e97 | ||
|
richjohnrambo
|
3fb95fd5d9 | ||
|
richjohnrambo
|
d96f661f30 | ||
|
richjohnrambo
|
704c65a7a4 | ||
|
richjohnrambo
|
188164ce82 | ||
|
richjohnrambo
|
d0e1e98f5f | ||
|
richjohnrambo
|
ecfe13ff0e |
@@ -41,7 +41,11 @@ jobs:
|
||||
CLUSTER_PROXY_TO_MASTER_WORKERS=16 \
|
||||
CLUSTER_PROXY_TO_MASTER_ENQUEUE_TIMEOUT_MS=300 \
|
||||
CLUSTER_ROUTE_RETRIES=3 \
|
||||
CLUSTER_ROUTE_RETRY_BACKOFF_MS=25
|
||||
CLUSTER_ROUTE_RETRY_BACKOFF_MS=25 \
|
||||
MYSQL_MAX_OPEN_CONNS=24 \
|
||||
MYSQL_MAX_IDLE_CONNS=8 \
|
||||
MYSQL_REGISTRY_MAX_OPEN_CONNS=4 \
|
||||
MYSQL_REGISTRY_MAX_IDLE_CONNS=2
|
||||
kubectl patch statefulset/im-server -n im-server-system --type=strategic -p \
|
||||
'{"spec":{"template":{"spec":{"containers":[{"name":"im-server","readinessProbe":{"httpGet":{"path":"/healthz","port":6060},"timeoutSeconds":3,"periodSeconds":10,"failureThreshold":3},"livenessProbe":{"httpGet":{"path":"/livez","port":6060},"timeoutSeconds":3,"periodSeconds":20,"failureThreshold":3}}]}}}}'
|
||||
if kubectl auth can-i get services -n im-server-system >/dev/null 2>&1 && \
|
||||
@@ -62,11 +66,32 @@ jobs:
|
||||
kubectl rollout restart statefulset/im-server -n im-server-system
|
||||
kubectl rollout status statefulset/im-server -n im-server-system --timeout=300s
|
||||
|
||||
resources="$(kubectl get statefulset/im-server -n im-server-system -o \
|
||||
jsonpath='{.spec.template.spec.containers[?(@.name=="im-server")].resources}')"
|
||||
echo "Waiting for ArgoCD reconciliation before validating effective settings"
|
||||
sleep 60
|
||||
request_cpu="$(kubectl get statefulset/im-server -n im-server-system -o \
|
||||
jsonpath='{.spec.template.spec.containers[?(@.name=="im-server")].resources.requests.cpu}')"
|
||||
request_memory="$(kubectl get statefulset/im-server -n im-server-system -o \
|
||||
jsonpath='{.spec.template.spec.containers[?(@.name=="im-server")].resources.requests.memory}')"
|
||||
limit_cpu="$(kubectl get statefulset/im-server -n im-server-system -o \
|
||||
jsonpath='{.spec.template.spec.containers[?(@.name=="im-server")].resources.limits.cpu}')"
|
||||
limit_memory="$(kubectl get statefulset/im-server -n im-server-system -o \
|
||||
jsonpath='{.spec.template.spec.containers[?(@.name=="im-server")].resources.limits.memory}')"
|
||||
liveness="$(kubectl get statefulset/im-server -n im-server-system -o \
|
||||
jsonpath='{.spec.template.spec.containers[?(@.name=="im-server")].livenessProbe.httpGet.path}')"
|
||||
echo "effective resources=${resources} liveness=${liveness}"
|
||||
if [ "${liveness}" != "/livez" ] || ! echo "${resources}" | grep -q '"cpu":"2"'; then
|
||||
echo "::warning::StatefulSet resources/probes were reconciled away; update the ArgoCD source kustomization"
|
||||
mysql_max_open="$(kubectl get statefulset/im-server -n im-server-system -o \
|
||||
jsonpath='{.spec.template.spec.containers[?(@.name=="im-server")].env[?(@.name=="MYSQL_MAX_OPEN_CONNS")].value}')"
|
||||
mysql_max_idle="$(kubectl get statefulset/im-server -n im-server-system -o \
|
||||
jsonpath='{.spec.template.spec.containers[?(@.name=="im-server")].env[?(@.name=="MYSQL_MAX_IDLE_CONNS")].value}')"
|
||||
mysql_registry_max_open="$(kubectl get statefulset/im-server -n im-server-system -o \
|
||||
jsonpath='{.spec.template.spec.containers[?(@.name=="im-server")].env[?(@.name=="MYSQL_REGISTRY_MAX_OPEN_CONNS")].value}')"
|
||||
mysql_registry_max_idle="$(kubectl get statefulset/im-server -n im-server-system -o \
|
||||
jsonpath='{.spec.template.spec.containers[?(@.name=="im-server")].env[?(@.name=="MYSQL_REGISTRY_MAX_IDLE_CONNS")].value}')"
|
||||
echo "effective requests=${request_cpu}/${request_memory} limits=${limit_cpu}/${limit_memory} liveness=${liveness} mysql=${mysql_max_open}/${mysql_max_idle} registry=${mysql_registry_max_open}/${mysql_registry_max_idle}"
|
||||
if [ "${request_cpu}" != "1" ] || [ "${request_memory}" != "512Mi" ] || \
|
||||
[ "${limit_cpu}" != "2" ] || [ "${limit_memory}" != "1Gi" ] || \
|
||||
[ "${liveness}" != "/livez" ] || [ "${mysql_max_open}" != "24" ] || \
|
||||
[ "${mysql_max_idle}" != "8" ] || [ "${mysql_registry_max_open}" != "4" ] || \
|
||||
[ "${mysql_registry_max_idle}" != "2" ]; then
|
||||
echo "::error::ArgoCD reverted StatefulSet resources/probes/MySQL pools. Apply deploy/argocd/im-server-test/statefulset-performance-patch.yaml to the im-server-test source."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
# im-server-test ArgoCD patch
|
||||
|
||||
The live StatefulSet is owned by the ArgoCD Application `im-server-test`.
|
||||
Copy `statefulset-performance-patch.yaml` into that application's Git source
|
||||
and reference it from the test overlay:
|
||||
|
||||
```yaml
|
||||
patches:
|
||||
- path: statefulset-performance-patch.yaml
|
||||
```
|
||||
|
||||
Also include `backend-debug-metrics-rbac.yaml` as a resource in the test
|
||||
overlay. It grants the existing `argocd/backend-debug` service account
|
||||
namespaced read access to Pod metrics so the load-test collector can run
|
||||
`kubectl top pods -n im-server-system`:
|
||||
|
||||
```yaml
|
||||
resources:
|
||||
- backend-debug-metrics-rbac.yaml
|
||||
```
|
||||
|
||||
After ArgoCD sync, verify the effective configuration:
|
||||
|
||||
```bash
|
||||
kubectl -n im-server-system get statefulset im-server -o jsonpath='{.spec.template.spec.containers[0].resources}'
|
||||
kubectl -n im-server-system get statefulset im-server -o jsonpath='{.spec.template.spec.containers[0].livenessProbe.httpGet.path}'
|
||||
kubectl -n im-server-system get statefulset im-server -o jsonpath='{.spec.template.spec.containers[0].env}'
|
||||
kubectl auth can-i list pods.metrics.k8s.io \
|
||||
--as=system:serviceaccount:argocd:backend-debug \
|
||||
-n im-server-system
|
||||
```
|
||||
|
||||
Expected values are requests `1 CPU / 512Mi`, limits `2 CPU / 1Gi`, and
|
||||
liveness path `/livez`. The MySQL pool environment must be
|
||||
`MYSQL_MAX_OPEN_CONNS=24`, `MYSQL_MAX_IDLE_CONNS=8`,
|
||||
`MYSQL_REGISTRY_MAX_OPEN_CONNS=4`, and `MYSQL_REGISTRY_MAX_IDLE_CONNS=2`.
|
||||
The application repository's deploy workflow waits for reconciliation and
|
||||
fails if ArgoCD restores different values.
|
||||
@@ -0,0 +1,27 @@
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: Role
|
||||
metadata:
|
||||
name: backend-debug-metrics-reader
|
||||
namespace: im-server-system
|
||||
rules:
|
||||
- apiGroups:
|
||||
- metrics.k8s.io
|
||||
resources:
|
||||
- pods
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata:
|
||||
name: backend-debug-metrics-reader
|
||||
namespace: im-server-system
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: Role
|
||||
name: backend-debug-metrics-reader
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: backend-debug
|
||||
namespace: argocd
|
||||
@@ -0,0 +1,50 @@
|
||||
apiVersion: apps/v1
|
||||
kind: StatefulSet
|
||||
metadata:
|
||||
name: im-server
|
||||
namespace: im-server-system
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: im-server
|
||||
env:
|
||||
- name: CLUSTER_PROXY_TO_MASTER_BUFFER
|
||||
value: "4096"
|
||||
- name: CLUSTER_PROXY_TO_MASTER_WORKERS
|
||||
value: "16"
|
||||
- name: CLUSTER_PROXY_TO_MASTER_ENQUEUE_TIMEOUT_MS
|
||||
value: "300"
|
||||
- name: CLUSTER_ROUTE_RETRIES
|
||||
value: "3"
|
||||
- name: CLUSTER_ROUTE_RETRY_BACKOFF_MS
|
||||
value: "25"
|
||||
- name: MYSQL_MAX_OPEN_CONNS
|
||||
value: "24"
|
||||
- name: MYSQL_MAX_IDLE_CONNS
|
||||
value: "8"
|
||||
- name: MYSQL_REGISTRY_MAX_OPEN_CONNS
|
||||
value: "4"
|
||||
- name: MYSQL_REGISTRY_MAX_IDLE_CONNS
|
||||
value: "2"
|
||||
resources:
|
||||
requests:
|
||||
cpu: "1"
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: "2"
|
||||
memory: 1Gi
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /healthz
|
||||
port: 6060
|
||||
timeoutSeconds: 3
|
||||
periodSeconds: 10
|
||||
failureThreshold: 3
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /livez
|
||||
port: 6060
|
||||
timeoutSeconds: 3
|
||||
periodSeconds: 20
|
||||
failureThreshold: 3
|
||||
@@ -79,8 +79,10 @@
|
||||
"mysql": {
|
||||
"database": "meeting_im",
|
||||
"dsn": "$MYSQL_DSN",
|
||||
"max_open_conns": 64,
|
||||
"max_idle_conns": 64,
|
||||
"max_open_conns": $MYSQL_MAX_OPEN_CONNS,
|
||||
"max_idle_conns": $MYSQL_MAX_IDLE_CONNS,
|
||||
"registry_max_open_conns": $MYSQL_REGISTRY_MAX_OPEN_CONNS,
|
||||
"registry_max_idle_conns": $MYSQL_REGISTRY_MAX_IDLE_CONNS,
|
||||
"conn_max_lifetime": 60,
|
||||
"sql_timeout": 10,
|
||||
"tenant_registry_db": "$TENANT_REGISTRY_DB",
|
||||
|
||||
@@ -18,6 +18,10 @@ else
|
||||
: "${CLUSTER_PROXY_TO_MASTER_ENQUEUE_TIMEOUT_MS:=300}"
|
||||
: "${CLUSTER_ROUTE_RETRIES:=3}"
|
||||
: "${CLUSTER_ROUTE_RETRY_BACKOFF_MS:=25}"
|
||||
: "${MYSQL_MAX_OPEN_CONNS:=24}"
|
||||
: "${MYSQL_MAX_IDLE_CONNS:=8}"
|
||||
: "${MYSQL_REGISTRY_MAX_OPEN_CONNS:=4}"
|
||||
: "${MYSQL_REGISTRY_MAX_IDLE_CONNS:=2}"
|
||||
if [ -z "$CLUSTER_NODES_JSON" ] ; then
|
||||
CLUSTER_NODES_JSON='[{"name":"tinode-0","addr":"tinode-0:12000"},{"name":"tinode-1","addr":"tinode-1:12001"},{"name":"tinode-2","addr":"tinode-2:12002"}]'
|
||||
fi
|
||||
|
||||
@@ -6,6 +6,8 @@
|
||||
|
||||
> 双机实际操作以 [`loadtest/im-pressure/README.md`](../loadtest/im-pressure/README.md) 为准。当前开发机固定为机器 A，负责数据准备、协调器、agent A、集中采集和最终报告；另一台机器固定为机器 B，只运行 agent B 并回传结果。
|
||||
|
||||
> 当前测试环境实际企业码为 `allrich`，沿用本方案最初为 006 商户定义的账号规模、场景和统计口径。已完成的 200 对复测、故障根因和修复证据见 [`docs/test/IM-单聊压测Pod日志根因分析与改进方案-20260927.md`](./test/IM-单聊压测Pod日志根因分析与改进方案-20260927.md)。
|
||||
|
||||
本次测试使用 006 商户的 2000 个账号，在两台压测机上各运行 1000 个在线 WebSocket 用户，验证多 Pod 场景下：
|
||||
|
||||
1. 1000 对 P2P 会话同时收发时，消息是否被接受、实时投递并最终持久化。
|
||||
@@ -128,7 +130,7 @@ Idempotency-Key: <unique key for write requests>
|
||||
1. `max_subscriber_count >= 2500`。仓库配置默认是 128，必须在所有 Pod 上统一调高并滚动重启。
|
||||
2. 006 商户 `max_group_members` 为 0（不限）或不小于 2000；目标群 `max_member_count` 同样满足要求。
|
||||
3. IM Server 所有 Pod 版本和配置一致，原生 Cluster 成员数与 Running Pod 数一致。
|
||||
4. Kafka 的 P2P、群消息持久化 Topic 可写，消费组无持续积压，死信在基线时为 0。
|
||||
4. Kafka 的 P2P、群消息持久化 Topic 可写，消费组无持续积压，死信在基线时为 0；测试环境的 `im.message.persist` 和 `im.group.message.persist` 均为 3 分区、3 副本。
|
||||
5. 两台压测机到目标入口网络稳定，文件描述符上限、临时端口和 NAT 容量足够。
|
||||
6. 两台压测机使用 chrony/NTP，同一时刻的时钟偏差绝对值小于 5ms，否则跨机器单向延迟无效。
|
||||
7. 每台压测机至少预留 2 CPU、4GB 内存和稳定的 100Mbps 网络；测试进程自身 CPU 不应长期超过 70%。
|
||||
@@ -357,8 +359,8 @@ e2e_realtime_latency = t_data - sent_at
|
||||
| 群实时投递丢失率 | <= 0.001%，发布版本验收目标为 0 |
|
||||
| 重复投递率 | 0 |
|
||||
| Topic `seq` 回退 | 0 |
|
||||
| ACK p95 / p99 | < 100ms / < 300ms |
|
||||
| 实时投递 p95 / p99 | < 200ms / < 500ms |
|
||||
| ACK p95 / p99 | <= 500ms / <= 1s；必须用原始 ACK 时间戳计算，不能用直方图桶上界代替 |
|
||||
| 实时投递 p95 / p99 | <= 1s / <= 2s；跨机器时要求时钟偏差达标 |
|
||||
| `deliver/hub_full`、WS 写队列满 | 0 |
|
||||
| Kafka producer/commit 错误 | 0 |
|
||||
| Kafka 消费重试、单条降级、死信新增 | 0 |
|
||||
@@ -787,8 +789,33 @@ results/<run_id>/report.md
|
||||
2. 完成 1000 对好友关系与 2000 人群，运行 `verify` 直到全部通过。
|
||||
3. 先用 20 对 P2P、100 人群做冒烟测试，验证统计口径。
|
||||
4. 执行 P0 热身。
|
||||
5. 执行 S1，等待服务端和 Kafka 恢复基线至少 5 分钟。
|
||||
6. 执行 S2，等待恢复基线至少 10 分钟。
|
||||
7. 执行 S3。
|
||||
5. P2P 按 200 对、500 对、1000 对逐级执行；每一级通过后才能进入下一级。
|
||||
6. 执行 S2 群聊，等待恢复基线至少 10 分钟。
|
||||
7. 执行 S3 混合场景。
|
||||
8. 合并结果，按实时链路与持久化链路分别给出结论。
|
||||
9. 全部稳定后再单独进行 Pod 重启、断网或 Kafka/Redis 故障注入，不与本轮无故障基线混跑。
|
||||
|
||||
### 14.1 P2P 容量阶梯
|
||||
|
||||
三个档位复用同一批账号和好友关系，只裁剪 runtime manifest，不重新注册或重复加好友。每档至少执行两轮；发布验收档建议执行三轮。任何一轮出现消息丢失、重复、乱序、非预期拒绝、数据库不一致或 Pod 重启，都必须停止升级并先定位。
|
||||
|
||||
| 档位 | A/B 用户 | WS 总数 | 每用户消息 | Publish 总数 | 目标 |
|
||||
|---|---:|---:|---:|---:|---|
|
||||
| P2P-200 | 各 200 | 400 | 10 | 4000 | 验证正确性、指标和基础延迟 |
|
||||
| P2P-500 | 各 500 | 1000 | 10 | 10000 | 验证 Kafka 三分区、MySQL 和 Cluster 中档容量 |
|
||||
| P2P-1000 | 各 1000 | 2000 | 10 | 20000 | 单聊发布验收 |
|
||||
|
||||
每档的执行流程固定为：生成唯一 `run_id`；开始前采集至少 2 分钟基线；启动 coordinator；A/B agent 全部 READY；统一时间发压；P2P 发送结束后继续监听至少 30 秒；等待 Kafka lag 归零；执行历史回查与 MySQL `client_mid` 精确核验；发送结束后继续采集至少 5 分钟；最后合并报告。
|
||||
|
||||
每档升级门槛：Ready、Publish ACK、实时到达、历史查询和 MySQL 精确匹配全部达到期望数；丢失、重复、乱序、断连、403/502/503、Cluster queue overflow、Kafka retry/dead letter、MySQL deadlock、Pod 重启均为 0；Kafka lag 在发送结束后 60 秒内归零；Pod 无持续 CPU throttling。ACK P95 超标时必须按压测机分别报告，并先排除 DNS、TCP、TLS、WebSocket RTT 和时钟偏差。
|
||||
|
||||
### 14.2 P2P 通过后的业务场景
|
||||
|
||||
1. **GROUP-2000**：2000 条 WS 全部加入同一群，A/B 各 500 个发送者；每人发送 2 条，两轮合计 2000 条。实时投递期望量按每条消息的实际 Ready 接收者计算，并保留 60 秒 drain。
|
||||
2. **MIXED**：300 对私聊用户双方各发 5 条，共 3000 条私聊 Publish；同时 A/B 各 250 个群发送者每人发 2 条，共 1000 条群 Publish。
|
||||
3. **稳定性复跑**：GROUP 和 MIXED 至少各两轮，确认无冷启动偶然性。
|
||||
4. **故障注入**：分别执行单 Pod 滚动重启、Kafka 短暂不可用与恢复、Redis 短暂不可用与恢复；故障轮只验证恢复能力，不与无故障性能基线混合统计。
|
||||
|
||||
### 14.3 当前下一步
|
||||
|
||||
当前已通过 P2P-200 的消息正确性、Kafka 60 秒排空、MySQL 精确落库、零乱序、零审计死锁和零 CPU throttling。下一步执行 **P2P-500**：A/B 各取账号索引 `1..500`，共 1000 条 WS、10000 条 Publish。若连续两轮满足上述升级门槛，再进入 P2P-1000。
|
||||
|
||||
+21
-8
@@ -398,6 +398,14 @@ X-MeetingIM-APIKey: <预置的普通 API Key>
|
||||
|
||||
`kp/kpa/kpv` 只表示输入态、录音态、视频输入态等短暂 UI 状态，不能作为长连接心跳。长连接保活必须依赖 0.5.1.1 中的 WebSocket Ping/Pong。
|
||||
|
||||
**单聊已读/已收回执协议约定（服务端已验证，2026-09-29）：**
|
||||
|
||||
1. **在线下行形态**：`info.topic` 为话题名（如 `p2p...`），`info.from` 为报读者，`info.what` 为 `recv`/`read`，`info.seq` 为水位。服务端跳过上报会话自身，推送给对端及本人其他在线设备（多 Pod 部署经 Kafka 同步，各 Pod 均会下发）。
|
||||
2. **离线补发形态**：接收方离线时收到 `{"info":{"topic":"me","src":"p2p...","from":"usr...","what":"recv","seq":12}}`，即 `topic` 固定为 `me`、`src` 才是话题名。客户端必须同时兼容上述两种形态。
|
||||
3. **序号单调递增**：服务端丢弃 `seq` 不大于当前水位的回执（`read`、`recv` 各自独立判断），`read` 回执会同步抬升 `recv`，恒有 `read ≥ recv`。客户端渲染一律取服务端最大游标，禁止用本地计数覆盖服务端值。
|
||||
4. **断线重连恢复**：`sub`/`get desc` 响应的 `desc` 中包含 `seq`、`read`、`recv`、`clear` 四个游标；`get {"what":"sub"}` 也返回各成员的 `read/recv`。重连后以这些游标恢复本地状态，不要依赖进程内存。
|
||||
5. **换设备登录**：回执水位按「用户 × 话题」在服务端持久化，与设备无关；新设备订阅后 `desc` 同时返回自己与对端的最新 `read/recv`，可完整恢复对方"已读/已收"状态。发送者自身的 `read/recv` 由服务端自动推进到其最新发出的 `seq`。
|
||||
|
||||
##### 0.5.4.5 Topic 资料、个人群设置和标签
|
||||
|
||||
| 场景 | 客户端发送 | 服务端返回/下行示例 |
|
||||
@@ -1140,7 +1148,9 @@ X-MeetingIM-APIKey: <预置的普通 API Key>
|
||||
|
||||
- 选择媒体后本地生成 `client_mid` 和稳定占位；调用文件上传，得到 `file_id` 后再 `pub x-im-type=image|video|voice`。
|
||||
- `content` 至少含 `file_id,name,size,mime,width,height,duration_ms,thumbnail_file_id` 中适用字段；上传进度是客户端任务状态，不写成聊天消息。
|
||||
- 服务端发布成功后关联文件与 Topic/消息；文件下载必须校验消息可见性。上传失败只重试上传，成功但 `pub` 失败时复用已上传 `file_id` 和 `client_mid`。
|
||||
- 服务端发布成功后关联文件与 Topic/消息；文件下载必须校验消息可见性。
|
||||
- 上传失败（网络中断、超时、5xx）按**整文件重传**处理：重新调用 `POST /v0/file/u/` 上传完整文件，不做分片/断点续传；每次重传得到新 `file_id`，旧 `file_id` 直接放弃、由服务端 TTL 回收，客户端不得再引用。
|
||||
- 上传成功但 `pub` 失败时不重新上传，复用已上传 `file_id` 和原 `client_mid` 重发消息，服务端按 `client_mid` 幂等去重。
|
||||
|
||||
### 3.7 个人对话—拉黑状态／已读状态.png
|
||||
|
||||
@@ -1294,7 +1304,9 @@ X-MeetingIM-APIKey: <预置的普通 API Key>
|
||||
|
||||
**状态：客户端任务管理新增；服务端上传需改造。**
|
||||
|
||||
- 每个待发送项保存 `client_mid,local_uri,type,upload_id,progress,state`；同一文件可分片/断点续传时使用稳定 `upload_id`。
|
||||
- 每个待发送项保存 `client_mid,local_uri,type,upload_id,progress,state`；`upload_id` 仅在单次上传任务内有效（进度上报与本地去重用）。
|
||||
- 重试即新建上传任务：`progress` 归零、生成新 `upload_id`，**整文件重传** `POST /v0/file/u/`，不做分片/断点续传；重传成功后消息体 `file_id` 等字段全部取最新一次上传响应。
|
||||
- 自动重试采用指数退避且不超过 3 次，之后转人工点按重试；App 冷启动扫描本地 `failed` 任务队列重新上传。
|
||||
- 上传完成后再发布消息；多任务可并发但必须各自保持原排序号。离开页面不取消，退出账号必须暂停并隔离任务。
|
||||
- 服务端返回 `FILE_TOO_LARGE`、`FILE_TYPE_NOT_ALLOWED`、`STORAGE_QUOTA_EXCEEDED` 时不可自动无限重试。
|
||||
|
||||
@@ -1410,10 +1422,10 @@ X-MeetingIM-APIKey: <预置的普通 API Key>
|
||||
| 接口 | 输入示例 | 成功 `data` 示例 |
|
||||
| --- | --- | --- |
|
||||
| `GET /v1/im/conversations/{topic}/context` | Path `p2p...` 或 `grp...` | `{"topic":"p2p...","type":"p2p","relationship":"friend","can_send":true,"capabilities":{"text":true,"image":true,"file":true,"voice":true,"can_call":true},"peer":{"user_id":"usr...","display_name":"张三","avatar_file_id":""}}` |
|
||||
| `POST /v1/im/conversations/p2p` 待实现 | `{"peer_user_id":"usr..."}` | `{"topic":"p2p...","relationship":"friend","can_send":true}` |
|
||||
| `GET /v1/im/conversations/{topic}/settings` 待实现 | Path topic | `{"pinned":false,"mute_notifications":false,"message_retention":"default","blocked":false,"remark":"","version":1,"permissions":{"can_update":true}}` |
|
||||
| `PATCH /v1/im/conversations/{topic}/settings` 待实现 | `{"pinned":true,"mute_notifications":false,"version":1}` | `{"pinned":true,"mute_notifications":false,"message_retention":"default","blocked":false,"remark":"","version":2}` |
|
||||
| `GET /v1/im/conversations/{topic}/messages` 待实现 | Query `?since=12&limit=20` | `{"topic":"p2p...","items":[{"seq":13,"from":"usr...","ts":"2026-08-20T09:00:00Z","head":{"x-im-type":"text"},"content":{"text":"你好"}}],"next_cursor":""}` |
|
||||
| `POST /v1/im/conversations/p2p` | `{"peer_user_id":"usr..."}` | `{"topic":"p2p...","relationship":"friend","can_send":true}` |
|
||||
| `GET /v1/im/conversations/{topic}/settings` | Path topic | `{"pinned":false,"mute_notifications":false,"message_retention":"default","blocked":false,"remark":"","version":1,"permissions":{"can_update":true}}` |
|
||||
| `PATCH /v1/im/conversations/{topic}/settings` | `{"pinned":true,"mute_notifications":false,"version":1}` | `{"pinned":true,"mute_notifications":false,"message_retention":"default","blocked":false,"remark":"","version":2}` |
|
||||
| `GET /v1/im/conversations/{topic}/messages` | Query `?since=12&limit=20` | `{"topic":"p2p...","items":[{"seq":13,"from":"usr...","ts":"2026-08-20T09:00:00Z","head":{"x-im-type":"text"},"content":{"text":"你好"}}],"next_cursor":""}` |
|
||||
| `PUT /v1/im/topics/{topic}/draft` 待实现 | `{"content":"未发送草稿","version":1}` | `{"topic":"grp...","content":"未发送草稿","version":2,"updated_at":"2026-08-20T09:00:00Z"}` |
|
||||
| `POST /v1/im/topics/{topic}/messages/{seq}/recall` | 单条撤回；`topic` 可传 `p2p...` 或 `grp...`；Body `{}`，可选 `{"version":1,"reason":"mistake"}` | `{"topic":"grp...","seq":12,"status":"recalled","version":2,"updated_at":"2026-08-20T09:00:00Z"}` |
|
||||
| `POST /v1/im/topics/{topic}/messages/recall` | 多选批量撤回；`{"items":[{"seq":12,"version":1},{"seq":13}],"reason":"mistake"}`；`items` 1～50 条，`version` 可选 | `{"topic":"grp...","items":[{"topic":"grp...","seq":12,"status":"recalled","version":2,"updated_at":"2026-08-20T09:00:00Z"},{"seq":13,"status":"failed","http":409,"code":30006,"reason":"VERSION_CONFLICT","message":"message version conflict"}]}` |
|
||||
@@ -1667,6 +1679,7 @@ GET /v1/im/topics/{topic}/message-mutations?since_version={localMutationVersion}
|
||||
|
||||
- `mutation_version` 是会话级水位，不是单条消息的 `version`。
|
||||
- 本地只做 `服务端 mutation_version > 本地 mutation_version` 比较，不要假设连续递增。
|
||||
- 补偿接口返回的是最新 `limit` 条（默认 50、上限 100）变更的**状态快照**，非严格增量；变更条数超过 `limit` 时更早的变更不再返回。撤回窗口 120 秒、编辑窗口 300 秒，超窗变更无需客户端处理。
|
||||
- 拉到 `message.recalled` 后，将本地 `topic + seq` 标记为撤回，并只在 `version` 更高时覆盖。
|
||||
- 拉到 `message.edited` 后，按 `seq + version` 拉取或覆盖消息内容。
|
||||
- 实时事件和补偿接口可能重复到达，客户端必须按 `topic + seq + version` 幂等处理。
|
||||
@@ -2162,7 +2175,7 @@ GET /v1/im/topics/{topic}/message-mutations?since_version={localMutationVersion}
|
||||
|
||||
- 先用 `purpose=group_file,topic=...` 上传，再 `POST /v1/im/groups/{topic}/files`，Body `{"file_id":"...","file_name":"方案.pdf","mime_type":"application/pdf","file_size":1024}` 建立群文件记录。
|
||||
- 服务端校验 `member_upload_files`、文件是否已上传完成、`file_name` 长度不超过 255、`mime_type` 长度不超过 128、`file_size` 非负且与上传记录一致、单文件上限、租户总存储和群成员身份；成功返回完整群文件项。
|
||||
- 上传失败可复用 `upload_id`；登记失败不得在列表显示，孤立文件按 TTL 回收。
|
||||
- 上传失败按整文件重传重新上传并取得新 `file_id`；登记失败不得在列表显示，孤立文件按 TTL 回收。
|
||||
|
||||
### 6.29 接口示例
|
||||
|
||||
@@ -2191,7 +2204,7 @@ GET /v1/im/topics/{topic}/message-mutations?since_version={localMutationVersion}
|
||||
| `GET /v1/im/groups/{topic}/members?query=&cursor=&mentionable=true` | Query only | `{"topic":"grp...","items":[{"user_id":"usr...","display_name":"张三","group_nickname":"小张","avatar_file_id":"","role":"member","member_status":"active","version":1,"joined_at":"2026-08-20T09:00:00Z","updated_at":"2026-08-20T09:00:00Z","mentionable":true,"acs":{"want":"JRWP","given":"JRWP","mode":"JRWP"}}],"next_cursor":"","sync_mode":"full","permissions":{"can_mention_all":true}}` |
|
||||
| `GET /v1/im/groups/{topic}/members/{user_id}` | Path topic/user | `{"topic":"grp...","user_id":"usr...","display_name":"张三","group_nickname":"小张","role":"member","member_status":"active","version":1,"joined_at":"2026-08-20T09:00:00Z","updated_at":"2026-08-20T09:00:00Z","sync_mode":"single","acs":{"want":"JRWP","given":"JRWP","mode":"JRWP"}}` |
|
||||
| `DELETE /v1/im/groups/{topic}/members/{user_id}` | Path topic/user；管理员可移除普通成员，群主可移除管理员和普通成员；不能移除群主或自己 | `{"topic":"grp...","user_id":"usr...","status":"left"}` |
|
||||
| `GET /v1/im/groups/{topic}/members/{user_id}/profile` | Path topic/user；`topic` 必须是 `grp...`，不能传 `usr...` | `{"topic":"grp...","user_id":"usr...","display_name":"张三","group_nickname":"小张","role":"member","relationship":"friend","capabilities":{"can_send":true,"can_call":true}}` |
|
||||
| `GET /v1/im/groups/{topic}/members/{user_id}/profile` | Path topic/user；`topic` 必须是 `grp...`，不能传 `usr...`；`user_id` 可传本人，返回 `relationship:"self"`、`capabilities.can_send=true` | `{"topic":"grp...","user_id":"usr...","display_name":"张三","group_nickname":"小张","role":"member","relationship":"friend","capabilities":{"can_send":true,"can_call":true}}` |
|
||||
| `POST /v1/im/groups/{topic}/qrcodes` | `{"expires_in_seconds":86400}` | `{"qr_token":"token...","qr_payload":"im://qr?token=...&purpose=join_group","expires_at":"2026-08-21T09:00:00Z","group":{"topic":"grp...","name":"项目群","member_count":2},"group_snapshot":{"topic":"grp...","name":"项目群","member_count":2}}` |
|
||||
| `POST /v1/im/groups/{topic}/invitations` | `{"user_ids":["usr..."],"message":"进群一起讨论"}` | `{"topic":"grp...","items":[{"user_id":"usr...","status":"joined","invitation_id":"ginv_1"}]}` |
|
||||
| `POST /v1/im/group-invitations/{invitation_id}/accept` 待实现 | `{}` | `{"invitation_id":"ginv_1","status":"joined","topic":"grp..."}` |
|
||||
|
||||
+1
-1
@@ -123,7 +123,7 @@ HEALTHCHECK --interval=1m --timeout=3s --start-period=20s \
|
||||
| 探针 | 路径 | 初始延迟 | 周期 | 超时 | 失败次数 |
|
||||
|---|---|---:|---:|---:|---:|
|
||||
| im-server readinessProbe | `/healthz` | 15s | 10s | 3s | 3 |
|
||||
| im-server livenessProbe | `/healthz` | 30s | 30s | 3s | 3 |
|
||||
| im-server livenessProbe | `/livez` | 15s | 20s | 3s | 3 |
|
||||
| im-manager readinessProbe | `/healthz` | 15s | 10s | 3s | 3 |
|
||||
| im-manager livenessProbe | `/healthz` | 30s | 30s | 3s | 3 |
|
||||
|
||||
|
||||
@@ -26,6 +26,7 @@ CREATE TABLE IF NOT EXISTS im_user_login_logs (
|
||||
created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
|
||||
PRIMARY KEY (id),
|
||||
INDEX idx_im_user_login_tenant_time (tenant_id, login_at DESC, id DESC),
|
||||
INDEX idx_im_user_login_session_close (tenant_id, session_id, logout_at),
|
||||
INDEX idx_im_user_login_user_time (user_id, login_at DESC, id DESC),
|
||||
INDEX idx_im_user_login_ip_time (ip, login_at DESC, id DESC),
|
||||
INDEX idx_im_user_login_device_time (device_id(64), login_at DESC),
|
||||
|
||||
@@ -16,37 +16,63 @@
|
||||
| 机器 A | 当前开发机，`192.168.1.135` |
|
||||
| 机器 B | 另一台压测机，运行 `-machine b` |
|
||||
|
||||
`config.json`、`.env.test`、`manifest.json` 均被 Git 忽略。manifest 含 2000 个账号 Token，只能通过受控内网传输。
|
||||
`config.json`、`.env.test`、`manifest.json` 均被 Git 忽略。manifest 含账号 Token，只能通过受控内网传输。
|
||||
|
||||
## 2. 数据准备结果
|
||||
|
||||
2026-09-27 已完成并通过硬校验：
|
||||
2026-09-27 首次准备曾按 2000 账号目标执行。当前分阶段压测使用的实际运行文件为：
|
||||
|
||||
- 账号：2000，机器 A/B 各 1000。
|
||||
- 用户名：`lta0001-lta1000`、`ltb0001-ltb1000`。
|
||||
- 测试手机号：A 使用配置前缀加 `0001-1000`，B 使用配置前缀加 `5001-6000`。
|
||||
- 好友关系：1000 对，序号相同的 A/B 用户互为好友。
|
||||
- P2P Topic：1000 个，全部写入 manifest。
|
||||
- 测试群：1 个，成员数 2000，Topic 仅在 manifest 中保存，文档只记录哈希 `98372d54fe65`。
|
||||
- `users_per_machine=500`。
|
||||
- 账号：1000，机器 A/B 各 500。
|
||||
- 好友关系与 P2P Topic：500 对。
|
||||
- 测试群：1 个，当前 manifest 的 `group_topic=grp3Ita5tyBIMs`。
|
||||
|
||||
正式的 2000 人群测试前，必须将 `users_per_machine` 恢复为 1000，在机器 A 重新执行 `prepare` 和 `verify`，并将新 manifest 同步给机器 B。不能用当前 1000 人清单宣称已完成 2000 人群压测。
|
||||
|
||||
准备过程观察到注册和好友同意接口存在少量瞬时 500/404，100 人群邀请批次存在 502、超过 2 分钟及 `IDEMPOTENCY_IN_PROGRESS`。工具使用相同业务幂等键和退避重试后全部恢复；这些属于准备链路现象，不计入正式消息到达率。
|
||||
|
||||
## 3. 当前运行状态
|
||||
## 3. 最新有效轮次
|
||||
|
||||
| 项目 | 值 |
|
||||
|---|---|
|
||||
| Run ID | `p2p-20260927-01` |
|
||||
| Run ID | `p2p500-r5-20260927-05` |
|
||||
| 场景 | `p2p` |
|
||||
| Coordinator | `http://192.168.1.135:8090` |
|
||||
| 机器 A agent | `1000/1000 READY`，意外断连 0 |
|
||||
| 机器 B agent | 等待启动 |
|
||||
| 远端 dev | `6d17154` 或更新版本 |
|
||||
| Git 提交 | `ea8732e` |
|
||||
| 连接 | A/B 各 500，合计 1000/1000 READY |
|
||||
| 发送 | 10000/10000 获得 ACK |
|
||||
| 实时到达 | 10000/10000，到达率 100% |
|
||||
| 历史落库 | 10000/10000，落库率 100% |
|
||||
| 异常 | 丢失 0、重复 0、乱序 0、断连 0、发送错误 0 |
|
||||
| ACK 延迟 | P50 约 650ms，P95 1313ms，最大 2075ms |
|
||||
| 同机 echo 投递 P95 | 2s 直方图桶 |
|
||||
| 报告门禁 | 正确性通过；延迟门禁未通过 |
|
||||
|
||||
机器 A 的 coordinator 和 agent A 已运行。机器 B 启动并达到 1000 READY 后，coordinator 自动下发统一开始时间，无需人工再次触发。
|
||||
本轮中 A/B 时钟未对齐，A 端记录到 4334 个跨机器负延迟样本，因此不使用跨机器单向延迟作为结论；延迟门禁使用发送机本机 ACK 和 echo 口径。
|
||||
|
||||
## 4. 机器 B 运行文件
|
||||
### 3.1 服务端阶段耗时
|
||||
|
||||
机器 A 已生成：
|
||||
| 阶段 | 平均耗时 | P95 所在桶 |
|
||||
|---|---:|---:|
|
||||
| `relationship_check` | 66.9ms | 250ms |
|
||||
| `redis_idempotency` | 21.1ms | 100ms |
|
||||
| `redis_sequence` | 62.7ms | 250ms |
|
||||
| `kafka_produce` | 11.0ms | 25ms |
|
||||
| `save` | 116.9ms | 500ms |
|
||||
|
||||
`meeting_im` 和 `hotgo` 连接池在该轮起压窗口的等待次数和等待时间增量均为 0，未出现连接池枯竭。
|
||||
|
||||
### 3.2 Kafka 和落库
|
||||
|
||||
- `im-server.message_persist` 的 consumer lag 峰值为 4663。
|
||||
- lag 在发压结束后约 2 分钟清零。
|
||||
- 测试窗口中批量持久化的单批平均耗时约 3.52s，P95 落在 10s 直方图桶。
|
||||
- agent 在等待 Kafka 追平后回查历史，10000 条 ACK 消息全部找到，没有持久化丢失。
|
||||
|
||||
结论是当前系统能在突发后最终消化积压，但前台 ACK 和异步落库延迟仍高，这两项应单独作为后续优化指标。
|
||||
|
||||
## 4. 历史机器 B 运行文件
|
||||
|
||||
以下为首轮历史运行包，不能继续用于新轮次：
|
||||
|
||||
```text
|
||||
/tmp/allrich-machine-b-runtime-p2p-20260927-01.zip
|
||||
@@ -89,3 +115,36 @@ go build -o im-pressure .
|
||||
## 5. 多 Pod 口径说明
|
||||
|
||||
当前 A/B 配置均使用 `wss://api.118im.com/v0/channels` 公共入口。该配置验证 LB 实际分配下的多 Pod 能力，但不能保证两台压测机固定落到不同 Pod。若验收要求强制指定不同 Pod，需要提供两个独立 WS 定向入口，分别配置到 `endpoints.a` 和 `endpoints.b`。
|
||||
|
||||
## 6. 后续测试顺序
|
||||
|
||||
### 6.1 P0：扩容到 2000 个用户
|
||||
|
||||
1. 将机器 A 的 `config.json` 修改为 `users_per_machine=1000`。
|
||||
2. 机器 A 执行 `prepare` 和 `verify`，确认 2000 账号、1000 对好友、1000 个 P2P Topic 和 2000 人群都就绪。
|
||||
3. 重新生成包含 `.env.test`、`config.json`、`manifest.json` 的 B 机运行包，B 机解压后执行 `verify`。
|
||||
4. A/B 都执行 NTP 校时，要求偏差小于 50ms，否则继续忽略跨机器单向延迟。
|
||||
|
||||
### 6.2 P1：2000 人群聊
|
||||
|
||||
使用 `scenario=group`：
|
||||
|
||||
- A/B 各建立 1000 个 WS，合计 2000 个在线群成员。
|
||||
- A/B 各取前 500 个发送者，每人 2 条，合计 2000 条群消息。
|
||||
- A/B 交错每 100ms 发送 1 条，单轮发送窗口约 100s。
|
||||
- 期望实时投递为 2000 条消息 x 2000 连接，即 400 万次有效到达。
|
||||
- 检查群消息 ACK、每个 WS 的到达数、重复、乱序、断连、`im.group.message.persist` lag 和群消息月表落库。
|
||||
|
||||
群聊是下一个应执行的场景，因为它直接验证本次多 Pod 改造的 Redis fanout 和群消息 Kafka 持久化。
|
||||
|
||||
### 6.3 P2：混合场景
|
||||
|
||||
群聊正确性通过后使用 `scenario=mixed`：
|
||||
|
||||
- 300 对私聊用户双向各发 5 条，合计 3000 条私聊消息。
|
||||
- A/B 各 250 个群发送者，每人 2 条，合计 1000 条群消息。
|
||||
- 同时观察私聊与群聊的到达率、两个 Kafka 持久化 Topic 的 lag，以及 Redis、MySQL 和 3 个 Pod 的资源争用。
|
||||
|
||||
### 6.4 P3：单聊完整规模回归
|
||||
|
||||
最后在 2000 连接下重跑 `p2p`，即 1000 对用户双向各发 10 条，共 20000 条。延迟门禁仍为 ACK P95 <= 500ms、同机 echo P95 <= 1000ms；若只有正确性通过而延迟未通过，结论必须标记为“可靠性通过，性能未达标”。
|
||||
|
||||
@@ -0,0 +1,145 @@
|
||||
# IM 2000 人群聊压测问题分析
|
||||
|
||||
日期：2026-09-27
|
||||
Run ID：`group2k-r1-20260927-01`
|
||||
场景：A/B 各 1000 个 WebSocket，各 500 个发送者发 2 条，计划发送 2000 条群消息。
|
||||
|
||||
## 1. 结论
|
||||
|
||||
本轮失败，且不能仅解读为 WebSocket 丢包。核心问题是 2000 人群每条消息触发 O(群成员数) 的额外用户 `me` Topic 路由，导致 Topic master、集群通道和 MySQL 连接池连锁积压。服务端处理吞吐下降后，发送端先经历 ACK 超时，随后 Topic 队列写满并返回 503。
|
||||
|
||||
同时，压测程序对 ACK 超时的口径不完整：超时后服务端仍可能接受、广播并落库，不能把 `ctrl_timeout` 直接当作明确拒绝。
|
||||
|
||||
## 2. 客户端结果
|
||||
|
||||
| 指标 | A | B | 合计 |
|
||||
|---|---:|---:|---:|
|
||||
| WS READY | 1000 | 1000 | 2000 |
|
||||
| 发送尝试 | 1000 | 1000 | 2000 |
|
||||
| 收到 2xx ACK | 16 | 19 | 35 |
|
||||
| 503 | 224 | 272 | 496 |
|
||||
| ACK 超时 | 732 | 709 | 1441 |
|
||||
| socket 错误 | 28 | 0 | 28 |
|
||||
| 断线 | 21 | 0 | 21 |
|
||||
| ACK 台账内实时缺失 | 735 | 0 | 735 |
|
||||
| ACK 台账内历史落库缺失 | 0 | 0 | 0 |
|
||||
| 未在 ACK 台账中的实时投递 | 368270 | 377000 | 745270 |
|
||||
|
||||
35 条 ACK 消息理论上应产生 70000 次本轮投递，实际共记录 69265 次，其中 A 端少 735 次。但这不是服务端实际接受的全量；额外 745270 次投递来自“未及时收到 ACK，但服务端已处理”的消息。
|
||||
|
||||
## 3. 服务端证据
|
||||
|
||||
### 3.1 实际接受和落库
|
||||
|
||||
- `im_message_publish_stage_duration_seconds_count{kind="group",stage="save",result="ok"}` 增加 488。
|
||||
- `im_kafka_message_persist_total{kind="group",result="batch_ok"}` 增加 488。
|
||||
- 群消息 Kafka 批持久化平均约 46.6ms，488 条全部 batch success。
|
||||
|
||||
因此，服务端至少接受并落库 488 条，而不是客户端 ACK 台账中的 35 条。其中至少 453 条属于“已接受但 ACK 未在 20 秒内返回”。
|
||||
|
||||
### 3.2 队列溢出
|
||||
|
||||
| Pod | 日志 | 次数 |
|
||||
|---|---|---:|
|
||||
| `im-server-2` | `s.publish: sub.broadcast channel full` | 496 |
|
||||
| `im-server-2` | `hub: topic's broadcast queue is full` | 1655 |
|
||||
| `im-server-1` | `cluster: proxy channel full` | 47 |
|
||||
|
||||
群 Topic `grp3Ita5tyBIMs` 的 master 在 `im-server-2`。其 `clientMsg` 队列容量只有 192，proxy 响应队列容量为 128。队列满时会立即返回 503 或丢弃 proxy 响应。
|
||||
|
||||
### 3.3 每条群消息的路由放大
|
||||
|
||||
`Topic.saveAndBroadcastMessage` 在每条消息中同步执行：
|
||||
|
||||
1. `presSubsOffline("msg", ..., offlineOnly=true)` 遍历 `t.perUser` 的 2000 个成员。
|
||||
2. 为每个成员构建一条用户 `me` Topic presence 消息。
|
||||
3. `routeUserServerMessage` 同时走 Tinode 原生 Cluster 路由和 Redis user fanout。
|
||||
4. 最后才向群 Topic 的会话广播 `{data}`。
|
||||
|
||||
488 条已接受群消息产生了约 98 万次额外用户路由：
|
||||
|
||||
- `realtime_fanout_events_total{stage="publish",target_type="user"}` 增加 979979。
|
||||
- `realtime_fanout_events_total{stage="receive",target_type="user"}` 增加 894360。
|
||||
- Redis 接收后真正本地投递只有 666 次，`no_local_session` 为 893694。
|
||||
- Cluster route 约增加 109 万次。
|
||||
|
||||
这证明绝大多数用户 fanout 在当前全员已订阅群 Topic 的场景中完全无效，却阻塞了 Topic master 的串行处理协程。
|
||||
|
||||
### 3.4 MySQL 连接等待
|
||||
|
||||
3 个 Pod 的 `meeting_im` 连接池在测试窗口中合计：
|
||||
|
||||
- 等待次数增加 4011。
|
||||
- 等待时间增加约 35399 秒。
|
||||
- DB `exec ok` 增加 3453，平均耗时约 501ms。
|
||||
|
||||
连接池每 Pod 最大 24 个连接。这不是本轮第一个堵点，但会进一步拖慢群权限检查、未读数更新和其他业务 SQL。仅提高连接数不会消除 O(成员数) 的放大。
|
||||
|
||||
## 4. 压测程序口径问题
|
||||
|
||||
1. `ctrl_timeout` 和 socket 错误属于发布结果未知，不是明确拒绝。
|
||||
2. recorder 能够统计 `unexpected_messages`，但 merge 报告没有汇总该字段，也没有将它加入门禁。
|
||||
3. 历史回查只检查 ACK 台账，因此只检查了 35 条，无法单独证明其他超时消息的落库情况。
|
||||
4. 群聊应增加“已 ACK”、“已明确拒绝”、“结果未知”、“未 ACK 但已观测广播”和“最终落库”五种状态。
|
||||
|
||||
## 5. 改进顺序
|
||||
|
||||
### P0
|
||||
|
||||
1. 停止在原生 Cluster 已开启时对同一条 user message 再做 Redis user fanout，避免双通道重复路由。
|
||||
2. 重构群消息的 unread presence：已订阅该群 Topic 的用户不再生成额外 `me` 通知；未订阅但在线的成员使用节点级批量 fanout，不得在 Topic master 协程中逐人跨 Pod 路由。
|
||||
3. 为 Topic `clientMsg`、proxy 响应队列和 session send queue 增加 length/capacity/drop Prometheus 指标。
|
||||
4. 压测 merge 报告增加结果未知及 unacked-observed 口径，并把 `unexpected_messages` 纳入失败门禁。
|
||||
|
||||
### P1
|
||||
|
||||
1. 检查群消息未读数更新和 Push receipt 是否对 2000 人逐人访问 Redis/MySQL，改为批处理或延迟合并。
|
||||
2. 为 `CanPost` 的全员禁言和单人禁言结果增加短 TTL 缓存，管理操作时精确失效。
|
||||
3. P0 完成后先跑 100、500、1000、2000 人群阶梯，每档检查 ACK P95、Topic 队列水位、Cluster route 数、DB pool wait 和 Kafka lag，不直接重跑全量。
|
||||
|
||||
## 6. 本轮验收结论
|
||||
|
||||
- 连接建立：通过，2000/2000 READY。
|
||||
- 发布成功率：失败。
|
||||
- ACK 延迟：失败，已超过 20 秒超时窗口。
|
||||
- 实时广播：失败，存在队列丢弃和断线。
|
||||
- Kafka 持久化：已处理的 488 条全部成功，但该结论不代表计划的 2000 条均被接受。
|
||||
- 总体：失败，必须完成 P0 后按阶梯重测。
|
||||
|
||||
## 7. P0 修复实现
|
||||
|
||||
本轮分析后的代码修复如下：
|
||||
|
||||
1. 群消息的 `{pres what=msg}` 不再由 Topic master 对每个成员逐条跨 Pod 路由。master 将收件人清单合并为一条 `group_unread/users` Redis Pub/Sub 消息；来源 Pod 本地投递一次，其他 Pod 各消费一次。
|
||||
2. 每个 Pod 只扫描一次本地 WebSocket：已订阅该群 Topic 的连接依靠正常 `{data}` 广播，不重复发送 `me` presence；同一用户未订阅该群的其他设备仍收到 `me` presence，保留多开语义。
|
||||
3. Redis fanout 未启用或批量发布失败时，自动回退到原来的逐用户 Topic 路由，避免静默漏掉未读通知。
|
||||
4. 原生 Tinode Cluster 开启时，普通 user message 不再重复发布 Redis user fanout；session control 仍保留 Redis 跨 Pod 广播。
|
||||
5. `routeSrv` 改为有界非阻塞写入，队列满时记录错误和指标，不再无限阻塞 Topic 处理链路。
|
||||
6. 新增 `im_internal_queue_enqueue_total`、`im_internal_queue_depth`、`im_internal_queue_capacity`，覆盖 `session_topic_publish`、`hub_route_server`、`topic_client`、`topic_server`、`cluster_topic_proxy`。
|
||||
|
||||
修复后的复杂度从“每条消息约 2000 次 Cluster route + 2000 次 Redis publish”降为“一次 Redis publish + 每 Pod 一次本地 session 扫描”。2000 人都订阅群 Topic 时，批量 unread 在本地扫描后不会产生重复 WebSocket 帧。
|
||||
|
||||
## 8. 复测观测项
|
||||
|
||||
复测继续使用 `group2k-r1` 相同发送模型，并重点检查：
|
||||
|
||||
```promql
|
||||
sum by (queue, result) (rate(im_internal_queue_enqueue_total[1m]))
|
||||
|
||||
histogram_quantile(
|
||||
0.95,
|
||||
sum by (le, queue) (rate(im_internal_queue_depth_bucket[1m]))
|
||||
)
|
||||
|
||||
sum by (stage, kind, target_type, status) (
|
||||
increase(realtime_fanout_events_total[5m])
|
||||
)
|
||||
```
|
||||
|
||||
验收要求：
|
||||
|
||||
- `session_topic_publish`、`topic_client`、`topic_server`、`cluster_topic_proxy` 的 `result="full"` 均为 0。
|
||||
- `group_unread/users` 的 publish 数应接近服务端实际接受的群消息数，不再乘以群成员数。
|
||||
- 群压测窗口内普通 `message/user` Redis publish 不再随群成员数增长。
|
||||
- `im_cluster_route_total` 不应再出现每条群消息约 2000 次的线性放大。
|
||||
- 2000 条计划消息需分别核对 ACK、未知结果、实时观测和最终落库，不能只按 ACK 台账判断落库率。
|
||||
@@ -0,0 +1,134 @@
|
||||
# IM 2000 人群聊 r4 复测分析
|
||||
|
||||
日期：2026-09-28
|
||||
|
||||
Run ID：`group2k-r4-20260928-01`
|
||||
|
||||
场景：机器 A/B 各 1000 个 WebSocket、各发送 1000 条，计划共发送 2000 条群消息。
|
||||
|
||||
## 1. 结论
|
||||
|
||||
本轮不能按合并报告中的 `14.39%` 实时丢失率直接判定服务端群消息丢失。两台压测机结果存在明显单边故障：
|
||||
|
||||
- 机器 B 的 1000 个连接全部稳定，1000 次发送全部收到 ACK，1497000 次 ACK 台账内投递全部到达，断线和实时缺失均为 0。
|
||||
- 机器 A 出现 654 次断线、314 次 socket 错误和 189 次 ACK 超时。A 的 TCP 源地址错误中包含 `broken pipe`、`websocket closed`，是本轮 430832 次实时缺失的主要来源。
|
||||
- 服务端实际接受并持久化 1601 条消息，其中客户端收到 ACK 1497 条，另外 104 条属于“客户端未收到 ACK，但服务端已接受”。B 恰好多收到 104000 次消息，即 104 条乘以 B 的 1000 个连接，账目完全吻合。
|
||||
- ACK 台账内 1497 条消息历史回查全部找到，`persistence_lost=0`；Kafka 群消息消费 lag 在采样点均为 0。
|
||||
- 上一轮的 O(群成员数) Redis fanout 放大已经消除：1601 条服务端消息只产生 1601 次 `group_unread/users` publish，每个远端 Pod 各 receive 1601 次，不再产生约 320 万次逐用户 Redis publish。
|
||||
- 实时发送与 60 秒收尾阶段没有新增 `topic_server` 或 `cluster_topic_proxy` 满队列。满队列发生在约 5 分钟后的并发历史回查阶段，属于另一项需要优化的问题，不能作为本轮实时丢失的原因。
|
||||
|
||||
因此，本轮结论为：**批量群未读 fanout 修复通过；B 侧实时投递通过；数据库持久化通过；A 侧网络稳定性失败；历史回查并发会打满 Topic/Proxy 队列。整体尚不能完成 2000 人群聊验收。**
|
||||
|
||||
## 2. 客户端结果
|
||||
|
||||
| 指标 | A | B | 合计 |
|
||||
|---|---:|---:|---:|
|
||||
| READY 连接 | 1000 | 1000 | 2000 |
|
||||
| 发送尝试 | 1000 | 1000 | 2000 |
|
||||
| 收到 2xx ACK | 497 | 1000 | 1497 |
|
||||
| 明确拒绝 | 0 | 0 | 0 |
|
||||
| ACK 超时 | 189 | 0 | 189 |
|
||||
| socket 结果未知 | 314 | 0 | 314 |
|
||||
| 断线 | 654 | 0 | 654 |
|
||||
| ACK 台账内应收 | 1497000 | 1497000 | 2994000 |
|
||||
| ACK 台账内实收 | 1066168 | 1497000 | 2563168 |
|
||||
| ACK 台账内实时缺失 | 430832 | 0 | 430832 |
|
||||
| ACK 台账外额外投递 | 68668 | 104000 | 172668 |
|
||||
| 重复 / 乱序 | 0 / 0 | 0 / 0 | 0 / 0 |
|
||||
| 历史检查 / 找到 | 1497 / 1497 | 0 / 0 | 1497 / 1497 |
|
||||
|
||||
合并报告按 ACK 台账计算出的到达率为 `85.61%`，但这只是 A/B 网络条件混合后的观测值，不代表服务端总体投递率。B 单机的 ACK 台账内到达率是 `100%`，A 单机是 `71.22%`。
|
||||
|
||||
A 端记录了 160074 个跨机器负延迟样本，说明两机时钟没有达到单向延迟统计要求。本轮不使用跨机器 P95/P99 判断性能，只使用 ACK、到达计数、断线、历史回查和服务端阶段指标判断正确性。
|
||||
|
||||
## 3. 未收到 ACK 的 503 条不是服务端拒绝
|
||||
|
||||
本轮没有任何明确的 HTTP/Tinode `503` 响应。报告字段 `publish_rejected=503` 实际由以下结果未知项组成：
|
||||
|
||||
- `ctrl_timeout=189`
|
||||
- `socket=314`
|
||||
|
||||
服务端持久化计数增加 1601，而 ACK 只有 1497，说明至少 104 条结果未知消息已被服务端成功接受。B 的 `unexpected_messages=104000` 与这 104 条消息对 1000 个 B 连接的广播数量完全一致。
|
||||
|
||||
后续报告应把字段继续显示为 `publish_unknown`，不能把它展示为 `rejected`，否则容易误判服务端主动拒绝。
|
||||
|
||||
## 4. 服务端与中间件
|
||||
|
||||
### 4.1 Kafka 与数据库落库
|
||||
|
||||
- `im_kafka_message_persist_total{kind="group",result="batch_ok"}` 增加 1601。
|
||||
- `im.group.message.persist` consumer lag 在采样点均为 0。
|
||||
- 1497 条 ACK 台账消息通过历史接口全部找到，持久化缺失为 0。
|
||||
- 服务端群消息 `save` 总耗时增加约 26.27s，平均约 16.4ms/条。
|
||||
- Kafka produce 总耗时增加约 19.15s，平均约 12.0ms/条。
|
||||
|
||||
这证明已接受消息能够及时进入 Kafka 并完成落库。104 条结果未知消息也计入 1601 条 batch success，但当前历史校验只以 ACK 台账为准，下一版应对“已观测但未 ACK”的消息也做 message ID 回查。
|
||||
|
||||
### 4.2 Redis fanout 修复验证
|
||||
|
||||
1601 条服务端接受消息对应：
|
||||
|
||||
- master Pod `group_unread/users publish ok` 增加 1601。
|
||||
- 两个远端 Pod 的 `group_unread/users receive ok` 各增加 1601。
|
||||
- 普通 `message/user` 在原生 Cluster 开启时继续使用 `skip_native_cluster`，没有恢复逐成员 Redis 发布。
|
||||
|
||||
这符合“一条群消息一次 Redis publish、每个 Pod 一次本地扫描”的设计，上一轮每条消息约 2000 次 Redis publish 的放大已经消失。
|
||||
|
||||
### 4.3 队列满发生在历史回查阶段
|
||||
|
||||
按时间序列拆分：
|
||||
|
||||
- `16:25:05Z` 开始发送。
|
||||
- `16:28:36Z` 最后一台压测机结束发送，之后进入收尾和历史等待。
|
||||
- 截至 `16:34:01Z`，`topic_server/full` 仍停留在测试前基线 1349，`cluster_topic_proxy/full` 仍为 0。
|
||||
- `16:34:25Z` 起，历史回查并发开始使 `topic_server/full` 增加。
|
||||
- 最终历史阶段新增 `topic_server/full=1355`、`cluster_topic_proxy/full=55`。
|
||||
|
||||
因此这些队列丢弃不是实时发送阶段 430832 次缺失的原因。历史回查一次并发拉取大量消息，经 Topic/Proxy 回送时会形成独立洪峰，应限制每机历史校验并发或分页节流，并单独验收历史吞吐。
|
||||
|
||||
## 5. A 机器故障判断
|
||||
|
||||
A 与 B 使用相同代码、相同公共入口和相同发送模型，但只有 A 发生 654 次断线。A 的错误包含：
|
||||
|
||||
- `websocket closed`
|
||||
- `write: broken pipe`
|
||||
- `ctrl timeout`
|
||||
- 到多个公网入口 IP 的 TCP 读写超时
|
||||
|
||||
这种强烈单边性不符合服务端全局过载的表现。当前最可能的故障域依次是 A 本机网络/NAT/安全软件、A 到公网 LB 的链路、A 的文件描述符或本地调度压力。由于 Kubernetes Metrics API 被 RBAC 拒绝，本轮没有取得 Pod CPU/内存数据，不能用资源指标进一步排除 Pod 饱和。
|
||||
|
||||
## 6. 下一轮执行顺序
|
||||
|
||||
1. 先做 A/B 角色互换：B 仍发 1000 条，A 暂时只保持 1000 个连接不发送。若 A 仍大量断线，可确认与服务端发送负载无关。
|
||||
2. 在 A 机同步采集 `ping`、TCP retransmit、连接跟踪、文件描述符、CPU、内存和进程调度数据；压测前检查 `ulimit -n`。
|
||||
3. 将历史回查改为分页和限并发，或先关闭自动历史回查，仅验证实时阶段；实时阶段结束后再启动独立历史校验。
|
||||
4. 修正报告字段，把超时/socket 明确归类为 `publish_unknown`，并把 104 条 unacked-observed 消息纳入历史校验。
|
||||
5. 补充 `backend-debug` 对 `metrics.k8s.io/pods` 的只读权限，否则下一轮仍无法采集 Pod CPU/内存。
|
||||
6. A 链路稳定后，使用相同 2000 连接、2000 条消息模型重新验收。门禁要求两侧断线为 0、明确拒绝为 0、实时到达率 100%、持久化丢失为 0，并分别报告实时阶段和历史阶段的队列指标。
|
||||
|
||||
## 7. 结果文件
|
||||
|
||||
本轮原始结果位于：
|
||||
|
||||
```text
|
||||
loadtest/im-pressure/results/group2k-r4-20260928-01/
|
||||
agent-a-summary.json
|
||||
agent-b-summary.json
|
||||
coordinator.json
|
||||
report.json
|
||||
pre-metrics.log
|
||||
pod-metrics.log
|
||||
post-metrics.log
|
||||
```
|
||||
|
||||
## 8. 已实施改造
|
||||
|
||||
本分析完成后，压测工具已实施以下改造：
|
||||
|
||||
1. agent 分开输出 `publish_rejected` 和 `publish_outcome_unknown`；前者只包含服务端明确拒绝，后者包含 ACK 超时和 socket 失败。
|
||||
2. A/B 在 drain 后向 coordinator 提交“实时已观测但未 ACK”的消息台账，coordinator 去重合并后将其加入历史落库核验。
|
||||
3. 报告新增 `history_unacked_observed_checked` 和 `history_unacked_observed_found`，可直接确认结果未知消息是否最终落库。
|
||||
4. r5 复测后，历史回查进一步调整为 1 个 worker、每页 20 条、全局每 500ms 最多发起一次查询，避免核验程序自身制造 Topic/Proxy 洪峰。
|
||||
5. 新增 `backend-debug-metrics-rbac.yaml`，为测试环境的 `argocd/backend-debug` ServiceAccount 提供 `metrics.k8s.io/pods` 的 namespace 只读权限。
|
||||
|
||||
上述改造只解决统计口径、历史查询压力和资源指标权限。机器 A 的公网连接异常仍需通过下一轮主机与网络观测定位。
|
||||
@@ -0,0 +1,92 @@
|
||||
# IM 2000 人群聊 r5 复测分析
|
||||
|
||||
日期：2026-09-28
|
||||
|
||||
Run ID：`group2k-r5-20260928-01`
|
||||
|
||||
## 1. 结论
|
||||
|
||||
本轮再次证明服务端对稳定客户端的群消息可靠性没有丢失，失败集中在机器 A 的本地代理/TUN 链路：
|
||||
|
||||
- B：1000 次发送全部 ACK，1426000 次 ACK 台账内投递全部到达，断线、重复、乱序和实时缺失均为 0。
|
||||
- A：574 次发布结果未知、657 次统计窗口内断线、464596 次 ACK 台账内实时缺失。
|
||||
- A 的 broken pipe 全部写向 `198.18.0.81:443`；本机 DNS 也将 `api.118im.com` 解析为 `198.18.0.81`，路由经 `utun19827 -> 172.19.0.1`。`198.18.0.0/15` 是代理/TUN 常用 Fake-IP 网段，不是 API 的真实公网地址。
|
||||
- 机器 A 压测进程 CPU 平均占用不高、内存峰值约 150MB，B 在同一服务端负载下完全稳定，不支持“服务端全局过载”解释。
|
||||
- 新增观测台账生效：574 条结果未知中，241 条被实时观测并加入历史核验，241 条全部找到。
|
||||
- 历史总核验 1667/1667，持久化丢失为 0。
|
||||
- 历史回查仍产生 1197 次 `topic broadcast queue full`，但全部发生在历史阶段，不在实时发送阶段。
|
||||
|
||||
总体结论：**群消息落库通过，B 侧实时可靠性通过；A 侧 TUN/Fake-IP 链路失败；历史查询限速仍需继续降低。**
|
||||
|
||||
## 2. 汇总结果
|
||||
|
||||
| 指标 | A | B | 合计 |
|
||||
|---|---:|---:|---:|
|
||||
| READY | 1000 | 1000 | 2000 |
|
||||
| 发送尝试 | 1000 | 1000 | 2000 |
|
||||
| ACK | 426 | 1000 | 1426 |
|
||||
| 明确拒绝 | 0 | 0 | 0 |
|
||||
| ACK 超时 | 444 | 0 | 444 |
|
||||
| socket 结果未知 | 130 | 0 | 130 |
|
||||
| 断线 | 657 | 0 | 657 |
|
||||
| ACK 台账内应收 | 1426000 | 1426000 | 2852000 |
|
||||
| ACK 台账内实收 | 961404 | 1426000 | 2387404 |
|
||||
| ACK 台账内实时缺失 | 464596 | 0 | 464596 |
|
||||
| ACK 台账外额外投递 | 185738 | 241000 | 426738 |
|
||||
| 历史核验 | 1667/1667 | 0/0 | 1667/1667 |
|
||||
| 未 ACK 观测消息核验 | 241/241 | 0/0 | 241/241 |
|
||||
|
||||
服务端实际处理消息至少为 `1426 + 241 = 1667` 条。其余 333 条结果未知既没有 ACK，也没有任何实时观测证据，不能证明服务端接受。
|
||||
|
||||
## 3. A 机器故障证据
|
||||
|
||||
### 3.1 网络路径
|
||||
|
||||
测试结束后的本机检查结果：
|
||||
|
||||
```text
|
||||
api.118im.com -> 198.18.0.81
|
||||
route: 198.18.0.81 -> gateway 172.19.0.1, interface utun19827
|
||||
broken pipe remote: 198.18.0.81:443
|
||||
```
|
||||
|
||||
A 的错误构成为：
|
||||
|
||||
- `ctrl_timeout=444`
|
||||
- `websocket_closed=103`
|
||||
- `broken_pipe=27`
|
||||
- 其他 1
|
||||
|
||||
连接从 `17:05:12 UTC` 开始下降，随后从 1000 持续降至约 333；B 同期保持 1000 连接。下一轮前必须关闭该代理/TUN，或至少将 `api.118im.com` 设置为 DIRECT，并确认 DNS 不再返回 `198.18.0.0/15` Fake-IP。
|
||||
|
||||
### 3.2 本机资源
|
||||
|
||||
- 进程 RSS 峰值约 150MB。
|
||||
- 压测期间累计 CPU 增量约 75 秒，对约 625 秒窗口而言不构成 CPU 饱和。
|
||||
- open FDs 在连接大量断开后仍约为 1011。`WSClient.readLoop` 在读失败返回时没有立即关闭底层连接，虽然进程结束时会统一关闭，但应修复运行期 FD 滞留。
|
||||
|
||||
## 4. 服务端和历史核验
|
||||
|
||||
实时发送和 drain 阶段没有新增以下错误：
|
||||
|
||||
- `sub.broadcast channel full`
|
||||
- `cluster: proxy channel full`
|
||||
- `topic's broadcast queue is full`
|
||||
- Redis fanout publish failure
|
||||
|
||||
历史核验约在 `17:13:15 UTC` 开始。`im-server-2` 从 `17:13:22` 到 `17:14:14` 记录 1197 次群 Topic broadcast queue full。当前 1 worker、每页 50 条、100ms 间隔仍会让单次历史回放快速灌入容量为 64 的 Topic 队列。
|
||||
|
||||
尽管重试后 1667 条全部找到，历史核验不应持续制造队列丢弃。建议调整为：
|
||||
|
||||
- `history_page_size=20`
|
||||
- `history_query_interval=500ms`
|
||||
- `history_workers=1`
|
||||
- 历史超时保持 3 分钟；1667 条约需 42 秒完成首轮查询。
|
||||
|
||||
## 5. 下一步
|
||||
|
||||
1. 修复 `WSClient.readLoop` 退出时未关闭底层连接的问题。
|
||||
2. 将历史分页调整为 20 条、查询间隔 500ms。
|
||||
3. 在 A 机关闭 TUN/代理，或配置 `api.118im.com` DIRECT；起压前检查 DNS 和路由不再进入 `198.18.0.0/15`、`utun*`。
|
||||
4. 测试环境尚未应用 metrics RBAC，`kubectl auth can-i list pods.metrics.k8s.io` 仍返回 `no`，因此本轮没有 Pod CPU/内存数据。需要将 `backend-debug-metrics-rbac.yaml` 加入实际 ArgoCD overlay。
|
||||
5. 完成上述三项后再运行同规模 r6。若 A/B 都稳定，再根据 B 本轮 ACK P95 约 1.14s、同机投递 P95 位于 2s 桶的结果继续做服务端延迟优化。
|
||||
@@ -0,0 +1,93 @@
|
||||
# IM 2000 人群聊 r6 复测分析
|
||||
|
||||
日期：2026-09-28
|
||||
|
||||
Run ID：`group2k-r6-20260928-01`
|
||||
|
||||
## 1. 结论
|
||||
|
||||
r6 比 r5 有改善，但仍未通过。A 去掉 Fake-IP 后 ACK 数从 426 增加到 758，统计窗口断线从 657 降到 213；然而 `api.118im.com` 的真实公网 IP 仍经 `utun4`，A 在正式开始前就发生 28 次连接抖动，正式开始后继续成批断线。后续排查确认该 `utun4` 由 OpenVPN Connect 的全局路由持有，而不是 LightXtreme。B 也出现 2 次断线和 1 次发布结果未知，不再是完全零错误。
|
||||
|
||||
历史落库仍然可靠：ACK 台账和未 ACK 观测台账合计核验 1870/1870，持久化丢失为 0。FD 修复已验证生效，A 的存活连接从 1000 降至 787 时，open FDs 从约 1013 同步降至约 799，不再滞留断开的 socket。
|
||||
|
||||
历史限速和退出行为仍会污染服务端队列指标：历史查询阶段约产生 436 次 Topic 队列满；A 结束时一次性关闭大量群订阅连接，又集中产生约 598 次队列满。
|
||||
|
||||
总体结论：**落库正确性通过；FD 修复通过；实时可靠性失败；A 的 OpenVPN/TUN 链路仍未绕过；历史回放与集中断连仍需改进。**
|
||||
|
||||
## 2. 汇总结果
|
||||
|
||||
| 指标 | A | B | 合计 |
|
||||
|---|---:|---:|---:|
|
||||
| READY | 1000 | 1000 | 2000 |
|
||||
| 发送尝试 | 1000 | 1000 | 2000 |
|
||||
| ACK | 758 | 999 | 1757 |
|
||||
| ACK 超时 | 122 | 0 | 122 |
|
||||
| socket/连接结果未知 | 120 | 1 | 121 |
|
||||
| 实际明确拒绝 | 0 | 0 | 0 |
|
||||
| 断线 | 213 | 2 | 215 |
|
||||
| ACK 台账内应收 | 1757000 | 1757000 | 3514000 |
|
||||
| ACK 台账内实收 | 1468873 | 1756514 | 3225387 |
|
||||
| ACK 台账内实时缺失 | 288127 | 486 | 288613 |
|
||||
| ACK 台账外额外投递 | 94965 | 112984 | 207949 |
|
||||
| 历史核验 | 1870/1870 | 0/0 | 1870/1870 |
|
||||
| 未 ACK 观测消息核验 | 113/113 | 0/0 | 113/113 |
|
||||
|
||||
服务端实际处理消息至少为 `1757 + 113 = 1870` 条。其余 130 条结果未知没有 ACK 或实时观测证据，不能证明服务端接受。
|
||||
|
||||
## 3. 报告口径修正
|
||||
|
||||
合并报告当前显示 91 次明确拒绝和 152 次结果未知，这是错误分类：
|
||||
|
||||
- A 的 91 次 `use of closed network connection` 被归类为 `other`。
|
||||
- 该错误是连接关闭后的写失败，应该归类为 `socket` 结果未知。
|
||||
- 修正后本轮明确拒绝为 0，结果未知为 243。
|
||||
|
||||
此外，群历史只由 A 执行。A 已证明全部 1757 条 ACK 消息落库，但 B 的 `realtime_lost_but_persisted` 无法读取 A 的历史结果，因此 B 的 486 次实时缺失没有计入“实时缺失但已落库”。合并报告需要共享全局历史核验结果后再计算该字段。
|
||||
|
||||
## 4. A 网络与 FD
|
||||
|
||||
本轮 DNS 返回真实地址：
|
||||
|
||||
```text
|
||||
64.118.158.131
|
||||
216.236.31.189
|
||||
216.236.31.191
|
||||
```
|
||||
|
||||
但到真实地址的系统路由仍为：
|
||||
|
||||
```text
|
||||
gateway: 172.27.240.1
|
||||
interface: utun4
|
||||
```
|
||||
|
||||
这说明 Fake-IP 已关闭，但流量仍进入 VPN/TUN。A 在 barrier 提交前持续掉线重连，正式开始后约 50 秒开始掉线，最终只剩约 787 个连接。后续确认 OpenVPN Connect 下发了 `0/1` 和 `128.0/1` 两条全局路由；LightXtreme 的 DIRECT 规则无法绕过另一套 OpenVPN TUN。r7 前已为 `api.118im.com` 当前三个 IP 添加经 `192.168.1.1/en1` 的 `/32` 主机路由，保留 OpenVPN 内网访问的同时让压测 HTTP/WS 直连。
|
||||
|
||||
FD 时间序列证明 read loop 修复有效：
|
||||
|
||||
- READY 1000 时 open FDs 约 1013。
|
||||
- READY 901 时 open FDs 约 913。
|
||||
- READY 787 时 open FDs 约 799。
|
||||
|
||||
连接每减少一个，FD 基本同步减少一个。
|
||||
|
||||
## 5. Topic 队列满
|
||||
|
||||
实时发送和 drain 阶段仍没有 Topic/Proxy 满队列。`im-server-2` 的 1034 次 `topic's broadcast queue is full` 分为：
|
||||
|
||||
- `00:35:59–00:36:11 UTC`：约 436 次，发生在历史回查阶段。
|
||||
- `00:37:19 UTC`：约 598 次，发生在 A agent 完成后批量关闭连接阶段。
|
||||
|
||||
20 条/500ms 虽有改善，但 Topic 历史数据仍以突发方式回送。建议：
|
||||
|
||||
1. 历史每页进一步降至 10 条，查询间隔保持 500ms。
|
||||
2. 每页请求后等待本页目标消息全部收到或超时，再发送下一页，而不是只按固定时间节流。
|
||||
3. agent 退出时按 10–20ms 间隔分批关闭连接，避免 1000 个群订阅同时离线。
|
||||
|
||||
## 6. 后续修复
|
||||
|
||||
1. 已将 `use of closed network connection` 分类为 socket 结果未知。
|
||||
2. 已在 coordinator 共享两机历史命中 ID，使 B 也能正确计算实时缺失但已落库。
|
||||
3. 历史回查已改为每页 10 条：同一 Topic 每页完成确认后再请求下一页，不同 Topic 最多 12 路并行。
|
||||
4. agent 已改为在 10 秒窗口内渐进关闭 WebSocket。
|
||||
5. r7 开始前重新解析 `api.118im.com`，确认每个 IP 都通过 `/32` 主机路由走 `en1`；OpenVPN 重连、DNS 变更或机器重启后必须重新核验。
|
||||
@@ -297,3 +297,219 @@ Run ID：`p2p-200-20260927-01`，A/B 两台压测机各模拟 200 个用户，
|
||||
- ACK P95 和到达 P95 均落在 2 秒直方图上界，未达 500ms / 1s 门槛。
|
||||
- ArgoCD 已将 StatefulSet 资源回滚为 `request 100m/128Mi`、`limit 500m/256Mi`，且 liveness 仍使用 `/healthz`。压测期间存在 CPU throttling 累计，部署配置必须在 GitOps 声明源中固化，否则应用后仍会被回滚。
|
||||
- 403 修复发布后应重跑 200 对；只有 ACK 成功率、实时到达率、MySQL 精确落库均达标，才进入 500 对档位。
|
||||
|
||||
## 12. 200 对第三轮复测结果
|
||||
|
||||
Run ID：`p2p-200-20260927-03`。压测前修复了压测工具复用固定 `Idempotency-Key` 而跳过好友关系校验的问题，并重新核对、修复 200 对测试账号。
|
||||
|
||||
| 指标 | 结果 |
|
||||
|---|---:|
|
||||
| WS Ready | 400/400 |
|
||||
| Publish 尝试 / ACK | 4000 / 4000 |
|
||||
| Publish 拒绝 / 403 / 502 / 503 | 0 / 0 / 0 / 0 |
|
||||
| ACK 后应到达 / 实际到达 | 4000 / 4000 |
|
||||
| 实时到达率 | 100% |
|
||||
| 历史查询 | 4000/4000 |
|
||||
| MySQL 精确匹配 | 4000/4000 |
|
||||
| MySQL 缺失 / 错配 / 重复 / 非 ACK 落库 | 0 / 0 / 0 / 0 |
|
||||
| 丢失 / 重复 / 断连 | 0 / 0 / 0 |
|
||||
| 乱序 | 1 |
|
||||
| Cluster 队列采样最大值 | 0 |
|
||||
| Kafka 峰值 lag / 最旧 backlog age / 最终 lag | 3028 / 92.624s / 0 |
|
||||
|
||||
结论：跨 Pod 发送、ACK 后实时投递、Kafka 最终消费和 MySQL 落库的数据正确性已通过 200 对验证。结果报告仍为不通过，原因是 B 端记录 1 次单连接 Topic seq 回退，以及 ACK P95 和投递 P95 均落入 1-2s 区间。
|
||||
|
||||
资源采样显示，三个 Pod 峰值内存低于 94MiB，但 Pod 0/1/2 在本轮窗口内的 CPU throttled 增量约为 7.62s / 0.78s / 8.51s。实际 StatefulSet 仍是 `request 100m/128Mi`、`limit 500m/256Mi`，且 liveness 仍是 `/healthz`。因此延迟和 Kafka 追平速度尚不能作为扩容到 500 对的验收基线。
|
||||
|
||||
已增加 `deploy/argocd/im-server-test/statefulset-performance-patch.yaml`，要求 ArgoCD 声明源固化 `request 1C/512Mi`、`limit 2C/1Gi`和 `/livez` liveness。部署流水线改为等待 ArgoCD 回写后严格校验，如果再被回滚则直接失败。压测工具保留乱序为失败门禁，并新增最多 100 条 `receiver/topic_hash/previous_seq/current_seq/client_mid` 明细，用于下轮精确定位。
|
||||
|
||||
## 13. GitOps 资源生效后的第四轮复测
|
||||
|
||||
Run ID：`p2p-200-20260927-04`。ArgoCD Application `im-server-test` 的真实配置仓库已固化 `request 1C/512Mi`、`limit 2C/1Gi` 和 `/livez`，三个 Pod 完成滚动后均为 Ready、零重启。
|
||||
|
||||
| 指标 | 结果 |
|
||||
|---|---:|
|
||||
| WS Ready | 400/400 |
|
||||
| Publish 尝试 / ACK | 4000 / 4000 |
|
||||
| Publish 拒绝 / 403 / 502 / 503 | 0 / 0 / 0 / 0 |
|
||||
| ACK 后应到达 / 实际到达 | 4000 / 4000 |
|
||||
| 实时丢失 / 重复 / 乱序 / 断连 | 0 / 0 / 0 / 0 |
|
||||
| 历史查询 | 4000/4000 |
|
||||
| MySQL 精确匹配 | 4000/4000 |
|
||||
| MySQL 缺失 / 错配 / 重复 / 非 ACK 落库 | 0 / 0 / 0 / 0 |
|
||||
| 精确 ACK P50 / P95 / P99 / Max | 368.189 / 566.683 / 725.654 / 1600.975 ms |
|
||||
| 投递 P95 直方图上界 | 1000 ms |
|
||||
| Cluster 队列采样最大值 | 0 |
|
||||
| Kafka message persist 峰值 lag / 归零耗时 | 2946 / 发送结束后 113s |
|
||||
| Pod 0/1/2 CPU throttled 增量 | 0 / 0 / 0 usec |
|
||||
| Pod 0/1/2 峰值内存 | 87.86 / 80.80 / 85.18 MiB |
|
||||
|
||||
结论：消息正确性全部通过，第三轮出现的 1 条 seq 回退没有复现。资源扩容完全消除了本轮 CPU throttling，证明旧的 500m CPU limit 是延迟放大因素之一；精确 ACK P95 从第三轮的 1006.288ms 降至 566.683ms，下降约 43.7%，但仍比 500ms 门槛高约 13.3%，因此报告只因 ACK 延迟门禁判定失败。
|
||||
|
||||
Kafka `im-server.message_persist` lag 在发送后峰值为 2946，并在发送结束后约 113 秒归零，超过 60 秒目标。历史 API 和 MySQL 最终均确认 4000 条完整落库，说明这是落库延迟而不是消息丢失。进入 500 对测试前，应优先检查 message persist consumer 的批量写入、worker/partition 并行度和 MySQL 写入耗时。
|
||||
|
||||
连接集中关闭期间，三个 Pod 分别记录 10、14、22 次 `failed to close login audit ... Deadlock found`，共 46 次。该问题未影响消息表，但会导致登录审计结束状态不完整。登录审计关闭更新应增加 MySQL 1213/1205 有界重试，并检查更新条件和索引以缩小锁范围。
|
||||
|
||||
## 14. 持久化并行与审计死锁修复后的第五轮复测
|
||||
|
||||
Run ID：`p2p-200-20260927-05`。本轮已将 `im.message.persist` 和 `im.group.message.persist` 固化为 3 分区、3 副本；同一 Topic 继续作为 Kafka key，保证会话内顺序。服务端同时按 batch 内最大 Seq 合并 Topic 和发送者已读位点更新，并为登录审计关闭增加复合索引和 MySQL 死锁重试。
|
||||
|
||||
| 指标 | 结果 |
|
||||
|---|---:|
|
||||
| WS Ready | 400/400 |
|
||||
| Publish 尝试 / ACK | 4000 / 4000 |
|
||||
| ACK 后应到达 / 实际到达 | 4000 / 4000 |
|
||||
| 丢失 / 重复 / 乱序 / 断连 | 0 / 0 / 0 / 0 |
|
||||
| 历史查询 / MySQL 精确匹配 | 4000/4000 / 4000/4000 |
|
||||
| MySQL 缺失 / 错配 / 重复 / 非 ACK 落库 | 0 / 0 / 0 / 0 |
|
||||
| 总体精确 ACK P50 / P95 / P99 / Max | 375.314 / 547.516 / 670.317 / 2845.347 ms |
|
||||
| A 机精确 ACK P95 | 408.155 ms |
|
||||
| B 机精确 ACK P95 | 594.533 ms |
|
||||
| Pod 0/1/2 Kafka 峰值 lag | 676 / 251 / 143 |
|
||||
| 最慢 Kafka 分区归零时间 | 发送结束后约 52.5s |
|
||||
| Cluster 队列峰值 / CPU throttled 增量 | 0 / 三 Pod 均为 0 |
|
||||
| 登录审计关闭死锁 | 三 Pod 均为 0 |
|
||||
| Kafka batch failure / Pod 重启 | 0 / 0 |
|
||||
|
||||
结论：三分区已由三个 Pod 实际并行消费，单 Pod 峰值 lag 从第四轮的 2946 降至 676，最慢消费者在 60 秒门槛内归零。消息正确性、最终落库、会话内顺序、CPU 限流和登录审计死锁均通过。
|
||||
|
||||
报告仍因 ACK P95 门禁显示失败。直方图只能给出 1000ms 上界，按 4000 条 ACK 原始时间戳和 nearest-rank 口径计算的实际 P95 为 547.516ms。A 机已经低于 500ms，而 B 机为 594.533ms，且 B 机存在 2.845s 最大长尾；两台机器使用相同公共入口和相同服务端版本，因此继续修改服务端前，应先采集 B 机到 `api.118im.com` 的 DNS、TCP、TLS、WebSocket RTT 与时钟偏差，并使用定向 Pod 入口做 A/B 对照，区分公网链路长尾和服务端处理耗时。
|
||||
|
||||
## 15. 500 对第一轮容量测试
|
||||
|
||||
Run ID：`p2p-500-20260927-01`。A/B 各建立 500 条 WebSocket，共 500 对、1000 条连接和 10000 次 Publish。正式发压前补齐了第 201–500 对中缺失的 9 对好友权限，两端预检和 Ready 均通过。
|
||||
|
||||
| 指标 | 结果 |
|
||||
|---|---:|
|
||||
| WS Ready | 1000/1000 |
|
||||
| Publish 尝试 / ACK | 10000 / 9946 |
|
||||
| Publish 失败 | 54（44 条服务端 500/other，10 条 socket error） |
|
||||
| Publish 成功率 | 99.46% |
|
||||
| ACK 后应到达 / 实际到达 | 9946 / 9936 |
|
||||
| 实时丢失 / 重复 / 乱序 / 断连 | 10 / 0 / 0 / 1 |
|
||||
| 实时到达率 | 99.8995% |
|
||||
| 总体精确 ACK P95 / Max | 2429.954 / 3636.045 ms |
|
||||
| 压测期历史回查 | 8580/9946 |
|
||||
| Kafka 完全排空后 MySQL 精确匹配 | 9946/9946 |
|
||||
| MySQL 缺失 / 错配 / 重复 / 非 ACK 落库 | 0 / 0 / 0 / 0 |
|
||||
| Pod 0/1/2 Kafka 峰值 lag | 2490 / 3234 / 2745 |
|
||||
| Kafka 归零时间 | 发送结束后约 4 分钟 |
|
||||
| Pod 0/1/2 峰值内存 | 148.1 / 130.5 / 122.4 MiB |
|
||||
| Pod 0/1/2 CPU throttled 增量 | 0 / 0 / 0 |
|
||||
| Pod 重启 | 0 |
|
||||
|
||||
本轮不通过，不得升级到 1000 对。根因是 MySQL 连接容量与 IM Server 连接池上限不匹配：
|
||||
|
||||
- MySQL `max_connections=170`，本轮 `Max_used_connections=172`，`Connection_errors_max_connections=27`。
|
||||
- IM Server 每 Pod `max_open_conns=64`，3 个 Pod 的理论连接池上限已达 192，还未计入 admin server、其他服务和运维连接。
|
||||
- 本轮 Pod 0/1 分别记录 153/19 次 `Error 1040: Too many connections`，包括 50 次 `failed to save message`和 44 次 P2P 关系检查失败；这些错误与 44 条服务端 500 同时出现。
|
||||
- A 机用户 190 的 WebSocket 断开，产生 10 次 socket error，并导致对端已 ACK 的 10 条消息未能实时投递到该连接。
|
||||
- 压测结束阶段 Pod 2 记录 282 次群 Topic broadcast queue full，发生在大量会话同时清理期，需要与消息发送链路分开处理。
|
||||
|
||||
压测阶段的 `history_found=8580`不是最终丢库：历史回查结束时 Kafka 仍在积压，等 lag 归零后从 Pod 内使用 ACK ledger 独立查询 `im_008_messages`，9946 条全部精确匹配。因此当前是“已接受消息最终零丢失，但持久化延迟超标”。
|
||||
|
||||
下一轮 500 对复测前必须先完成：
|
||||
|
||||
1. 统一规划 MySQL 全局连接预算，覆盖 IM Server、admin server 及其他共库服务；单个服务的连接池上限不能独立设置到总库容量。
|
||||
2. 修正 IM Server 每 Pod 的 `max_open_conns/max_idle_conns`，或在验证 MySQL CPU、内存、线程和 PXC 限制后提高 `max_connections`；必须保留至少 20% 全局余量。
|
||||
3. 压测中增加 DB pool `OpenConnections/InUse/Idle/WaitCount/WaitDuration`和 MySQL `Threads_connected/Threads_running/Max_used_connections/Connection_errors_max_connections` 指标。
|
||||
4. 复测要求 Publish ACK 10000/10000、实时丢失和断连为 0、Kafka 60 秒内归零、MySQL 1040 和清理期 queue full 为 0，然后才能进入 1000 对。
|
||||
|
||||
### 15.1 为什么 Kafka 可以积压，但本轮仍会报错
|
||||
|
||||
Kafka 积压只能保护“已经成功写入 Kafka”的消息。正常情况下，consumer 落库变慢只会使 lag 上升，消息会在 Kafka 中等待，不应导致 Publish 报错。本轮的 54 条失败发生在 Kafka 之前：
|
||||
|
||||
1. 普通文本消息先同步查询青少年模式设置，但当前规则无论查询结果都允许消息继续进入内容审核。
|
||||
2. P2P 发布在写 Kafka 前调用 `CanSendP2P`，旧实现即使 Redis 已命中好友关系，仍会读取 MySQL 用户和好友记录。
|
||||
3. 入口校验、Kafka consumer、历史查询和登录审计共用 MySQL 连接预算。连接耗尽后，入口校验直接返回 500，这些消息没有进入 Kafka，因此无法通过“后续慢慢消费”恢复。
|
||||
|
||||
### 15.2 IM 侧改造
|
||||
|
||||
2026-09-27 已完成以下修复：
|
||||
|
||||
- 普通消息不再读取青少年模式表；通话的夜间禁用规则仍保留数据库查询。
|
||||
- P2P 热路径先查双向黑名单缓存，再使用好友正缓存；缓存未命中或 Redis 异常时才回源 MySQL。新增拉黑会同时清理双向好友正缓存，避免安全状态陈旧。
|
||||
- `meeting_im` 每 Pod 连接池设为 `max_open=24/max_idle=8`，`hotgo` 注册表库独立设为 `max_open=4/max_idle=2`。3 个 Pod 的 IM 理论总上限由 384 降为 84；结合 Workbench 30、Manager 20 的上限，整套服务理论上限为 134，在当前 MySQL 170 连接上限下保留 36 条全局余量。
|
||||
- 连接池参数改为环境变量，并同时固化到应用仓库及 ArgoCD 真实配置源。发布流水线会在 ArgoCD 回写后校验四个连接池参数。
|
||||
- 新增 `im_db_pool_*` 指标，包括 max/open/in-use/idle/wait count/wait duration/关闭计数，并为连接池 85%/95% 使用率配置预警和严重告警。
|
||||
|
||||
这些修复的目标是先让入口稳定完成缓存校验和 Kafka enqueue，把 MySQL 慢转化为可观测、可恢复的 Kafka lag，而不是 Publish 500。如果复测时 `im_db_pool_wait_total` 持续增长或 Kafka 仍无法在 60 秒内归零，再根据 PXC 的 CPU/内存/线程容量将 MySQL `max_connections` 持久化提高到 300–400，不使用临时 `SET GLOBAL`代替配置源。
|
||||
|
||||
## 16. 500 对连接池改造后复测
|
||||
|
||||
Run ID：`p2p500-fix-20260927-02`。A/B 各建立 500 条 WebSocket，共 500 对、1000 条连接和 10000 次 Publish。三个 IM Pod 均已使用 `meeting_im=24/8`、`hotgo=4/2` 的连接池配置。
|
||||
|
||||
| 指标 | 改造前 | 本轮 |
|
||||
|---|---:|---:|
|
||||
| WS Ready | 1000/1000 | 1000/1000 |
|
||||
| Publish ACK | 9946/10000 | 10000/10000 |
|
||||
| 服务端 500 / socket error | 44 / 10 | 0 / 0 |
|
||||
| 实时到达 | 9936/9946 | 10000/10000 |
|
||||
| 实时丢失 / 重复 / 乱序 / 断连 | 10 / 0 / 0 / 1 | 0 / 0 / 0 / 0 |
|
||||
| ACK P50 / P95 / P99 / Max | - / 2429.954 / - / 3636.045 ms | 662.062 / 1567.569 / 2620.878 / 3156.025 ms |
|
||||
| 压测期历史回查 | 8580/9946 | 6738/10000 |
|
||||
| Kafka 排空后 MySQL 精确匹配 | 9946/9946 | 10000/10000 |
|
||||
| MySQL 缺失 / 错配 / 重复 / 非 ACK 落库 | 0 / 0 / 0 / 0 | 0 / 0 / 0 / 0 |
|
||||
| Pod 0/1/2 Kafka 峰值 lag | 2490 / 3234 / 2745 | 2259 / 2875 / 2326 |
|
||||
| Kafka 最慢归零 | 约 4 分钟 | 发送结束后约 4 分 30 秒 |
|
||||
| MySQL 1040 | 172 次 Pod 日志记录 | 0 |
|
||||
| Pod 重启 | 0 | 0 |
|
||||
|
||||
结论分为两部分：
|
||||
|
||||
1. 入口和实时正确性修复有效。10000 次 Publish 全部被接受，实时投递、最终落库、顺序、重复和断连门禁全部通过；连接池上限把全局 MySQL 连接耗尽转化成了进程内排队，没有再出现 `Too many connections`。
|
||||
2. 容量门禁仍未通过，暂不升级到 1000 对。ACK P95 仍为 1567.569ms，Kafka 未在 60 秒内归零，压测期历史回查因落库未追平只看到 6738 条。
|
||||
|
||||
本轮新增指标进一步定位出两个瓶颈：
|
||||
|
||||
- 发送阶段 `hotgo` 注册库连接池出现 5841 次新增等待，累计等待约 210.25 秒。`TenantAdapterFor` 当前在命中已创建的租户 adapter 之前仍执行 `tenantByID`，使共库路由的高频操作重复查询 `im_tenant`；4 条注册库连接因此成为 ACK 热路径瓶颈。
|
||||
- 发送结束 30 秒后，两台 agent 同时以默认 48 worker 执行历史核验，形成 96 路历史查询。三个 `meeting_im` 连接池随后全部达到 24/24；主库新增 6780 次连接等待，累计等待约 11010.16 秒。历史查询与 Kafka persistence 共用连接池，直接延长了 lag 排空时间。
|
||||
- Pod 2 有一个 100 条批次因 `context deadline exceeded` 降级为逐条写入，100 条最终均成功；三个 Pod 共记录 8 个数据库超时相关日志，没有死信，最终数据库精确核验为 10000/10000。
|
||||
|
||||
下一轮前应完成：
|
||||
|
||||
1. 为租户元数据增加带失效机制的进程内缓存；`TenantAdapterFor` 命中有效缓存时不得每次读取 `hotgo.im_tenant`，租户停用或配置变更通过现有配置事件主动失效，并保留短 TTL 兜底。
|
||||
2. 压测程序把实时发压和持久化核验拆成两个阶段：先观察 Kafka 自然排空，再以每机 8–16 个 worker 执行历史核验，避免核验流量反向干扰被测落库吞吐。同时另设“历史读取并发压测”场景验证读写混合容量。
|
||||
3. persistence 与在线历史查询至少增加独立并发预算；若继续共用连接池，应为 persistence 保留连接或限制历史查询并发，避免读请求耗尽全部 24 条连接。
|
||||
4. 完成上述改造后仍以 500 对复测，门禁保持 ACK 10000/10000、实时错误为 0、Kafka 60 秒内归零、最终数据库 10000/10000，再决定是否升级到 1000 对。
|
||||
|
||||
### 16.1 路由缓存和分阶段核验改造
|
||||
|
||||
2026-09-27 已完成下一轮复测所需代码调整：
|
||||
|
||||
- MySQL `TenantAdapterFor` 增加 1 分钟租户路由有效期；有效期内直接复用 tenant adapter，不再读取 `hotgo.im_tenant`。
|
||||
- 同一租户的缓存刷新使用 singleflight 合并，TTL 到期瞬间最多执行一次注册库查询，避免并发击穿。
|
||||
- `tenant.updated`、`tenant.status.changed`、`tenant.limits.updated`、`tenant.features.updated` 和 `cache.tenant.invalidate` 继续使用现有 Kafka 广播事件；`InvalidateTenantConfigCache` 现在同时使数据库路由缓存失效。失效与刷新并发时通过 generation 校验阻止旧查询重新写入有效缓存。
|
||||
- 压测配置新增 `history_settle`，默认 5 分钟。实时 drain 完成后先等待该窗口，再发起历史查询，从而单独测量 Kafka 自然排空时间。
|
||||
- 历史查询默认并发从每机 48 降为 12；历史读取高并发能力后续使用独立读写混合场景验证，不再污染消息落库吞吐结果。
|
||||
|
||||
## 17. 路由缓存发布后 500 对复测与延迟改造
|
||||
|
||||
Run ID：`p2p500-r3-20260927-03`。A/B 各 500 条 WebSocket、每人发送 10 条消息，总计 10000 次 Publish。
|
||||
|
||||
| 指标 | 结果 |
|
||||
|---|---:|
|
||||
| WS Ready | 1000/1000 |
|
||||
| Publish ACK | 10000/10000 |
|
||||
| 实时到达 / 丢失 | 10000/10000 / 0 |
|
||||
| 历史回查 / 落库丢失 | 10000/10000 / 0 |
|
||||
| 重复 / 乱序 / 断连 | 0 / 0 / 0 |
|
||||
| 总体精确 ACK P95 | 1308.150 ms |
|
||||
| A / B 精确 ACK P95 | 1121.2 / 1519.2 ms |
|
||||
| 原报告投递 P95 | 2000 ms（受双机时钟偏差污染） |
|
||||
| Kafka 最终 lag | 0 |
|
||||
| `hotgo` pool wait | 0 |
|
||||
|
||||
租户路由缓存已经消除 `hotgo` 热路径等待，消息正确性和最终落库全部通过。剩余 ACK 延迟集中在消息 4–6 的突发积压阶段。发压窗口内，`meeting_im` 连接池出现明显等待；同时 Kafka persistence consumer 正在使用同一连接池。Cluster Route RPC 平均约 1–2ms，P2M 队列始终为 0，因此跨 Pod RPC 不是秒级延迟来源。
|
||||
|
||||
原投递 P95 不能作为服务端结论：压测程序使用发送机绝对时间戳在接收机计算单向耗时。A 只接受了 5777 个非负样本，而 B 接受了全部 10000 个样本，说明 B 时钟领先 A；A 丢弃负值，B 则把时钟偏差计入延迟。
|
||||
|
||||
本次完成以下改造：
|
||||
|
||||
1. 压测程序新增同机与跨机两个投递直方图；`delivery_local_p95_ms` 作为门禁，跨机负延迟计入 `cross_machine_negative_latency_samples`，不再静默丢弃后继续生成误导性 P95。
|
||||
2. 压测消息显式携带 `sender_machine`，旧消息仍可从 `client_mid` 兼容识别来源。
|
||||
3. 好友关系首次数据库确认后双向回填 Redis，减少双向聊天后续回源。
|
||||
4. 新增 `im_message_publish_stage_duration_seconds{kind,stage,result}`，覆盖 `relationship_check`、`redis_idempotency`、`redis_sequence`、`kafka_produce` 和整体 `save` 阶段。
|
||||
5. 新增 `log_client_packets` 配置，默认关闭逐包 INFO 日志，避免每条消息额外 JSON 解析和容器 stdout I/O；需要诊断时才短期开启。
|
||||
|
||||
下一轮仍使用 500 对复测。重点比较 `relationship_check`、`kafka_produce` 和 `save` 的 P95，并以 `delivery_local_p95_ms` 判断投递延迟；跨机延迟只有在 chrony/NTP 偏差确认小于 5ms 后才用于验收。数据库连接池暂不扩大，避免再次超过 MySQL 全局 170 连接预算；若 `relationship_check` 已稳定但 pool wait 与 Kafka lag 仍持续增长，再设计 persistence 独立连接预算或批量更新方案。
|
||||
|
||||
@@ -357,6 +357,14 @@ X-MeetingIM-APIKey: <预置的普通 API Key>
|
||||
| 删除单条/多条消息 | `{"del":{"id":"del-1","topic":"grp...","what":"msg","delseq":[{"low":12,"hi":13}],"hard":false}}` | `{"ctrl":{"id":"del-1","code":200,"text":"ok","params":{"del":3}}}`；其他端可能收到 `{"pres":{"topic":"grp...","what":"del","seq":3}}` |
|
||||
| 清空当前用户视图 | `{"del":{"id":"del-all-1","topic":"p2p...","what":"msg","delseq":[{"low":1,"hi":126}],"hard":false}}` | `{"ctrl":{"id":"del-all-1","code":200,"text":"ok","params":{"del":4}}}` |
|
||||
|
||||
**单聊已读/已收回执协议约定（服务端已验证，2026-09-29）：**
|
||||
|
||||
1. **在线下行形态**：`info.topic` 为话题名（如 `p2p...`），`info.from` 为报读者，`info.what` 为 `recv`/`read`，`info.seq` 为水位。服务端跳过上报会话自身，推送给对端及本人其他在线设备（多 Pod 部署经 Kafka 同步，各 Pod 均会下发）。
|
||||
2. **离线补发形态**：接收方离线时收到 `{"info":{"topic":"me","src":"p2p...","from":"usr...","what":"recv","seq":12}}`，即 `topic` 固定为 `me`、`src` 才是话题名。客户端必须同时兼容上述两种形态。
|
||||
3. **序号单调递增**：服务端丢弃 `seq` 不大于当前水位的回执（`read`、`recv` 各自独立判断），`read` 回执会同步抬升 `recv`，恒有 `read ≥ recv`。客户端渲染一律取服务端最大游标，禁止用本地计数覆盖服务端值。
|
||||
4. **断线重连恢复**：`sub`/`get desc` 响应的 `desc` 中包含 `seq`、`read`、`recv`、`clear` 四个游标；`get {"what":"sub"}` 也返回各成员的 `read/recv`。重连后以这些游标恢复本地状态，不要依赖进程内存。
|
||||
5. **换设备登录**：回执水位按「用户 × 话题」在服务端持久化，与设备无关；新设备订阅后 `desc` 同时返回自己与对端的最新 `read/recv`，可完整恢复对方"已读/已收"状态。发送者自身的 `read/recv` 由服务端自动推进到其最新发出的 `seq`。
|
||||
|
||||
##### 0.5.4.5 Topic 资料、个人群设置和标签
|
||||
|
||||
| 场景 | 客户端发送 | 服务端返回/下行示例 |
|
||||
@@ -1105,7 +1113,9 @@ X-MeetingIM-APIKey: <预置的普通 API Key>
|
||||
|
||||
- 选择媒体后本地生成 `client_mid` 和稳定占位；调用文件上传，得到 `file_id` 后再 `pub x-im-type=image|video|voice`。
|
||||
- `content` 至少含 `file_id,name,size,mime,width,height,duration_ms,thumbnail_file_id` 中适用字段；上传进度是客户端任务状态，不写成聊天消息。
|
||||
- 服务端发布成功后关联文件与 Topic/消息；文件下载必须校验消息可见性。上传失败只重试上传，成功但 `pub` 失败时复用已上传 `file_id` 和 `client_mid`。
|
||||
- 服务端发布成功后关联文件与 Topic/消息；文件下载必须校验消息可见性。
|
||||
- 上传失败（网络中断、超时、5xx）按**整文件重传**处理：重新调用 `POST /v0/file/u/` 上传完整文件，不做分片/断点续传；每次重传得到新 `file_id`，旧 `file_id` 直接放弃、由服务端 TTL 回收，客户端不得再引用。
|
||||
- 上传成功但 `pub` 失败时不重新上传，复用已上传 `file_id` 和原 `client_mid` 重发消息，服务端按 `client_mid` 幂等去重。
|
||||
|
||||
### 3.7 个人对话—拉黑状态／已读状态.png
|
||||
|
||||
@@ -1259,7 +1269,9 @@ X-MeetingIM-APIKey: <预置的普通 API Key>
|
||||
|
||||
**状态：客户端任务管理新增；服务端上传需改造。**
|
||||
|
||||
- 每个待发送项保存 `client_mid,local_uri,type,upload_id,progress,state`；同一文件可分片/断点续传时使用稳定 `upload_id`。
|
||||
- 每个待发送项保存 `client_mid,local_uri,type,upload_id,progress,state`；`upload_id` 仅在单次上传任务内有效（进度上报与本地去重用）。
|
||||
- 重试即新建上传任务：`progress` 归零、生成新 `upload_id`，**整文件重传** `POST /v0/file/u/`，不做分片/断点续传；重传成功后消息体 `file_id` 等字段全部取最新一次上传响应。
|
||||
- 自动重试采用指数退避且不超过 3 次，之后转人工点按重试；App 冷启动扫描本地 `failed` 任务队列重新上传。
|
||||
- 上传完成后再发布消息；多任务可并发但必须各自保持原排序号。离开页面不取消，退出账号必须暂停并隔离任务。
|
||||
- 服务端返回 `FILE_TOO_LARGE`、`FILE_TYPE_NOT_ALLOWED`、`STORAGE_QUOTA_EXCEEDED` 时不可自动无限重试。
|
||||
|
||||
@@ -1374,10 +1386,10 @@ X-MeetingIM-APIKey: <预置的普通 API Key>
|
||||
| 接口 | 输入示例 | 成功 `data` 示例 |
|
||||
| --- | --- | --- |
|
||||
| `GET /v1/im/conversations/{topic}/context` | Path `p2p...` 或 `grp...` | `{"topic":"p2p...","type":"p2p","relationship":"friend","can_send":true,"capabilities":{"text":true,"image":true,"file":true,"voice":true,"can_call":true},"peer":{"user_id":"usr...","display_name":"张三","avatar_file_id":""}}` |
|
||||
| `POST /v1/im/conversations/p2p` 待实现 | `{"peer_user_id":"usr..."}` | `{"topic":"p2p...","relationship":"friend","can_send":true}` |
|
||||
| `GET /v1/im/conversations/{topic}/settings` 待实现 | Path topic | `{"pinned":false,"mute_notifications":false,"message_retention":"default","blocked":false,"remark":"","version":1,"permissions":{"can_update":true}}` |
|
||||
| `PATCH /v1/im/conversations/{topic}/settings` 待实现 | `{"pinned":true,"mute_notifications":false,"version":1}` | `{"pinned":true,"mute_notifications":false,"message_retention":"default","blocked":false,"remark":"","version":2}` |
|
||||
| `GET /v1/im/conversations/{topic}/messages` 待实现 | Query `?since=12&limit=20` | `{"topic":"p2p...","items":[{"seq":13,"from":"usr...","ts":"2026-08-20T09:00:00Z","head":{"x-im-type":"text"},"content":{"text":"你好"}}],"next_cursor":""}` |
|
||||
| `POST /v1/im/conversations/p2p` | `{"peer_user_id":"usr..."}` | `{"topic":"p2p...","relationship":"friend","can_send":true}` |
|
||||
| `GET /v1/im/conversations/{topic}/settings` | Path topic | `{"pinned":false,"mute_notifications":false,"message_retention":"default","blocked":false,"remark":"","version":1,"permissions":{"can_update":true}}` |
|
||||
| `PATCH /v1/im/conversations/{topic}/settings` | `{"pinned":true,"mute_notifications":false,"version":1}` | `{"pinned":true,"mute_notifications":false,"message_retention":"default","blocked":false,"remark":"","version":2}` |
|
||||
| `GET /v1/im/conversations/{topic}/messages` | Query `?since=12&limit=20` | `{"topic":"p2p...","items":[{"seq":13,"from":"usr...","ts":"2026-08-20T09:00:00Z","head":{"x-im-type":"text"},"content":{"text":"你好"}}],"next_cursor":""}` |
|
||||
| `PUT /v1/im/topics/{topic}/draft` 待实现 | `{"content":"未发送草稿","version":1}` | `{"topic":"grp...","content":"未发送草稿","version":2,"updated_at":"2026-08-20T09:00:00Z"}` |
|
||||
| `POST /v1/im/topics/{topic}/messages/{seq}/recall` | 单条撤回；`topic` 可传 `p2p...` 或 `grp...`；Body `{}`，可选 `{"version":1,"reason":"mistake"}` | `{"topic":"grp...","seq":12,"status":"recalled","version":2,"updated_at":"2026-08-20T09:00:00Z"}` |
|
||||
| `POST /v1/im/topics/{topic}/messages/recall` | 多选批量撤回；`{"items":[{"seq":12,"version":1},{"seq":13}],"reason":"mistake"}`；`items` 1～50 条，`version` 可选 | `{"topic":"grp...","items":[{"topic":"grp...","seq":12,"status":"recalled","version":2,"updated_at":"2026-08-20T09:00:00Z"},{"seq":13,"status":"failed","http":409,"code":30006,"reason":"VERSION_CONFLICT","message":"message version conflict"}]}` |
|
||||
@@ -1553,6 +1565,8 @@ Body：
|
||||
|
||||
撤回、编辑属于 `seq` 不变的原地变更。在线且已订阅 Topic 时，客户端会收到 `message.recalled` / `message.edited` 实时事件；离线或未订阅时，需要靠会话级水位补偿。
|
||||
|
||||
单聊场景中，客户端在线但未订阅该 P2P 时，服务端还会向双方用户通道下发 `message.mutation` 脏标记事件（`what:"im"`、`event:"message.mutation"`，payload 含 `topic/seq/version/event`），客户端收到后同样按下方水位补偿流程拉取；群聊不下发脏标记，仅靠会话列表水位 + 拉取补偿。
|
||||
|
||||
会话列表同步时，`me.meta.sub[].private` 会带：
|
||||
|
||||
```json
|
||||
@@ -1606,6 +1620,7 @@ GET /v1/im/topics/{topic}/message-mutations?since_version={localMutationVersion}
|
||||
|
||||
- `mutation_version` 是会话级水位，不是单条消息的 `version`。
|
||||
- 本地只做 `服务端 mutation_version > 本地 mutation_version` 比较，不要假设连续递增。
|
||||
- 补偿接口返回的是最新 `limit` 条（默认 50、上限 100）变更的**状态快照**，非严格增量；变更条数超过 `limit` 时更早的变更不再返回。撤回窗口 120 秒、编辑窗口 300 秒，超窗变更无需客户端处理。
|
||||
- 拉到 `message.recalled` 后，将本地 `topic + seq` 标记为撤回，并只在 `version` 更高时覆盖。
|
||||
- 拉到 `message.edited` 后，按 `seq + version` 拉取或覆盖消息内容。
|
||||
- 实时事件和补偿接口可能重复到达，客户端必须按 `topic + seq + version` 幂等处理。
|
||||
@@ -2100,7 +2115,7 @@ GET /v1/im/topics/{topic}/message-mutations?since_version={localMutationVersion}
|
||||
|
||||
- 先用 `purpose=group_file,topic=...` 上传，再 `POST /v1/im/groups/{topic}/files`，Body `{"file_id":"..."}` 建立群文件记录。
|
||||
- 服务端校验 `member_upload_files`、单文件上限、租户总存储和群成员身份；成功返回完整群文件项。
|
||||
- 上传失败可复用 `upload_id`；登记失败不得在列表显示，孤立文件按 TTL 回收。
|
||||
- 上传失败按整文件重传重新上传并取得新 `file_id`；登记失败不得在列表显示，孤立文件按 TTL 回收。
|
||||
|
||||
### 6.29 接口示例
|
||||
|
||||
@@ -2127,7 +2142,7 @@ GET /v1/im/topics/{topic}/message-mutations?since_version={localMutationVersion}
|
||||
| `PUT /v1/im/groups/{topic}/members/me/profile` | `{"nickname":"我在群里的昵称","version":1}` | `{"nickname":"我在群里的昵称","version":2,"updated_at":"2026-08-20T09:00:00Z"}` |
|
||||
| `POST /v1/im/groups/{topic}/leave` | `{}` | `{"topic":"grp...","status":"left"}` |
|
||||
| `GET /v1/im/groups/{topic}/members?query=&cursor=&mentionable=true` | Query only | `{"topic":"grp...","items":[{"user_id":"usr...","display_name":"张三","avatar_file_id":"","role":"member","mentionable":true,"acs":{"want":"JRWP","given":"JRWP","mode":"JRWP"}}],"next_cursor":"","permissions":{"can_mention_all":true}}` |
|
||||
| `GET /v1/im/groups/{topic}/members/{user_id}/profile` | Path topic/user；`topic` 必须是 `grp...`，不能传 `usr...` | `{"topic":"grp...","user_id":"usr...","display_name":"张三","group_nickname":"小张","role":"member","relationship":"friend","capabilities":{"can_send":true,"can_call":true}}` |
|
||||
| `GET /v1/im/groups/{topic}/members/{user_id}/profile` | Path topic/user；`topic` 必须是 `grp...`，不能传 `usr...`；`user_id` 可传本人，返回 `relationship:"self"`、`capabilities.can_send=true` | `{"topic":"grp...","user_id":"usr...","display_name":"张三","group_nickname":"小张","role":"member","relationship":"friend","capabilities":{"can_send":true,"can_call":true}}` |
|
||||
| `POST /v1/im/groups/{topic}/qrcodes` | `{"expires_in_seconds":86400}` | `{"qr_token":"token...","qr_payload":"im://qr?token=...&purpose=join_group","expires_at":"2026-08-21T09:00:00Z","group":{"topic":"grp...","name":"项目群","member_count":2},"group_snapshot":{"topic":"grp...","name":"项目群","member_count":2}}` |
|
||||
| `POST /v1/im/groups/{topic}/invitations` | `{"user_ids":["usr..."],"message":"进群一起讨论"}` | `{"topic":"grp...","items":[{"user_id":"usr...","status":"joined","invitation_id":"ginv_1"}]}` |
|
||||
| `POST /v1/im/group-invitations/{invitation_id}/accept` 待实现 | `{}` | `{"invitation_id":"ginv_1","status":"joined","topic":"grp..."}` |
|
||||
|
||||
@@ -36,6 +36,7 @@ require (
|
||||
go.mongodb.org/mongo-driver v1.17.9
|
||||
golang.org/x/crypto v0.53.0
|
||||
golang.org/x/oauth2 v0.36.0
|
||||
golang.org/x/sync v0.21.0
|
||||
golang.org/x/text v0.38.0
|
||||
google.golang.org/api v0.287.0
|
||||
google.golang.org/grpc v1.82.0
|
||||
@@ -139,7 +140,6 @@ require (
|
||||
github.com/xdg-go/scram v1.2.0 // indirect
|
||||
github.com/xdg-go/stringprep v1.0.4 // indirect
|
||||
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
|
||||
golang.org/x/sync v0.21.0 // indirect
|
||||
golang.org/x/sys v0.46.0 // indirect
|
||||
golang.org/x/time v0.15.0 // indirect
|
||||
google.golang.org/appengine v1.6.8 // indirect
|
||||
|
||||
@@ -8,37 +8,39 @@
|
||||
|
||||
## 当前 allrich 测试批次
|
||||
|
||||
以下参数适用于 2026-09-27 正在执行的第一轮单聊压测：
|
||||
以下参数适用于 2026-09-27 的 2000 人群聊压测：
|
||||
|
||||
| 参数 | 值 |
|
||||
|---|---|
|
||||
| Git 分支 | `dev` |
|
||||
| 最低提交 | `6d17154` |
|
||||
| 最低提交 | `ea8732e` |
|
||||
| 企业码 | `allrich` |
|
||||
| HTTP | `https://api.118im.com` |
|
||||
| WebSocket | `wss://api.118im.com/v0/channels` |
|
||||
| 机器 A 内网地址 | `192.168.1.135` |
|
||||
| Coordinator | `http://192.168.1.135:8090` |
|
||||
| Run ID | `p2p-20260927-01` |
|
||||
| 场景 | `p2p` |
|
||||
| 机器 A 状态 | `1000/1000 READY`，等待机器 B |
|
||||
| Run ID | `group2k-r1-20260927-01` |
|
||||
| 场景 | `group` |
|
||||
| 机器 A 状态 | 等待启动 coordinator 和 agent A |
|
||||
|
||||
机器 A 已生成安全运行包：
|
||||
|
||||
```text
|
||||
/tmp/allrich-machine-b-runtime-p2p-20260927-01.zip
|
||||
SHA-256: 479f119e7d01e1829c7262cace282c72a5f53ad8e7f5128abc2ade75d046db64
|
||||
/tmp/group2k-r1-20260927-01.zip
|
||||
SHA-256: 21e65b19476e64348b9214322021d880b231de1f9c9f685ea43d0082103e9cd1
|
||||
```
|
||||
|
||||
运行包文件名固定为 `/tmp/<RUN_ID>.zip`。它包含被 Git 忽略的密钥和 Token，因此 `git pull` 不会下载该文件，必须从机器 A 的 `/tmp` 目录单独复制。
|
||||
|
||||
运行包内只有三个运行时文件：`.env.test`、`config.json`、`manifest.json`。其中包含 API Key、测试账号密码、固定测试验证码和账号 Token，不进入 Git，必须通过受控内网传输。测试环境验证码为 `123456`，正式环境禁止使用固定验证码。
|
||||
|
||||
因此，机器 B 只执行 `git pull` 后看不到这三个文件是正常现象。机器 B 需要从机器 A 单独复制运行包：
|
||||
|
||||
```bash
|
||||
scp youshunlai@192.168.1.135:/tmp/allrich-machine-b-runtime-p2p-20260927-01.zip /tmp/
|
||||
scp youshunlai@192.168.1.135:/tmp/group2k-r1-20260927-01.zip /tmp/
|
||||
|
||||
shasum -a 256 /tmp/allrich-machine-b-runtime-p2p-20260927-01.zip
|
||||
# 必须输出：479f119e7d01e1829c7262cace282c72a5f53ad8e7f5128abc2ade75d046db64
|
||||
shasum -a 256 /tmp/group2k-r1-20260927-01.zip
|
||||
# 必须输出：21e65b19476e64348b9214322021d880b231de1f9c9f685ea43d0082103e9cd1
|
||||
```
|
||||
|
||||
如果机器 B 无法主动连接 A，也可以由机器 A 执行 `scp` 推送；目标机器的用户名和地址由 B 机提供。禁止通过 Git、公开网盘或无访问控制的 HTTP 服务传输该运行包。
|
||||
@@ -76,7 +78,7 @@ ulimit -n 100000
|
||||
在机器 B 的仓库根目录执行：
|
||||
|
||||
```bash
|
||||
unzip -o /tmp/allrich-machine-b-runtime-p2p-20260927-01.zip \
|
||||
unzip -o /tmp/group2k-r1-20260927-01.zip \
|
||||
-d loadtest/im-pressure
|
||||
chmod 600 \
|
||||
loadtest/im-pressure/.env.test \
|
||||
@@ -108,7 +110,7 @@ source loadtest/im-pressure/.env.test
|
||||
set +a
|
||||
|
||||
export MACHINE_A_IP='192.168.1.135'
|
||||
export RUN_ID='p2p-20260927-01'
|
||||
export RUN_ID='group2k-r1-20260927-01'
|
||||
|
||||
./loadtest/im-pressure/im-pressure verify \
|
||||
-config loadtest/im-pressure/config.json \
|
||||
@@ -144,9 +146,10 @@ mkdir -p "loadtest/im-pressure/results/${RUN_ID}"
|
||||
1. 机器 B 建立 1000 条 WebSocket 连接并订阅对应 P2P Topic 和群 Topic。
|
||||
2. agent 向机器 A coordinator 报告 READY。
|
||||
3. 如果机器 A 尚未 READY，当前命令会等待，这是正常现象。
|
||||
4. coordinator 下发 `run_id`、场景和统一 UTC 开始时间。
|
||||
5. agent 完成发送、WS 到达统计、drain 和历史持久化回查。
|
||||
6. agent 写出 summary、上报 coordinator，然后正常退出。
|
||||
4. 等待期间连接断开会自动重连并重新订阅；在 A/B 都实时保持 1000 条连接并完成最终屏障前，agent 不会退出或开始发压。
|
||||
5. coordinator 下发 `run_id`、场景和统一 UTC 开始时间。
|
||||
6. agent 完成发送和 WS 到达统计后，先向协调器提交未 ACK 但已观测的消息台账；等待 `history_settle`（默认 5 分钟）后，以同 Topic 串行、不同 Topic 并行的限速分页方式执行历史持久化回查。等待期间不要停止进程。
|
||||
7. 两机历史回查完成后会通过 coordinator 汇总命中消息，B 也会据此计算 `realtime_lost_but_persisted`；随后 agent 在默认 10 秒窗口内渐进关闭连接、写出 summary 并正常退出。
|
||||
|
||||
机器 B 不需要传场景参数。`p2p`、`group` 或 `mixed` 由机器 A coordinator 决定。
|
||||
|
||||
|
||||
@@ -40,17 +40,24 @@
|
||||
- 如果由机器 A 集中采集 agent 指标，机器 A 必须能访问 `http://机器B内网IP:9105/metrics`。
|
||||
- 两台机器都必须能访问配置中的 HTTP API 和各自的 WS 地址。
|
||||
- `8090` 是无鉴权的压测协调端口，只允许压测内网访问，不要暴露到公网。
|
||||
- 两台机器必须使用 NTP/chrony 同步时间；端到端延迟依赖两边时钟一致。
|
||||
- 两台机器必须使用 NTP/chrony 同步时间。报告会分别输出同机时钟延迟和跨机器单向延迟；跨机器出现负延迟样本时，后者仅用于诊断，不参与门禁。
|
||||
- agent 默认每 20 秒发送 WebSocket Ping，覆盖等待另一台机器、发压、drain 和历史核验阶段；`ws_ping_interval` 必须小于公网入口的空闲连接超时。
|
||||
- 正式开始前 agent 会持续检查实际存活连接；连接断开后自动重新登录并订阅 P2P、群 Topic。初次连接不齐或等待另一台机器期间掉线不会退出。
|
||||
- coordinator 使用两阶段屏障：两边实时连接数都达到配置值后进入验证倒计时；倒计时内任一方掉线就撤销屏障。两边再次确认满连接后才提交最终开始时间。
|
||||
- 压测机如保持 OpenVPN 等全局 TUN，仅在另一套代理中设置 `DIRECT` 无法绕过它。起压前必须确认 `http_base` 和 WS 域名的每个当前 IP 都走物理网卡；需保留 VPN 访问内网时，可为这些 IP 配置经局域网关的 `/32` 主机路由。VPN 重连、DNS 变更或机器重启后需重新核验。
|
||||
|
||||
## 统计口径
|
||||
|
||||
- 发送成功以 `{ctrl code=2xx, params.seq>0}` 为准，形成两台机器共享的 ACK 台账。
|
||||
- 实时到达只统计同时匹配本轮 `run_id` 和 `x-im-client-mid` 的 `{data}`。
|
||||
- 群消息设置 `noecho=false`，每条服务端接受的群消息期望 2000 个连接各收到一次。
|
||||
- 实时 drain 结束后通过 `{get what=data}` 按 ACK 的 Topic/Seq 回查历史，区分跨 Pod 实时漏投与数据库持久化缺失。
|
||||
- 实时 drain 结束后，两机先汇总 ACK 台账和“已实时观测但未收到 ACK”的消息台账；等待 `history_settle`（默认 5 分钟）让 Kafka 自然排空，再通过 `{get what=data}` 按 Topic/Seq 回查两类消息。
|
||||
- 每轮建立 WS 后、正式计时前，以 32 个 worker 对本机 1000 个账号做好友关系、P2P Topic 和发布权限预检；任一异常会终止本轮。
|
||||
- 历史回查默认使用 48 个 worker、总超时 3 分钟，并对缺失项再并发重试两轮。
|
||||
- publish 错误完整分类为 `403`、`502`、`503`、`ctrl_timeout`、`socket_error` 和 `other`，不受文本错误样本上限影响。
|
||||
- 历史回查默认每机最多使用 12 个 worker、每页 10 条、同一 Topic 两页之间至少间隔 500ms，并在每页请求后等待目标消息全部返回（最长 3 秒）再继续。同一 Topic 严格串行，不同 Topic 才并行；因此群聊只会有 1 个活跃回查 worker，单聊可并行核验不同 Topic。总超时默认 3 分钟，并在超时内持续重试缺失区间。该限速用于避免正确性核验本身制造 Topic/Proxy 洪峰；历史读取吞吐应使用独立场景测试。
|
||||
- agent 完成后默认在 10 秒内均匀关闭本机 WebSocket，避免 1000 个群订阅同时离线冲击 Topic 队列。两机退出时间可能相差数秒，协调器完成以两份 summary 都已提交为准。
|
||||
- publish 错误完整分类为 `403`、`502`、`503`、`ctrl_timeout`、`socket_error` 和 `other`。`publish_rejected` 只统计明确拒绝，超时和 socket 错误单独计入 `publish_outcome_unknown`。
|
||||
- `delivery_local_p95_ms` 使用发送方本机收到 echo 的同机时钟样本，是投递延迟门禁口径；兼容字段 `delivery_p95_ms` 与它保持一致。
|
||||
- `delivery_cross_machine_p95_ms` 是跨机器单向观测值，只有两机时钟同步达标时才有意义；`cross_machine_negative_latency_samples > 0` 明确表示存在时钟偏差。
|
||||
- `manifest.json` 含登录 token，文件权限为 `0600`，不得提交到 Git 或放入结果包。
|
||||
|
||||
## 两台机器初始化
|
||||
|
||||
+327
-51
@@ -24,10 +24,10 @@ type agentRun struct {
|
||||
runID string
|
||||
scenario string
|
||||
clients map[int]*WSClient
|
||||
clientsMu sync.RWMutex
|
||||
recorder *deliveryRecorder
|
||||
ack *DurationHistogram
|
||||
attempted atomic.Int64
|
||||
rejected atomic.Int64
|
||||
ready atomic.Int64
|
||||
publish403 atomic.Int64
|
||||
publish502 atomic.Int64
|
||||
@@ -132,24 +132,59 @@ func runAgent(ctx context.Context, cfg *Config, manifest *Manifest, machine, coo
|
||||
return err
|
||||
}
|
||||
defer stopMetrics()
|
||||
if err = run.connect(ctx, accounts, manifest.GroupTopic, apiKey, password); err != nil {
|
||||
run.close()
|
||||
return err
|
||||
for {
|
||||
err = run.connect(ctx, accounts, manifest.GroupTopic, apiKey, password)
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
run.close()
|
||||
return ctx.Err()
|
||||
}
|
||||
log.Printf("agent=%s initial connection set incomplete; retrying: %v", machine, err)
|
||||
if err = waitContext(ctx, 2*time.Second); err != nil {
|
||||
run.close()
|
||||
return err
|
||||
}
|
||||
}
|
||||
defer run.close()
|
||||
start, err := coord.ready(ctx, AgentReady{Machine: machine, ReadyConnections: len(run.clients)})
|
||||
maintenanceCtx, stopMaintenance := context.WithCancel(ctx)
|
||||
maintenanceDone := make(chan struct{})
|
||||
go func() {
|
||||
defer close(maintenanceDone)
|
||||
run.maintainConnections(maintenanceCtx, accounts, manifest.GroupTopic, apiKey, password)
|
||||
}()
|
||||
start, err := coord.ready(ctx, func() AgentReady {
|
||||
return AgentReady{
|
||||
Machine: machine, ReadyConnections: run.aliveConnections(), ExpectedConnections: len(accounts),
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
stopMaintenance()
|
||||
<-maintenanceDone
|
||||
return err
|
||||
}
|
||||
if !runIDPattern.MatchString(start.RunID) || !validScenario(start.Scenario) {
|
||||
stopMaintenance()
|
||||
<-maintenanceDone
|
||||
return fmt.Errorf("invalid coordinator start: %+v", start)
|
||||
}
|
||||
run.runID = start.RunID
|
||||
run.scenario = start.Scenario
|
||||
run.recorder.setRunID(start.RunID)
|
||||
if err = waitUntil(ctx, start.StartAt); err != nil {
|
||||
stopMaintenance()
|
||||
<-maintenanceDone
|
||||
return err
|
||||
}
|
||||
stopMaintenance()
|
||||
<-maintenanceDone
|
||||
if alive := run.aliveConnections(); alive != len(accounts) {
|
||||
return fmt.Errorf("connections alive at scheduled start=%d expected=%d", alive, len(accounts))
|
||||
}
|
||||
// Connection churn while waiting for the other machine is outside the load
|
||||
// measurement window. From this point onward every disconnect is a failure.
|
||||
run.recorder.resetDisconnects()
|
||||
startedAt := time.Now().UTC()
|
||||
log.Printf("agent=%s scenario=%s started", machine, start.Scenario)
|
||||
if err = run.execute(ctx, start.StartAt, manifest.GroupTopic); err != nil {
|
||||
@@ -171,11 +206,21 @@ func runAgent(ctx context.Context, cfg *Config, manifest *Manifest, machine, coo
|
||||
if err = waitContext(ctx, drain); err != nil {
|
||||
return err
|
||||
}
|
||||
observedLocal := run.recorder.unackedObservedMessages(ledger, start.Scenario)
|
||||
if err = coord.submitObserved(ctx, machine, observedLocal); err != nil {
|
||||
return err
|
||||
}
|
||||
observedLedger, err := coord.observed(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
verificationLedger := mergeMessageLedgers(ledger, observedLedger)
|
||||
base := AgentSummary{
|
||||
RunID: start.RunID, Scenario: start.Scenario, Machine: machine, StartedAt: startedAt,
|
||||
LoadEndedAt: loadEndedAt, FinishedAt: time.Now().UTC(), ExpectedConnections: cfg.UsersPerMachine,
|
||||
ReadyConnections: len(run.clients),
|
||||
PublishAttempted: run.attempted.Load(), PublishAccepted: int64(len(local)), PublishRejected: run.rejected.Load(),
|
||||
ReadyConnections: run.clientCount(),
|
||||
PublishAttempted: run.attempted.Load(), PublishAccepted: int64(len(local)),
|
||||
PublishRejected: run.explicitRejected(), PublishOutcomeUnknown: run.outcomeUnknown(),
|
||||
PublishErrors: run.publishErrorCounts(),
|
||||
AckLatency: run.ack.Snapshot(), AcceptedLocal: local,
|
||||
}
|
||||
@@ -183,11 +228,26 @@ func runAgent(ctx context.Context, cfg *Config, manifest *Manifest, machine, coo
|
||||
base.Errors = append(base.Errors, run.errors...)
|
||||
run.mu.Unlock()
|
||||
summary := run.recorder.summarize(base, ledger)
|
||||
checked, found, persistenceMissing := run.reconcileHistory(ctx, ledger)
|
||||
summary.HistoryChecked = int64(checked)
|
||||
summary.HistoryFound = int64(found)
|
||||
historySettle, _ := configDuration(cfg.HistorySettle)
|
||||
log.Printf("agent=%s waiting %s before history verification", machine, historySettle)
|
||||
if err = waitContext(ctx, historySettle); err != nil {
|
||||
return err
|
||||
}
|
||||
history := run.reconcileHistory(ctx, verificationLedger, ledger)
|
||||
if err = coord.submitHistory(ctx, machine, history.foundIDs); err != nil {
|
||||
return err
|
||||
}
|
||||
storedMessageIDs, err := coord.history(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
summary.HistoryChecked = int64(history.checked)
|
||||
summary.HistoryFound = int64(history.found)
|
||||
summary.HistoryUnackedChecked = int64(history.unackedChecked)
|
||||
summary.HistoryUnackedFound = int64(history.unackedFound)
|
||||
persistenceMissing := history.missing
|
||||
summary.PersistenceLost = int64(len(persistenceMissing))
|
||||
summary.RealtimeLostStored = run.recorder.countRealtimeLostStored(ledger)
|
||||
summary.RealtimeLostStored = run.recorder.countRealtimeLostStored(ledger, storedMessageIDs)
|
||||
if len(persistenceMissing) > 10000 {
|
||||
summary.PersistenceMissing = append([]string(nil), persistenceMissing[:10000]...)
|
||||
summary.Errors = append(summary.Errors, "persistence missing details truncated at 10000")
|
||||
@@ -208,15 +268,29 @@ func runAgent(ctx context.Context, cfg *Config, manifest *Manifest, machine, coo
|
||||
}
|
||||
|
||||
type historyQuery struct {
|
||||
client *WSClient
|
||||
topic string
|
||||
since int64
|
||||
before int64
|
||||
limit int
|
||||
client *WSClient
|
||||
topic string
|
||||
since int64
|
||||
before int64
|
||||
limit int
|
||||
expected []string
|
||||
}
|
||||
|
||||
func (r *agentRun) reconcileHistory(ctx context.Context, ledger []AcceptedMessage) (int, int, []string) {
|
||||
type historyResult struct {
|
||||
checked int
|
||||
found int
|
||||
unackedChecked int
|
||||
unackedFound int
|
||||
missing []string
|
||||
foundIDs []string
|
||||
}
|
||||
|
||||
func (r *agentRun) reconcileHistory(ctx context.Context, ledger, ackLedger []AcceptedMessage) historyResult {
|
||||
expected := make(map[string]AcceptedMessage)
|
||||
acked := make(map[string]bool, len(ackLedger))
|
||||
for _, message := range ackLedger {
|
||||
acked[message.ClientMid] = true
|
||||
}
|
||||
byTopic := make(map[string][]AcceptedMessage)
|
||||
for _, message := range ledger {
|
||||
if message.RunID != r.runID {
|
||||
@@ -231,27 +305,27 @@ func (r *agentRun) reconcileHistory(ctx context.Context, ledger []AcceptedMessag
|
||||
}
|
||||
}
|
||||
if len(expected) == 0 {
|
||||
return 0, 0, nil
|
||||
return historyResult{}
|
||||
}
|
||||
r.recorder.beginHistory()
|
||||
queries := make([]historyQuery, 0, len(byTopic))
|
||||
for topic, messages := range byTopic {
|
||||
sort.Slice(messages, func(i, j int) bool { return messages[i].Seq < messages[j].Seq })
|
||||
client := r.clients[messages[0].PeerIndex]
|
||||
client := r.client(messages[0].PeerIndex)
|
||||
if messages[0].Kind == "group" {
|
||||
client = r.clients[1]
|
||||
client = r.client(1)
|
||||
}
|
||||
if client == nil {
|
||||
r.addError(fmt.Sprintf("history topic=%s has no verifier client", topicHash(topic)))
|
||||
continue
|
||||
}
|
||||
for start := 0; start < len(messages); start += 200 {
|
||||
end := start + 200
|
||||
pageSize := r.historyPageSize()
|
||||
for start := 0; start < len(messages); start += pageSize {
|
||||
end := start + pageSize
|
||||
if end > len(messages) {
|
||||
end = len(messages)
|
||||
}
|
||||
queries = append(queries, historyQuery{client: client, topic: topic,
|
||||
since: messages[start].Seq, before: messages[end-1].Seq + 1, limit: 256})
|
||||
queries = append(queries, newHistoryQuery(client, topic, messages[start:end], pageSize))
|
||||
}
|
||||
}
|
||||
historyTimeout, _ := configDuration(r.cfg.HistoryTimeout)
|
||||
@@ -274,7 +348,22 @@ func (r *agentRun) reconcileHistory(ctx context.Context, ledger []AcceptedMessag
|
||||
missingIDs = append(missingIDs, message.ClientMid)
|
||||
}
|
||||
sort.Strings(missingIDs)
|
||||
return len(expected), len(expected) - len(missing), missingIDs
|
||||
result := historyResult{checked: len(expected), found: len(expected) - len(missing), missing: missingIDs,
|
||||
foundIDs: make([]string, 0, len(expected)-len(missing))}
|
||||
for mid := range expected {
|
||||
if r.recorder.historyFound(mid) {
|
||||
result.foundIDs = append(result.foundIDs, mid)
|
||||
}
|
||||
if acked[mid] {
|
||||
continue
|
||||
}
|
||||
result.unackedChecked++
|
||||
if r.recorder.historyFound(mid) {
|
||||
result.unackedFound++
|
||||
}
|
||||
}
|
||||
sort.Strings(result.foundIDs)
|
||||
return result
|
||||
}
|
||||
|
||||
func (r *agentRun) historyQueriesForMessages(messages []AcceptedMessage) []historyQuery {
|
||||
@@ -285,49 +374,75 @@ func (r *agentRun) historyQueriesForMessages(messages []AcceptedMessage) []histo
|
||||
queries := make([]historyQuery, 0, len(byTopic))
|
||||
for topic, topicMessages := range byTopic {
|
||||
sort.Slice(topicMessages, func(i, j int) bool { return topicMessages[i].Seq < topicMessages[j].Seq })
|
||||
client := r.clients[topicMessages[0].PeerIndex]
|
||||
client := r.client(topicMessages[0].PeerIndex)
|
||||
if topicMessages[0].Kind == "group" {
|
||||
client = r.clients[1]
|
||||
client = r.client(1)
|
||||
}
|
||||
if client == nil {
|
||||
continue
|
||||
}
|
||||
for start := 0; start < len(topicMessages); start += 200 {
|
||||
end := start + 200
|
||||
pageSize := r.historyPageSize()
|
||||
for start := 0; start < len(topicMessages); start += pageSize {
|
||||
end := start + pageSize
|
||||
if end > len(topicMessages) {
|
||||
end = len(topicMessages)
|
||||
}
|
||||
queries = append(queries, historyQuery{client: client, topic: topic,
|
||||
since: topicMessages[start].Seq, before: topicMessages[end-1].Seq + 1, limit: 256})
|
||||
queries = append(queries, newHistoryQuery(client, topic, topicMessages[start:end], pageSize))
|
||||
}
|
||||
}
|
||||
return queries
|
||||
}
|
||||
|
||||
func (r *agentRun) issueHistoryQueries(ctx context.Context, queries []historyQuery) {
|
||||
byTopic := make(map[string][]historyQuery)
|
||||
for _, query := range queries {
|
||||
byTopic[query.topic] = append(byTopic[query.topic], query)
|
||||
}
|
||||
workers := r.cfg.HistoryWorkers
|
||||
if workers > len(queries) {
|
||||
workers = len(queries)
|
||||
if workers > len(byTopic) {
|
||||
workers = len(byTopic)
|
||||
}
|
||||
if workers == 0 {
|
||||
return
|
||||
}
|
||||
jobs := make(chan historyQuery)
|
||||
jobs := make(chan []historyQuery)
|
||||
interval := 500 * time.Millisecond
|
||||
if r.cfg != nil && r.cfg.HistoryInterval != "" {
|
||||
interval = mustDuration(r.cfg.HistoryInterval)
|
||||
}
|
||||
pageWait := 3 * time.Second
|
||||
if r.cfg != nil && r.cfg.HistoryPageWait != "" {
|
||||
pageWait = mustDuration(r.cfg.HistoryPageWait)
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < workers; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for query := range jobs {
|
||||
if err := query.client.GetData(ctx, query.topic, query.since, query.before, query.limit); err != nil && ctx.Err() == nil {
|
||||
r.addError(fmt.Sprintf("history get topic=%s range=%d-%d: %v", topicHash(query.topic), query.since, query.before, err))
|
||||
for topicQueries := range jobs {
|
||||
for index, query := range topicQueries {
|
||||
if index > 0 {
|
||||
if err := waitContext(ctx, interval); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
if err := query.client.GetData(ctx, query.topic, query.since, query.before, query.limit); err != nil && ctx.Err() == nil {
|
||||
r.addError(fmt.Sprintf("history get topic=%s range=%d-%d: %v", topicHash(query.topic), query.since, query.before, err))
|
||||
continue
|
||||
}
|
||||
r.waitHistoryPage(ctx, query.expected, pageWait)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
for _, query := range queries {
|
||||
topics := make([]string, 0, len(byTopic))
|
||||
for topic := range byTopic {
|
||||
topics = append(topics, topic)
|
||||
}
|
||||
sort.Strings(topics)
|
||||
for _, topic := range topics {
|
||||
select {
|
||||
case jobs <- query:
|
||||
case jobs <- byTopic[topic]:
|
||||
case <-ctx.Done():
|
||||
close(jobs)
|
||||
wg.Wait()
|
||||
@@ -338,6 +453,41 @@ func (r *agentRun) issueHistoryQueries(ctx context.Context, queries []historyQue
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func newHistoryQuery(client *WSClient, topic string, messages []AcceptedMessage, limit int) historyQuery {
|
||||
expected := make([]string, 0, len(messages))
|
||||
for _, message := range messages {
|
||||
expected = append(expected, message.ClientMid)
|
||||
}
|
||||
return historyQuery{client: client, topic: topic, since: messages[0].Seq,
|
||||
before: messages[len(messages)-1].Seq + 1, limit: limit, expected: expected}
|
||||
}
|
||||
|
||||
func (r *agentRun) waitHistoryPage(ctx context.Context, expected []string, timeout time.Duration) {
|
||||
deadline := time.NewTimer(timeout)
|
||||
defer deadline.Stop()
|
||||
ticker := time.NewTicker(25 * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
if r.recorder.historyFoundAll(expected) {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-deadline.C:
|
||||
return
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (r *agentRun) historyPageSize() int {
|
||||
if r.cfg != nil && r.cfg.HistoryPageSize > 0 {
|
||||
return r.cfg.HistoryPageSize
|
||||
}
|
||||
return 10
|
||||
}
|
||||
|
||||
func missingHistory(recorder *deliveryRecorder, expected map[string]AcceptedMessage) []AcceptedMessage {
|
||||
missing := make([]AcceptedMessage, 0)
|
||||
for mid, message := range expected {
|
||||
@@ -348,6 +498,33 @@ func missingHistory(recorder *deliveryRecorder, expected map[string]AcceptedMess
|
||||
return missing
|
||||
}
|
||||
|
||||
func mergeMessageLedgers(accepted, observed []AcceptedMessage) []AcceptedMessage {
|
||||
byID := make(map[string]AcceptedMessage, len(accepted)+len(observed))
|
||||
for _, message := range observed {
|
||||
if message.ClientMid == "" {
|
||||
continue
|
||||
}
|
||||
current, exists := byID[message.ClientMid]
|
||||
currentIsPeer := current.PeerMachine != "" && current.PeerMachine != current.SenderMachine
|
||||
candidateIsPeer := message.PeerMachine != "" && message.PeerMachine != message.SenderMachine
|
||||
if !exists || (!currentIsPeer && candidateIsPeer) {
|
||||
byID[message.ClientMid] = message
|
||||
}
|
||||
}
|
||||
// ACK-derived records carry authoritative sender and timing fields.
|
||||
for _, message := range accepted {
|
||||
if message.ClientMid != "" {
|
||||
byID[message.ClientMid] = message
|
||||
}
|
||||
}
|
||||
merged := make([]AcceptedMessage, 0, len(byID))
|
||||
for _, message := range byID {
|
||||
merged = append(merged, message)
|
||||
}
|
||||
sort.Slice(merged, func(i, j int) bool { return merged[i].ClientMid < merged[j].ClientMid })
|
||||
return merged
|
||||
}
|
||||
|
||||
func (r *agentRun) addError(message string) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
@@ -364,9 +541,14 @@ func (r *agentRun) connect(ctx context.Context, accounts []AccountRecord, groupT
|
||||
interval := ramp / time.Duration(len(accounts))
|
||||
var wg sync.WaitGroup
|
||||
errCh := make(chan error, len(accounts))
|
||||
var mu sync.Mutex
|
||||
for _, account := range accounts {
|
||||
account := account
|
||||
if client := r.client(account.Index); client != nil {
|
||||
if client.alive() {
|
||||
continue
|
||||
}
|
||||
client.Close()
|
||||
}
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
@@ -375,9 +557,7 @@ func (r *agentRun) connect(ctx context.Context, accounts []AccountRecord, groupT
|
||||
errCh <- connectErr
|
||||
return
|
||||
}
|
||||
mu.Lock()
|
||||
r.clients[account.Index] = client
|
||||
mu.Unlock()
|
||||
r.setClient(account.Index, client)
|
||||
r.ready.Add(1)
|
||||
}()
|
||||
if interval > 0 {
|
||||
@@ -394,12 +574,55 @@ func (r *agentRun) connect(ctx context.Context, accounts []AccountRecord, groupT
|
||||
failures = append(failures, connectErr.Error())
|
||||
}
|
||||
}
|
||||
if len(r.clients) != len(accounts) {
|
||||
return fmt.Errorf("ready connections=%d expected=%d failures=%v", len(r.clients), len(accounts), failures)
|
||||
if r.aliveConnections() != len(accounts) {
|
||||
return fmt.Errorf("ready connections=%d expected=%d failures=%v", r.aliveConnections(), len(accounts), failures)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *agentRun) maintainConnections(ctx context.Context, accounts []AccountRecord, groupTopic, apiKey, password string) {
|
||||
byIndex := make(map[int]AccountRecord, len(accounts))
|
||||
for _, account := range accounts {
|
||||
byIndex[account.Index] = account
|
||||
}
|
||||
ticker := time.NewTicker(time.Second)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
for index, account := range byIndex {
|
||||
current := r.client(index)
|
||||
if current != nil && current.alive() {
|
||||
continue
|
||||
}
|
||||
wg.Add(1)
|
||||
go func(index int, account AccountRecord, previous *WSClient) {
|
||||
defer wg.Done()
|
||||
if previous != nil {
|
||||
previous.Close()
|
||||
}
|
||||
client, err := r.connectAccountWithRetry(ctx, account, groupTopic, apiKey, password)
|
||||
if err != nil {
|
||||
if ctx.Err() == nil {
|
||||
log.Printf("account=%s pre-start reconnect failed: %v", account.Username, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
r.setClient(index, client)
|
||||
log.Printf("account=%s pre-start connection restored", account.Username)
|
||||
}(index, account, current)
|
||||
}
|
||||
wg.Wait()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (r *agentRun) connectAccountWithRetry(ctx context.Context, account AccountRecord, groupTopic, apiKey, password string) (*WSClient, error) {
|
||||
var lastErr error
|
||||
for attempt := 1; attempt <= 6; attempt++ {
|
||||
@@ -459,7 +682,7 @@ func (r *agentRun) execute(ctx context.Context, start time.Time, groupTopic stri
|
||||
func (r *agentRun) scheduleP2P(ctx context.Context, start time.Time, users, messages int, interval time.Duration) error {
|
||||
var wg sync.WaitGroup
|
||||
for index := 1; index <= users; index++ {
|
||||
client := r.clients[index]
|
||||
client := r.client(index)
|
||||
if client == nil {
|
||||
return fmt.Errorf("missing client %d", index)
|
||||
}
|
||||
@@ -491,7 +714,7 @@ func (r *agentRun) scheduleP2P(ctx context.Context, start time.Time, users, mess
|
||||
}
|
||||
|
||||
func (r *agentRun) scheduleGroup(ctx context.Context, start time.Time, topic string, first, last, rounds int) error {
|
||||
if first < 1 || last > len(r.clients) || first > last {
|
||||
if first < 1 || last > r.clientCount() || first > last {
|
||||
return fmt.Errorf("invalid group publisher range %d-%d", first, last)
|
||||
}
|
||||
count := last - first + 1
|
||||
@@ -502,7 +725,7 @@ func (r *agentRun) scheduleGroup(ctx context.Context, start time.Time, topic str
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
for index := first; index <= last; index++ {
|
||||
client := r.clients[index]
|
||||
client := r.client(index)
|
||||
position := index - first
|
||||
wg.Add(1)
|
||||
go func(index, position int, client *WSClient) {
|
||||
@@ -528,7 +751,6 @@ func (r *agentRun) publish(ctx context.Context, client *WSClient, topic, mid str
|
||||
r.attempted.Add(1)
|
||||
accepted, err := client.Publish(ctx, topic, mid, r.runID, r.scenario, messageIndex)
|
||||
if err != nil {
|
||||
r.rejected.Add(1)
|
||||
r.classifyPublishError(err)
|
||||
r.mu.Lock()
|
||||
if len(r.errors) < 1000 {
|
||||
@@ -553,7 +775,7 @@ func (r *agentRun) classifyPublishError(err error) {
|
||||
case strings.Contains(err.Error(), "ctrl timeout") || errors.Is(err, context.DeadlineExceeded):
|
||||
r.publishTimeout.Add(1)
|
||||
case strings.Contains(err.Error(), "websocket") || strings.Contains(err.Error(), "broken pipe") ||
|
||||
strings.Contains(err.Error(), "connection reset"):
|
||||
strings.Contains(err.Error(), "connection reset") || strings.Contains(err.Error(), "closed network connection"):
|
||||
r.publishSocket.Add(1)
|
||||
default:
|
||||
r.publishOther.Add(1)
|
||||
@@ -567,6 +789,14 @@ func (r *agentRun) publishErrorCounts() map[string]int64 {
|
||||
}
|
||||
}
|
||||
|
||||
func (r *agentRun) explicitRejected() int64 {
|
||||
return r.publish403.Load() + r.publish502.Load() + r.publish503.Load() + r.publishOther.Load()
|
||||
}
|
||||
|
||||
func (r *agentRun) outcomeUnknown() int64 {
|
||||
return r.publishTimeout.Load() + r.publishSocket.Load()
|
||||
}
|
||||
|
||||
func (r *agentRun) addAccepted(message AcceptedMessage) {
|
||||
r.mu.Lock()
|
||||
r.accepted = append(r.accepted, message)
|
||||
@@ -581,6 +811,44 @@ func (r *agentRun) acceptedCopy() []AcceptedMessage {
|
||||
return out
|
||||
}
|
||||
|
||||
func (r *agentRun) aliveConnections() int {
|
||||
alive := 0
|
||||
for _, client := range r.clientsSnapshot() {
|
||||
if client.alive() {
|
||||
alive++
|
||||
}
|
||||
}
|
||||
return alive
|
||||
}
|
||||
|
||||
func (r *agentRun) client(index int) *WSClient {
|
||||
r.clientsMu.RLock()
|
||||
defer r.clientsMu.RUnlock()
|
||||
return r.clients[index]
|
||||
}
|
||||
|
||||
func (r *agentRun) setClient(index int, client *WSClient) {
|
||||
r.clientsMu.Lock()
|
||||
r.clients[index] = client
|
||||
r.clientsMu.Unlock()
|
||||
}
|
||||
|
||||
func (r *agentRun) clientCount() int {
|
||||
r.clientsMu.RLock()
|
||||
defer r.clientsMu.RUnlock()
|
||||
return len(r.clients)
|
||||
}
|
||||
|
||||
func (r *agentRun) clientsSnapshot() []*WSClient {
|
||||
r.clientsMu.RLock()
|
||||
defer r.clientsMu.RUnlock()
|
||||
clients := make([]*WSClient, 0, len(r.clients))
|
||||
for _, client := range r.clients {
|
||||
clients = append(clients, client)
|
||||
}
|
||||
return clients
|
||||
}
|
||||
|
||||
func (r *agentRun) drainDuration() (time.Duration, error) {
|
||||
if r.scenario == "p2p" {
|
||||
return configDuration(r.cfg.DrainP2P)
|
||||
@@ -589,8 +857,16 @@ func (r *agentRun) drainDuration() (time.Duration, error) {
|
||||
}
|
||||
|
||||
func (r *agentRun) close() {
|
||||
for _, client := range r.clients {
|
||||
clients := r.clientsSnapshot()
|
||||
var interval time.Duration
|
||||
if r.cfg != nil && r.cfg.CloseRamp != "" && len(clients) > 1 {
|
||||
interval = mustDuration(r.cfg.CloseRamp) / time.Duration(len(clients)-1)
|
||||
}
|
||||
for index, client := range clients {
|
||||
client.Close()
|
||||
if interval > 0 && index < len(clients)-1 {
|
||||
time.Sleep(interval)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,11 +1,14 @@
|
||||
package main
|
||||
|
||||
import "testing"
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestHistoryQueriesForMessagesGroupsRangesByTopic(t *testing.T) {
|
||||
p2pClient := &WSClient{}
|
||||
groupClient := &WSClient{}
|
||||
run := &agentRun{clients: map[int]*WSClient{1: groupClient, 7: p2pClient}}
|
||||
run := &agentRun{cfg: &Config{HistoryPageSize: 200}, clients: map[int]*WSClient{1: groupClient, 7: p2pClient}}
|
||||
messages := make([]AcceptedMessage, 0, 207)
|
||||
for seq := int64(205); seq >= 1; seq-- {
|
||||
messages = append(messages, AcceptedMessage{Kind: "p2p", Topic: "p2p-a", PeerIndex: 7, Seq: seq})
|
||||
@@ -25,7 +28,7 @@ func TestHistoryQueriesForMessagesGroupsRangesByTopic(t *testing.T) {
|
||||
switch query.topic {
|
||||
case "p2p-a":
|
||||
p2pQueries++
|
||||
if query.client != p2pClient || query.limit != 256 {
|
||||
if query.client != p2pClient || query.limit != 200 || len(query.expected) == 0 {
|
||||
t.Fatalf("unexpected p2p query: %#v", query)
|
||||
}
|
||||
case "grp-a":
|
||||
@@ -39,3 +42,26 @@ func TestHistoryQueriesForMessagesGroupsRangesByTopic(t *testing.T) {
|
||||
t.Fatalf("p2p queries=%d group queries=%d", p2pQueries, groupQueries)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifyPublishErrorTreatsClosedConnectionAsUnknownSocketError(t *testing.T) {
|
||||
run := &agentRun{}
|
||||
run.classifyPublishError(errors.New("write tcp: use of closed network connection"))
|
||||
if run.publishSocket.Load() != 1 || run.publishOther.Load() != 0 {
|
||||
t.Fatalf("socket=%d other=%d", run.publishSocket.Load(), run.publishOther.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeMessageLedgersPrefersPeerObservationAndAck(t *testing.T) {
|
||||
senderEcho := AcceptedMessage{ClientMid: "mid-1", SenderMachine: "a", PeerMachine: "a", Seq: 10}
|
||||
peerDelivery := AcceptedMessage{ClientMid: "mid-1", SenderMachine: "a", PeerMachine: "b", PeerIndex: 7, Seq: 10}
|
||||
merged := mergeMessageLedgers(nil, []AcceptedMessage{senderEcho, peerDelivery})
|
||||
if len(merged) != 1 || merged[0].PeerMachine != "b" || merged[0].PeerIndex != 7 {
|
||||
t.Fatalf("peer observation was not preferred: %+v", merged)
|
||||
}
|
||||
ack := AcceptedMessage{ClientMid: "mid-1", SenderMachine: "a", PeerMachine: "b", PeerIndex: 7, Seq: 10,
|
||||
Scenario: "p2p"}
|
||||
merged = mergeMessageLedgers([]AcceptedMessage{ack}, []AcceptedMessage{peerDelivery})
|
||||
if len(merged) != 1 || merged[0].Scenario != "p2p" {
|
||||
t.Fatalf("ACK ledger was not authoritative: %+v", merged)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,7 +26,7 @@ func startAgentMetrics(address string, run *agentRun) (func(), error) {
|
||||
collectors.NewGoCollector(),
|
||||
collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),
|
||||
prometheus.NewGaugeFunc(prometheus.GaugeOpts{Name: "im_load_agent_ready_connections", Help: "Current ready WebSocket connections.", ConstLabels: labels}, func() float64 {
|
||||
return float64(run.ready.Load())
|
||||
return float64(run.aliveConnections())
|
||||
}),
|
||||
prometheus.NewCounterFunc(prometheus.CounterOpts{Name: "im_load_agent_publish_attempted_total", Help: "Publish commands attempted.", ConstLabels: labels}, func() float64 {
|
||||
return float64(run.attempted.Load())
|
||||
@@ -36,8 +36,11 @@ func startAgentMetrics(address string, run *agentRun) (func(), error) {
|
||||
defer run.mu.Unlock()
|
||||
return float64(len(run.accepted))
|
||||
}),
|
||||
prometheus.NewCounterFunc(prometheus.CounterOpts{Name: "im_load_agent_publish_rejected_total", Help: "Publish commands rejected or timed out.", ConstLabels: labels}, func() float64 {
|
||||
return float64(run.rejected.Load())
|
||||
prometheus.NewCounterFunc(prometheus.CounterOpts{Name: "im_load_agent_publish_rejected_total", Help: "Publish commands explicitly rejected by the server.", ConstLabels: labels}, func() float64 {
|
||||
return float64(run.explicitRejected())
|
||||
}),
|
||||
prometheus.NewCounterFunc(prometheus.CounterOpts{Name: "im_load_agent_publish_outcome_unknown_total", Help: "Publish commands with ACK timeout or socket failure and therefore unknown outcome.", ConstLabels: labels}, func() float64 {
|
||||
return float64(run.outcomeUnknown())
|
||||
}),
|
||||
prometheus.NewCounterFunc(prometheus.CounterOpts{Name: "im_load_agent_publish_403_total", Help: "Publish commands rejected with 403.", ConstLabels: labels}, func() float64 { return float64(run.publish403.Load()) }),
|
||||
prometheus.NewCounterFunc(prometheus.CounterOpts{Name: "im_load_agent_publish_502_total", Help: "Publish commands rejected with 502.", ConstLabels: labels}, func() float64 { return float64(run.publish502.Load()) }),
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/websocket"
|
||||
)
|
||||
|
||||
func TestMaintainConnectionsRestoresClosedClient(t *testing.T) {
|
||||
var connections atomic.Int64
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
conn, err := (&websocket.Upgrader{}).Upgrade(w, r, nil)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
connections.Add(1)
|
||||
defer conn.Close()
|
||||
for {
|
||||
var packet map[string]json.RawMessage
|
||||
if err = conn.ReadJSON(&packet); err != nil {
|
||||
return
|
||||
}
|
||||
for _, raw := range packet {
|
||||
var body struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
if json.Unmarshal(raw, &body) != nil || body.ID == "" {
|
||||
continue
|
||||
}
|
||||
response := map[string]any{"ctrl": map[string]any{
|
||||
"id": body.ID, "code": 200, "text": "ok",
|
||||
"params": map[string]any{"user": "usrReconnectTest"},
|
||||
}}
|
||||
if err = conn.WriteJSON(response); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
account := AccountRecord{
|
||||
Machine: "a", Index: 1, Username: "lta0001", Token: "token",
|
||||
P2PTopic: "p2pReconnectTest", Endpoint: "ws" + strings.TrimPrefix(server.URL, "http"),
|
||||
}
|
||||
cfg := &Config{Tenant: "allrich", CommandTimeout: "1s", WSPingInterval: "1h"}
|
||||
run := &agentRun{
|
||||
cfg: cfg, machine: "a", clients: make(map[int]*WSClient),
|
||||
recorder: newDeliveryRecorder("pending", "a", 1), ack: NewDurationHistogram(),
|
||||
}
|
||||
initial, err := run.connectAccountWithRetry(context.Background(), account, "grpReconnectTest", "api-key", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
run.setClient(account.Index, initial)
|
||||
initial.Close()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
run.maintainConnections(ctx, []AccountRecord{account}, "grpReconnectTest", "api-key", "")
|
||||
close(done)
|
||||
}()
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if replacement := run.client(account.Index); replacement != initial && replacement.alive() {
|
||||
cancel()
|
||||
<-done
|
||||
run.close()
|
||||
if got := connections.Load(); got < 2 {
|
||||
t.Fatalf("websocket connections = %d, want at least 2", got)
|
||||
}
|
||||
return
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
cancel()
|
||||
<-done
|
||||
run.close()
|
||||
t.Fatal("closed client was not restored before deadline")
|
||||
}
|
||||
@@ -14,10 +14,16 @@
|
||||
"prepare_workers": 20,
|
||||
"connect_ramp": "20s",
|
||||
"command_timeout": "20s",
|
||||
"ws_ping_interval": "20s",
|
||||
"preflight_workers": 32,
|
||||
"skip_p2p_preflight": false,
|
||||
"history_workers": 48,
|
||||
"history_workers": 12,
|
||||
"history_page_size": 10,
|
||||
"history_query_interval": "500ms",
|
||||
"history_page_wait": "3s",
|
||||
"history_settle": "5m",
|
||||
"history_timeout": "3m",
|
||||
"close_ramp": "10s",
|
||||
"p2p_messages_per_user": 10,
|
||||
"p2p_interval": "1s",
|
||||
"drain_p2p": "30s",
|
||||
|
||||
@@ -16,23 +16,30 @@ import (
|
||||
)
|
||||
|
||||
type StartInfo struct {
|
||||
RunID string `json:"run_id"`
|
||||
Scenario string `json:"scenario"`
|
||||
StartAt time.Time `json:"start_at"`
|
||||
RunID string `json:"run_id"`
|
||||
Scenario string `json:"scenario"`
|
||||
StartAt time.Time `json:"start_at"`
|
||||
Committed bool `json:"committed"`
|
||||
}
|
||||
|
||||
type AgentReady struct {
|
||||
Machine string `json:"machine"`
|
||||
ReadyConnections int `json:"ready_connections"`
|
||||
Machine string `json:"machine"`
|
||||
ReadyConnections int `json:"ready_connections"`
|
||||
ExpectedConnections int `json:"expected_connections"`
|
||||
Armed bool `json:"armed"`
|
||||
}
|
||||
|
||||
type coordinatorState struct {
|
||||
mu sync.Mutex
|
||||
start StartInfo
|
||||
startDelay time.Duration
|
||||
commitDelay time.Duration
|
||||
expectedAgents int
|
||||
ready map[string]AgentReady
|
||||
armed map[string]bool
|
||||
ledgers map[string][]AcceptedMessage
|
||||
observed map[string][]AcceptedMessage
|
||||
history map[string][]string
|
||||
summaries map[string]AgentSummary
|
||||
complete chan struct{}
|
||||
completeOnce sync.Once
|
||||
@@ -49,14 +56,18 @@ func runCoordinator(ctx context.Context, listen, runID, scenario, output string,
|
||||
expectedAgents = 2
|
||||
}
|
||||
state := &coordinatorState{
|
||||
start: StartInfo{RunID: runID, Scenario: scenario}, startDelay: startDelay, expectedAgents: expectedAgents,
|
||||
ready: make(map[string]AgentReady), ledgers: make(map[string][]AcceptedMessage),
|
||||
summaries: make(map[string]AgentSummary), complete: make(chan struct{}),
|
||||
start: StartInfo{RunID: runID, Scenario: scenario}, startDelay: startDelay, commitDelay: 5 * time.Second,
|
||||
expectedAgents: expectedAgents,
|
||||
ready: make(map[string]AgentReady), ledgers: make(map[string][]AcceptedMessage),
|
||||
observed: make(map[string][]AcceptedMessage), history: make(map[string][]string),
|
||||
armed: make(map[string]bool), summaries: make(map[string]AgentSummary), complete: make(chan struct{}),
|
||||
}
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/v1/ready", state.handleReady)
|
||||
mux.HandleFunc("/v1/start", state.handleStart)
|
||||
mux.HandleFunc("/v1/ledger", state.handleLedger)
|
||||
mux.HandleFunc("/v1/observed", state.handleObserved)
|
||||
mux.HandleFunc("/v1/history", state.handleHistory)
|
||||
mux.HandleFunc("/v1/summary", state.handleSummary)
|
||||
mux.HandleFunc("/v1/status", state.handleStatus)
|
||||
server := &http.Server{Addr: listen, Handler: mux, ReadHeaderTimeout: 5 * time.Second}
|
||||
@@ -85,23 +96,138 @@ func runCoordinator(ctx context.Context, listen, runID, scenario, output string,
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *coordinatorState) handleHistory(w http.ResponseWriter, r *http.Request) {
|
||||
machine := strings.ToLower(strings.TrimSpace(r.URL.Query().Get("machine")))
|
||||
switch r.Method {
|
||||
case http.MethodPost:
|
||||
if machine != "a" && machine != "b" {
|
||||
http.Error(w, "invalid machine", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
var messageIDs []string
|
||||
if !decodeLimitedJSON(w, r, &messageIDs) {
|
||||
return
|
||||
}
|
||||
s.mu.Lock()
|
||||
if s.history == nil {
|
||||
s.history = make(map[string][]string)
|
||||
}
|
||||
s.history[machine] = messageIDs
|
||||
count := len(s.history)
|
||||
s.mu.Unlock()
|
||||
writeJSON(w, http.StatusOK, map[string]int{"submitted_agents": count})
|
||||
case http.MethodGet:
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if len(s.history) < s.expectedAgents {
|
||||
writeJSON(w, http.StatusAccepted, map[string]int{"submitted_agents": len(s.history), "expected_agents": s.expectedAgents})
|
||||
return
|
||||
}
|
||||
unique := make(map[string]bool)
|
||||
for _, messageIDs := range s.history {
|
||||
for _, mid := range messageIDs {
|
||||
if mid != "" {
|
||||
unique[mid] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
merged := make([]string, 0, len(unique))
|
||||
for mid := range unique {
|
||||
merged = append(merged, mid)
|
||||
}
|
||||
sort.Strings(merged)
|
||||
writeJSON(w, http.StatusOK, merged)
|
||||
default:
|
||||
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *coordinatorState) handleObserved(w http.ResponseWriter, r *http.Request) {
|
||||
machine := strings.ToLower(strings.TrimSpace(r.URL.Query().Get("machine")))
|
||||
switch r.Method {
|
||||
case http.MethodPost:
|
||||
if machine != "a" && machine != "b" {
|
||||
http.Error(w, "invalid machine", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
var messages []AcceptedMessage
|
||||
if !decodeLimitedJSON(w, r, &messages) {
|
||||
return
|
||||
}
|
||||
s.mu.Lock()
|
||||
if s.observed == nil {
|
||||
s.observed = make(map[string][]AcceptedMessage)
|
||||
}
|
||||
s.observed[machine] = messages
|
||||
count := len(s.observed)
|
||||
s.mu.Unlock()
|
||||
writeJSON(w, http.StatusOK, map[string]int{"submitted_agents": count})
|
||||
case http.MethodGet:
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if len(s.observed) < s.expectedAgents {
|
||||
writeJSON(w, http.StatusAccepted, map[string]int{"submitted_agents": len(s.observed), "expected_agents": s.expectedAgents})
|
||||
return
|
||||
}
|
||||
var merged []AcceptedMessage
|
||||
for _, messages := range s.observed {
|
||||
merged = append(merged, messages...)
|
||||
}
|
||||
merged = mergeMessageLedgers(nil, merged)
|
||||
writeJSON(w, http.StatusOK, merged)
|
||||
default:
|
||||
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *coordinatorState) handleReady(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPost {
|
||||
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
var ready AgentReady
|
||||
if !decodeLimitedJSON(w, r, &ready) || (ready.Machine != "a" && ready.Machine != "b") || ready.ReadyConnections <= 0 {
|
||||
if !decodeLimitedJSON(w, r, &ready) || (ready.Machine != "a" && ready.Machine != "b") ||
|
||||
ready.ExpectedConnections <= 0 || ready.ReadyConnections < 0 || ready.ReadyConnections > ready.ExpectedConnections {
|
||||
if ready.Machine != "a" && ready.Machine != "b" {
|
||||
http.Error(w, "invalid machine", http.StatusBadRequest)
|
||||
} else {
|
||||
http.Error(w, "invalid connection counts", http.StatusBadRequest)
|
||||
}
|
||||
return
|
||||
}
|
||||
s.mu.Lock()
|
||||
s.ready[ready.Machine] = ready
|
||||
if len(s.ready) == s.expectedAgents && s.start.StartAt.IsZero() {
|
||||
s.start.StartAt = time.Now().UTC().Add(s.startDelay)
|
||||
log.Printf("all agents ready; start_at=%s", s.start.StartAt.Format(time.RFC3339Nano))
|
||||
now := time.Now().UTC()
|
||||
allStable := len(s.ready) == s.expectedAgents
|
||||
for _, agent := range s.ready {
|
||||
if agent.ReadyConnections != agent.ExpectedConnections {
|
||||
allStable = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if !allStable && (!s.start.Committed || now.Before(s.start.StartAt)) {
|
||||
log.Printf("agent readiness dropped before start; resetting start barrier")
|
||||
s.start.StartAt = time.Time{}
|
||||
s.start.Committed = false
|
||||
clear(s.armed)
|
||||
}
|
||||
if allStable && s.start.StartAt.IsZero() {
|
||||
s.start.StartAt = now.Add(s.startDelay)
|
||||
s.start.Committed = false
|
||||
clear(s.armed)
|
||||
log.Printf("all agents ready; validation_at=%s", s.start.StartAt.Format(time.RFC3339Nano))
|
||||
}
|
||||
if allStable && !s.start.Committed && ready.Armed && !now.Before(s.start.StartAt) {
|
||||
s.armed[ready.Machine] = true
|
||||
}
|
||||
if allStable && !s.start.Committed && len(s.armed) == s.expectedAgents {
|
||||
delay := s.commitDelay
|
||||
if delay <= 0 {
|
||||
delay = 5 * time.Second
|
||||
}
|
||||
s.start.StartAt = now.Add(delay)
|
||||
s.start.Committed = true
|
||||
log.Printf("all agents validated; committed start_at=%s", s.start.StartAt.Format(time.RFC3339Nano))
|
||||
}
|
||||
start := s.start
|
||||
s.mu.Unlock()
|
||||
@@ -185,7 +311,9 @@ func (s *coordinatorState) handleStatus(w http.ResponseWriter, _ *http.Request)
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
writeJSON(w, http.StatusOK, map[string]any{
|
||||
"start": s.start, "ready": s.ready, "ledger_agents": len(s.ledgers), "summary_agents": len(s.summaries),
|
||||
"start": s.start, "ready": s.ready, "armed": s.armed,
|
||||
"ledger_agents": len(s.ledgers), "observed_agents": len(s.observed),
|
||||
"history_agents": len(s.history), "summary_agents": len(s.summaries),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -195,8 +323,13 @@ func (s *coordinatorState) writeOutput(path string) error {
|
||||
Start StartInfo `json:"start"`
|
||||
Ready map[string]AgentReady `json:"ready"`
|
||||
Ledgers map[string][]AcceptedMessage `json:"ledgers"`
|
||||
Observed map[string][]AcceptedMessage `json:"observed"`
|
||||
History map[string][]string `json:"history"`
|
||||
Summaries map[string]AgentSummary `json:"summaries"`
|
||||
}{s.start, s.ready, s.ledgers, s.summaries}
|
||||
}{
|
||||
Start: s.start, Ready: s.ready, Ledgers: s.ledgers, Observed: s.observed,
|
||||
History: s.history, Summaries: s.summaries,
|
||||
}
|
||||
s.mu.Unlock()
|
||||
raw, err := json.MarshalIndent(data, "", " ")
|
||||
if err != nil {
|
||||
@@ -232,24 +365,37 @@ func newCoordinatorClient(base string) *coordinatorClient {
|
||||
return &coordinatorClient{base: strings.TrimRight(base, "/"), client: &http.Client{Timeout: 35 * time.Second}}
|
||||
}
|
||||
|
||||
func (c *coordinatorClient) ready(ctx context.Context, ready AgentReady) (StartInfo, error) {
|
||||
func (c *coordinatorClient) ready(ctx context.Context, current func() AgentReady) (StartInfo, error) {
|
||||
var start StartInfo
|
||||
if _, err := c.request(ctx, http.MethodPost, "/v1/ready", ready, &start); err != nil {
|
||||
return start, err
|
||||
}
|
||||
for start.StartAt.IsZero() {
|
||||
armed := false
|
||||
for {
|
||||
state := current()
|
||||
state.Armed = armed
|
||||
if _, err := c.request(ctx, http.MethodPost, "/v1/ready", state, &start); err != nil {
|
||||
return start, err
|
||||
}
|
||||
if start.StartAt.IsZero() {
|
||||
armed = false
|
||||
} else {
|
||||
remaining := time.Until(start.StartAt)
|
||||
if start.Committed && remaining <= 0 {
|
||||
return start, nil
|
||||
}
|
||||
if !start.Committed && remaining <= 0 {
|
||||
armed = true
|
||||
continue
|
||||
}
|
||||
if remaining < time.Second {
|
||||
if err := waitContext(ctx, remaining); err != nil {
|
||||
return start, err
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
if err := waitContext(ctx, time.Second); err != nil {
|
||||
return start, err
|
||||
}
|
||||
status, err := c.request(ctx, http.MethodGet, "/v1/start", nil, &start)
|
||||
if err != nil {
|
||||
return start, err
|
||||
}
|
||||
if status != http.StatusOK {
|
||||
start = StartInfo{}
|
||||
}
|
||||
}
|
||||
return start, nil
|
||||
}
|
||||
|
||||
func (c *coordinatorClient) submitLedger(ctx context.Context, machine string, messages []AcceptedMessage) error {
|
||||
@@ -273,6 +419,48 @@ func (c *coordinatorClient) ledger(ctx context.Context) ([]AcceptedMessage, erro
|
||||
}
|
||||
}
|
||||
|
||||
func (c *coordinatorClient) submitObserved(ctx context.Context, machine string, messages []AcceptedMessage) error {
|
||||
_, err := c.request(ctx, http.MethodPost, "/v1/observed?machine="+machine, messages, nil)
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *coordinatorClient) observed(ctx context.Context) ([]AcceptedMessage, error) {
|
||||
for {
|
||||
var messages []AcceptedMessage
|
||||
status, err := c.request(ctx, http.MethodGet, "/v1/observed", nil, &messages)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if status == http.StatusOK {
|
||||
return messages, nil
|
||||
}
|
||||
if err = waitContext(ctx, time.Second); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *coordinatorClient) submitHistory(ctx context.Context, machine string, messageIDs []string) error {
|
||||
_, err := c.request(ctx, http.MethodPost, "/v1/history?machine="+machine, messageIDs, nil)
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *coordinatorClient) history(ctx context.Context) ([]string, error) {
|
||||
for {
|
||||
var messageIDs []string
|
||||
status, err := c.request(ctx, http.MethodGet, "/v1/history", nil, &messageIDs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if status == http.StatusOK {
|
||||
return messageIDs, nil
|
||||
}
|
||||
if err = waitContext(ctx, time.Second); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *coordinatorClient) submitSummary(ctx context.Context, summary AgentSummary) error {
|
||||
_, err := c.request(ctx, http.MethodPost, "/v1/summary", summary, nil)
|
||||
return err
|
||||
|
||||
@@ -1,18 +1,57 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestCoordinatorResetsBarrierWhenConnectionCountDrops(t *testing.T) {
|
||||
state := &coordinatorState{
|
||||
start: StartInfo{RunID: "run-reset", Scenario: "group"}, startDelay: time.Minute,
|
||||
commitDelay: time.Second, expectedAgents: 2, ready: make(map[string]AgentReady),
|
||||
armed: make(map[string]bool), ledgers: make(map[string][]AcceptedMessage),
|
||||
summaries: make(map[string]AgentSummary), complete: make(chan struct{}),
|
||||
}
|
||||
postReady := func(ready AgentReady) StartInfo {
|
||||
raw, err := json.Marshal(ready)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/ready", bytes.NewReader(raw))
|
||||
w := httptest.NewRecorder()
|
||||
state.handleReady(w, req)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("ready status=%d body=%s", w.Code, w.Body.String())
|
||||
}
|
||||
var start StartInfo
|
||||
if err = json.Unmarshal(w.Body.Bytes(), &start); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return start
|
||||
}
|
||||
fullA := AgentReady{Machine: "a", ReadyConnections: 1000, ExpectedConnections: 1000}
|
||||
fullB := AgentReady{Machine: "b", ReadyConnections: 1000, ExpectedConnections: 1000}
|
||||
postReady(fullA)
|
||||
if start := postReady(fullB); start.StartAt.IsZero() {
|
||||
t.Fatal("full agents must create a validation barrier")
|
||||
}
|
||||
fullA.ReadyConnections = 999
|
||||
if start := postReady(fullA); !start.StartAt.IsZero() || start.Committed {
|
||||
t.Fatalf("connection drop must reset barrier: %+v", start)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCoordinatorClientWaitsForBothAgentsAndMergesLedger(t *testing.T) {
|
||||
state := &coordinatorState{
|
||||
start: StartInfo{RunID: "run1", Scenario: "p2p"}, startDelay: time.Second, expectedAgents: 2,
|
||||
start: StartInfo{RunID: "run1", Scenario: "p2p"}, startDelay: 50 * time.Millisecond,
|
||||
commitDelay: 50 * time.Millisecond, expectedAgents: 2,
|
||||
ready: make(map[string]AgentReady), ledgers: make(map[string][]AcceptedMessage),
|
||||
summaries: make(map[string]AgentSummary), complete: make(chan struct{}),
|
||||
armed: make(map[string]bool), summaries: make(map[string]AgentSummary), complete: make(chan struct{}),
|
||||
}
|
||||
server := httptest.NewServer(testCoordinatorMux(state))
|
||||
defer server.Close()
|
||||
@@ -20,12 +59,16 @@ func TestCoordinatorClientWaitsForBothAgentsAndMergesLedger(t *testing.T) {
|
||||
|
||||
aDone := make(chan StartInfo, 1)
|
||||
go func() {
|
||||
start, _ := client.ready(context.Background(), AgentReady{Machine: "a", ReadyConnections: 1000})
|
||||
start, _ := client.ready(context.Background(), func() AgentReady {
|
||||
return AgentReady{Machine: "a", ReadyConnections: 1000, ExpectedConnections: 1000}
|
||||
})
|
||||
aDone <- start
|
||||
}()
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
startB, err := client.ready(context.Background(), AgentReady{Machine: "b", ReadyConnections: 1000})
|
||||
if err != nil || startB.StartAt.IsZero() {
|
||||
startB, err := client.ready(context.Background(), func() AgentReady {
|
||||
return AgentReady{Machine: "b", ReadyConnections: 1000, ExpectedConnections: 1000}
|
||||
})
|
||||
if err != nil || startB.StartAt.IsZero() || !startB.Committed {
|
||||
t.Fatalf("machine B ready: start=%+v err=%v", startB, err)
|
||||
}
|
||||
select {
|
||||
@@ -47,6 +90,26 @@ func TestCoordinatorClientWaitsForBothAgentsAndMergesLedger(t *testing.T) {
|
||||
if err != nil || len(ledger) != 2 || ledger[0].ClientMid != "a" || ledger[1].ClientMid != "b" {
|
||||
t.Fatalf("ledger=%+v err=%v", ledger, err)
|
||||
}
|
||||
if err = client.submitObserved(context.Background(), "a", []AcceptedMessage{{ClientMid: "unknown", Seq: 10}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = client.submitObserved(context.Background(), "b", []AcceptedMessage{{ClientMid: "unknown", Seq: 10}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
observed, err := client.observed(context.Background())
|
||||
if err != nil || len(observed) != 1 || observed[0].ClientMid != "unknown" {
|
||||
t.Fatalf("observed=%+v err=%v", observed, err)
|
||||
}
|
||||
if err = client.submitHistory(context.Background(), "a", []string{"mid-2", "mid-1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = client.submitHistory(context.Background(), "b", []string{"mid-2", "mid-3"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
history, err := client.history(context.Background())
|
||||
if err != nil || len(history) != 3 || history[0] != "mid-1" || history[1] != "mid-2" || history[2] != "mid-3" {
|
||||
t.Fatalf("history=%+v err=%v", history, err)
|
||||
}
|
||||
}
|
||||
|
||||
func testCoordinatorMux(state *coordinatorState) *http.ServeMux {
|
||||
@@ -54,6 +117,8 @@ func testCoordinatorMux(state *coordinatorState) *http.ServeMux {
|
||||
mux.HandleFunc("/v1/ready", state.handleReady)
|
||||
mux.HandleFunc("/v1/start", state.handleStart)
|
||||
mux.HandleFunc("/v1/ledger", state.handleLedger)
|
||||
mux.HandleFunc("/v1/observed", state.handleObserved)
|
||||
mux.HandleFunc("/v1/history", state.handleHistory)
|
||||
mux.HandleFunc("/v1/summary", state.handleSummary)
|
||||
mux.HandleFunc("/v1/status", state.handleStatus)
|
||||
return mux
|
||||
|
||||
@@ -197,15 +197,17 @@ func (a *apiClient) findPendingFriendRequest(ctx context.Context, target Account
|
||||
func (a *apiClient) createFriendRequest(ctx context.Context, source, target AccountRecord) (*friendRequestResponse, error) {
|
||||
var result friendRequestResponse
|
||||
body := map[string]any{"target_user_id": target.UserID, "message": "loadtest", "source": "search", "source_topic": ""}
|
||||
idempotencyKey := fmt.Sprintf("loadtest-friend-%d-%s-%04d", time.Now().UnixNano(), source.Machine, source.Index)
|
||||
err := a.do(ctx, http.MethodPost, "/v1/im/friend-requests", source.Token,
|
||||
fmt.Sprintf("loadtest-friend-%04d", source.Index), body, &result)
|
||||
idempotencyKey, body, &result)
|
||||
return &result, err
|
||||
}
|
||||
|
||||
func (a *apiClient) acceptFriendRequest(ctx context.Context, target AccountRecord, requestID string) (*friendRequestResponse, error) {
|
||||
var result friendRequestResponse
|
||||
idempotencyKey := fmt.Sprintf("loadtest-friend-accept-%d", time.Now().UnixNano())
|
||||
err := a.do(ctx, http.MethodPost, "/v1/im/friend-requests/"+url.PathEscape(requestID)+"/accept", target.Token,
|
||||
"loadtest-friend-accept-"+requestID, map[string]any{}, &result)
|
||||
idempotencyKey, map[string]any{}, &result)
|
||||
return &result, err
|
||||
}
|
||||
|
||||
@@ -216,11 +218,12 @@ type p2pResponse struct {
|
||||
|
||||
func (a *apiClient) ensureP2P(ctx context.Context, source, target AccountRecord) (*p2pResponse, error) {
|
||||
var result p2pResponse
|
||||
// EnsureP2P is itself idempotent and must execute on every preflight so it
|
||||
// can repair subscriptions and refresh a loaded topic. A stable HTTP
|
||||
// idempotency key would replay an old response and skip that repair.
|
||||
// EnsureP2P must execute on every preflight so it can repair subscriptions
|
||||
// and refresh a loaded topic. Use a fresh key instead of replaying an old
|
||||
// successful response from a previous load-test run.
|
||||
idempotencyKey := fmt.Sprintf("loadtest-p2p-%d-%s-%04d", time.Now().UnixNano(), source.Machine, source.Index)
|
||||
err := a.do(ctx, http.MethodPost, "/v1/im/conversations/p2p", source.Token,
|
||||
"",
|
||||
idempotencyKey,
|
||||
map[string]any{"peer_user_id": target.UserID}, &result)
|
||||
return &result, err
|
||||
}
|
||||
|
||||
@@ -10,34 +10,43 @@ import (
|
||||
)
|
||||
|
||||
type CombinedReport struct {
|
||||
RunID string `json:"run_id"`
|
||||
Scenario string `json:"scenario"`
|
||||
GeneratedAt time.Time `json:"generated_at"`
|
||||
Agents []AgentSummary `json:"agents"`
|
||||
ExpectedConnections int `json:"expected_connections"`
|
||||
ReadyConnections int `json:"ready_connections"`
|
||||
PublishAttempted int64 `json:"publish_attempted"`
|
||||
PublishAccepted int64 `json:"publish_accepted"`
|
||||
PublishRejected int64 `json:"publish_rejected"`
|
||||
PublishErrors map[string]int64 `json:"publish_errors,omitempty"`
|
||||
PublishSuccessRate float64 `json:"publish_success_rate"`
|
||||
AckP95MS float64 `json:"ack_p95_ms"`
|
||||
DeliveryP95MS float64 `json:"delivery_p95_ms"`
|
||||
ExpectedDeliveries int64 `json:"expected_deliveries"`
|
||||
ReceivedDeliveries int64 `json:"received_deliveries"`
|
||||
RealtimeLost int64 `json:"realtime_lost"`
|
||||
RealtimeLostStored int64 `json:"realtime_lost_but_persisted"`
|
||||
ArrivalRate float64 `json:"arrival_rate"`
|
||||
LossRate float64 `json:"loss_rate"`
|
||||
Duplicates int64 `json:"duplicates"`
|
||||
OutOfOrder int64 `json:"out_of_order"`
|
||||
Disconnects int64 `json:"disconnects"`
|
||||
HistoryChecked int64 `json:"history_checked"`
|
||||
HistoryFound int64 `json:"history_found"`
|
||||
PersistenceLost int64 `json:"persistence_lost"`
|
||||
Missing []MissingDelivery `json:"missing,omitempty"`
|
||||
Passed bool `json:"passed"`
|
||||
FailureReasons []string `json:"failure_reasons,omitempty"`
|
||||
RunID string `json:"run_id"`
|
||||
Scenario string `json:"scenario"`
|
||||
GeneratedAt time.Time `json:"generated_at"`
|
||||
Agents []AgentSummary `json:"agents"`
|
||||
ExpectedConnections int `json:"expected_connections"`
|
||||
ReadyConnections int `json:"ready_connections"`
|
||||
PublishAttempted int64 `json:"publish_attempted"`
|
||||
PublishAccepted int64 `json:"publish_accepted"`
|
||||
PublishRejected int64 `json:"publish_rejected"`
|
||||
PublishExplicitReject int64 `json:"publish_explicitly_rejected"`
|
||||
PublishOutcomeUnknown int64 `json:"publish_outcome_unknown"`
|
||||
PublishErrors map[string]int64 `json:"publish_errors,omitempty"`
|
||||
PublishSuccessRate float64 `json:"publish_success_rate"`
|
||||
AckP95MS float64 `json:"ack_p95_ms"`
|
||||
DeliveryP95MS float64 `json:"delivery_p95_ms"`
|
||||
DeliveryLocalP95MS float64 `json:"delivery_local_p95_ms"`
|
||||
DeliveryCrossP95MS float64 `json:"delivery_cross_machine_p95_ms"`
|
||||
CrossClockSkewSamples int64 `json:"cross_machine_negative_latency_samples"`
|
||||
ExpectedDeliveries int64 `json:"expected_deliveries"`
|
||||
ReceivedDeliveries int64 `json:"received_deliveries"`
|
||||
RealtimeLost int64 `json:"realtime_lost"`
|
||||
RealtimeLostStored int64 `json:"realtime_lost_but_persisted"`
|
||||
ArrivalRate float64 `json:"arrival_rate"`
|
||||
LossRate float64 `json:"loss_rate"`
|
||||
Duplicates int64 `json:"duplicates"`
|
||||
OutOfOrder int64 `json:"out_of_order"`
|
||||
OrderingViolations []OrderingViolation `json:"ordering_violations,omitempty"`
|
||||
Disconnects int64 `json:"disconnects"`
|
||||
UnexpectedMessages int64 `json:"unexpected_messages"`
|
||||
HistoryChecked int64 `json:"history_checked"`
|
||||
HistoryFound int64 `json:"history_found"`
|
||||
HistoryUnackedChecked int64 `json:"history_unacked_observed_checked"`
|
||||
HistoryUnackedFound int64 `json:"history_unacked_observed_found"`
|
||||
PersistenceLost int64 `json:"persistence_lost"`
|
||||
Missing []MissingDelivery `json:"missing,omitempty"`
|
||||
Passed bool `json:"passed"`
|
||||
FailureReasons []string `json:"failure_reasons,omitempty"`
|
||||
}
|
||||
|
||||
func runMerge(aPath, bPath, output string) error {
|
||||
@@ -63,20 +72,33 @@ func runMerge(aPath, bPath, output string) error {
|
||||
report.ReadyConnections += summary.ReadyConnections
|
||||
report.PublishAttempted += summary.PublishAttempted
|
||||
report.PublishAccepted += summary.PublishAccepted
|
||||
report.PublishRejected += summary.PublishRejected
|
||||
for category, count := range summary.PublishErrors {
|
||||
report.PublishErrors[category] += count
|
||||
}
|
||||
unknown := summary.PublishOutcomeUnknown
|
||||
if unknown == 0 {
|
||||
unknown = summary.PublishErrors["ctrl_timeout"] + summary.PublishErrors["socket"]
|
||||
}
|
||||
explicit := summary.PublishErrors["403"] + summary.PublishErrors["502"] +
|
||||
summary.PublishErrors["503"] + summary.PublishErrors["other"]
|
||||
report.PublishOutcomeUnknown += unknown
|
||||
report.PublishExplicitReject += explicit
|
||||
report.PublishRejected += explicit
|
||||
report.ExpectedDeliveries += summary.ExpectedPeer + summary.ExpectedGroup
|
||||
report.ReceivedDeliveries += summary.ReceivedPeer + summary.ReceivedGroup
|
||||
report.RealtimeLost += summary.RealtimeLost
|
||||
report.RealtimeLostStored += summary.RealtimeLostStored
|
||||
report.Duplicates += summary.Duplicates
|
||||
report.OutOfOrder += summary.OutOfOrder
|
||||
report.OrderingViolations = append(report.OrderingViolations, summary.OrderingViolations...)
|
||||
report.Disconnects += summary.Disconnects
|
||||
report.UnexpectedMessages += summary.UnexpectedMessages
|
||||
report.HistoryChecked += summary.HistoryChecked
|
||||
report.HistoryFound += summary.HistoryFound
|
||||
report.HistoryUnackedChecked += summary.HistoryUnackedChecked
|
||||
report.HistoryUnackedFound += summary.HistoryUnackedFound
|
||||
report.PersistenceLost += summary.PersistenceLost
|
||||
report.CrossClockSkewSamples += summary.CrossClockSkewSamples
|
||||
report.Missing = append(report.Missing, summary.Missing...)
|
||||
if len(summary.Errors) > 0 {
|
||||
report.FailureReasons = append(report.FailureReasons, fmt.Sprintf("agent %s errors=%d", summary.Machine, len(summary.Errors)))
|
||||
@@ -85,8 +107,15 @@ func runMerge(aPath, bPath, output string) error {
|
||||
if report.PublishAttempted > 0 {
|
||||
report.PublishSuccessRate = float64(report.PublishAccepted) / float64(report.PublishAttempted)
|
||||
}
|
||||
report.AckP95MS = mergedHistogramP95(report.Agents, func(summary AgentSummary) HistogramSnapshot { return summary.AckLatency })
|
||||
report.DeliveryP95MS = mergedHistogramP95(report.Agents, func(summary AgentSummary) HistogramSnapshot { return summary.DeliveryLatency })
|
||||
report.AckP95MS = mergedAckP95(report.Agents)
|
||||
report.DeliveryLocalP95MS = mergedHistogramP95(report.Agents, func(summary AgentSummary) HistogramSnapshot { return summary.DeliveryLatencyLocal })
|
||||
report.DeliveryCrossP95MS = mergedHistogramP95(report.Agents, func(summary AgentSummary) HistogramSnapshot { return summary.DeliveryLatencyCross })
|
||||
// delivery_p95_ms remains the compatibility field, but now uses only same-clock
|
||||
// samples. Cross-machine wall clocks cannot produce a valid one-way latency.
|
||||
report.DeliveryP95MS = report.DeliveryLocalP95MS
|
||||
if report.DeliveryP95MS == 0 {
|
||||
report.DeliveryP95MS = mergedHistogramP95(report.Agents, func(summary AgentSummary) HistogramSnapshot { return summary.DeliveryLatency })
|
||||
}
|
||||
if report.ExpectedDeliveries > 0 {
|
||||
report.ArrivalRate = float64(report.ReceivedDeliveries) / float64(report.ExpectedDeliveries)
|
||||
report.LossRate = float64(report.RealtimeLost) / float64(report.ExpectedDeliveries)
|
||||
@@ -99,14 +128,22 @@ func runMerge(aPath, bPath, output string) error {
|
||||
report.FailureReasons = append(report.FailureReasons,
|
||||
fmt.Sprintf("publish success rate=%.6f expected>=0.999900", report.PublishSuccessRate))
|
||||
}
|
||||
if report.PublishOutcomeUnknown > 0 {
|
||||
report.FailureReasons = append(report.FailureReasons,
|
||||
fmt.Sprintf("publish outcome unknown=%d (ACK timeout or socket failure)", report.PublishOutcomeUnknown))
|
||||
}
|
||||
if report.UnexpectedMessages > 0 {
|
||||
report.FailureReasons = append(report.FailureReasons,
|
||||
fmt.Sprintf("deliveries observed without ACK ledger=%d", report.UnexpectedMessages))
|
||||
}
|
||||
if report.AckP95MS > 500 {
|
||||
report.FailureReasons = append(report.FailureReasons, fmt.Sprintf("ack p95=%.0fms expected<=500ms", report.AckP95MS))
|
||||
report.FailureReasons = append(report.FailureReasons, fmt.Sprintf("ack p95=%.3fms expected<=500ms", report.AckP95MS))
|
||||
}
|
||||
if report.DeliveryP95MS > 1000 {
|
||||
report.FailureReasons = append(report.FailureReasons,
|
||||
fmt.Sprintf("delivery p95=%.0fms expected<=1000ms", report.DeliveryP95MS))
|
||||
}
|
||||
if report.PublishRejected > 0 || report.RealtimeLost > 0 || report.PersistenceLost > 0 || report.Duplicates > 0 || report.OutOfOrder > 0 || report.Disconnects > 0 {
|
||||
if report.PublishRejected > 0 || report.RealtimeLost > 0 || report.PersistenceLost > 0 || report.Duplicates > 0 || report.OutOfOrder > 0 || report.Disconnects > 0 || report.UnexpectedMessages > 0 {
|
||||
report.FailureReasons = append(report.FailureReasons, "message correctness gate failed")
|
||||
}
|
||||
report.Passed = len(report.FailureReasons) == 0
|
||||
@@ -119,6 +156,26 @@ func runMerge(aPath, bPath, output string) error {
|
||||
return writeJSONFile(output, report)
|
||||
}
|
||||
|
||||
func mergedAckP95(agents []AgentSummary) float64 {
|
||||
var expected int64
|
||||
latencies := make([]float64, 0)
|
||||
for _, agent := range agents {
|
||||
expected += agent.AckLatency.Count
|
||||
for _, message := range agent.AcceptedLocal {
|
||||
if message.SentAt.IsZero() || message.AckAt.Before(message.SentAt) {
|
||||
continue
|
||||
}
|
||||
latencies = append(latencies, float64(message.AckAt.Sub(message.SentAt))/float64(time.Millisecond))
|
||||
}
|
||||
}
|
||||
if expected == 0 || int64(len(latencies)) != expected {
|
||||
return mergedHistogramP95(agents, func(summary AgentSummary) HistogramSnapshot { return summary.AckLatency })
|
||||
}
|
||||
sort.Float64s(latencies)
|
||||
index := (len(latencies)*95 + 99) / 100
|
||||
return latencies[index-1]
|
||||
}
|
||||
|
||||
func mergedHistogramP95(agents []AgentSummary, selectHistogram func(AgentSummary) HistogramSnapshot) float64 {
|
||||
var count int64
|
||||
buckets := make([]int64, len(latencyBounds)+1)
|
||||
|
||||
@@ -5,18 +5,21 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestRunMergeComputesArrivalAndPersistenceGate(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
a := AgentSummary{RunID: "run1", Scenario: "p2p", Machine: "a", ReadyConnections: 1000,
|
||||
ExpectedConnections: 1000,
|
||||
PublishAttempted: 10, PublishAccepted: 10, ExpectedPeer: 10, ReceivedPeer: 9,
|
||||
RealtimeLost: 1, HistoryChecked: 10, HistoryFound: 10}
|
||||
PublishAttempted: 10, PublishAccepted: 10, ExpectedPeer: 10, ReceivedPeer: 9, UnexpectedMessages: 42,
|
||||
PublishErrors: map[string]int64{"ctrl_timeout": 3, "socket": 2, "503": 1},
|
||||
RealtimeLost: 1, HistoryChecked: 10, HistoryFound: 10}
|
||||
b := AgentSummary{RunID: "run1", Scenario: "p2p", Machine: "b", ReadyConnections: 1000,
|
||||
ExpectedConnections: 1000,
|
||||
PublishAttempted: 10, PublishAccepted: 10, ExpectedPeer: 10, ReceivedPeer: 10,
|
||||
HistoryChecked: 10, HistoryFound: 9, PersistenceLost: 1}
|
||||
HistoryChecked: 10, HistoryFound: 9, PersistenceLost: 1, OutOfOrder: 1,
|
||||
OrderingViolations: []OrderingViolation{{ReceiverMachine: "b", ReceiverIndex: 7, PreviousSeq: 12, CurrentSeq: 11}}}
|
||||
aPath, bPath, reportPath := filepath.Join(dir, "a.json"), filepath.Join(dir, "b.json"), filepath.Join(dir, "report.json")
|
||||
writeSummaryForTest(t, aPath, a)
|
||||
writeSummaryForTest(t, bPath, b)
|
||||
@@ -37,6 +40,13 @@ func TestRunMergeComputesArrivalAndPersistenceGate(t *testing.T) {
|
||||
if report.Passed || report.PersistenceLost != 1 {
|
||||
t.Fatalf("correctness gate did not fail: %+v", report)
|
||||
}
|
||||
if report.UnexpectedMessages != 42 || report.PublishOutcomeUnknown != 5 ||
|
||||
report.PublishExplicitReject != 1 || report.PublishRejected != 1 {
|
||||
t.Fatalf("ambiguous publish outcomes were not merged: %+v", report)
|
||||
}
|
||||
if len(report.OrderingViolations) != 1 || report.OrderingViolations[0].ReceiverIndex != 7 {
|
||||
t.Fatalf("ordering violations were not merged: %+v", report.OrderingViolations)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergedHistogramP95(t *testing.T) {
|
||||
@@ -49,6 +59,63 @@ func TestMergedHistogramP95(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergedAckP95UsesExactAcceptedTimestamps(t *testing.T) {
|
||||
base := time.Date(2026, 9, 27, 8, 0, 0, 0, time.UTC)
|
||||
agents := []AgentSummary{{
|
||||
AckLatency: HistogramSnapshot{Count: 4, MaxMS: 900, Buckets: map[string]int64{"1s": 4}},
|
||||
AcceptedLocal: []AcceptedMessage{
|
||||
{SentAt: base, AckAt: base.Add(100 * time.Millisecond)},
|
||||
{SentAt: base, AckAt: base.Add(200 * time.Millisecond)},
|
||||
{SentAt: base, AckAt: base.Add(300 * time.Millisecond)},
|
||||
{SentAt: base, AckAt: base.Add(400 * time.Millisecond)},
|
||||
},
|
||||
}}
|
||||
if got := mergedAckP95(agents); got != 400 {
|
||||
t.Fatalf("exact p95=%v want=400", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergedAckP95FallsBackWhenRawSamplesAreIncomplete(t *testing.T) {
|
||||
agents := []AgentSummary{{
|
||||
AckLatency: HistogramSnapshot{Count: 2, MaxMS: 900, Buckets: map[string]int64{"1s": 2}},
|
||||
AcceptedLocal: []AcceptedMessage{{SentAt: time.Now(), AckAt: time.Now().Add(100 * time.Millisecond)}},
|
||||
}}
|
||||
if got := mergedAckP95(agents); got != 1000 {
|
||||
t.Fatalf("fallback p95=%v want=1000", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunMergeUsesSameClockDeliveryLatencyForGate(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
local := HistogramSnapshot{Count: 10, MaxMS: 200, Buckets: map[string]int64{"200ms": 10}}
|
||||
cross := HistogramSnapshot{Count: 10, MaxMS: 5000, Buckets: map[string]int64{"5s": 10}}
|
||||
a := AgentSummary{RunID: "run1", Scenario: "p2p", Machine: "a", ExpectedConnections: 1,
|
||||
ReadyConnections: 1, PublishAttempted: 1, PublishAccepted: 1, ExpectedPeer: 1, ReceivedPeer: 1,
|
||||
DeliveryLatencyLocal: local, DeliveryLatencyCross: cross, CrossClockSkewSamples: 3}
|
||||
b := a
|
||||
b.Machine = "b"
|
||||
aPath, bPath, reportPath := filepath.Join(dir, "a.json"), filepath.Join(dir, "b.json"), filepath.Join(dir, "report.json")
|
||||
writeSummaryForTest(t, aPath, a)
|
||||
writeSummaryForTest(t, bPath, b)
|
||||
if err := runMerge(aPath, bPath, reportPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var report CombinedReport
|
||||
raw, err := os.ReadFile(reportPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = json.Unmarshal(raw, &report); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if report.DeliveryP95MS != 200 || report.DeliveryLocalP95MS != 200 || report.DeliveryCrossP95MS != 5000 {
|
||||
t.Fatalf("unexpected delivery percentiles: %+v", report)
|
||||
}
|
||||
if report.CrossClockSkewSamples != 6 || !report.Passed {
|
||||
t.Fatalf("cross-machine clock skew must not fail the same-clock latency gate: %+v", report)
|
||||
}
|
||||
}
|
||||
|
||||
func writeSummaryForTest(t *testing.T, path string, summary AgentSummary) {
|
||||
t.Helper()
|
||||
raw, err := json.Marshal(summary)
|
||||
|
||||
@@ -25,6 +25,7 @@ type Config struct {
|
||||
PrepareWorkers int `json:"prepare_workers"`
|
||||
ConnectRamp string `json:"connect_ramp"`
|
||||
CommandTimeout string `json:"command_timeout"`
|
||||
WSPingInterval string `json:"ws_ping_interval"`
|
||||
P2PMessages int `json:"p2p_messages_per_user"`
|
||||
P2PInterval string `json:"p2p_interval"`
|
||||
DrainP2P string `json:"drain_p2p"`
|
||||
@@ -32,7 +33,12 @@ type Config struct {
|
||||
PhonePrefix string `json:"registration_phone_prefix"`
|
||||
VerifyCodeEnv string `json:"verification_code_env"`
|
||||
HistoryWorkers int `json:"history_workers"`
|
||||
HistoryPageSize int `json:"history_page_size"`
|
||||
HistoryInterval string `json:"history_query_interval"`
|
||||
HistoryPageWait string `json:"history_page_wait"`
|
||||
HistorySettle string `json:"history_settle"`
|
||||
HistoryTimeout string `json:"history_timeout"`
|
||||
CloseRamp string `json:"close_ramp"`
|
||||
PreflightWorkers int `json:"preflight_workers"`
|
||||
SkipP2PPreflight bool `json:"skip_p2p_preflight"`
|
||||
}
|
||||
@@ -61,6 +67,12 @@ func loadConfig(path string) (*Config, error) {
|
||||
if cfg.CommandTimeout == "" {
|
||||
cfg.CommandTimeout = "20s"
|
||||
}
|
||||
if cfg.WSPingInterval == "" {
|
||||
cfg.WSPingInterval = "20s"
|
||||
}
|
||||
if _, err = configDuration(cfg.WSPingInterval); err != nil {
|
||||
return nil, fmt.Errorf("ws_ping_interval: %w", err)
|
||||
}
|
||||
if cfg.P2PMessages <= 0 {
|
||||
cfg.P2PMessages = 10
|
||||
}
|
||||
@@ -74,17 +86,47 @@ func loadConfig(path string) (*Config, error) {
|
||||
cfg.DrainGroup = "60s"
|
||||
}
|
||||
if cfg.HistoryWorkers <= 0 {
|
||||
cfg.HistoryWorkers = 48
|
||||
cfg.HistoryWorkers = 12
|
||||
}
|
||||
if cfg.HistoryWorkers > 128 {
|
||||
return nil, errors.New("history_workers must not exceed 128")
|
||||
}
|
||||
if cfg.HistoryPageSize <= 0 {
|
||||
cfg.HistoryPageSize = 10
|
||||
}
|
||||
if cfg.HistoryPageSize > 200 {
|
||||
return nil, errors.New("history_page_size must not exceed 200")
|
||||
}
|
||||
if cfg.HistoryInterval == "" {
|
||||
cfg.HistoryInterval = "500ms"
|
||||
}
|
||||
if _, err = configDuration(cfg.HistoryInterval); err != nil {
|
||||
return nil, fmt.Errorf("history_query_interval: %w", err)
|
||||
}
|
||||
if cfg.HistoryPageWait == "" {
|
||||
cfg.HistoryPageWait = "3s"
|
||||
}
|
||||
if _, err = configDuration(cfg.HistoryPageWait); err != nil {
|
||||
return nil, fmt.Errorf("history_page_wait: %w", err)
|
||||
}
|
||||
if cfg.HistoryTimeout == "" {
|
||||
cfg.HistoryTimeout = "3m"
|
||||
}
|
||||
if cfg.HistorySettle == "" {
|
||||
cfg.HistorySettle = "5m"
|
||||
}
|
||||
if _, err = configDuration(cfg.HistorySettle); err != nil {
|
||||
return nil, fmt.Errorf("history_settle: %w", err)
|
||||
}
|
||||
if _, err = configDuration(cfg.HistoryTimeout); err != nil {
|
||||
return nil, fmt.Errorf("history_timeout: %w", err)
|
||||
}
|
||||
if cfg.CloseRamp == "" {
|
||||
cfg.CloseRamp = "10s"
|
||||
}
|
||||
if _, err = configDuration(cfg.CloseRamp); err != nil {
|
||||
return nil, fmt.Errorf("close_ramp: %w", err)
|
||||
}
|
||||
if cfg.PreflightWorkers <= 0 {
|
||||
cfg.PreflightWorkers = 32
|
||||
}
|
||||
@@ -343,39 +385,55 @@ type AcceptedMessage struct {
|
||||
}
|
||||
|
||||
type AgentSummary struct {
|
||||
RunID string `json:"run_id"`
|
||||
Scenario string `json:"scenario"`
|
||||
Machine string `json:"machine"`
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
LoadEndedAt time.Time `json:"load_ended_at"`
|
||||
FinishedAt time.Time `json:"finished_at"`
|
||||
ExpectedConnections int `json:"expected_connections"`
|
||||
ReadyConnections int `json:"ready_connections"`
|
||||
PublishAttempted int64 `json:"publish_attempted"`
|
||||
PublishAccepted int64 `json:"publish_accepted"`
|
||||
PublishRejected int64 `json:"publish_rejected"`
|
||||
PublishErrors map[string]int64 `json:"publish_errors,omitempty"`
|
||||
ExpectedPeer int64 `json:"expected_peer"`
|
||||
ReceivedPeer int64 `json:"received_peer"`
|
||||
ExpectedEcho int64 `json:"expected_echo"`
|
||||
ReceivedEcho int64 `json:"received_echo"`
|
||||
ExpectedGroup int64 `json:"expected_group"`
|
||||
ReceivedGroup int64 `json:"received_group"`
|
||||
RealtimeLost int64 `json:"realtime_lost"`
|
||||
RealtimeLostStored int64 `json:"realtime_lost_but_persisted"`
|
||||
Duplicates int64 `json:"duplicates"`
|
||||
OutOfOrder int64 `json:"out_of_order"`
|
||||
Disconnects int64 `json:"disconnects"`
|
||||
UnexpectedMessages int64 `json:"unexpected_messages"`
|
||||
HistoryChecked int64 `json:"history_checked"`
|
||||
HistoryFound int64 `json:"history_found"`
|
||||
PersistenceLost int64 `json:"persistence_lost"`
|
||||
AckLatency HistogramSnapshot `json:"ack_latency"`
|
||||
DeliveryLatency HistogramSnapshot `json:"delivery_latency"`
|
||||
Errors []string `json:"errors,omitempty"`
|
||||
Missing []MissingDelivery `json:"missing,omitempty"`
|
||||
PersistenceMissing []string `json:"persistence_missing,omitempty"`
|
||||
AcceptedLocal []AcceptedMessage `json:"accepted_local,omitempty"`
|
||||
RunID string `json:"run_id"`
|
||||
Scenario string `json:"scenario"`
|
||||
Machine string `json:"machine"`
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
LoadEndedAt time.Time `json:"load_ended_at"`
|
||||
FinishedAt time.Time `json:"finished_at"`
|
||||
ExpectedConnections int `json:"expected_connections"`
|
||||
ReadyConnections int `json:"ready_connections"`
|
||||
PublishAttempted int64 `json:"publish_attempted"`
|
||||
PublishAccepted int64 `json:"publish_accepted"`
|
||||
PublishRejected int64 `json:"publish_rejected"`
|
||||
PublishOutcomeUnknown int64 `json:"publish_outcome_unknown"`
|
||||
PublishErrors map[string]int64 `json:"publish_errors,omitempty"`
|
||||
ExpectedPeer int64 `json:"expected_peer"`
|
||||
ReceivedPeer int64 `json:"received_peer"`
|
||||
ExpectedEcho int64 `json:"expected_echo"`
|
||||
ReceivedEcho int64 `json:"received_echo"`
|
||||
ExpectedGroup int64 `json:"expected_group"`
|
||||
ReceivedGroup int64 `json:"received_group"`
|
||||
RealtimeLost int64 `json:"realtime_lost"`
|
||||
RealtimeLostStored int64 `json:"realtime_lost_but_persisted"`
|
||||
Duplicates int64 `json:"duplicates"`
|
||||
OutOfOrder int64 `json:"out_of_order"`
|
||||
OrderingViolations []OrderingViolation `json:"ordering_violations,omitempty"`
|
||||
Disconnects int64 `json:"disconnects"`
|
||||
UnexpectedMessages int64 `json:"unexpected_messages"`
|
||||
HistoryChecked int64 `json:"history_checked"`
|
||||
HistoryFound int64 `json:"history_found"`
|
||||
HistoryUnackedChecked int64 `json:"history_unacked_observed_checked"`
|
||||
HistoryUnackedFound int64 `json:"history_unacked_observed_found"`
|
||||
PersistenceLost int64 `json:"persistence_lost"`
|
||||
AckLatency HistogramSnapshot `json:"ack_latency"`
|
||||
DeliveryLatency HistogramSnapshot `json:"delivery_latency"`
|
||||
DeliveryLatencyLocal HistogramSnapshot `json:"delivery_latency_local"`
|
||||
DeliveryLatencyCross HistogramSnapshot `json:"delivery_latency_cross_machine"`
|
||||
CrossClockSkewSamples int64 `json:"cross_machine_negative_latency_samples"`
|
||||
Errors []string `json:"errors,omitempty"`
|
||||
Missing []MissingDelivery `json:"missing,omitempty"`
|
||||
PersistenceMissing []string `json:"persistence_missing,omitempty"`
|
||||
AcceptedLocal []AcceptedMessage `json:"accepted_local,omitempty"`
|
||||
}
|
||||
|
||||
type OrderingViolation struct {
|
||||
ReceiverMachine string `json:"receiver_machine"`
|
||||
ReceiverIndex int `json:"receiver_index"`
|
||||
TopicHash string `json:"topic_hash"`
|
||||
PreviousSeq int64 `json:"previous_seq"`
|
||||
CurrentSeq int64 `json:"current_seq"`
|
||||
ClientMid string `json:"client_mid"`
|
||||
}
|
||||
|
||||
type MissingDelivery struct {
|
||||
|
||||
@@ -16,7 +16,10 @@ func TestLoadConfigAppliesBoundedHistoryDefaults(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if cfg.HistoryWorkers != 48 || cfg.HistoryTimeout != "3m" || cfg.PreflightWorkers != 32 {
|
||||
if cfg.HistoryWorkers != 12 || cfg.HistoryPageSize != 10 || cfg.HistoryInterval != "500ms" ||
|
||||
cfg.HistoryPageWait != "3s" || cfg.CloseRamp != "10s" ||
|
||||
cfg.HistorySettle != "5m" || cfg.HistoryTimeout != "3m" ||
|
||||
cfg.PreflightWorkers != 32 || cfg.WSPingInterval != "20s" {
|
||||
t.Fatalf("unexpected defaults: %+v", cfg)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -127,12 +127,23 @@ func prepareFriendships(ctx context.Context, cfg *Config, manifest *Manifest, ap
|
||||
err := parallelAccounts(ctx, pairs, cfg.PrepareWorkers, func(a AccountRecord) error {
|
||||
return retryPrepareOperation(ctx, fmt.Sprintf("friend-pair-%04d", a.Index), func() error {
|
||||
b := *accounts[accountKey("b", a.Index)]
|
||||
if a.P2PTopic != "" && b.P2PTopic == a.P2PTopic {
|
||||
return nil
|
||||
}
|
||||
if a.UserID == "" || b.UserID == "" {
|
||||
return fmt.Errorf("pair %d has missing user ID", a.Index)
|
||||
}
|
||||
conversation, conversationErr := api.ensureP2P(ctx, a, b)
|
||||
if conversationErr == nil && conversation.Relationship == "friend" && conversation.Topic != "" {
|
||||
mu.Lock()
|
||||
accounts[accountKey("a", a.Index)].P2PTopic = conversation.Topic
|
||||
accounts[accountKey("b", a.Index)].P2PTopic = conversation.Topic
|
||||
mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
if conversationErr != nil {
|
||||
var apiErr *apiError
|
||||
if !errors.As(conversationErr, &apiErr) || apiErr.Status != http.StatusForbidden {
|
||||
return fmt.Errorf("check friend pair %d: %w", a.Index, conversationErr)
|
||||
}
|
||||
}
|
||||
request, requestErr := api.createFriendRequest(ctx, a, b)
|
||||
if requestErr == nil && request.Status == "pending" {
|
||||
if _, requestErr = api.acceptFriendRequest(ctx, b, request.RequestID); requestErr != nil {
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"math/bits"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
@@ -11,27 +12,35 @@ import (
|
||||
)
|
||||
|
||||
type observation struct {
|
||||
topic string
|
||||
seq int64
|
||||
bits []uint64
|
||||
topic string
|
||||
seq int64
|
||||
senderMachine string
|
||||
senderIndex int
|
||||
scenario string
|
||||
sentAt int64
|
||||
bits []uint64
|
||||
}
|
||||
|
||||
type deliveryRecorder struct {
|
||||
machine string
|
||||
localUsers int
|
||||
latency *DurationHistogram
|
||||
disconnects atomic.Int64
|
||||
received atomic.Int64
|
||||
runID atomic.Value
|
||||
historyMode atomic.Bool
|
||||
machine string
|
||||
localUsers int
|
||||
latency *DurationHistogram
|
||||
localLatency *DurationHistogram
|
||||
crossLatency *DurationHistogram
|
||||
disconnects atomic.Int64
|
||||
received atomic.Int64
|
||||
runID atomic.Value
|
||||
historyMode atomic.Bool
|
||||
|
||||
mu sync.Mutex
|
||||
observed map[string]*observation
|
||||
lastSeq []map[string]int64
|
||||
duplicates int64
|
||||
outOfOrder int64
|
||||
malformed int64
|
||||
history map[string]bool
|
||||
mu sync.Mutex
|
||||
observed map[string]*observation
|
||||
lastSeq []map[string]int64
|
||||
duplicates int64
|
||||
outOfOrder int64
|
||||
ordering []OrderingViolation
|
||||
malformed int64
|
||||
crossClockSkewSamples int64
|
||||
history map[string]bool
|
||||
}
|
||||
|
||||
func newDeliveryRecorder(runID, machine string, localUsers int) *deliveryRecorder {
|
||||
@@ -41,6 +50,7 @@ func newDeliveryRecorder(runID, machine string, localUsers int) *deliveryRecorde
|
||||
}
|
||||
recorder := &deliveryRecorder{
|
||||
machine: machine, localUsers: localUsers, latency: NewDurationHistogram(),
|
||||
localLatency: NewDurationHistogram(), crossLatency: NewDurationHistogram(),
|
||||
observed: make(map[string]*observation), lastSeq: lastSeq, history: make(map[string]bool),
|
||||
}
|
||||
recorder.runID.Store(runID)
|
||||
@@ -49,6 +59,8 @@ func newDeliveryRecorder(runID, machine string, localUsers int) *deliveryRecorde
|
||||
|
||||
func (r *deliveryRecorder) setRunID(runID string) { r.runID.Store(runID) }
|
||||
|
||||
func (r *deliveryRecorder) resetDisconnects() { r.disconnects.Store(0) }
|
||||
|
||||
func (r *deliveryRecorder) onData(account AccountRecord, data dataPacket) {
|
||||
runID, _ := r.runID.Load().(string)
|
||||
mid, _ := data.Head["x-im-client-mid"].(string)
|
||||
@@ -56,8 +68,11 @@ func (r *deliveryRecorder) onData(account AccountRecord, data dataPacket) {
|
||||
return
|
||||
}
|
||||
var content struct {
|
||||
RunID string `json:"run_id"`
|
||||
SentAt int64 `json:"sent_at_unix_nano"`
|
||||
RunID string `json:"run_id"`
|
||||
SenderMachine string `json:"sender_machine"`
|
||||
SenderIndex int `json:"sender_index"`
|
||||
Scenario string `json:"scenario"`
|
||||
SentAt int64 `json:"sent_at_unix_nano"`
|
||||
}
|
||||
if json.Unmarshal(data.Content, &content) != nil || content.RunID != runID {
|
||||
r.mu.Lock()
|
||||
@@ -79,7 +94,9 @@ func (r *deliveryRecorder) onData(account AccountRecord, data dataPacket) {
|
||||
defer r.mu.Unlock()
|
||||
obs := r.observed[mid]
|
||||
if obs == nil {
|
||||
obs = &observation{topic: data.Topic, seq: data.Seq, bits: make([]uint64, (r.localUsers+63)/64)}
|
||||
obs = &observation{topic: data.Topic, seq: data.Seq, senderMachine: content.SenderMachine,
|
||||
senderIndex: content.SenderIndex, scenario: content.Scenario, sentAt: content.SentAt,
|
||||
bits: make([]uint64, (r.localUsers+63)/64)}
|
||||
r.observed[mid] = obs
|
||||
}
|
||||
word, bit := index/64, uint(index%64)
|
||||
@@ -92,15 +109,46 @@ func (r *deliveryRecorder) onData(account AccountRecord, data dataPacket) {
|
||||
r.received.Add(1)
|
||||
if previous := r.lastSeq[index][data.Topic]; previous > 0 && data.Seq < previous {
|
||||
r.outOfOrder++
|
||||
if len(r.ordering) < 100 {
|
||||
r.ordering = append(r.ordering, OrderingViolation{
|
||||
ReceiverMachine: r.machine,
|
||||
ReceiverIndex: account.Index,
|
||||
TopicHash: topicHash(data.Topic),
|
||||
PreviousSeq: previous,
|
||||
CurrentSeq: data.Seq,
|
||||
ClientMid: mid,
|
||||
})
|
||||
}
|
||||
}
|
||||
if data.Seq > r.lastSeq[index][data.Topic] {
|
||||
r.lastSeq[index][data.Topic] = data.Seq
|
||||
}
|
||||
if content.SentAt > 0 {
|
||||
r.latency.Observe(time.Since(time.Unix(0, content.SentAt)))
|
||||
delay := time.Since(time.Unix(0, content.SentAt))
|
||||
r.latency.Observe(delay)
|
||||
senderMachine := content.SenderMachine
|
||||
if senderMachine == "" {
|
||||
senderMachine = senderMachineFromClientMid(runID, mid)
|
||||
}
|
||||
if senderMachine == r.machine {
|
||||
r.localLatency.Observe(delay)
|
||||
} else if delay < 0 {
|
||||
r.crossClockSkewSamples++
|
||||
} else {
|
||||
r.crossLatency.Observe(delay)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func senderMachineFromClientMid(runID, mid string) string {
|
||||
prefix := strings.TrimPrefix(mid, runID+"-")
|
||||
parts := strings.SplitN(prefix, "-", 3)
|
||||
if len(parts) < 2 {
|
||||
return ""
|
||||
}
|
||||
return parts[1]
|
||||
}
|
||||
|
||||
func (r *deliveryRecorder) metricCounts() (received, duplicates, outOfOrder, disconnects int64) {
|
||||
r.mu.Lock()
|
||||
duplicates = r.duplicates
|
||||
@@ -117,12 +165,67 @@ func (r *deliveryRecorder) historyFound(mid string) bool {
|
||||
return r.history[mid]
|
||||
}
|
||||
|
||||
func (r *deliveryRecorder) historyFoundAll(messageIDs []string) bool {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
for _, mid := range messageIDs {
|
||||
if !r.history[mid] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (r *deliveryRecorder) unackedObservedMessages(ledger []AcceptedMessage, scenario string) []AcceptedMessage {
|
||||
accepted := make(map[string]bool, len(ledger))
|
||||
for _, message := range ledger {
|
||||
accepted[message.ClientMid] = true
|
||||
}
|
||||
runID, _ := r.runID.Load().(string)
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
messages := make([]AcceptedMessage, 0)
|
||||
for mid, obs := range r.observed {
|
||||
if accepted[mid] || obs.seq <= 0 || obs.topic == "" {
|
||||
continue
|
||||
}
|
||||
kind := "p2p"
|
||||
if strings.HasPrefix(obs.topic, "grp") {
|
||||
kind = "group"
|
||||
}
|
||||
observedScenario := obs.scenario
|
||||
if observedScenario == "" {
|
||||
observedScenario = scenario
|
||||
}
|
||||
message := AcceptedMessage{RunID: runID, Scenario: observedScenario, Kind: kind,
|
||||
ClientMid: mid, Topic: obs.topic, Seq: obs.seq, SenderMachine: obs.senderMachine,
|
||||
SenderIndex: obs.senderIndex, PeerMachine: r.machine, PeerIndex: firstObservedIndex(obs.bits)}
|
||||
if obs.sentAt > 0 {
|
||||
message.SentAt = time.Unix(0, obs.sentAt).UTC()
|
||||
}
|
||||
messages = append(messages, message)
|
||||
}
|
||||
sort.Slice(messages, func(i, j int) bool { return messages[i].ClientMid < messages[j].ClientMid })
|
||||
return messages
|
||||
}
|
||||
|
||||
func firstObservedIndex(words []uint64) int {
|
||||
for wordIndex, word := range words {
|
||||
if word != 0 {
|
||||
return wordIndex*64 + bits.TrailingZeros64(word) + 1
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (r *deliveryRecorder) onDisconnect(_ AccountRecord, _ error) {
|
||||
r.disconnects.Add(1)
|
||||
}
|
||||
|
||||
func (r *deliveryRecorder) summarize(base AgentSummary, ledger []AcceptedMessage) AgentSummary {
|
||||
base.DeliveryLatency = r.latency.Snapshot()
|
||||
base.DeliveryLatencyLocal = r.localLatency.Snapshot()
|
||||
base.DeliveryLatencyCross = r.crossLatency.Snapshot()
|
||||
base.Disconnects = r.disconnects.Load()
|
||||
runID, _ := r.runID.Load().(string)
|
||||
accepted := make(map[string]AcceptedMessage, len(ledger))
|
||||
@@ -132,6 +235,8 @@ func (r *deliveryRecorder) summarize(base AgentSummary, ledger []AcceptedMessage
|
||||
defer r.mu.Unlock()
|
||||
base.Duplicates = r.duplicates
|
||||
base.OutOfOrder = r.outOfOrder
|
||||
base.CrossClockSkewSamples = r.crossClockSkewSamples
|
||||
base.OrderingViolations = append([]OrderingViolation(nil), r.ordering...)
|
||||
if r.malformed > 0 {
|
||||
base.Errors = append(base.Errors, fmt.Sprintf("malformed run messages=%d", r.malformed))
|
||||
}
|
||||
@@ -180,13 +285,17 @@ func (r *deliveryRecorder) summarize(base AgentSummary, ledger []AcceptedMessage
|
||||
return base
|
||||
}
|
||||
|
||||
func (r *deliveryRecorder) countRealtimeLostStored(ledger []AcceptedMessage) int64 {
|
||||
func (r *deliveryRecorder) countRealtimeLostStored(ledger []AcceptedMessage, storedMessageIDs []string) int64 {
|
||||
runID, _ := r.runID.Load().(string)
|
||||
stored := make(map[string]bool, len(storedMessageIDs))
|
||||
for _, mid := range storedMessageIDs {
|
||||
stored[mid] = true
|
||||
}
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
var count int64
|
||||
for _, message := range ledger {
|
||||
if message.RunID != runID || !r.history[message.ClientMid] {
|
||||
if message.RunID != runID || !stored[message.ClientMid] {
|
||||
continue
|
||||
}
|
||||
obs := r.observed[message.ClientMid]
|
||||
|
||||
@@ -43,13 +43,78 @@ func TestDeliveryRecorderSeparatesHistoryFromRealtime(t *testing.T) {
|
||||
if !recorder.historyFound(message.ClientMid) {
|
||||
t.Fatal("history message was not recorded")
|
||||
}
|
||||
if got := recorder.countRealtimeLostStored([]AcceptedMessage{message}); got != 1 {
|
||||
if got := recorder.countRealtimeLostStored([]AcceptedMessage{message}, []string{message.ClientMid}); got != 1 {
|
||||
t.Fatalf("realtime lost but persisted=%d", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeliveryRecorderUsesSharedHistoryForRealtimeLostStored(t *testing.T) {
|
||||
recorder := newDeliveryRecorder("run1", "b", 1)
|
||||
message := AcceptedMessage{RunID: "run1", Kind: "group", ClientMid: "run1-g-a-0001-01", Topic: "grp-one", Seq: 10}
|
||||
|
||||
if got := recorder.countRealtimeLostStored([]AcceptedMessage{message}, []string{message.ClientMid}); got != 1 {
|
||||
t.Fatalf("shared history count=%d want=1", got)
|
||||
}
|
||||
if got := recorder.countRealtimeLostStored([]AcceptedMessage{message}, nil); got != 0 {
|
||||
t.Fatalf("empty shared history count=%d want=0", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeliveryRecorderExportsObservedMessagesWithoutAck(t *testing.T) {
|
||||
recorder := newDeliveryRecorder("run1", "b", 2)
|
||||
acked := AcceptedMessage{RunID: "run1", Scenario: "group", Kind: "group", ClientMid: "run1-g-a-0001-01",
|
||||
Topic: "grp-one", Seq: 10, SenderMachine: "a", SenderIndex: 1}
|
||||
unacked := AcceptedMessage{RunID: "run1", Scenario: "group", Kind: "group", ClientMid: "run1-g-a-0002-01",
|
||||
Topic: "grp-one", Seq: 11, SenderMachine: "a", SenderIndex: 2}
|
||||
recorder.onData(AccountRecord{Index: 1}, testData(acked, time.Now()))
|
||||
recorder.onData(AccountRecord{Index: 2}, testData(unacked, time.Now()))
|
||||
|
||||
messages := recorder.unackedObservedMessages([]AcceptedMessage{acked}, "group")
|
||||
if len(messages) != 1 || messages[0].ClientMid != unacked.ClientMid || messages[0].Seq != 11 ||
|
||||
messages[0].PeerMachine != "b" || messages[0].PeerIndex != 2 || messages[0].SenderIndex != 2 {
|
||||
t.Fatalf("unexpected observed ledger: %+v", messages)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeliveryRecorderCapturesOrderingViolationDetails(t *testing.T) {
|
||||
recorder := newDeliveryRecorder("run1", "b", 1)
|
||||
newer := AcceptedMessage{RunID: "run1", ClientMid: "run1-p-a-0001-02", Topic: "p2p-one", Seq: 12}
|
||||
older := AcceptedMessage{RunID: "run1", ClientMid: "run1-p-a-0001-01", Topic: "p2p-one", Seq: 11}
|
||||
recorder.onData(AccountRecord{Index: 1}, testData(newer, time.Now()))
|
||||
recorder.onData(AccountRecord{Index: 1}, testData(older, time.Now()))
|
||||
|
||||
summary := recorder.summarize(AgentSummary{}, nil)
|
||||
if summary.OutOfOrder != 1 || len(summary.OrderingViolations) != 1 {
|
||||
t.Fatalf("out_of_order=%d violations=%+v", summary.OutOfOrder, summary.OrderingViolations)
|
||||
}
|
||||
violation := summary.OrderingViolations[0]
|
||||
if violation.ReceiverMachine != "b" || violation.ReceiverIndex != 1 ||
|
||||
violation.PreviousSeq != 12 || violation.CurrentSeq != 11 || violation.ClientMid != older.ClientMid {
|
||||
t.Fatalf("unexpected violation: %+v", violation)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeliveryRecorderSeparatesLocalAndCrossMachineLatency(t *testing.T) {
|
||||
recorder := newDeliveryRecorder("run1", "a", 1)
|
||||
local := AcceptedMessage{RunID: "run1", ClientMid: "run1-p-a-0001-01", Topic: "p2p-one", Seq: 1}
|
||||
cross := AcceptedMessage{RunID: "run1", ClientMid: "run1-p-b-0001-01", Topic: "p2p-one", Seq: 2}
|
||||
recorder.onData(AccountRecord{Index: 1}, testData(local, time.Now().Add(-10*time.Millisecond)))
|
||||
recorder.onData(AccountRecord{Index: 1}, testData(cross, time.Now().Add(time.Second)))
|
||||
|
||||
summary := recorder.summarize(AgentSummary{}, nil)
|
||||
if summary.DeliveryLatencyLocal.Count != 1 || summary.DeliveryLatencyCross.Count != 0 {
|
||||
t.Fatalf("unexpected split latency counts: local=%d cross=%d",
|
||||
summary.DeliveryLatencyLocal.Count, summary.DeliveryLatencyCross.Count)
|
||||
}
|
||||
if summary.CrossClockSkewSamples != 1 {
|
||||
t.Fatalf("cross clock skew samples=%d", summary.CrossClockSkewSamples)
|
||||
}
|
||||
}
|
||||
|
||||
func testData(message AcceptedMessage, sentAt time.Time) dataPacket {
|
||||
content, _ := json.Marshal(map[string]any{"run_id": message.RunID, "sent_at_unix_nano": sentAt.UnixNano()})
|
||||
content, _ := json.Marshal(map[string]any{"run_id": message.RunID, "scenario": message.Scenario,
|
||||
"sender_machine": message.SenderMachine, "sender_index": message.SenderIndex,
|
||||
"sent_at_unix_nano": sentAt.UnixNano()})
|
||||
return dataPacket{Topic: message.Topic, Seq: message.Seq,
|
||||
Head: map[string]any{"x-im-client-mid": message.ClientMid}, Content: content}
|
||||
}
|
||||
|
||||
@@ -116,6 +116,12 @@ func connectWSClient(ctx context.Context, cfg *Config, account AccountRecord, ap
|
||||
return nil, fmt.Errorf("login %s: %w", account.Username, err)
|
||||
}
|
||||
client.applyLogin(ctrl)
|
||||
pingInterval, err := configDuration(cfg.WSPingInterval)
|
||||
if err != nil {
|
||||
client.Close()
|
||||
return nil, err
|
||||
}
|
||||
client.startKeepalive(pingInterval)
|
||||
return client, nil
|
||||
}
|
||||
|
||||
@@ -245,7 +251,8 @@ func (c *WSClient) Publish(ctx context.Context, topic, clientMid, runID, scenari
|
||||
"id": id, "topic": topic, "noecho": false,
|
||||
"head": map[string]any{"mime": "application/json", "x-im-schema": 1, "x-im-type": "text", "x-im-client-mid": clientMid},
|
||||
"content": map[string]any{"text": "loadtest", "run_id": runID, "scenario": scenario,
|
||||
"sender_index": c.account.Index, "message_index": messageIndex, "sent_at_unix_nano": sentAt.UnixNano()},
|
||||
"sender_machine": c.account.Machine, "sender_index": c.account.Index,
|
||||
"message_index": messageIndex, "sent_at_unix_nano": sentAt.UnixNano()},
|
||||
}})
|
||||
if err != nil {
|
||||
return AcceptedMessage{}, err
|
||||
@@ -337,7 +344,10 @@ func (c *WSClient) removeWait(id string) {
|
||||
}
|
||||
|
||||
func (c *WSClient) readLoop() {
|
||||
defer close(c.done)
|
||||
defer func() {
|
||||
_ = c.conn.Close()
|
||||
close(c.done)
|
||||
}()
|
||||
for {
|
||||
var envelope map[string]json.RawMessage
|
||||
if err := c.conn.ReadJSON(&envelope); err != nil {
|
||||
@@ -368,6 +378,37 @@ func (c *WSClient) readLoop() {
|
||||
}
|
||||
}
|
||||
|
||||
func (c *WSClient) startKeepalive(interval time.Duration) {
|
||||
go func() {
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
deadline := time.Now().Add(interval / 2)
|
||||
if err := c.conn.WriteControl(websocket.PingMessage, nil, deadline); err != nil {
|
||||
_ = c.conn.Close()
|
||||
return
|
||||
}
|
||||
case <-c.done:
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func (c *WSClient) alive() bool {
|
||||
if c == nil || c.closed.Load() {
|
||||
return false
|
||||
}
|
||||
select {
|
||||
case <-c.done:
|
||||
return false
|
||||
default:
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
func (c *WSClient) Close() {
|
||||
if c == nil || !c.closed.CompareAndSwap(false, true) {
|
||||
return
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/websocket"
|
||||
)
|
||||
|
||||
func TestWSClientKeepaliveSendsPing(t *testing.T) {
|
||||
ping := make(chan struct{}, 1)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
conn, err := (&websocket.Upgrader{}).Upgrade(w, r, nil)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer conn.Close()
|
||||
conn.SetPingHandler(func(string) error {
|
||||
select {
|
||||
case ping <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
return conn.WriteControl(websocket.PongMessage, nil, time.Now().Add(time.Second))
|
||||
})
|
||||
for {
|
||||
if _, _, err = conn.ReadMessage(); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
conn, _, err := websocket.DefaultDialer.Dial("ws"+strings.TrimPrefix(server.URL, "http"), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client := &WSClient{conn: conn, waits: make(map[string]chan ctrlPacket), done: make(chan struct{})}
|
||||
go client.readLoop()
|
||||
client.startKeepalive(10 * time.Millisecond)
|
||||
defer client.Close()
|
||||
|
||||
select {
|
||||
case <-ping:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("websocket keepalive ping was not received")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWSClientReadLoopClosesConnectionAfterDecodeError(t *testing.T) {
|
||||
serverSawClose := make(chan struct{}, 1)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
conn, err := (&websocket.Upgrader{}).Upgrade(w, r, nil)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer conn.Close()
|
||||
if err = conn.WriteMessage(websocket.TextMessage, []byte("{")); err != nil {
|
||||
return
|
||||
}
|
||||
if _, _, err = conn.ReadMessage(); err != nil {
|
||||
serverSawClose <- struct{}{}
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
conn, _, err := websocket.DefaultDialer.Dial("ws"+strings.TrimPrefix(server.URL, "http"), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client := &WSClient{conn: conn, waits: make(map[string]chan ctrlPacket), done: make(chan struct{})}
|
||||
go client.readLoop()
|
||||
|
||||
select {
|
||||
case <-client.done:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("read loop did not exit after malformed frame")
|
||||
}
|
||||
select {
|
||||
case <-serverSawClose:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("read loop exited without closing the underlying connection")
|
||||
}
|
||||
}
|
||||
@@ -49,6 +49,20 @@ spec:
|
||||
summary: "IM WebSocket connections are imbalanced across pods"
|
||||
- name: im-server-persistence
|
||||
rules:
|
||||
- alert: IMDatabasePoolHigh
|
||||
expr: im_db_pool_in_use_connections / clamp_min(im_db_pool_max_open_connections, 1) > 0.85
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "IM database pool usage is above 85%"
|
||||
- alert: IMDatabasePoolCritical
|
||||
expr: im_db_pool_in_use_connections / clamp_min(im_db_pool_max_open_connections, 1) > 0.95
|
||||
for: 30s
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "IM database pool usage is above 95%"
|
||||
- alert: IMKafkaPersistDeadLetter
|
||||
expr: sum by (kind, reason) (increase(im_kafka_dead_letters_total[5m])) > 0
|
||||
labels:
|
||||
|
||||
@@ -743,7 +743,9 @@ func (Cluster) TopicProxy(msg *ClusterResp, unused *bool) error {
|
||||
msg.SrvMsg.uid = types.ParseUserId(msg.SrvMsg.AsUser)
|
||||
select {
|
||||
case t.proxy <- msg:
|
||||
immetrics.ObserveInternalQueueEnqueue("cluster_topic_proxy", len(t.proxy), cap(t.proxy), true)
|
||||
default:
|
||||
immetrics.ObserveInternalQueueEnqueue("cluster_topic_proxy", len(t.proxy), cap(t.proxy), false)
|
||||
logs.Warn.Printf("cluster: proxy channel full, topic %s", msg.RcptTo)
|
||||
}
|
||||
} else {
|
||||
|
||||
@@ -260,6 +260,12 @@ type TenantRoutedAdapter interface {
|
||||
TenantAdapterFor(t.TenantID) (TenantBusinessAdapter, TenantAuthAdapter, error)
|
||||
}
|
||||
|
||||
// TenantRouteCacheInvalidator clears cached tenant routing metadata. A zero
|
||||
// tenant ID clears every cached route.
|
||||
type TenantRouteCacheInvalidator interface {
|
||||
InvalidateTenantRoute(t.TenantID)
|
||||
}
|
||||
|
||||
type TenantMessageBatchAdapter interface {
|
||||
TenantMessageSaveBatch(t.TenantID, []*t.Message) error
|
||||
TenantGroupMessageSaveBatch(t.TenantID, []*t.Message) error
|
||||
|
||||
+105
-40
@@ -21,8 +21,10 @@ import (
|
||||
"github.com/tinode/chat/server/auth"
|
||||
dbadapter "github.com/tinode/chat/server/db"
|
||||
"github.com/tinode/chat/server/db/common"
|
||||
immetrics "github.com/tinode/chat/server/metrics"
|
||||
"github.com/tinode/chat/server/store"
|
||||
t "github.com/tinode/chat/server/store/types"
|
||||
"golang.org/x/sync/singleflight"
|
||||
)
|
||||
|
||||
// adapter holds MySQL connection data.
|
||||
@@ -42,14 +44,17 @@ type adapter struct {
|
||||
// DB transaction timeout.
|
||||
txTimeout time.Duration
|
||||
|
||||
baseConfig *configType
|
||||
tenantDBRouting bool
|
||||
tenantScopedDB bool
|
||||
tenantID t.TenantID
|
||||
tenantRegistryDB string
|
||||
tenantAdapters map[t.TenantID]*adapter
|
||||
tenantMu sync.Mutex
|
||||
groupShardWindow string
|
||||
baseConfig *configType
|
||||
tenantDBRouting bool
|
||||
tenantScopedDB bool
|
||||
tenantID t.TenantID
|
||||
tenantRegistryDB string
|
||||
tenantAdapters map[t.TenantID]*adapter
|
||||
tenantRouteExpiry map[t.TenantID]time.Time
|
||||
tenantRouteGen uint64
|
||||
tenantRouteRefresh singleflight.Group
|
||||
tenantMu sync.Mutex
|
||||
groupShardWindow string
|
||||
}
|
||||
|
||||
const (
|
||||
@@ -62,6 +67,7 @@ const (
|
||||
defaultMaxResults = 1024
|
||||
// This is capped by the Session's send queue limit (128).
|
||||
defaultMaxMessageResults = 100
|
||||
tenantRouteCacheTTL = time.Minute
|
||||
|
||||
// If DB request timeout is specified,
|
||||
// we allocate txTimeoutMultiplier times more time for transactions.
|
||||
@@ -109,6 +115,9 @@ type configType struct {
|
||||
MaxOpenConns int `json:"max_open_conns,omitempty"`
|
||||
// Maximum number of connections in the idle connection pool.
|
||||
MaxIdleConns int `json:"max_idle_conns,omitempty"`
|
||||
// Registry database has a separate, much smaller workload and pool budget.
|
||||
RegistryMaxOpenConns int `json:"registry_max_open_conns,omitempty"`
|
||||
RegistryMaxIdleConns int `json:"registry_max_idle_conns,omitempty"`
|
||||
// Maximum amount of time a connection may be reused (in seconds).
|
||||
ConnMaxLifetime int `json:"conn_max_lifetime,omitempty"`
|
||||
|
||||
@@ -185,6 +194,7 @@ func openMySQLDB(config configType, dsn, dbName string) (*sqlx.DB, string, error
|
||||
if config.ConnMaxLifetime > 0 {
|
||||
db.SetConnMaxLifetime(time.Duration(config.ConnMaxLifetime) * time.Second)
|
||||
}
|
||||
immetrics.RegisterDBPool(dbName, db.DB)
|
||||
return db, dbName, nil
|
||||
}
|
||||
|
||||
@@ -257,6 +267,7 @@ func (a *adapter) Open(jsonconfig json.RawMessage) error {
|
||||
a.tenantID = defaultTenantID
|
||||
a.tenantRegistryDB = strings.TrimSpace(config.TenantRegistryDB)
|
||||
a.tenantAdapters = make(map[t.TenantID]*adapter)
|
||||
a.tenantRouteExpiry = make(map[t.TenantID]time.Time)
|
||||
|
||||
if a.maxResults <= 0 {
|
||||
a.maxResults = defaultMaxResults
|
||||
@@ -281,7 +292,14 @@ func (a *adapter) Open(jsonconfig json.RawMessage) error {
|
||||
a.txTimeout = time.Duration(float64(config.SqlTimeout)*txTimeoutMultiplier) * time.Second
|
||||
}
|
||||
if a.tenantRegistryDB != "" && a.tenantRegistryDB != a.dbName {
|
||||
a.registryDB, _, err = openMySQLDB(config, a.dsn, a.tenantRegistryDB)
|
||||
registryConfig := config
|
||||
if config.RegistryMaxOpenConns > 0 {
|
||||
registryConfig.MaxOpenConns = config.RegistryMaxOpenConns
|
||||
}
|
||||
if config.RegistryMaxIdleConns > 0 {
|
||||
registryConfig.MaxIdleConns = config.RegistryMaxIdleConns
|
||||
}
|
||||
a.registryDB, _, err = openMySQLDB(registryConfig, a.dsn, a.tenantRegistryDB)
|
||||
if err == nil {
|
||||
err = a.registryDB.Ping()
|
||||
}
|
||||
@@ -308,6 +326,8 @@ func (a *adapter) Close() error {
|
||||
}
|
||||
delete(a.tenantAdapters, tenantID)
|
||||
}
|
||||
a.tenantRouteExpiry = make(map[t.TenantID]time.Time)
|
||||
a.tenantRouteGen++
|
||||
a.tenantMu.Unlock()
|
||||
if a.registryDB != nil {
|
||||
if closeErr := a.registryDB.Close(); err == nil {
|
||||
@@ -463,35 +483,62 @@ func (a *adapter) TenantListActive(limit int) ([]t.Tenant, error) {
|
||||
}
|
||||
|
||||
func (a *adapter) TenantAdapterFor(tenantID t.TenantID) (dbadapter.TenantBusinessAdapter, dbadapter.TenantAuthAdapter, error) {
|
||||
if !a.tenantDBRouting {
|
||||
if a.tenantID != defaultTenantID && a.tenantID != tenantID {
|
||||
return nil, nil, t.ErrMalformed
|
||||
}
|
||||
if a.registryDB != nil {
|
||||
tenant, err := a.tenantByID(tenantID)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if tenant.State != t.TenantStateActive {
|
||||
return nil, nil, t.ErrPermissionDenied
|
||||
}
|
||||
}
|
||||
return a, a, nil
|
||||
if !tenantID.IsValid() || (!a.tenantDBRouting && a.tenantID != defaultTenantID && a.tenantID != tenantID) {
|
||||
return nil, nil, t.ErrMalformed
|
||||
}
|
||||
tenant, err := a.tenantByID(tenantID)
|
||||
if tenantAdp, ok := a.cachedTenantAdapter(tenantID, time.Now()); ok {
|
||||
return tenantAdp, tenantAdp, nil
|
||||
}
|
||||
|
||||
resolved, err, _ := a.tenantRouteRefresh.Do(strconv.FormatInt(int64(tenantID), 10), func() (any, error) {
|
||||
if tenantAdp, ok := a.cachedTenantAdapter(tenantID, time.Now()); ok {
|
||||
return tenantAdp, nil
|
||||
}
|
||||
generation := a.tenantRouteGeneration()
|
||||
if a.registryDB != nil {
|
||||
if _, err := a.tenantByID(tenantID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
tenantAdp := a.tenantAdapter(tenantID)
|
||||
a.cacheTenantAdapter(tenantID, tenantAdp, generation, time.Now().Add(tenantRouteCacheTTL))
|
||||
return tenantAdp, nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if tenant.State != t.TenantStateActive {
|
||||
return nil, nil, t.ErrPermissionDenied
|
||||
tenantAdp := resolved.(*adapter)
|
||||
return tenantAdp, tenantAdp, nil
|
||||
}
|
||||
|
||||
func (a *adapter) cachedTenantAdapter(tenantID t.TenantID, now time.Time) (*adapter, bool) {
|
||||
a.tenantMu.Lock()
|
||||
defer a.tenantMu.Unlock()
|
||||
if expiresAt, ok := a.tenantRouteExpiry[tenantID]; !ok || !now.Before(expiresAt) {
|
||||
return nil, false
|
||||
}
|
||||
if !a.tenantDBRouting {
|
||||
return a, true
|
||||
}
|
||||
tenantAdp := a.tenantAdapters[tenantID]
|
||||
return tenantAdp, tenantAdp != nil
|
||||
}
|
||||
|
||||
func (a *adapter) tenantRouteGeneration() uint64 {
|
||||
a.tenantMu.Lock()
|
||||
defer a.tenantMu.Unlock()
|
||||
return a.tenantRouteGen
|
||||
}
|
||||
|
||||
func (a *adapter) tenantAdapter(tenantID t.TenantID) *adapter {
|
||||
if !a.tenantDBRouting {
|
||||
return a
|
||||
}
|
||||
a.tenantMu.Lock()
|
||||
if tenantAdp := a.tenantAdapters[tenantID]; tenantAdp != nil {
|
||||
a.tenantMu.Unlock()
|
||||
return tenantAdp, tenantAdp, nil
|
||||
defer a.tenantMu.Unlock()
|
||||
if existing := a.tenantAdapters[tenantID]; existing != nil {
|
||||
return existing
|
||||
}
|
||||
a.tenantMu.Unlock()
|
||||
|
||||
registryDB := a.registryDB
|
||||
if registryDB == nil {
|
||||
registryDB = a.db
|
||||
@@ -513,14 +560,32 @@ func (a *adapter) TenantAdapterFor(tenantID t.TenantID) (dbadapter.TenantBusines
|
||||
// explicit tenant_id writes enabled for logical isolation.
|
||||
tenantScopedDB: false,
|
||||
}
|
||||
a.tenantMu.Lock()
|
||||
if existing := a.tenantAdapters[tenantID]; existing != nil {
|
||||
a.tenantMu.Unlock()
|
||||
return existing, existing, nil
|
||||
}
|
||||
a.tenantAdapters[tenantID] = tenantAdp
|
||||
a.tenantMu.Unlock()
|
||||
return tenantAdp, tenantAdp, nil
|
||||
return tenantAdp
|
||||
}
|
||||
|
||||
func (a *adapter) cacheTenantAdapter(tenantID t.TenantID, tenantAdp *adapter, generation uint64, expiresAt time.Time) {
|
||||
a.tenantMu.Lock()
|
||||
defer a.tenantMu.Unlock()
|
||||
if generation != a.tenantRouteGen || tenantAdp == nil {
|
||||
return
|
||||
}
|
||||
if a.tenantRouteExpiry == nil {
|
||||
a.tenantRouteExpiry = make(map[t.TenantID]time.Time)
|
||||
}
|
||||
a.tenantRouteExpiry[tenantID] = expiresAt
|
||||
}
|
||||
|
||||
// InvalidateTenantRoute forces the next operation to revalidate tenant state.
|
||||
func (a *adapter) InvalidateTenantRoute(tenantID t.TenantID) {
|
||||
a.tenantMu.Lock()
|
||||
defer a.tenantMu.Unlock()
|
||||
a.tenantRouteGen++
|
||||
if tenantID.IsValid() {
|
||||
delete(a.tenantRouteExpiry, tenantID)
|
||||
return
|
||||
}
|
||||
a.tenantRouteExpiry = make(map[t.TenantID]time.Time)
|
||||
}
|
||||
|
||||
func (a *adapter) tenantByID(tenantID t.TenantID) (*t.Tenant, error) {
|
||||
@@ -4671,7 +4736,7 @@ func (a *adapter) FileUserHasAccess(fid string, uid t.Uid) (bool, error) {
|
||||
return a.db.GetContext(ctx, &allowed, query, args...)
|
||||
}
|
||||
err := run()
|
||||
for attempt := 0; isRetryableMySQLReadErr(err) && attempt < 2; attempt++ {
|
||||
for attempt := 0; isRetryableMySQLError(err) && attempt < 2; attempt++ {
|
||||
time.Sleep(time.Duration(attempt+1) * 25 * time.Millisecond)
|
||||
err = run()
|
||||
}
|
||||
@@ -4813,7 +4878,7 @@ func isIDOutOfRange(err error) bool {
|
||||
return ok && myerr.Number == 1264 && strings.Contains(strings.ToLower(myerr.Message), "id")
|
||||
}
|
||||
|
||||
func isRetryableMySQLReadErr(err error) bool {
|
||||
func isRetryableMySQLError(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
//go:build mysql
|
||||
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/tinode/chat/server/store/types"
|
||||
)
|
||||
|
||||
func TestTenantRouteCacheExpiresAndInvalidates(t *testing.T) {
|
||||
now := time.Now()
|
||||
tenantID := types.TenantID(8)
|
||||
tenantAdp := &adapter{tenantID: tenantID}
|
||||
root := &adapter{
|
||||
tenantDBRouting: true,
|
||||
tenantAdapters: map[types.TenantID]*adapter{tenantID: tenantAdp},
|
||||
tenantRouteExpiry: make(map[types.TenantID]time.Time),
|
||||
}
|
||||
|
||||
root.cacheTenantAdapter(tenantID, tenantAdp, root.tenantRouteGeneration(), now.Add(time.Minute))
|
||||
if got, ok := root.cachedTenantAdapter(tenantID, now); !ok || got != tenantAdp {
|
||||
t.Fatalf("cached route = %p, %v; want %p, true", got, ok, tenantAdp)
|
||||
}
|
||||
if got, _, err := root.TenantAdapterFor(tenantID); err != nil || got != tenantAdp {
|
||||
t.Fatalf("TenantAdapterFor cached route = %p, %v; want %p, nil", got, err, tenantAdp)
|
||||
}
|
||||
|
||||
staleGeneration := root.tenantRouteGeneration()
|
||||
root.InvalidateTenantRoute(tenantID)
|
||||
if _, ok := root.cachedTenantAdapter(tenantID, now); ok {
|
||||
t.Fatal("invalidated tenant route remained cached")
|
||||
}
|
||||
root.cacheTenantAdapter(tenantID, tenantAdp, staleGeneration, now.Add(time.Minute))
|
||||
if _, ok := root.cachedTenantAdapter(tenantID, now); ok {
|
||||
t.Fatal("an in-flight stale refresh repopulated the route cache")
|
||||
}
|
||||
|
||||
root.cacheTenantAdapter(tenantID, tenantAdp, root.tenantRouteGeneration(), now.Add(-time.Second))
|
||||
if _, ok := root.cachedTenantAdapter(tenantID, now); ok {
|
||||
t.Fatal("expired tenant route remained cached")
|
||||
}
|
||||
}
|
||||
|
||||
func TestInvalidateAllTenantRoutes(t *testing.T) {
|
||||
now := time.Now()
|
||||
root := &adapter{
|
||||
tenantDBRouting: true,
|
||||
tenantAdapters: map[types.TenantID]*adapter{
|
||||
8: {tenantID: 8},
|
||||
9: {tenantID: 9},
|
||||
},
|
||||
tenantRouteExpiry: map[types.TenantID]time.Time{
|
||||
8: now.Add(time.Minute),
|
||||
9: now.Add(time.Minute),
|
||||
},
|
||||
}
|
||||
|
||||
root.InvalidateTenantRoute(0)
|
||||
for _, tenantID := range []types.TenantID{8, 9} {
|
||||
if _, ok := root.cachedTenantAdapter(tenantID, now); ok {
|
||||
t.Fatalf("tenant %d route remained cached after global invalidation", tenantID)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -17,6 +17,7 @@ func ensureIMAdminGovernanceTables(exec sqlExecutor) error {
|
||||
request_id VARCHAR(64) NOT NULL DEFAULT '', login_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
|
||||
logout_at DATETIME(3), metadata JSON, created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
|
||||
PRIMARY KEY(id), INDEX idx_im_user_login_tenant_time(tenant_id, login_at DESC, id DESC),
|
||||
INDEX idx_im_user_login_session_close(tenant_id, session_id, logout_at),
|
||||
INDEX idx_im_user_login_user_time(user_id, login_at DESC, id DESC),
|
||||
INDEX idx_im_user_login_ip_time(ip, login_at DESC, id DESC),
|
||||
INDEX idx_im_user_login_device_time(device_id(64), login_at DESC),
|
||||
@@ -53,5 +54,9 @@ func ensureIMAdminGovernanceTables(exec sqlExecutor) error {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := execIgnoreMySQLError(exec,
|
||||
`ALTER TABLE im_user_login_logs ADD INDEX idx_im_user_login_session_close(tenant_id,session_id,logout_at)`, 1061); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -13,8 +13,8 @@ func TestEnsureIMAdminGovernanceTablesDefinesLoginAuditTenantScope(t *testing.T)
|
||||
if err := ensureIMAdminGovernanceTables(exec); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(exec.statements) != 3 {
|
||||
t.Fatalf("created %d admin governance tables, want 3", len(exec.statements))
|
||||
if len(exec.statements) != 4 {
|
||||
t.Fatalf("executed %d admin governance schema statements, want 4", len(exec.statements))
|
||||
}
|
||||
joined := strings.ToLower(strings.Join(exec.statements, "\n"))
|
||||
for _, table := range []string{"im_user_login_logs", "im_message_admin_actions", "im_user_ip_whitelists"} {
|
||||
@@ -38,4 +38,8 @@ func TestEnsureIMAdminGovernanceTablesDefinesLoginAuditTenantScope(t *testing.T)
|
||||
if end < 0 || !strings.Contains(joined[start:start+end], "tenant_id bigint not null") {
|
||||
t.Fatal("im_user_login_logs must declare tenant_id")
|
||||
}
|
||||
if !strings.Contains(joined, "idx_im_user_login_session_close(tenant_id, session_id, logout_at)") &&
|
||||
!strings.Contains(joined, "idx_im_user_login_session_close(tenant_id,session_id,logout_at)") {
|
||||
t.Fatal("im_user_login_logs must index tenant-scoped session close updates")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3305,7 +3305,7 @@ func (a *adapter) tenantP2PMessageMutations(ctx context.Context, id t.TenantID,
|
||||
fmt.Sprintf("FROM %s AS m ", tenantMessagesTable(id))+
|
||||
"LEFT JOIN im_dellog AS d ON d.tenant_id=m.tenant_id AND d.topic=m.topic AND m.seqid BETWEEN d.low AND d.hi-1 AND d.deletedfor=? "+
|
||||
"LEFT JOIN im_message_revisions AS rev ON rev.tenant_id=m.tenant_id AND rev.topic=m.topic AND rev.seqid=m.seqid AND rev.version=m.version-1 "+
|
||||
"WHERE m.tenant_id=? AND m.topic=? AND d.deletedfor IS NULL AND (m.version>1 OR m.recalled_at IS NOT NULL OR m.deletedat IS NOT NULL) "+
|
||||
"WHERE m.tenant_id=? AND m.topic=? AND d.deletedfor IS NULL AND (m.version>1 OR m.recalled_at IS NOT NULL) "+
|
||||
"ORDER BY m.updatedat DESC,m.seqid DESC LIMIT ?",
|
||||
decodeUID(uid), id, topic, limit+1)
|
||||
if err != nil {
|
||||
@@ -3327,7 +3327,7 @@ func (a *adapter) tenantGroupMessageMutations(ctx context.Context, id t.TenantID
|
||||
fmt.Sprintf("FROM %s AS m ", table)+
|
||||
"LEFT JOIN im_dellog AS d ON d.tenant_id=m.tenant_id AND d.topic COLLATE utf8mb4_unicode_ci=m.topic AND m.seqid BETWEEN d.low AND d.hi-1 AND d.deletedfor=? "+
|
||||
"LEFT JOIN im_message_revisions AS rev ON rev.tenant_id=m.tenant_id AND rev.topic=m.topic AND rev.seqid=m.seqid AND rev.version=m.version-1 "+
|
||||
"WHERE m.tenant_id=? AND m.topic=? AND d.deletedfor IS NULL AND (m.version>1 OR m.recalled_at IS NOT NULL OR m.deleted_at IS NOT NULL OR m.status='recalled')")
|
||||
"WHERE m.tenant_id=? AND m.topic=? AND d.deletedfor IS NULL AND (m.version>1 OR m.recalled_at IS NOT NULL OR m.status='recalled')")
|
||||
args = append(args, decodeUID(uid), id, topic)
|
||||
}
|
||||
args = append(args, limit+1)
|
||||
|
||||
@@ -93,7 +93,18 @@ func (a *adapter) TenantLoginAuditClose(id t.TenantID, sessionID string, at time
|
||||
if tenantAdp != a {
|
||||
return tenantAdp.TenantLoginAuditClose(id, sessionID, at)
|
||||
}
|
||||
_, err = a.db.Exec("UPDATE im_user_login_logs SET logout_at=? WHERE tenant_id=? AND session_id=? AND logout_at IS NULL", at, int64(id), sessionID)
|
||||
return retryMySQLDeadlock(func() error {
|
||||
_, execErr := a.db.Exec("UPDATE im_user_login_logs SET logout_at=? WHERE tenant_id=? AND session_id=? AND logout_at IS NULL", at, int64(id), sessionID)
|
||||
return execErr
|
||||
}, time.Sleep)
|
||||
}
|
||||
|
||||
func retryMySQLDeadlock(operation func() error, sleep func(time.Duration)) error {
|
||||
err := operation()
|
||||
for attempt := 1; isRetryableMySQLError(err) && attempt < 4; attempt++ {
|
||||
sleep(time.Duration(attempt) * 25 * time.Millisecond)
|
||||
err = operation()
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
|
||||
@@ -3,9 +3,13 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
mysqldriver "github.com/go-sql-driver/mysql"
|
||||
)
|
||||
|
||||
func TestLoginAuditQueriesAreTenantScoped(t *testing.T) {
|
||||
@@ -24,3 +28,48 @@ func TestLoginAuditQueriesAreTenantScoped(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetryMySQLDeadlockEventuallySucceeds(t *testing.T) {
|
||||
calls := 0
|
||||
var delays []time.Duration
|
||||
err := retryMySQLDeadlock(func() error {
|
||||
calls++
|
||||
if calls < 3 {
|
||||
return &mysqldriver.MySQLError{Number: 1213, SQLState: [5]byte{'4', '0', '0', '0', '1'}}
|
||||
}
|
||||
return nil
|
||||
}, func(delay time.Duration) { delays = append(delays, delay) })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if calls != 3 {
|
||||
t.Fatalf("operation calls=%d want=3", calls)
|
||||
}
|
||||
want := []time.Duration{25 * time.Millisecond, 50 * time.Millisecond}
|
||||
if len(delays) != len(want) || delays[0] != want[0] || delays[1] != want[1] {
|
||||
t.Fatalf("retry delays=%v want=%v", delays, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetryMySQLDeadlockStopsOnNonRetryableError(t *testing.T) {
|
||||
want := errors.New("invalid update")
|
||||
calls := 0
|
||||
err := retryMySQLDeadlock(func() error {
|
||||
calls++
|
||||
return want
|
||||
}, func(time.Duration) { t.Fatal("non-retryable error must not sleep") })
|
||||
if !errors.Is(err, want) || calls != 1 {
|
||||
t.Fatalf("err=%v calls=%d want=%v/1", err, calls, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetryMySQLDeadlockStopsAfterFourAttempts(t *testing.T) {
|
||||
calls := 0
|
||||
err := retryMySQLDeadlock(func() error {
|
||||
calls++
|
||||
return &mysqldriver.MySQLError{Number: 1205}
|
||||
}, func(time.Duration) {})
|
||||
if err == nil || calls != 4 {
|
||||
t.Fatalf("err=%v calls=%d want non-nil/4", err, calls)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
|
||||
"github.com/tinode/chat/server/auth"
|
||||
"github.com/tinode/chat/server/logs"
|
||||
immetrics "github.com/tinode/chat/server/metrics"
|
||||
"github.com/tinode/chat/server/store"
|
||||
"github.com/tinode/chat/server/store/types"
|
||||
)
|
||||
@@ -247,7 +248,9 @@ func (h *Hub) run() {
|
||||
if dst.clientMsg != nil {
|
||||
select {
|
||||
case dst.clientMsg <- msg:
|
||||
immetrics.ObserveInternalQueueEnqueue("topic_client", len(dst.clientMsg), cap(dst.clientMsg), true)
|
||||
default:
|
||||
immetrics.ObserveInternalQueueEnqueue("topic_client", len(dst.clientMsg), cap(dst.clientMsg), false)
|
||||
logs.Err.Println("hub: topic's broadcast queue is full", dst.name)
|
||||
}
|
||||
} else {
|
||||
@@ -275,7 +278,9 @@ func (h *Hub) run() {
|
||||
// Everything is OK, sending packet to known topic
|
||||
select {
|
||||
case dst.serverMsg <- msg:
|
||||
immetrics.ObserveInternalQueueEnqueue("topic_server", len(dst.serverMsg), cap(dst.serverMsg), true)
|
||||
default:
|
||||
immetrics.ObserveInternalQueueEnqueue("topic_server", len(dst.serverMsg), cap(dst.serverMsg), false)
|
||||
logs.Err.Println("hub: topic's broadcast queue is full", dst.name)
|
||||
}
|
||||
} else if (strings.HasPrefix(msg.RcptTo, "usr") || strings.HasPrefix(msg.RcptTo, "grp")) &&
|
||||
|
||||
+38
-4
@@ -3133,6 +3133,25 @@ func serveIMTopicCalls(wrt http.ResponseWriter, req *http.Request) {
|
||||
writeIMSuccess(wrt, req, map[string]any{"topic": topic, "items": callRecordPayloads(items, principal.Uid), "next_cursor": nextCursor})
|
||||
}
|
||||
|
||||
// serveIMCallsByTopic serves GET /v1/im/calls/{topic} — 对接文档契约路径，按会话查通话记录。
|
||||
func serveIMCallsByTopic(wrt http.ResponseWriter, req *http.Request) {
|
||||
if req.Method != http.MethodGet {
|
||||
writeIMError(wrt, req, http.StatusMethodNotAllowed, 30002, imReasonMethodNotAllowed, "method not allowed", nil)
|
||||
return
|
||||
}
|
||||
principal, topic, ok := authenticateTopicReader(wrt, req, topicFromCallsByTopicPath(req.URL.Path))
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
limit := parseIMLimit(req.URL.Query().Get("limit"), 50, 100)
|
||||
items, nextCursor, err := store.Calls.List(principal.TenantID, principal.Uid, topic, strings.TrimSpace(req.URL.Query().Get("cursor")), limit)
|
||||
if err != nil {
|
||||
writeIMRelationError(wrt, req, err)
|
||||
return
|
||||
}
|
||||
writeIMSuccess(wrt, req, map[string]any{"topic": topic, "items": callRecordPayloads(items, principal.Uid), "next_cursor": nextCursor})
|
||||
}
|
||||
|
||||
func serveIMMessageSearch(wrt http.ResponseWriter, req *http.Request) {
|
||||
if req.Method != http.MethodGet {
|
||||
writeIMError(wrt, req, http.StatusMethodNotAllowed, 30002, imReasonMethodNotAllowed, "method not allowed", nil)
|
||||
@@ -5794,10 +5813,16 @@ func serveIMGroupMemberProfile(wrt http.ResponseWriter, req *http.Request, topic
|
||||
writeIMError(wrt, req, http.StatusInternalServerError, 90000, imReasonInternal, "internal error", nil)
|
||||
return
|
||||
}
|
||||
ctx, err := store.Relations.GetRelationshipContext(principal.TenantID, principal.Uid, memberID, principal.Uid.P2PName(memberID))
|
||||
if err != nil {
|
||||
writeIMRelationError(wrt, req, err)
|
||||
return
|
||||
// 查询本人时 GetRelationshipContext 不接受 userID==peerID（会报 ErrMalformed），
|
||||
// 直接构造 self 上下文，群内本人资料返回 200。
|
||||
ctx := &types.RelationshipContext{Relationship: types.RelationshipSelf, CanSend: true}
|
||||
if memberID != principal.Uid {
|
||||
var err error
|
||||
ctx, err = store.Relations.GetRelationshipContext(principal.TenantID, principal.Uid, memberID, principal.Uid.P2PName(memberID))
|
||||
if err != nil {
|
||||
writeIMRelationError(wrt, req, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
writeIMSuccess(wrt, req, groupMemberProfilePayload(topic, memberSub, user, ctx))
|
||||
}
|
||||
@@ -11006,6 +11031,15 @@ func topicFromCallsPath(path string) string {
|
||||
return topicFromPathWithSuffix(path, prefix, suffix)
|
||||
}
|
||||
|
||||
func topicFromCallsByTopicPath(path string) string {
|
||||
const prefix = "/v1/im/calls/"
|
||||
idx := strings.Index(path, prefix)
|
||||
if idx < 0 {
|
||||
return ""
|
||||
}
|
||||
return strings.Trim(strings.TrimSpace(path[idx+len(prefix):]), "/")
|
||||
}
|
||||
|
||||
func topicFromDraftPath(path string) string {
|
||||
const prefix = "/v1/im/topics/"
|
||||
const suffix = "/draft"
|
||||
|
||||
@@ -47,6 +47,12 @@ func imYouthModePolicyDecision(tenantID types.TenantID, actor types.Uid, action
|
||||
if store.Security == nil || actor.IsZero() {
|
||||
return imAllowDecision()
|
||||
}
|
||||
// Message content filtering is enforced by moderation below the policy
|
||||
// check. Youth mode currently has no message-level deny rule, so avoid a
|
||||
// database lookup on every ordinary publish.
|
||||
if strings.TrimSpace(action) == imYouthActionMessage {
|
||||
return imAllowDecision()
|
||||
}
|
||||
settings, err := store.Security.GetYouthMode(tenantID, actor)
|
||||
if err != nil {
|
||||
if !errors.Is(err, types.ErrNotFound) && !errors.Is(err, types.ErrUnsupported) {
|
||||
@@ -65,10 +71,6 @@ func imYouthModePolicyDecision(tenantID types.TenantID, actor types.Uid, action
|
||||
"policy_version": settings.PolicyVersion,
|
||||
})
|
||||
}
|
||||
case imYouthActionMessage:
|
||||
// content_filter is enforced by routing messages through content moderation
|
||||
// before persistence; the decision remains allow unless moderation rejects it.
|
||||
return imAllowDecision()
|
||||
}
|
||||
return imAllowDecision()
|
||||
}
|
||||
|
||||
@@ -33,3 +33,18 @@ func TestYouthModeNightQuietRestrictsCalls(t *testing.T) {
|
||||
t.Fatalf("daytime call should be allowed: %#v", decision)
|
||||
}
|
||||
}
|
||||
|
||||
func TestYouthModeOrdinaryMessageDoesNotReadDatabase(t *testing.T) {
|
||||
oldSecurity := store.Security
|
||||
security := &testSecurityStore{err: types.ErrInternal}
|
||||
store.Security = security
|
||||
defer func() { store.Security = oldSecurity }()
|
||||
|
||||
decision := imYouthModePolicyDecision(7, types.Uid(42), imYouthActionMessage, time.Now())
|
||||
if !decision.Allowed {
|
||||
t.Fatalf("ordinary message should continue to moderation: %#v", decision)
|
||||
}
|
||||
if security.youthModeReads != 0 {
|
||||
t.Fatalf("ordinary message read youth settings %d times", security.youthModeReads)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -53,7 +53,9 @@ func saveMessageMaybeKafka(ctx context.Context, tenantID types.TenantID, msg *ty
|
||||
msg.ClientMsgID = clientMsgID
|
||||
fromUid := types.ParseUid(msg.From)
|
||||
if clientMsgID != "" {
|
||||
started := time.Now()
|
||||
claim, claimed, runtimeReady, claimErr := store.ClaimMessageUnique(tenantID, msg.Topic, fromUid, clientMsgID)
|
||||
immetrics.ObserveMessagePublishStage(messageKind(msg.Topic), "redis_idempotency", started, claimErr)
|
||||
if claimErr != nil {
|
||||
return nil, claimErr
|
||||
}
|
||||
@@ -69,12 +71,16 @@ func saveMessageMaybeKafka(ctx context.Context, tenantID types.TenantID, msg *ty
|
||||
}
|
||||
}
|
||||
candidateMonth := msg.CreatedAt.UTC().Format("200601")
|
||||
if seq, shardMonth, runtimeReady, seqErr := store.ReserveMessageSeq(tenantID, msg.Topic, msg.SeqId-1, candidateMonth); seqErr != nil {
|
||||
seqStarted := time.Now()
|
||||
seq, shardMonth, runtimeReady, seqErr := store.ReserveMessageSeq(tenantID, msg.Topic, msg.SeqId-1, candidateMonth)
|
||||
immetrics.ObserveMessagePublishStage(messageKind(msg.Topic), "redis_sequence", seqStarted, seqErr)
|
||||
if seqErr != nil {
|
||||
if clientMsgID != "" {
|
||||
_ = store.MarkMessageUniqueFailed(tenantID, msg.Topic, fromUid, clientMsgID)
|
||||
}
|
||||
return nil, seqErr
|
||||
} else if runtimeReady {
|
||||
}
|
||||
if runtimeReady {
|
||||
msg.SeqId = seq
|
||||
if isGroupMessage {
|
||||
msg.ShardMonth = shardMonth
|
||||
@@ -83,7 +89,9 @@ func saveMessageMaybeKafka(ctx context.Context, tenantID types.TenantID, msg *ty
|
||||
if msg.ShardMonth == "" && isGroupMessage {
|
||||
msg.ShardMonth = candidateMonth
|
||||
}
|
||||
kafkaStarted := time.Now()
|
||||
if err = enqueueKafkaMessagePersist(ctx, tenantID, msg, attachmentURLs, readBySender); err != nil {
|
||||
immetrics.ObserveMessagePublishStage(messageKind(msg.Topic), "kafka_produce", kafkaStarted, err)
|
||||
if clientMsgID != "" {
|
||||
if markErr := store.MarkMessageUniquePending(tenantID, msg.Topic, fromUid, clientMsgID, msg.SeqId); markErr != nil {
|
||||
logs.Warn.Printf("kafka message persist: failed to mark message unique key pending topic=%s seq=%d: %v", msg.Topic, msg.SeqId, markErr)
|
||||
@@ -92,6 +100,7 @@ func saveMessageMaybeKafka(ctx context.Context, tenantID types.TenantID, msg *ty
|
||||
logs.Warn.Printf("kafka message persist: publish state unknown topic=%s seq=%d client_mid=%s err=%v", msg.Topic, msg.SeqId, clientMsgID, err)
|
||||
return &types.MessageSaveResult{SeqId: msg.SeqId, Pending: true, MarkedReadBySender: readBySender && !fromUid.IsZero()}, err
|
||||
}
|
||||
immetrics.ObserveMessagePublishStage(messageKind(msg.Topic), "kafka_produce", kafkaStarted, nil)
|
||||
if clientMsgID != "" {
|
||||
if err = store.MarkMessageUniquePending(tenantID, msg.Topic, fromUid, clientMsgID, msg.SeqId); err != nil {
|
||||
logs.Warn.Printf("kafka message persist: failed to mark message unique key pending topic=%s seq=%d: %v", msg.Topic, msg.SeqId, err)
|
||||
@@ -105,11 +114,19 @@ func saveMessageMaybeKafka(ctx context.Context, tenantID types.TenantID, msg *ty
|
||||
return &types.MessageSaveResult{SeqId: msg.SeqId, Pending: true, MarkedReadBySender: readBySender && !fromUid.IsZero()}, nil
|
||||
}
|
||||
|
||||
func enqueueKafkaMessagePersist(ctx context.Context, tenantID types.TenantID, msg *types.Message, attachmentURLs []string, readBySender bool) error {
|
||||
kind := "p2p"
|
||||
if types.GetTopicCat(msg.Topic) == types.TopicCatGrp {
|
||||
kind = "group"
|
||||
func messageKind(topic string) string {
|
||||
switch types.GetTopicCat(topic) {
|
||||
case types.TopicCatGrp:
|
||||
return "group"
|
||||
case types.TopicCatP2P:
|
||||
return "p2p"
|
||||
default:
|
||||
return "other"
|
||||
}
|
||||
}
|
||||
|
||||
func enqueueKafkaMessagePersist(ctx context.Context, tenantID types.TenantID, msg *types.Message, attachmentURLs []string, readBySender bool) error {
|
||||
kind := messageKind(msg.Topic)
|
||||
payload, err := json.Marshal(kafkaMessagePersistPayload{
|
||||
Message: *msg,
|
||||
AttachmentURLs: attachmentURLs,
|
||||
|
||||
@@ -182,6 +182,8 @@ var globals struct {
|
||||
tlsRedirectHTTP string
|
||||
// Maximum message size allowed from peer.
|
||||
maxMessageSize int64
|
||||
// Log complete client packets. Disabled by default because it is expensive on the publish hot path.
|
||||
logClientPackets bool
|
||||
// Maximum number of group topic subscribers.
|
||||
maxSubscriberCount int
|
||||
// Maximum number of indexable tags.
|
||||
@@ -335,6 +337,8 @@ type configType struct {
|
||||
// Maximum message size allowed from client. Intended to prevent malicious client from sending
|
||||
// very large files inband (does not affect out of band uploads).
|
||||
MaxMessageSize int `json:"max_message_size"`
|
||||
// Log every decoded client packet. Intended only for short diagnostic sessions.
|
||||
LogClientPackets bool `json:"log_client_packets"`
|
||||
// Maximum number of group topic subscribers.
|
||||
MaxSubscriberCount int `json:"max_subscriber_count"`
|
||||
// Masked tags namespaces: tags immutable on User (mask), mutable on Topic only within the mask.
|
||||
@@ -670,6 +674,7 @@ func main() {
|
||||
if globals.maxMessageSize <= 0 {
|
||||
globals.maxMessageSize = defaultMaxMessageSize
|
||||
}
|
||||
globals.logClientPackets = config.LogClientPackets
|
||||
// Maximum number of group topic subscribers
|
||||
globals.maxSubscriberCount = config.MaxSubscriberCount
|
||||
if globals.maxSubscriberCount <= 1 {
|
||||
@@ -929,6 +934,7 @@ func main() {
|
||||
mux.HandleFunc(config.ApiPath+"v1/im/favorites", serveIMFavorites)
|
||||
mux.HandleFunc(config.ApiPath+"v1/im/favorites/", serveIMFavoriteResource)
|
||||
mux.HandleFunc(config.ApiPath+"v1/im/calls/rtc-join", serveIMCallRTCJoin)
|
||||
mux.HandleFunc(config.ApiPath+"v1/im/calls/", serveIMCallsByTopic)
|
||||
mux.HandleFunc(config.ApiPath+"v1/im/calls", serveIMCalls)
|
||||
mux.HandleFunc(config.ApiPath+"v1/im/topics/", serveIMTopicMessageAction)
|
||||
mux.HandleFunc(config.ApiPath+"v1/im/conversations/", serveIMConversationContext)
|
||||
|
||||
@@ -10,6 +10,7 @@
|
||||
|
||||
"api_key_salt": "T713/rYYgW7g4m3vG6zGRh7+FM1t0T8j13koXScOAj4=",
|
||||
"max_message_size": 131072,
|
||||
"log_client_packets": false,
|
||||
"max_subscriber_count": 128,
|
||||
"max_tag_count": 16,
|
||||
"permanent_accounts": false,
|
||||
@@ -84,8 +85,10 @@
|
||||
"dsn": "mysql://root:oWxz*2=sb),ouW%p@tcp(mysql.118im.com:30306)/meeting_im?parseTime=true&collation=utf8mb4_0900_ai_ci",
|
||||
"tenant_registry_db": "hotgo",
|
||||
"tenant_db_routing": true,
|
||||
"max_open_conns": 64,
|
||||
"max_idle_conns": 64,
|
||||
"max_open_conns": 24,
|
||||
"max_idle_conns": 8,
|
||||
"registry_max_open_conns": 4,
|
||||
"registry_max_idle_conns": 2,
|
||||
"conn_max_lifetime": 60,
|
||||
"sql_timeout": 10
|
||||
}
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
)
|
||||
|
||||
type dbStatsProvider interface {
|
||||
Stats() sql.DBStats
|
||||
}
|
||||
|
||||
type dbPoolCollector struct {
|
||||
pools sync.Map
|
||||
desc map[string]*prometheus.Desc
|
||||
}
|
||||
|
||||
func newDBPoolCollector() *dbPoolCollector {
|
||||
labels := []string{"pod", "pool"}
|
||||
return &dbPoolCollector{desc: map[string]*prometheus.Desc{
|
||||
"max_open": prometheus.NewDesc("im_db_pool_max_open_connections", "Configured maximum open database connections.", labels, nil),
|
||||
"open": prometheus.NewDesc("im_db_pool_open_connections", "Current open database connections.", labels, nil),
|
||||
"in_use": prometheus.NewDesc("im_db_pool_in_use_connections", "Current database connections in use.", labels, nil),
|
||||
"idle": prometheus.NewDesc("im_db_pool_idle_connections", "Current idle database connections.", labels, nil),
|
||||
"wait": prometheus.NewDesc("im_db_pool_wait_total", "Total waits for a database connection.", labels, nil),
|
||||
"wait_duration": prometheus.NewDesc("im_db_pool_wait_duration_seconds_total", "Total time spent waiting for database connections.", labels, nil),
|
||||
"max_idle_closed": prometheus.NewDesc("im_db_pool_max_idle_closed_total", "Connections closed because the idle limit was exceeded.", labels, nil),
|
||||
"max_lifetime_closed": prometheus.NewDesc("im_db_pool_max_lifetime_closed_total", "Connections closed because their lifetime expired.", labels, nil),
|
||||
}}
|
||||
}
|
||||
|
||||
func (c *dbPoolCollector) Describe(ch chan<- *prometheus.Desc) {
|
||||
for _, desc := range c.desc {
|
||||
ch <- desc
|
||||
}
|
||||
}
|
||||
|
||||
func (c *dbPoolCollector) Collect(ch chan<- prometheus.Metric) {
|
||||
pod := metricPodName()
|
||||
c.pools.Range(func(key, value any) bool {
|
||||
pool, ok := key.(string)
|
||||
provider, providerOK := value.(dbStatsProvider)
|
||||
if !ok || !providerOK {
|
||||
return true
|
||||
}
|
||||
stats := provider.Stats()
|
||||
labels := []string{pod, pool}
|
||||
ch <- prometheus.MustNewConstMetric(c.desc["max_open"], prometheus.GaugeValue, float64(stats.MaxOpenConnections), labels...)
|
||||
ch <- prometheus.MustNewConstMetric(c.desc["open"], prometheus.GaugeValue, float64(stats.OpenConnections), labels...)
|
||||
ch <- prometheus.MustNewConstMetric(c.desc["in_use"], prometheus.GaugeValue, float64(stats.InUse), labels...)
|
||||
ch <- prometheus.MustNewConstMetric(c.desc["idle"], prometheus.GaugeValue, float64(stats.Idle), labels...)
|
||||
ch <- prometheus.MustNewConstMetric(c.desc["wait"], prometheus.CounterValue, float64(stats.WaitCount), labels...)
|
||||
ch <- prometheus.MustNewConstMetric(c.desc["wait_duration"], prometheus.CounterValue, stats.WaitDuration.Seconds(), labels...)
|
||||
ch <- prometheus.MustNewConstMetric(c.desc["max_idle_closed"], prometheus.CounterValue, float64(stats.MaxIdleClosed), labels...)
|
||||
ch <- prometheus.MustNewConstMetric(c.desc["max_lifetime_closed"], prometheus.CounterValue, float64(stats.MaxLifetimeClosed), labels...)
|
||||
return true
|
||||
})
|
||||
}
|
||||
|
||||
var dbPools = newDBPoolCollector()
|
||||
|
||||
// RegisterDBPool exposes database/sql pool pressure for one logical database.
|
||||
// Registering the same name again replaces the provider, which keeps tests and
|
||||
// adapter reopen operations safe.
|
||||
func RegisterDBPool(name string, provider dbStatsProvider) {
|
||||
if provider == nil {
|
||||
return
|
||||
}
|
||||
name = strings.TrimSpace(name)
|
||||
if name == "" {
|
||||
name = "default"
|
||||
}
|
||||
dbPools.pools.Store(name, provider)
|
||||
}
|
||||
|
||||
func init() {
|
||||
Registry.MustRegister(dbPools)
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type fixedDBStats struct {
|
||||
stats sql.DBStats
|
||||
}
|
||||
|
||||
func (s fixedDBStats) Stats() sql.DBStats { return s.stats }
|
||||
|
||||
func TestDBPoolCollector(t *testing.T) {
|
||||
RegisterDBPool("test", fixedDBStats{stats: sql.DBStats{
|
||||
MaxOpenConnections: 32,
|
||||
OpenConnections: 12,
|
||||
InUse: 7,
|
||||
Idle: 5,
|
||||
WaitCount: 9,
|
||||
WaitDuration: 1500 * time.Millisecond,
|
||||
}})
|
||||
|
||||
families, err := Registry.Gather()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := map[string]bool{
|
||||
"im_db_pool_max_open_connections": false,
|
||||
"im_db_pool_open_connections": false,
|
||||
"im_db_pool_in_use_connections": false,
|
||||
"im_db_pool_idle_connections": false,
|
||||
"im_db_pool_wait_total": false,
|
||||
"im_db_pool_wait_duration_seconds_total": false,
|
||||
}
|
||||
for _, family := range families {
|
||||
if _, ok := want[family.GetName()]; ok {
|
||||
want[family.GetName()] = true
|
||||
}
|
||||
}
|
||||
for name, found := range want {
|
||||
if !found {
|
||||
t.Errorf("metric %s was not gathered", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -102,11 +102,29 @@ var (
|
||||
Name: "im_cluster_proxy_response_total",
|
||||
Help: "Cluster proxy responses by pod and result.",
|
||||
}, []string{"pod", "result"})
|
||||
InternalQueueEnqueueTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "im_internal_queue_enqueue_total",
|
||||
Help: "Internal message queue enqueue attempts by pod, queue, and result.",
|
||||
}, []string{"pod", "queue", "result"})
|
||||
InternalQueueDepth = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Name: "im_internal_queue_depth",
|
||||
Help: "Internal message queue depth observed after enqueue attempts by pod and queue.",
|
||||
Buckets: []float64{0, 1, 4, 16, 32, 64, 128, 192, 256, 512, 1024, 2048, 4096, 8192},
|
||||
}, []string{"pod", "queue"})
|
||||
InternalQueueCapacity = prometheus.NewGaugeVec(prometheus.GaugeOpts{
|
||||
Name: "im_internal_queue_capacity",
|
||||
Help: "Configured internal message queue capacity by pod and queue.",
|
||||
}, []string{"pod", "queue"})
|
||||
MessagePersistDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Name: "im_message_persist_duration_seconds",
|
||||
Help: "Kafka message persistence duration by message kind and result.",
|
||||
Buckets: []float64{0.001, 0.003, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10},
|
||||
}, []string{"kind", "result"})
|
||||
MessagePublishStageDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Name: "im_message_publish_stage_duration_seconds",
|
||||
Help: "Message publish hot-path duration by message kind, stage, and result.",
|
||||
Buckets: []float64{0.001, 0.003, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10},
|
||||
}, []string{"kind", "stage", "result"})
|
||||
KafkaMessagePersistTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "im_kafka_message_persist_total",
|
||||
Help: "Kafka message persistence outcomes by message kind and result.",
|
||||
@@ -130,7 +148,9 @@ func init() {
|
||||
HTTPRequestsTotal, HTTPRequestDuration, DBQueryDuration,
|
||||
RealtimeFanoutEventsTotal, RealtimeFanoutDeliveryDuration, ClusterP2MQueueLength,
|
||||
ClusterP2MQueueCapacity, ClusterP2MEnqueueTotal, ClusterRPCInflight, ClusterRPCDuration,
|
||||
ClusterRouteTotal, ClusterProxyResponseTotal, MessagePersistDuration, KafkaMessagePersistTotal)
|
||||
ClusterRouteTotal, ClusterProxyResponseTotal, InternalQueueEnqueueTotal, InternalQueueDepth,
|
||||
InternalQueueCapacity, MessagePersistDuration, MessagePublishStageDuration,
|
||||
KafkaMessagePersistTotal)
|
||||
Registry.MustRegister(KafkaConsumerLag, KafkaBacklogOldestAge, KafkaDeadLettersTotal)
|
||||
}
|
||||
|
||||
@@ -149,6 +169,15 @@ func ObserveDB(driverName, operation string, started time.Time, err error) {
|
||||
DBQueryDuration.WithLabelValues(driverName, operation, status).Observe(time.Since(started).Seconds())
|
||||
}
|
||||
|
||||
func ObserveMessagePublishStage(kind, stage string, started time.Time, err error) {
|
||||
status := "ok"
|
||||
if err != nil {
|
||||
status = "error"
|
||||
}
|
||||
MessagePublishStageDuration.WithLabelValues(normalizeLabel(kind), normalizeLabel(stage), status).
|
||||
Observe(time.Since(started).Seconds())
|
||||
}
|
||||
|
||||
func IncRealtimeFanout(stage, kind, targetType, status string) {
|
||||
RealtimeFanoutEventsTotal.WithLabelValues(normalizeLabel(stage), normalizeLabel(kind),
|
||||
normalizeLabel(targetType), normalizeLabel(status)).Inc()
|
||||
@@ -162,6 +191,18 @@ func ObserveRealtimeFanoutDelivery(kind, targetType string, timestamp time.Time)
|
||||
Observe(time.Since(timestamp).Seconds())
|
||||
}
|
||||
|
||||
func ObserveInternalQueueEnqueue(queue string, depth, capacity int, accepted bool) {
|
||||
result := "ok"
|
||||
if !accepted {
|
||||
result = "full"
|
||||
}
|
||||
pod := metricPodName()
|
||||
queue = normalizeLabel(queue)
|
||||
InternalQueueEnqueueTotal.WithLabelValues(pod, queue, result).Inc()
|
||||
InternalQueueDepth.WithLabelValues(pod, queue).Observe(float64(depth))
|
||||
InternalQueueCapacity.WithLabelValues(pod, queue).Set(float64(capacity))
|
||||
}
|
||||
|
||||
func normalizeLabel(value string) string {
|
||||
if value == "" {
|
||||
return "unknown"
|
||||
|
||||
@@ -90,6 +90,7 @@ func TestServeMetricsIncludesWSConnections(t *testing.T) {
|
||||
immetrics.KafkaMessagePersistTotal.WithLabelValues("p2p", "batch_ok").Add(0)
|
||||
immetrics.KafkaConsumerLag.WithLabelValues("test", "topic", "group").Set(0)
|
||||
immetrics.KafkaDeadLettersTotal.WithLabelValues("p2p", "test").Add(0)
|
||||
immetrics.ObserveInternalQueueEnqueue("test", 0, 1, true)
|
||||
wrt := httptest.NewRecorder()
|
||||
serveMetrics(wrt, httptest.NewRequest(http.MethodGet, "/metrics", nil))
|
||||
|
||||
@@ -102,6 +103,7 @@ func TestServeMetricsIncludesWSConnections(t *testing.T) {
|
||||
}
|
||||
for _, metric := range []string{"im_ws_connections_active", "im_cluster_p2m_queue_length",
|
||||
"im_cluster_p2m_enqueue_total", "im_cluster_rpc_duration_seconds", "im_cluster_route_total",
|
||||
"im_internal_queue_enqueue_total", "im_internal_queue_depth", "im_internal_queue_capacity",
|
||||
"im_kafka_message_persist_total", "im_kafka_consumer_lag", "im_kafka_dead_letters_total"} {
|
||||
if !strings.Contains(body, "# HELP "+metric+" ") {
|
||||
t.Fatalf("metric %s was not exported", metric)
|
||||
|
||||
+24
-3
@@ -5,6 +5,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/tinode/chat/server/logs"
|
||||
immetrics "github.com/tinode/chat/server/metrics"
|
||||
"github.com/tinode/chat/server/store"
|
||||
"github.com/tinode/chat/server/store/types"
|
||||
)
|
||||
@@ -466,12 +467,26 @@ func (t *Topic) presSubsOffline(what string, params *presParams,
|
||||
if offlineOnly {
|
||||
skipTopic = t.name
|
||||
}
|
||||
if t.cat == types.TopicCatGrp && offlineOnly && what == "msg" && store.RealtimeFanoutEnabled() {
|
||||
recipients := make([]realtimeGroupUnreadRecipient, 0, len(t.perUser))
|
||||
for uid, pud := range t.perUser {
|
||||
if pud.deleted || !presOfflineFilter(pud.modeGiven&pud.modeWant, what, filterSource) {
|
||||
continue
|
||||
}
|
||||
recipients = append(recipients, realtimeGroupUnreadRecipient{
|
||||
User: uid.UserId(),
|
||||
Source: t.original(uid),
|
||||
})
|
||||
}
|
||||
if publishRealtimeGroupUnread(t.tenantID, t.name, params.actor, params.seqID, recipients) {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
for uid, pud := range t.perUser {
|
||||
if pud.deleted || !presOfflineFilter(pud.modeGiven&pud.modeWant, what, filterSource) {
|
||||
continue
|
||||
}
|
||||
|
||||
user := uid.UserId()
|
||||
actor := params.actor
|
||||
target := params.target
|
||||
@@ -693,10 +708,16 @@ func presSingleUserOfflineOffline(tenantID types.TenantID, uid types.Uid, origin
|
||||
}
|
||||
|
||||
func routeUserServerMessage(uid types.Uid, msg *ServerComMessage) {
|
||||
if msg == nil || uid.IsZero() {
|
||||
if msg == nil || uid.IsZero() || globals.hub == nil {
|
||||
return
|
||||
}
|
||||
globals.hub.routeSrv <- msg
|
||||
select {
|
||||
case globals.hub.routeSrv <- msg:
|
||||
immetrics.ObserveInternalQueueEnqueue("hub_route_server", len(globals.hub.routeSrv), cap(globals.hub.routeSrv), true)
|
||||
default:
|
||||
immetrics.ObserveInternalQueueEnqueue("hub_route_server", len(globals.hub.routeSrv), cap(globals.hub.routeSrv), false)
|
||||
logs.Err.Printf("route user message: hub routeSrv queue full, user=%s", uid.UserId())
|
||||
}
|
||||
publishRealtimeUserMessage(uid, msg)
|
||||
}
|
||||
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
|
||||
const (
|
||||
realtimeKindMessage = "message"
|
||||
realtimeKindGroupUnread = "group_unread"
|
||||
realtimeKindSessionControl = "session_control"
|
||||
realtimeKindCallEvent = "call_event"
|
||||
|
||||
@@ -23,8 +24,21 @@ const (
|
||||
realtimeTargetUser = "user"
|
||||
realtimeTargetDevice = "device"
|
||||
realtimeTargetPlatf = "platform"
|
||||
realtimeTargetUsers = "users"
|
||||
)
|
||||
|
||||
type realtimeGroupUnreadRecipient struct {
|
||||
User string `json:"user"`
|
||||
Source string `json:"source"`
|
||||
}
|
||||
|
||||
type realtimeGroupUnreadMessage struct {
|
||||
Topic string `json:"topic"`
|
||||
Actor string `json:"actor,omitempty"`
|
||||
SeqID int64 `json:"seq_id"`
|
||||
Recipients []realtimeGroupUnreadRecipient `json:"recipients"`
|
||||
}
|
||||
|
||||
type realtimeCallEventMessage struct {
|
||||
Topic string `json:"topic"`
|
||||
From string `json:"from"`
|
||||
@@ -68,6 +82,12 @@ func publishRealtimeUserMessage(uid types.Uid, msg *ServerComMessage) {
|
||||
immetrics.IncRealtimeFanout("publish", realtimeKindMessage, realtimeTargetUser, "skip_invalid")
|
||||
return
|
||||
}
|
||||
// Native cluster routing already forwards user topics between pods. Publishing
|
||||
// the same message through Redis duplicates work and may deliver it twice.
|
||||
if globals.cluster != nil {
|
||||
immetrics.IncRealtimeFanout("publish", realtimeKindMessage, realtimeTargetUser, "skip_native_cluster")
|
||||
return
|
||||
}
|
||||
dedupKey := realtimeMessageDedupKey(uid.UserId(), msg)
|
||||
if err := store.PublishRealtimeFanout(currentNodeID(), msg.TenantID, realtimeKindMessage, realtimeTargetUser, uid.UserId(), msg.SkipSid, dedupKey, msg); err != nil {
|
||||
immetrics.IncRealtimeFanout("publish", realtimeKindMessage, realtimeTargetUser, "error")
|
||||
@@ -77,6 +97,29 @@ func publishRealtimeUserMessage(uid types.Uid, msg *ServerComMessage) {
|
||||
immetrics.IncRealtimeFanout("publish", realtimeKindMessage, realtimeTargetUser, "ok")
|
||||
}
|
||||
|
||||
func publishRealtimeGroupUnread(tenantID types.TenantID, topic, actor string, seqID int64,
|
||||
recipients []realtimeGroupUnreadRecipient) bool {
|
||||
if !tenantID.IsValid() || topic == "" || seqID <= 0 || len(recipients) == 0 || !store.RealtimeFanoutEnabled() {
|
||||
return false
|
||||
}
|
||||
payload := &realtimeGroupUnreadMessage{
|
||||
Topic: topic,
|
||||
Actor: actor,
|
||||
SeqID: seqID,
|
||||
Recipients: recipients,
|
||||
}
|
||||
dedupKey := "group-unread:" + topic + ":" + strconv.FormatInt(seqID, 10)
|
||||
if err := store.PublishRealtimeFanout(currentNodeID(), tenantID, realtimeKindGroupUnread,
|
||||
realtimeTargetUsers, topic, "", dedupKey, payload); err != nil {
|
||||
immetrics.IncRealtimeFanout("publish", realtimeKindGroupUnread, realtimeTargetUsers, "error")
|
||||
logs.Warn.Printf("redis realtime fanout: publish group unread topic=%s seq=%d failed: %v", topic, seqID, err)
|
||||
return false
|
||||
}
|
||||
immetrics.IncRealtimeFanout("publish", realtimeKindGroupUnread, realtimeTargetUsers, "ok")
|
||||
deliverRealtimeGroupUnread(tenantID, payload)
|
||||
return true
|
||||
}
|
||||
|
||||
func publishRealtimeCallEvent(msg *ClientComMessage) {
|
||||
if msg == nil || msg.Note == nil || !msg.TenantID.IsValid() ||
|
||||
msg.RcptTo == "" || msg.AsUser == "" || msg.Note.What != "call" ||
|
||||
@@ -193,6 +236,16 @@ func deliverRealtimeFanoutEnvelope(envelope *store.RealtimeFanoutEnvelope) {
|
||||
return
|
||||
}
|
||||
immetrics.IncRealtimeFanout("receive", envelope.Kind, envelope.TargetType, "ok")
|
||||
if envelope.Kind == realtimeKindGroupUnread {
|
||||
var batch realtimeGroupUnreadMessage
|
||||
if err := json.Unmarshal(envelope.Message, &batch); err != nil {
|
||||
immetrics.IncRealtimeFanout("receive", envelope.Kind, envelope.TargetType, "malformed")
|
||||
logs.Warn.Println("redis realtime fanout: malformed group unread batch", err)
|
||||
return
|
||||
}
|
||||
deliverRealtimeGroupUnread(envelope.TenantID, &batch)
|
||||
return
|
||||
}
|
||||
var msg ServerComMessage
|
||||
if err := json.Unmarshal(envelope.Message, &msg); err != nil {
|
||||
immetrics.IncRealtimeFanout("receive", envelope.Kind, envelope.TargetType, "malformed")
|
||||
@@ -224,6 +277,51 @@ func deliverRealtimeFanoutEnvelope(envelope *store.RealtimeFanoutEnvelope) {
|
||||
}
|
||||
}
|
||||
|
||||
func deliverRealtimeGroupUnread(tenantID types.TenantID, batch *realtimeGroupUnreadMessage) {
|
||||
if globals.sessionStore == nil || batch == nil || batch.Topic == "" || batch.SeqID <= 0 || len(batch.Recipients) == 0 {
|
||||
immetrics.IncRealtimeFanout("deliver", realtimeKindGroupUnread, realtimeTargetUsers, "skip_invalid")
|
||||
return
|
||||
}
|
||||
recipients := make(map[types.Uid]string, len(batch.Recipients))
|
||||
for _, recipient := range batch.Recipients {
|
||||
uid := types.ParseUserId(recipient.User)
|
||||
if !uid.IsZero() {
|
||||
recipients[uid] = recipient.Source
|
||||
}
|
||||
}
|
||||
delivered := 0
|
||||
globals.sessionStore.Range(func(_ string, sess *Session) bool {
|
||||
if sess == nil || sess.isCluster() || sess.isProxy() || sess.tenantID != tenantID {
|
||||
return true
|
||||
}
|
||||
source, ok := recipients[sess.uid]
|
||||
if !ok || sessionHasTopicOrP2PAlias(sess, batch.Topic) {
|
||||
return true
|
||||
}
|
||||
actor := batch.Actor
|
||||
if actor == sess.uid.UserId() {
|
||||
actor = ""
|
||||
}
|
||||
msg := &ServerComMessage{
|
||||
TenantID: tenantID,
|
||||
RcptTo: sess.uid.UserId(),
|
||||
Pres: &MsgServerPres{
|
||||
Topic: "me",
|
||||
What: "msg",
|
||||
Src: source,
|
||||
AcsActor: actor,
|
||||
SeqId: batch.SeqID,
|
||||
SkipTopic: batch.Topic,
|
||||
},
|
||||
}
|
||||
if sess.queueOut(msg) {
|
||||
delivered++
|
||||
}
|
||||
return true
|
||||
})
|
||||
recordRealtimeFanoutDeliverResult(realtimeKindGroupUnread, realtimeTargetUsers, delivered)
|
||||
}
|
||||
|
||||
func deliverRealtimeCallEvent(envelope *store.RealtimeFanoutEnvelope) {
|
||||
var event realtimeCallEventMessage
|
||||
if err := json.Unmarshal(envelope.Message, &event); err != nil {
|
||||
|
||||
@@ -109,6 +109,60 @@ func TestRealtimeUserPresenceSkipsSessionsAttachedToTopic(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRealtimeGroupUnreadBatchPreservesDetachedMultiDeviceNotification(t *testing.T) {
|
||||
oldStore := globals.sessionStore
|
||||
defer func() { globals.sessionStore = oldStore }()
|
||||
|
||||
tenantID := types.TenantID(1)
|
||||
recipient := types.Uid(1002)
|
||||
other := types.Uid(1003)
|
||||
topic := "grpUnreadBatchTest"
|
||||
globals.sessionStore = realtimeTestSessionStore(
|
||||
realtimeTestSession("sid-attached", tenantID, recipient, topic),
|
||||
realtimeTestSession("sid-detached-1", tenantID, recipient, ""),
|
||||
realtimeTestSession("sid-detached-2", tenantID, recipient, ""),
|
||||
realtimeTestSession("sid-other", tenantID, other, ""),
|
||||
)
|
||||
payload := realtimeGroupUnreadMessage{
|
||||
Topic: topic,
|
||||
Actor: types.Uid(1001).UserId(),
|
||||
SeqID: 51,
|
||||
Recipients: []realtimeGroupUnreadRecipient{
|
||||
{User: recipient.UserId(), Source: topic},
|
||||
},
|
||||
}
|
||||
raw, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
deliverRealtimeFanoutEnvelope(&store.RealtimeFanoutEnvelope{
|
||||
TenantID: tenantID,
|
||||
Kind: realtimeKindGroupUnread,
|
||||
TargetType: realtimeTargetUsers,
|
||||
Target: topic,
|
||||
Message: raw,
|
||||
})
|
||||
|
||||
if len(globals.sessionStore.Get("sid-attached").send) != 0 {
|
||||
t.Fatal("attached session must rely on the topic data broadcast")
|
||||
}
|
||||
for _, sid := range []string{"sid-detached-1", "sid-detached-2"} {
|
||||
select {
|
||||
case got := <-globals.sessionStore.Get(sid).send:
|
||||
pres := got.(*ServerComMessage).Pres
|
||||
if pres == nil || pres.What != "msg" || pres.Src != topic || pres.SeqId != 51 || pres.SkipTopic != topic {
|
||||
t.Fatalf("%s got unexpected presence: %#v", sid, pres)
|
||||
}
|
||||
default:
|
||||
t.Fatalf("detached session %s must receive unread presence", sid)
|
||||
}
|
||||
}
|
||||
if len(globals.sessionStore.Get("sid-other").send) != 0 {
|
||||
t.Fatal("non-recipient session received unread presence")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRealtimeP2PTopicFallbackRewritesVisibleTopic(t *testing.T) {
|
||||
oldStore := globals.sessionStore
|
||||
oldHub := globals.hub
|
||||
|
||||
+11
-6
@@ -25,6 +25,7 @@ import (
|
||||
"github.com/tinode/chat/pbx"
|
||||
"github.com/tinode/chat/server/auth"
|
||||
"github.com/tinode/chat/server/logs"
|
||||
immetrics "github.com/tinode/chat/server/metrics"
|
||||
"github.com/tinode/chat/server/store"
|
||||
"github.com/tinode/chat/server/store/types"
|
||||
|
||||
@@ -519,13 +520,15 @@ func (s *Session) dispatchRaw(raw []byte) {
|
||||
s.queueOut(ErrMalformed("", "", now))
|
||||
return
|
||||
}
|
||||
toLog := redactClientMessageForLog(raw)
|
||||
truncated := ""
|
||||
if len(toLog) > 512 {
|
||||
toLog = toLog[:512]
|
||||
truncated = "<...>"
|
||||
if globals.logClientPackets {
|
||||
toLog := redactClientMessageForLog(raw)
|
||||
truncated := ""
|
||||
if len(toLog) > 512 {
|
||||
toLog = toLog[:512]
|
||||
truncated = "<...>"
|
||||
}
|
||||
logs.Info.Printf("in: '%s%s' sid='%s' uid='%s'", toLog, truncated, s.sid, s.uid)
|
||||
}
|
||||
logs.Info.Printf("in: '%s%s' sid='%s' uid='%s'", toLog, truncated, s.sid, s.uid)
|
||||
|
||||
s.dispatch(&msg)
|
||||
}
|
||||
@@ -858,7 +861,9 @@ func (s *Session) publish(msg *ClientComMessage) {
|
||||
// This is a post to a subscribed topic. The message is sent to the topic only
|
||||
select {
|
||||
case sub.broadcast <- msg:
|
||||
immetrics.ObserveInternalQueueEnqueue("session_topic_publish", len(sub.broadcast), cap(sub.broadcast), true)
|
||||
default:
|
||||
immetrics.ObserveInternalQueueEnqueue("session_topic_publish", len(sub.broadcast), cap(sub.broadcast), false)
|
||||
// Reply with a 503 to the user.
|
||||
s.queueOut(ErrServiceUnavailableReply(msg, msg.Timestamp))
|
||||
logs.Err.Println("s.publish: sub.broadcast channel full, topic ", msg.RcptTo, s.sid)
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
adapter "github.com/tinode/chat/server/db"
|
||||
"github.com/tinode/chat/server/store/types"
|
||||
)
|
||||
|
||||
@@ -152,6 +153,9 @@ func InvalidateTenantConfigCache(code string, id types.TenantID) {
|
||||
tenantByIDCache.clear()
|
||||
realNamePolicyCache.clear()
|
||||
}
|
||||
if invalidator, ok := adp.(adapter.TenantRouteCacheInvalidator); ok {
|
||||
invalidator.InvalidateTenantRoute(id)
|
||||
}
|
||||
}
|
||||
|
||||
// InvalidateRegistrationConfigCache clears cached global registration config.
|
||||
|
||||
@@ -392,6 +392,13 @@ func StartRealtimeFanoutSubscriber(ctx context.Context, nodeID string, handler f
|
||||
return groupMsgRuntime.StartRealtimeFanoutSubscriber(ctx, nodeID, handler)
|
||||
}
|
||||
|
||||
// RealtimeFanoutEnabled reports whether Redis Pub/Sub fanout is configured.
|
||||
// Callers use it to retain the native routing fallback when Redis is disabled.
|
||||
func RealtimeFanoutEnabled() bool {
|
||||
runtime, ok := groupMsgRuntime.(*redisGroupMessageRuntime)
|
||||
return ok && runtime != nil && runtime.cfg.GroupFanout
|
||||
}
|
||||
|
||||
func CacheObject(tenantID types.TenantID, namespace, id string, value any) error {
|
||||
if groupMsgRuntime == nil || !tenantID.IsValid() || namespace == "" || id == "" || value == nil {
|
||||
return nil
|
||||
|
||||
@@ -17,6 +17,7 @@ type testGroupMessageRuntime struct {
|
||||
shortStates map[string]bool
|
||||
shortValues map[string]string
|
||||
objects map[string][]byte
|
||||
sets map[string]map[string]bool
|
||||
deletedPrefixes []string
|
||||
deletedObjects []string
|
||||
}
|
||||
@@ -28,6 +29,7 @@ func newTestGroupMessageRuntime() *testGroupMessageRuntime {
|
||||
shortStates: make(map[string]bool),
|
||||
shortValues: make(map[string]string),
|
||||
objects: make(map[string][]byte),
|
||||
sets: make(map[string]map[string]bool),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -186,16 +188,24 @@ func (r *testGroupMessageRuntime) DeleteCachedObjectsByPrefix(tenantID types.Ten
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *testGroupMessageRuntime) AddCachedSetMember(types.TenantID, string, string, string) error {
|
||||
func (r *testGroupMessageRuntime) AddCachedSetMember(tenantID types.TenantID, namespace, id, member string) error {
|
||||
key := strconv.FormatInt(int64(tenantID), 10) + ":" + namespace + ":" + id
|
||||
if r.sets[key] == nil {
|
||||
r.sets[key] = make(map[string]bool)
|
||||
}
|
||||
r.sets[key][member] = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *testGroupMessageRuntime) RemoveCachedSetMember(types.TenantID, string, string, string) error {
|
||||
func (r *testGroupMessageRuntime) RemoveCachedSetMember(tenantID types.TenantID, namespace, id, member string) error {
|
||||
key := strconv.FormatInt(int64(tenantID), 10) + ":" + namespace + ":" + id
|
||||
delete(r.sets[key], member)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *testGroupMessageRuntime) CachedSetContains(types.TenantID, string, string, string) (bool, error) {
|
||||
return false, nil
|
||||
func (r *testGroupMessageRuntime) CachedSetContains(tenantID types.TenantID, namespace, id, member string) (bool, error) {
|
||||
key := strconv.FormatInt(int64(tenantID), 10) + ":" + namespace + ":" + id
|
||||
return r.sets[key][member], nil
|
||||
}
|
||||
|
||||
func (r *testGroupMessageRuntime) AllowRate(types.TenantID, string, int, time.Duration) (bool, int, time.Duration, error) {
|
||||
@@ -355,6 +365,29 @@ func TestMessageRuntimeSeqAndClientMidIdempotency(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCanSendP2PUsesWarmRelationshipCache(t *testing.T) {
|
||||
oldRuntime := groupMsgRuntime
|
||||
runtime := newTestGroupMessageRuntime()
|
||||
groupMsgRuntime = runtime
|
||||
t.Cleanup(func() { groupMsgRuntime = oldRuntime })
|
||||
|
||||
tenantID := types.TenantID(7)
|
||||
userID := types.Uid(42)
|
||||
peerID := types.Uid(43)
|
||||
cacheFriendPair(tenantID, userID, peerID)
|
||||
|
||||
allowed, relationship, err := (relationsMapper{}).CanSendP2P(tenantID, userID, peerID)
|
||||
if err != nil || !allowed || relationship != types.RelationshipFriend {
|
||||
t.Fatalf("warm friend cache was not used: allowed=%v relationship=%q err=%v", allowed, relationship, err)
|
||||
}
|
||||
|
||||
cacheBlockOneWay(tenantID, peerID, userID)
|
||||
allowed, relationship, err = (relationsMapper{}).CanSendP2P(tenantID, userID, peerID)
|
||||
if err != nil || allowed || relationship != types.RelationshipBlockedByPeer {
|
||||
t.Fatalf("reverse block cache was not enforced: allowed=%v relationship=%q err=%v", allowed, relationship, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBumpMessageSeqFloorNeverMovesBackward(t *testing.T) {
|
||||
oldRuntime := groupMsgRuntime
|
||||
runtime := newTestGroupMessageRuntime()
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/tinode/chat/server/store/types"
|
||||
)
|
||||
|
||||
func TestAggregatePreparedMessageBatchUpdatesKeepsLatestSeq(t *testing.T) {
|
||||
user1 := types.Uid(1)
|
||||
user2 := types.Uid(2)
|
||||
items := []PreparedMessageBatchItem{
|
||||
{Message: &types.Message{Topic: "p2p-b", From: user1.String(), SeqId: 3}, ReadBySender: true},
|
||||
{Message: &types.Message{Topic: "p2p-a", From: user1.String(), SeqId: 2}, ReadBySender: true},
|
||||
{Message: &types.Message{Topic: "p2p-a", From: user2.String(), SeqId: 5}, ReadBySender: true},
|
||||
{Message: &types.Message{Topic: "p2p-a", From: user1.String(), SeqId: 6}, ReadBySender: true},
|
||||
{Message: &types.Message{Topic: "p2p-b", From: user1.String(), SeqId: 7}, ReadBySender: false},
|
||||
{},
|
||||
}
|
||||
|
||||
topics, reads := aggregatePreparedMessageBatchUpdates(items)
|
||||
if len(topics) != 2 || topics[0].topic != "p2p-a" || topics[0].message.SeqId != 6 ||
|
||||
topics[1].topic != "p2p-b" || topics[1].message.SeqId != 7 {
|
||||
t.Fatalf("unexpected topic updates: %+v", topics)
|
||||
}
|
||||
if len(reads) != 3 {
|
||||
t.Fatalf("read updates=%d want=3: %+v", len(reads), reads)
|
||||
}
|
||||
got := make(map[string]int64, len(reads))
|
||||
for _, update := range reads {
|
||||
got[update.topic+":"+update.from.String()] = update.message.SeqId
|
||||
}
|
||||
for key, want := range map[string]int64{
|
||||
"p2p-a:" + user1.String(): 6,
|
||||
"p2p-a:" + user2.String(): 5,
|
||||
"p2p-b:" + user1.String(): 3,
|
||||
} {
|
||||
if got[key] != want {
|
||||
t.Fatalf("read update %s=%d want=%d; all=%v", key, got[key], want, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
+95
-14
@@ -3147,6 +3147,29 @@ func (relationsMapper) GetRelationshipContext(tenantID types.TenantID, userID, p
|
||||
}
|
||||
|
||||
func (relationsMapper) CanSendP2P(tenantID types.TenantID, userID, peerID types.Uid) (bool, string, error) {
|
||||
if userID.IsZero() || peerID.IsZero() || userID == peerID {
|
||||
return false, "", types.ErrMalformed
|
||||
}
|
||||
// A warm relationship cache is sufficient for the publish hot path. Block
|
||||
// entries are checked first in both directions; friendship and block writes
|
||||
// keep these sets coherent. Redis errors and cache misses fall back to the
|
||||
// authoritative database path below.
|
||||
blockCacheReady := true
|
||||
if blocked, err := cachedBlockContains(tenantID, userID, peerID); err != nil {
|
||||
blockCacheReady = false
|
||||
} else if blocked {
|
||||
return false, types.RelationshipBlockedByMe, nil
|
||||
}
|
||||
if blocked, err := cachedBlockContains(tenantID, peerID, userID); err != nil {
|
||||
blockCacheReady = false
|
||||
} else if blocked {
|
||||
return false, types.RelationshipBlockedByPeer, nil
|
||||
}
|
||||
if blockCacheReady {
|
||||
if friend, err := cachedFriendContains(tenantID, userID, peerID); err == nil && friend {
|
||||
return true, types.RelationshipFriend, nil
|
||||
}
|
||||
}
|
||||
ctx, err := (relationsMapper{}).GetRelationshipContext(tenantID, userID, peerID, "")
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
@@ -3168,6 +3191,9 @@ func (relationsMapper) BlockUser(tenantID types.TenantID, blockerID, blockedID t
|
||||
block, err := tenantAdp.TenantUserBlockUpsert(tenantID, blockerID, blockedID, now)
|
||||
if err == nil {
|
||||
cacheBlockOneWay(tenantID, blockerID, blockedID)
|
||||
// A positive friend cache must never outlive a new block. Removing both
|
||||
// directions also makes a later block-cache eviction fall back to MySQL.
|
||||
uncacheFriendPair(tenantID, blockerID, blockedID)
|
||||
}
|
||||
return block, err
|
||||
}
|
||||
@@ -3347,7 +3373,7 @@ func relationshipContext(tenantAdp adapter.TenantBusinessAdapter, tenantID types
|
||||
if friendship, err := tenantAdp.TenantFriendshipGet(tenantID, userID, peerID); err != nil {
|
||||
return nil, err
|
||||
} else if friendship != nil && friendship.Status == types.RelationshipFriend {
|
||||
cacheFriendOneWay(tenantID, userID, peerID)
|
||||
cacheFriendPair(tenantID, userID, peerID)
|
||||
ctx.Relationship = types.RelationshipFriend
|
||||
ctx.CanSend = true
|
||||
ctx.Remark = friendship.Remark
|
||||
@@ -3682,6 +3708,58 @@ type PreparedMessageBatchItem struct {
|
||||
ReadBySender bool
|
||||
}
|
||||
|
||||
type preparedMessageBatchTopicUpdate struct {
|
||||
topic string
|
||||
message *types.Message
|
||||
}
|
||||
|
||||
type preparedMessageBatchReadUpdate struct {
|
||||
topic string
|
||||
from types.Uid
|
||||
message *types.Message
|
||||
}
|
||||
|
||||
func aggregatePreparedMessageBatchUpdates(items []PreparedMessageBatchItem) ([]preparedMessageBatchTopicUpdate, []preparedMessageBatchReadUpdate) {
|
||||
topics := make(map[string]*types.Message)
|
||||
reads := make(map[string]preparedMessageBatchReadUpdate)
|
||||
for _, item := range items {
|
||||
msg := item.Message
|
||||
if msg == nil {
|
||||
continue
|
||||
}
|
||||
if previous := topics[msg.Topic]; previous == nil || msg.SeqId > previous.SeqId {
|
||||
topics[msg.Topic] = msg
|
||||
}
|
||||
from := types.ParseUid(msg.From)
|
||||
if !item.ReadBySender || from.IsZero() {
|
||||
continue
|
||||
}
|
||||
key := msg.Topic + "\x00" + from.String()
|
||||
if previous, ok := reads[key]; !ok || msg.SeqId > previous.message.SeqId {
|
||||
reads[key] = preparedMessageBatchReadUpdate{topic: msg.Topic, from: from, message: msg}
|
||||
}
|
||||
}
|
||||
topicKeys := make([]string, 0, len(topics))
|
||||
for topic := range topics {
|
||||
topicKeys = append(topicKeys, topic)
|
||||
}
|
||||
sort.Strings(topicKeys)
|
||||
topicUpdates := make([]preparedMessageBatchTopicUpdate, 0, len(topicKeys))
|
||||
for _, topic := range topicKeys {
|
||||
topicUpdates = append(topicUpdates, preparedMessageBatchTopicUpdate{topic: topic, message: topics[topic]})
|
||||
}
|
||||
readKeys := make([]string, 0, len(reads))
|
||||
for key := range reads {
|
||||
readKeys = append(readKeys, key)
|
||||
}
|
||||
sort.Strings(readKeys)
|
||||
readUpdates := make([]preparedMessageBatchReadUpdate, 0, len(readKeys))
|
||||
for _, key := range readKeys {
|
||||
readUpdates = append(readUpdates, reads[key])
|
||||
}
|
||||
return topicUpdates, readUpdates
|
||||
}
|
||||
|
||||
func PersistPreparedMessageBatch(tenantID types.TenantID, items []PreparedMessageBatchItem) error {
|
||||
if len(items) == 0 {
|
||||
return nil
|
||||
@@ -3725,14 +3803,17 @@ func PersistPreparedMessageBatch(tenantID types.TenantID, items []PreparedMessag
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, item := range items {
|
||||
msg := item.Message
|
||||
from := types.ParseUid(msg.From)
|
||||
if err = tenantAdp.TenantTopicUpdateOnMessage(tenantID, msg.Topic, msg); err != nil {
|
||||
topicUpdates, readUpdates := aggregatePreparedMessageBatchUpdates(items)
|
||||
for _, update := range topicUpdates {
|
||||
if err = tenantAdp.TenantTopicUpdateOnMessage(tenantID, update.topic, update.message); err != nil {
|
||||
return err
|
||||
}
|
||||
invalidateTopicCache(tenantID, msg.Topic)
|
||||
if isGroup && groupMsgRuntime != nil {
|
||||
invalidateTopicCache(tenantID, update.topic)
|
||||
}
|
||||
if isGroup && groupMsgRuntime != nil {
|
||||
for _, item := range items {
|
||||
msg := item.Message
|
||||
from := types.ParseUid(msg.From)
|
||||
if msg.ClientMsgID != "" {
|
||||
_ = groupMsgRuntime.MarkDone(tenantID, msg.Topic, from, msg.ClientMsgID, msg.SeqId, msg.Uid().String())
|
||||
}
|
||||
@@ -3741,13 +3822,13 @@ func PersistPreparedMessageBatch(tenantID types.TenantID, items []PreparedMessag
|
||||
})
|
||||
_ = groupMsgRuntime.CacheHotMessage(tenantID, msg)
|
||||
}
|
||||
if item.ReadBySender && !from.IsZero() {
|
||||
if updateErr := tenantAdp.TenantSubsUpdate(tenantID, msg.Topic, from, map[string]any{
|
||||
"RecvSeqId": msg.SeqId, "ReadSeqId": msg.SeqId,
|
||||
}); updateErr == nil {
|
||||
invalidateSubscriptionCache(tenantID, msg.Topic, from)
|
||||
DeleteCachedUserUnreadCount(tenantID, from)
|
||||
}
|
||||
}
|
||||
for _, update := range readUpdates {
|
||||
if updateErr := tenantAdp.TenantSubsUpdate(tenantID, update.topic, update.from, map[string]any{
|
||||
"RecvSeqId": update.message.SeqId, "ReadSeqId": update.message.SeqId,
|
||||
}); updateErr == nil {
|
||||
invalidateSubscriptionCache(tenantID, update.topic, update.from)
|
||||
DeleteCachedUserUnreadCount(tenantID, update.from)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
|
||||
@@ -9,6 +9,7 @@ const (
|
||||
FriendRequestExpired = "expired"
|
||||
|
||||
RelationshipNone = "none"
|
||||
RelationshipSelf = "self"
|
||||
RelationshipFriend = "friend"
|
||||
RelationshipPendingInbound = "pending_inbound"
|
||||
RelationshipPendingOutbound = "pending_outbound"
|
||||
|
||||
+4
-2
@@ -84,8 +84,10 @@
|
||||
"dsn": "mysql://root:oWxz*2=sb),ouW%p@tcp(mysql.118im.com:30306)/meeting_im?parseTime=true&collation=utf8mb4_0900_ai_ci",
|
||||
"tenant_registry_db": "hotgo",
|
||||
"tenant_db_routing": true,
|
||||
"max_open_conns": 64,
|
||||
"max_idle_conns": 64,
|
||||
"max_open_conns": 24,
|
||||
"max_idle_conns": 8,
|
||||
"registry_max_open_conns": 4,
|
||||
"registry_max_idle_conns": 2,
|
||||
"conn_max_lifetime": 60,
|
||||
"sql_timeout": 10
|
||||
}
|
||||
|
||||
@@ -20,6 +20,7 @@ import (
|
||||
|
||||
"github.com/tinode/chat/server/auth"
|
||||
"github.com/tinode/chat/server/logs"
|
||||
immetrics "github.com/tinode/chat/server/metrics"
|
||||
"github.com/tinode/chat/server/moderation"
|
||||
"github.com/tinode/chat/server/store"
|
||||
"github.com/tinode/chat/server/store/types"
|
||||
@@ -1362,7 +1363,9 @@ func (t *Topic) saveAndBroadcastMessage(msg *ClientComMessage, asUid types.Uid,
|
||||
store.Relations != nil && store.Store != nil && store.Store.IsOpen() {
|
||||
peer := t.p2pOtherUser(asUid)
|
||||
if !peer.IsZero() {
|
||||
started := time.Now()
|
||||
canSend, _, err := store.Relations.CanSendP2P(t.tenantID, asUid, peer)
|
||||
immetrics.ObserveMessagePublishStage("p2p", "relationship_check", started, err)
|
||||
if err != nil {
|
||||
logs.Warn.Printf("topic[%s]: failed to check p2p relationship: %v", t.name, err)
|
||||
msg.sess.queueOut(ErrUnknown(msg.Id, t.original(asUid), msg.Timestamp))
|
||||
@@ -1375,6 +1378,7 @@ func (t *Topic) saveAndBroadcastMessage(msg *ClientComMessage, asUid types.Uid,
|
||||
}
|
||||
}
|
||||
|
||||
persistStarted := time.Now()
|
||||
saveResult, err := saveMessageMaybeKafka(context.Background(), t.tenantID,
|
||||
&types.Message{
|
||||
ObjHeader: types.ObjHeader{CreatedAt: msg.Timestamp},
|
||||
@@ -1385,6 +1389,13 @@ func (t *Topic) saveAndBroadcastMessage(msg *ClientComMessage, asUid types.Uid,
|
||||
Head: head,
|
||||
Content: content,
|
||||
}, attachments, (pud.modeGiven & pud.modeWant).IsReader())
|
||||
kind := "other"
|
||||
if t.cat == types.TopicCatP2P {
|
||||
kind = "p2p"
|
||||
} else if t.cat == types.TopicCatGrp {
|
||||
kind = "group"
|
||||
}
|
||||
immetrics.ObserveMessagePublishStage(kind, "save", persistStarted, err)
|
||||
if err != nil {
|
||||
logs.Warn.Printf("topic[%s]: failed to save message: %v", t.name, err)
|
||||
msg.sess.queueOut(ErrUnknown(msg.Id, t.original(asUid), msg.Timestamp))
|
||||
|
||||
@@ -272,6 +272,7 @@ type testSecurityStore struct {
|
||||
consumedTarget string
|
||||
independentPassword *types.IndependentPassword
|
||||
youthMode *types.YouthModeSettings
|
||||
youthModeReads int
|
||||
attempts int
|
||||
lockedUntil *time.Time
|
||||
notifications []types.SystemNotification
|
||||
@@ -379,6 +380,7 @@ func (s *testSecurityStore) UpdateIndependentPasswordAttempts(_ types.TenantID,
|
||||
}
|
||||
|
||||
func (s *testSecurityStore) GetYouthMode(_ types.TenantID, uid types.Uid) (*types.YouthModeSettings, error) {
|
||||
s.youthModeReads++
|
||||
if s.err != nil {
|
||||
return nil, s.err
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user