Compare commits
25
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 richjohnrambo
|
7b25365ec3 | ||
|
richjohnrambo
|
746f75ab0f | ||
|
richjohnrambo
|
db92c41241 | ||
|
richjohnrambo
|
80808349fb | ||
|
richjohnrambo
|
a406d63154 | ||
|
richjohnrambo
|
d43b1b63d3 | ||
|
richjohnrambo
|
dccd0969d8 | ||
|
richjohnrambo
|
73a594146f | ||
|
richjohnrambo
|
19584d0939 | ||
|
richjohnrambo
|
2b631ab176 | ||
|
richjohnrambo
|
b83800c0f3 | ||
|
richjohnrambo
|
05b3da55ac | ||
|
richjohnrambo
|
99ae6328ff | ||
|
richjohnrambo
|
7685fb0c02 | ||
|
richjohnrambo
|
097b028a19 | ||
|
richjohnrambo
|
dcb131dcea | ||
|
richjohnrambo
|
d75c6fa0b4 | ||
|
richjohnrambo
|
06e08a65e4 | ||
|
richjohnrambo
|
9f30995787 | ||
|
richjohnrambo
|
da8efb6e38 | ||
|
richjohnrambo
|
b1545b6876 | ||
|
richjohnrambo
|
4b00b5ae37 | ||
|
Austin
|
ab73b69870 | ||
|
Austin
|
cf9d559944 | ||
|
Austin
|
1a8043bee2 |
@@ -25,6 +25,7 @@ After ArgoCD sync, verify the effective configuration:
|
||||
kubectl -n im-server-system get statefulset im-server -o jsonpath='{.spec.template.spec.containers[0].resources}'
|
||||
kubectl -n im-server-system get statefulset im-server -o jsonpath='{.spec.template.spec.containers[0].livenessProbe.httpGet.path}'
|
||||
kubectl -n im-server-system get statefulset im-server -o jsonpath='{.spec.template.spec.containers[0].env}'
|
||||
kubectl -n im-server-system get statefulset im-server -o jsonpath='{.spec.template.spec.dnsConfig.options}'
|
||||
kubectl auth can-i list pods.metrics.k8s.io \
|
||||
--as=system:serviceaccount:argocd:backend-debug \
|
||||
-n im-server-system
|
||||
@@ -34,5 +35,26 @@ Expected values are requests `1 CPU / 512Mi`, limits `2 CPU / 1Gi`, and
|
||||
liveness path `/livez`. The MySQL pool environment must be
|
||||
`MYSQL_MAX_OPEN_CONNS=24`, `MYSQL_MAX_IDLE_CONNS=8`,
|
||||
`MYSQL_REGISTRY_MAX_OPEN_CONNS=4`, and `MYSQL_REGISTRY_MAX_IDLE_CONNS=2`.
|
||||
The WebSocket heartbeat environment must be `WS_PING_PERIOD_SECONDS=15` and
|
||||
`WS_PONG_WAIT_SECONDS=90`.
|
||||
The Pod DNS options must be `ndots:1`, `timeout:2`, and `attempts:2`. Keeping
|
||||
`ClusterFirst` preserves short Kubernetes service-name resolution, while the
|
||||
lower `ndots` value prevents fully qualified internal and external dependency
|
||||
names from being expanded through every cluster search suffix first.
|
||||
The application repository's deploy workflow waits for reconciliation and
|
||||
fails if ArgoCD restores different values.
|
||||
|
||||
After the rollout, verify the generated resolver configuration on every Pod:
|
||||
|
||||
```bash
|
||||
for pod in im-server-0 im-server-1 im-server-2; do
|
||||
echo "--- ${pod}"
|
||||
kubectl -n im-server-system exec "${pod}" -- cat /etc/resolv.conf
|
||||
done
|
||||
```
|
||||
|
||||
The output must retain the cluster DNS nameserver and search domains, with
|
||||
`options ndots:1 timeout:2 attempts:2`. This mitigation reduces amplified DNS
|
||||
traffic from the application Pods. If lookup timeouts remain under load,
|
||||
CoreDNS capacity or NodeLocal DNSCache must be changed in the cluster
|
||||
infrastructure repository by an administrator with `kube-system` access.
|
||||
|
||||
@@ -6,9 +6,22 @@ metadata:
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
dnsPolicy: ClusterFirst
|
||||
dnsConfig:
|
||||
options:
|
||||
- name: ndots
|
||||
value: "1"
|
||||
- name: timeout
|
||||
value: "2"
|
||||
- name: attempts
|
||||
value: "2"
|
||||
containers:
|
||||
- name: im-server
|
||||
env:
|
||||
- name: WS_PING_PERIOD_SECONDS
|
||||
value: "15"
|
||||
- name: WS_PONG_WAIT_SECONDS
|
||||
value: "90"
|
||||
- name: CLUSTER_PROXY_TO_MASTER_BUFFER
|
||||
value: "4096"
|
||||
- name: CLUSTER_PROXY_TO_MASTER_WORKERS
|
||||
|
||||
@@ -194,6 +194,8 @@ ENV TNPG_ORG=
|
||||
ENV JPUSH_PUSH_ENABLED=false
|
||||
ENV JPUSH_API_DOMAIN=https://api.jpush.cn
|
||||
ENV JPUSH_PUSH_PATH=/v3/push
|
||||
ENV JPUSH_WORKERS=8
|
||||
ENV JPUSH_DEVICE_CACHE_TTL_SECONDS=30
|
||||
ENV JPUSH_DEFAULT_TTL_SECONDS=86400
|
||||
ENV JPUSH_CALL_TTL_SECONDS=60
|
||||
ENV JPUSH_APNS_PRODUCTION=false
|
||||
|
||||
@@ -110,6 +110,8 @@ ENV TNPG_PUSH_ENABLED=false
|
||||
ENV JPUSH_PUSH_ENABLED=false
|
||||
ENV JPUSH_API_DOMAIN=https://api.jpush.cn
|
||||
ENV JPUSH_PUSH_PATH=/v3/push
|
||||
ENV JPUSH_WORKERS=8
|
||||
ENV JPUSH_DEVICE_CACHE_TTL_SECONDS=30
|
||||
ENV JPUSH_DEFAULT_TTL_SECONDS=86400
|
||||
ENV JPUSH_CALL_TTL_SECONDS=60
|
||||
ENV JPUSH_APNS_PRODUCTION=false
|
||||
|
||||
@@ -5,6 +5,8 @@
|
||||
"static_mount": "/",
|
||||
"grpc_listen": ":16060",
|
||||
"grpc_keepalive_enabled": true,
|
||||
"ws_ping_period_seconds": $WS_PING_PERIOD_SECONDS,
|
||||
"ws_pong_wait_seconds": $WS_PONG_WAIT_SECONDS,
|
||||
"api_key_salt": "$API_KEY_SALT",
|
||||
"max_message_size": 4194304,
|
||||
"max_subscriber_count": 128,
|
||||
@@ -83,7 +85,7 @@
|
||||
"max_idle_conns": $MYSQL_MAX_IDLE_CONNS,
|
||||
"registry_max_open_conns": $MYSQL_REGISTRY_MAX_OPEN_CONNS,
|
||||
"registry_max_idle_conns": $MYSQL_REGISTRY_MAX_IDLE_CONNS,
|
||||
"conn_max_lifetime": 60,
|
||||
"conn_max_lifetime": $MYSQL_CONN_MAX_LIFETIME_SECONDS,
|
||||
"sql_timeout": 10,
|
||||
"tenant_registry_db": "$TENANT_REGISTRY_DB",
|
||||
"tenant_db_routing": true
|
||||
@@ -133,7 +135,7 @@
|
||||
"write_timeout_ms": 3000,
|
||||
"read_timeout_ms": 3000,
|
||||
"batch_timeout_ms": 10,
|
||||
"required_acks": 1,
|
||||
"required_acks": -1,
|
||||
"allow_auto_topic_creation": false,
|
||||
"tls_enabled": false,
|
||||
"sasl_enabled": false,
|
||||
@@ -264,6 +266,8 @@
|
||||
"enabled": $JPUSH_PUSH_ENABLED,
|
||||
"api_domain": "$JPUSH_API_DOMAIN",
|
||||
"push_path": "$JPUSH_PUSH_PATH",
|
||||
"workers": $JPUSH_WORKERS,
|
||||
"device_cache_ttl_seconds": $JPUSH_DEVICE_CACHE_TTL_SECONDS,
|
||||
"default_ttl_seconds": $JPUSH_DEFAULT_TTL_SECONDS,
|
||||
"call_ttl_seconds": $JPUSH_CALL_TTL_SECONDS,
|
||||
"apns_production": $JPUSH_APNS_PRODUCTION,
|
||||
|
||||
@@ -13,6 +13,8 @@ else
|
||||
CONFIG=working.config
|
||||
|
||||
: "${REDIS_KEY_PREFIX:=im}"
|
||||
: "${WS_PING_PERIOD_SECONDS:=15}"
|
||||
: "${WS_PONG_WAIT_SECONDS:=90}"
|
||||
: "${CLUSTER_PROXY_TO_MASTER_BUFFER:=4096}"
|
||||
: "${CLUSTER_PROXY_TO_MASTER_WORKERS:=16}"
|
||||
: "${CLUSTER_PROXY_TO_MASTER_ENQUEUE_TIMEOUT_MS:=300}"
|
||||
@@ -22,6 +24,7 @@ else
|
||||
: "${MYSQL_MAX_IDLE_CONNS:=8}"
|
||||
: "${MYSQL_REGISTRY_MAX_OPEN_CONNS:=4}"
|
||||
: "${MYSQL_REGISTRY_MAX_IDLE_CONNS:=2}"
|
||||
: "${MYSQL_CONN_MAX_LIFETIME_SECONDS:=1800}"
|
||||
if [ -z "$CLUSTER_NODES_JSON" ] ; then
|
||||
CLUSTER_NODES_JSON='[{"name":"tinode-0","addr":"tinode-0:12000"},{"name":"tinode-1","addr":"tinode-1:12001"},{"name":"tinode-2","addr":"tinode-2:12002"}]'
|
||||
fi
|
||||
|
||||
@@ -737,7 +737,7 @@ loadtest/im-pressure/internal/collector/
|
||||
loadtest/im-pressure/internal/report/
|
||||
```
|
||||
|
||||
协调方式：Machine A 启动 coordinator，等待 A/B 两个 agent 都上报 1000 个 READY，然后发布一个至少晚于当前时间 30 秒的 UTC `start_at`。两台机器按相同 `run_id` 和确定性调度执行。任何 agent 失联都将本轮标记为 `INVALID`。
|
||||
协调方式：Machine A 启动 coordinator，等待 A/B 两个 agent 都上报 1000 个 READY，然后发布一个至少晚于当前时间 30 秒的 UTC `start_at`。到达 `start_at` 后，A/B 必须在前后 3 秒内向 coordinator 完成最终起跑确认；只有双方都确认后才开始发送，任一方迟到则两边共同终止并将本轮标记为 `INVALID`。两台机器按相同 `run_id` 和确定性调度执行，任何 agent 失联同样判定为 `INVALID`。
|
||||
|
||||
`agent` 自身暴露 `/metrics`，提供连接状态、发布、接收、重复、乱序、断连及 Go/process 指标；ACK 和投递延迟分布写入最终 summary。`collect` 只执行只读查询，并把各数据源的原始快照与查询时间一起保存；连接凭证通过环境变量传入，不写入结果目录。
|
||||
|
||||
|
||||
+37
-4
@@ -154,6 +154,12 @@ WebSocket Upgrade 阶段由 `server/hdl_websock.go` 先调用 `checkAPIKey`。
|
||||
|
||||
客户端只使用稳定的 `reason` 分支和本地化文案，不解析 `message`。ID、Topic、Seq 游标在客户端均按字符串处理；时间统一为 UTC RFC3339 毫秒。分页统一使用 `limit` 与不透明 `cursor`，返回 `items`、`next_cursor`、`has_more`。
|
||||
|
||||
资源不存在类错误统一语义（2026-09-29 修订）：
|
||||
|
||||
- 好友申请 `accept/reject` 在申请不存在时统一返回 `404/20004 RESOURCE_NOT_FOUND`，两个入口语义一致。
|
||||
- 群类接口（`DELETE /v1/im/groups/{topic}`、`POST /v1/im/groups/{topic}/leave`、`POST /v1/im/groups/{topic}/qrcodes`、`PUT /v1/im/groups/{topic}/members/me/profile`、`GET/PATCH /v1/im/groups/{topic}/settings` 等经群成员校验的路径）在群不存在时返回 `404/30004 RESOURCE_NOT_FOUND`；群已解散时返回 `410/30004 GROUP_DISSOLVED`；群存在但当前用户非成员仍是 `403/30005 PERMISSION_DENIED`。
|
||||
- 服务端对不存在的资源不再返回 5xx 或网关错误。
|
||||
|
||||
### 0.5 WebSocket 建连、登录和恢复顺序
|
||||
|
||||
#### 0.5.1 现有服务端可直接执行的冷启动流程
|
||||
@@ -236,6 +242,8 @@ WebSocket Upgrade 阶段由 `server/hdl_websock.go` 先调用 `checkAPIKey`。
|
||||
{"sub":{"id":"sub-topic-1","topic":"grp...","get":{"what":"desc sub data del","data":{"since":125,"limit":50},"del":{"since":1,"limit":50}}}}
|
||||
```
|
||||
|
||||
**向上翻页（加载更早历史）最终参数约定（后端已确认，2026-09-29）**：复用同一 `get data`，最终参数为 `before`（排他上界，返回 `seq < before`），**没有 REST cursor 语义**。聊天页滚动到顶部时发 `data:{"before":<当前已加载最小 seq>,"limit":50}`（`since` 可省略，默认 0）。服务端返回该锚点之前**最接近 `before` 的 `limit` 条**，`{data}` 按 `seq` 从大到小下发（首条即锚点前最新一条）；收到 `ctrl 204 no content`（`params.what=="data"`）表示已到会话顶部。`data` 参数全集：`since`（含下界）、`before`（排他上界）、`limit`、`ranges`。P2P 与群聊（含热缓存 fallback）语义一致。REST `GET /v1/im/conversations/{topic}/messages` 当前为空占位（见 4.21.1），不得用于历史分页。
|
||||
|
||||
订阅成功后才允许 `{pub}`、`note read/recv` 等会话操作。断线恢复必须重新执行 `WebSocket Upgrade -> hi -> token login -> sub me -> 恢复 Topic`，同一连接内不得跳过 `hi` 或切换企业码。
|
||||
|
||||
##### 0.5.1.1 WebSocket 心跳与在线状态
|
||||
@@ -935,7 +943,7 @@ X-MeetingIM-APIKey: <预置的普通 API Key>
|
||||
|
||||
**状态：新增。**
|
||||
|
||||
- 敏感操作先调用 `POST /v1/im/security/challenges`；开启青少年模式设置独立密码时 Body 为 `{"scene":"set_independent_password","channel":"sms"}`，关闭青少年模式时 Body 为 `{"scene":"close_youth_mode","channel":"sms"}`；群主转让使用 `{"scene":"group_owner_transfer","channel":"sms"}`；解散群聊使用 `{"scene":"group_dissolve","channel":"sms"}`。手机号由登录身份读取，客户端不提交目标手机号。
|
||||
- 敏感操作先调用 `POST /v1/im/security/challenges`；开启青少年模式设置独立密码时 Body 为 `{"scene":"set_independent_password","channel":"sms"}`，关闭青少年模式时 Body 为 `{"scene":"close_youth_mode","channel":"sms"}`；群主转让使用 `{"scene":"group_owner_transfer","channel":"sms"}`；解散群聊使用 `{"scene":"group_dissolve","channel":"sms"}`；注销账号使用 `{"scene":"delete_account","channel":"sms"}`。手机号由登录身份读取，客户端不提交目标手机号。
|
||||
- 发码/校验：服务端返回 `challenge_id` 和脱敏手机号；提交 `POST /v1/im/security/challenges/{id}/verify`，Body `{"code":"..."}`。
|
||||
- 重复发码：同一用户、同一手机号、同一敏感场景存在未过期 challenge 且仍在 `resend_after_seconds` 冷却期内时，服务端返回 HTTP `429 VERIFICATION_RATE_LIMITED`，`details` 带 `challenge_id` 和 `retry_after_seconds`。客户端应复用该 `challenge_id` 继续倒计时，不要当成接口不可达或重新登录。
|
||||
- 小时限流：超过一小时创建次数上限时同样返回 HTTP `429 VERIFICATION_RATE_LIMITED`，`details.retry_after_seconds` 可能大于 60 秒且不一定带 `challenge_id`。客户端只展示倒计时，倒计时结束后允许用户重新获取验证码。
|
||||
@@ -961,7 +969,7 @@ X-MeetingIM-APIKey: <预置的普通 API Key>
|
||||
|
||||
- 页面只调用 `GET /v1/im/security/realname`。`verified` 时返回如 `masked_name:"张*"`、`masked_id_number:"110***********1234"`、`verified_at`。
|
||||
- 前端只读展示，不缓存原始证件信息，不提供直接修改接口。
|
||||
- 更正信息使用独立申诉 `POST /v1/im/security/realname/appeals`，需二次身份验证；不复用本页保存按钮。
|
||||
- 更正信息使用独立申诉 `POST /v1/im/security/realname/appeals`，Body `{"reason":"...","attachments":[]}`（`reason` 必填、≤500 字；`attachments` 最多 9 个 file_id），Bearer 即可，当前无需二次身份验证；不复用本页保存按钮。
|
||||
|
||||
### 1.10 实名认证（安卓）.png
|
||||
|
||||
@@ -1425,7 +1433,7 @@ X-MeetingIM-APIKey: <预置的普通 API Key>
|
||||
| `POST /v1/im/conversations/p2p` | `{"peer_user_id":"usr..."}` | `{"topic":"p2p...","relationship":"friend","can_send":true}` |
|
||||
| `GET /v1/im/conversations/{topic}/settings` | Path topic | `{"pinned":false,"mute_notifications":false,"message_retention":"default","blocked":false,"remark":"","version":1,"permissions":{"can_update":true}}` |
|
||||
| `PATCH /v1/im/conversations/{topic}/settings` | `{"pinned":true,"mute_notifications":false,"version":1}` | `{"pinned":true,"mute_notifications":false,"message_retention":"default","blocked":false,"remark":"","version":2}` |
|
||||
| `GET /v1/im/conversations/{topic}/messages` | Query `?since=12&limit=20` | `{"topic":"p2p...","items":[{"seq":13,"from":"usr...","ts":"2026-08-20T09:00:00Z","head":{"x-im-type":"text"},"content":{"text":"你好"}}],"next_cursor":""}` |
|
||||
| `GET /v1/im/conversations/{topic}/messages` | **服务端当前为空占位**：GET 恒返回空列表，勿用 `?since=&limit=` 调它；历史消息（首屏与向上翻页）一律走 WS `get data` 的 `since`/`before`/`limit`（向上翻页见 0.5.1 约定） | `{"topic":"p2p...","items":[],"next_cursor":"","has_more":false}` |
|
||||
| `PUT /v1/im/topics/{topic}/draft` 待实现 | `{"content":"未发送草稿","version":1}` | `{"topic":"grp...","content":"未发送草稿","version":2,"updated_at":"2026-08-20T09:00:00Z"}` |
|
||||
| `POST /v1/im/topics/{topic}/messages/{seq}/recall` | 单条撤回；`topic` 可传 `p2p...` 或 `grp...`；Body `{}`，可选 `{"version":1,"reason":"mistake"}` | `{"topic":"grp...","seq":12,"status":"recalled","version":2,"updated_at":"2026-08-20T09:00:00Z"}` |
|
||||
| `POST /v1/im/topics/{topic}/messages/recall` | 多选批量撤回；`{"items":[{"seq":12,"version":1},{"seq":13}],"reason":"mistake"}`；`items` 1～50 条，`version` 可选 | `{"topic":"grp...","items":[{"topic":"grp...","seq":12,"status":"recalled","version":2,"updated_at":"2026-08-20T09:00:00Z"},{"seq":13,"status":"failed","http":409,"code":30006,"reason":"VERSION_CONFLICT","message":"message version conflict"}]}` |
|
||||
@@ -1706,6 +1714,7 @@ GET /v1/im/topics/{topic}/message-mutations?since_version={localMutationVersion}
|
||||
- 列表：`GET /v1/im/friend-requests?direction=inbound&status=all&cursor=`，返回 `request_id,applicant,message,source,status,created_at,handled_at`。
|
||||
- 同意：`POST /v1/im/friend-requests/{request_id}/accept`；拒绝：`POST /v1/im/friend-requests/{request_id}/reject`，均带幂等键和可选 `version`。
|
||||
- 成功返回最终 `status` 和 P2P `topic`。重复审批返回原结果；并发冲突返回 `REQUEST_ALREADY_HANDLED` 并附当前状态，前端刷新该项。
|
||||
- 申请不存在时 `accept/reject` 统一返回 `404/20004 RESOURCE_NOT_FOUND`；通用语义见 0.4。
|
||||
- 主动下行：创建申请时被申请人收到 `info.what="im", event="friend.request.created"`；通过/拒绝后申请人与处理人本人都收到 `friend.request.accepted/rejected`。前端收到后刷新新朋友列表、好友列表、会话 context 和 `bootstrap.entry_badges`。当前系统通知列表只给被申请人/申请人写记录，处理人本人主要靠实时业务事件刷新。
|
||||
|
||||
### 5.3 添加好友.png
|
||||
@@ -1960,6 +1969,7 @@ GET /v1/im/topics/{topic}/message-mutations?since_version={localMutationVersion}
|
||||
- `GET /v1/im/groups/{topic}/settings` 返回 `my_role,permissions,profile,announcement,member_count,pinned,mute_notifications` 和群治理开关。
|
||||
- 个人置顶/免打扰通过 `PATCH /v1/im/groups/{topic}/member-settings`；群级设置必须使用 6.17，不混在个人设置里。
|
||||
- 退出：`POST /v1/im/groups/{topic}/leave`；群主必须先转让或解散。清记录、举报分别见 5.14、11.3。
|
||||
- 群不存在返回 `404/30004 RESOURCE_NOT_FOUND`；群已解散返回 `410/30004 GROUP_DISSOLVED`；非群成员返回 `403/30005 PERMISSION_DENIED`。通用语义见 0.4。
|
||||
|
||||
### 6.11 群资料.png
|
||||
|
||||
@@ -1989,6 +1999,7 @@ GET /v1/im/topics/{topic}/message-mutations?since_version={localMutationVersion}
|
||||
|
||||
- `PUT /v1/im/groups/{topic}/members/me/profile`，Body `{"nickname":"...","version":n}`。
|
||||
- 服务端校验成员资格、长度和敏感词，返回 `nickname,version,updated_at` 并推送成员资料变化。
|
||||
- 群不存在返回 `404/30004 RESOURCE_NOT_FOUND`；群已解散返回 `410/30004 GROUP_DISSOLVED`；非群成员返回 `403/30005 PERMISSION_DENIED`。通用语义见 0.4。
|
||||
- 该昵称仅作用当前群，不能修改全局用户昵称或其他群资料。
|
||||
- 群成员列表和群成员资料中的 `display_name` 仍由服务端按当前登录人计算，好友备注优先；群内昵称单独返回为 `group_nickname`，前端不要用 `nickname/group_nickname` 覆盖 `display_name`。
|
||||
|
||||
@@ -2057,6 +2068,7 @@ GET /v1/im/topics/{topic}/message-mutations?since_version={localMutationVersion}
|
||||
**状态：新增。**
|
||||
|
||||
- `POST /v1/im/groups/{topic}/qrcodes`，Body `{"expires_in_seconds":86400}`，返回签名 `qr_token,qr_payload,expires_at,group,group_snapshot`；`group` 与 `group_snapshot` 内容一致，前端优先使用 `group`。
|
||||
- 群不存在返回 `404/30004 RESOURCE_NOT_FOUND`；群已解散返回 `410/30004 GROUP_DISSOLVED`；非群主/管理员返回 `403/30005 PERMISSION_DENIED`。通用语义见 0.4。
|
||||
- 扫码调用 `POST /v1/im/qrcodes/resolve`，`purpose:"join_group"`；随后 `POST /v1/im/groups/{topic}/join-requests` 并携带 `qr_token`。
|
||||
- 群解散、二维码过期或管理员撤销后必须失效；二维码不得直接承载内部群权限。
|
||||
|
||||
@@ -2069,6 +2081,7 @@ GET /v1/im/topics/{topic}/message-mutations?since_version={localMutationVersion}
|
||||
- 查询/更新：`GET/PATCH /v1/im/groups/{topic}/governance`，字段为 `join_approval,mute_all,member_upload_files,member_private_chat,member_invite` 和 `version`。
|
||||
- 每个字段返回 `editable`；PATCH 仅群主或被授权管理员可用，记录操作者、旧值、新值和 Request ID。
|
||||
- 解散：`DELETE /v1/im/groups/{topic}`，需二次 `X-Action-Token` 和幂等键；成功后撤销 Topic、二维码、邀请和文件新增权限。
|
||||
- 群不存在返回 `404/30004 RESOURCE_NOT_FOUND`；已解散重复解散返回 `410/30004 GROUP_DISSOLVED`；非群主返回 `403/30005 PERMISSION_DENIED`。通用语义见 0.4。
|
||||
|
||||
### 6.18 设置群管理员.png
|
||||
|
||||
@@ -3166,9 +3179,29 @@ GET /v1/im/topics/{topic}/message-mutations?since_version={localMutationVersion}
|
||||
| `POST /v1/im/security/challenges/{id}/verify` 关闭青少年模式 | `{"code":"123456"}` | `{"action_token":"close_token","action":"close_youth_mode","expires_at":"2026-08-20T09:05:00Z"}` |
|
||||
| `GET /v1/im/security/realname` | Query only | `{"status":"unverified","can_submit":true}` |
|
||||
| `POST /v1/im/security/realname/submissions` | `{"name":"张三","id_type":"cn_id","id_number":"110101199001011234"}` | `{"submission_id":"rn_1","status":"pending","can_submit":false}` |
|
||||
| `POST /v1/im/security/realname/appeals` 待实现 | `{"reason":"证件变更","description":"需要更正实名信息","action_token":"action_token"}` | `{"appeal_id":"rna_1","status":"pending","submitted_at":"2026-08-20T09:00:00Z"}` |
|
||||
| `POST /v1/im/security/realname/appeals` | `{"reason":"证件变更需要更正实名信息","attachments":["file_1"]}` | `{"appeal_id":"rna_1","reason":"...","attachments":["file_1"],"status":"pending"}`；当前无需二次身份验证（如后续启用，将复用 1.7 的 action_token 机制并在此处更新） |
|
||||
| `PUT /v1/im/devices/{device_id}/push-token` | `{"provider":"jpush","token":"registration_id","platform":"android","app_version":"1.0.0","push_env":"test","sdk_version":"5.0.0","notification_permission":"granted"}` | `{"device_id":"device_1","provider":"jpush","platform":"android","app_version":"1.0.0","updated_at":"2026-08-20T09:00:00Z"}` |
|
||||
| `DELETE /v1/im/security/sessions/{session_id}` 待实现 | Header `X-Action-Token: action_token` | `{"session_id":"sess_1","revoked":true}` |
|
||||
| `POST /v1/im/me/password` | `{"old_password":"OldPass123","new_password":"NewPass123"}` | `{"status":"ok"}`；旧密码错误返回 HTTP `401`：`{"code":30001,"reason":"AUTH_REQUIRED","message":"invalid credentials"}`；新密码不满足策略返回 HTTP `400`：`{"code":30040,"reason":"PASSWORD_POLICY_VIOLATION"}` |
|
||||
| `POST /v1/im/security/challenges` 注销账号发码 | Header `Idempotency-Key`；Body `{"scene":"delete_account","channel":"sms"}` | `{"challenge_id":"sec_chg_9","phone_mask":"+861****000","expires_in_seconds":300,"resend_after_seconds":60}` |
|
||||
| `POST /v1/im/security/challenges/{id}/verify` 注销账号验证 | `{"code":"123456"}` | `{"action_token":"del_token","action":"delete_account","expires_at":"2026-08-20T09:05:00Z"}` |
|
||||
| `POST /v1/im/me/deletion` | Header `X-Action-Token: del_token`；Body `{}` | `{"status":"ok"}` |
|
||||
|
||||
#### 10.22.2 账号安全：修改密码与注销账号
|
||||
|
||||
**修改密码 `POST /v1/im/me/password`（已实现）：**
|
||||
|
||||
- 已登录（Bearer）用户提交 `old_password`、`new_password`；两者必填，缺一返回 `400 MALFORMED_REQUEST`。
|
||||
- 服务端先用旧密码走 `basic` 认证校验，失败返回 HTTP `401`：`{"code":30001,"reason":"AUTH_REQUIRED","message":"invalid credentials"}`，不泄露账号状态。
|
||||
- 新密码不满足密码策略返回 HTTP `400`：`{"code":30040,"reason":"PASSWORD_POLICY_VIOLATION"}`。
|
||||
- 成功后服务端递增 `auth_version` 并撤销该用户全部在线 Session（广播 `password_changed`）；包括当前请求在内的所有旧 Token 全部失效，客户端收到成功响应后必须清理本租户凭证并跳转登录页。
|
||||
|
||||
**注销账号 `POST /v1/im/me/deletion`（已实现）：**
|
||||
|
||||
- 两步挑战复用 1.7 敏感操作基建：`POST /v1/im/security/challenges`（Body `{"scene":"delete_account","channel":"sms"}`，发码到账号绑定手机号）→ `POST /v1/im/security/challenges/{id}/verify`（返回一次性 `action_token`）→ `POST /v1/im/me/deletion`（Header `X-Action-Token` 携带）。
|
||||
- 服务端配置 `permanent_accounts=true` 时拒绝注销：HTTP `403` `{"code":30040,"reason":"FORBIDDEN","message":"account deletion is disabled"}`。
|
||||
- 执行成功为**软删除**：撤销全部认证记录（含登录密码）、踢出所有在线 Session（`reason=user_deleted`）、退出并停用其拥有的群与 P2P 会话、向关注该用户的好友下发 `gone` 在线状态；账号不再可登录。客户端收到成功响应后清理本地数据并返回企业入口。
|
||||
- 挑战阶段限流与错误码同 1.7（`VERIFICATION_RATE_LIMITED`、`CHALLENGE_EXPIRED`、`CODE_INVALID`、`TOO_MANY_ATTEMPTS`）；action token 缺失/无效返回 HTTP `403` `{"code":30042,"reason":"ACTION_TOKEN_INVALID"}`。
|
||||
|
||||
## 11. 反诈安全、举报与意见反馈
|
||||
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
-- Persist client-provided upload filename for downstream business modules.
|
||||
-- Apply before deploying im_server changes which read/write im_fileuploads.original_name.
|
||||
|
||||
ALTER TABLE im_fileuploads
|
||||
ADD COLUMN original_name VARCHAR(255) NOT NULL DEFAULT '' AFTER location;
|
||||
@@ -0,0 +1,73 @@
|
||||
# GROUP-2000 R9 双机群聊压测分析与改进
|
||||
|
||||
## 1. 结论
|
||||
|
||||
`group2k-r9-20261003-01` 不能作为 2000 人群聊容量结论。消息最终落库完整，但实时阶段同时发生机器 B 迟到、Pod 重启和集群 DNS 拒绝查询，导致大量连接断开和跨机器投递缺失。
|
||||
|
||||
- 发送尝试 2000 条，数据库精确找到 1774 条服务端已确认消息，缺失、重复、字段不一致均为 0。
|
||||
- 实时期望投递 3,548,000 次，实际收到 1,384,872 次，到达率约 39.03%。
|
||||
- 机器 B 比 coordinator 的计划起跑时间晚约 7 分 55 秒；B 未收到任何 A 消息，本轮并非有效的同步双机负载。
|
||||
- `im-server-1` 在负载中退出，容器退出码 137；现有权限无法读取 Event 和 Pod metrics，尚不能把原因唯一归为 OOM、liveness 或节点压力。
|
||||
- 三个 Pod 合计约 1715 次依赖域名解析失败，日志包含对集群 DNS `10.43.0.10:53` 的 `connection refused`。
|
||||
- JPush 设备查询产生 792 次数据库错误；用户更新队列满 1132 次，属于依赖抖动和连接重建后的次生压力。
|
||||
|
||||
Kafka 持久化链路本轮表现正确：各 Pod 批量持久化计数合计 1774，与数据库精确核验一致，之前的失败消息保留修复已生效。
|
||||
|
||||
## 2. 根因链路
|
||||
|
||||
### 2.1 JPush 查询按群消息放大
|
||||
|
||||
JPush 使用 registration ID 模式时，每条群消息都会按全部收件人重新查询 `im_device_push_tokens`。2000 人群和 2000 条消息会形成约 400 万个用户查询输入；测试账号没有推送设备，旧实现仍重复查询空结果。JPush 回执又以无限 goroutine 并发处理，使查询、对象分配和内存压力同时放大。
|
||||
|
||||
### 2.2 MySQL 连接每 60 秒回收
|
||||
|
||||
旧配置的 `conn_max_lifetime=60` 会在持续压测期间频繁重建连接。每次建连都重新解析 `mysql.118im.com`；当集群 DNS 短时拒绝请求时，业务查询、JPush 查询和 Kafka 持久化会一起失败或等待。
|
||||
|
||||
### 2.3 双机只有计划时间，没有最终到达屏障
|
||||
|
||||
旧协议在双方 READY 后下发统一时间，但 agent 到达该时间后直接发送。机器 B 晚启动时，机器 A 无法知道对方没有真正进入发送阶段，因此形成两段错开的单机流量并生成无效的跨机器统计。
|
||||
|
||||
## 3. 已完成改进
|
||||
|
||||
### 3.1 JPush
|
||||
|
||||
- 设备查询结果按租户、provider、用户缓存 30 秒，包含“无设备”结果。
|
||||
- 并发首次查询被合并，同一批群成员只访问一次数据库。
|
||||
- 查询失败后退避 1 秒，已有缓存可在 5 分钟陈旧窗口内继续使用。
|
||||
- 缓存定期清理，避免用户集合长期增长。
|
||||
- JPush 回执限制为 8 个工作协程，取消无限 goroutine，队列负责形成背压。
|
||||
|
||||
该改动不删除在线多端静默推送语义。新注册或更新的设备最迟约 30 秒进入推送查询结果。
|
||||
|
||||
### 3.2 MySQL 与 DNS
|
||||
|
||||
- 容器默认 `conn_max_lifetime` 从 60 秒提高到 1800 秒，并支持 `MYSQL_CONN_MAX_LIFETIME_SECONDS` 覆盖。
|
||||
- 测试环境 GitOps 显式设置 1800 秒，降低持续负载中的连接重建和 DNS 查询频率。
|
||||
- 保留现有 `ndots:1`、`timeout:2`、`attempts:2`。
|
||||
|
||||
这会显著减少应用制造的 DNS 压力，但不能替代集群 DNS 修复。若复测仍出现 `10.43.0.10:53 connection refused`，需要集群管理员检查 CoreDNS 副本、CPU、转发上游、conntrack，并评估与当前 K3s 网络模式匹配的 NodeLocal DNSCache；不能在未知网络模式下直接套用 DaemonSet。
|
||||
|
||||
### 3.3 压测同步
|
||||
|
||||
- coordinator 增加最终起跑屏障。
|
||||
- A/B 必须在计划时间前后 3 秒内到达，双方都到达后才开始发送。
|
||||
- 任一机器迟到时，两边收到同一失败原因并终止本轮。
|
||||
- coordinator 输出保存双方实际到达时间和 `start_error`。
|
||||
|
||||
### 3.4 诊断权限
|
||||
|
||||
`backend-debug` 只读权限增加 Event 和 `metrics.k8s.io/pods`，用于下一轮区分 OOM、探针失败、驱逐和节点资源压力。若集群未安装 metrics-server，Pod metrics 仍不可用，但 Event 读取不受影响。
|
||||
|
||||
## 4. 复测门禁
|
||||
|
||||
下一轮使用新 `run_id`，A/B 必须更新到包含起跑屏障的同一提交，并在应用和 GitOps 配置都发布后执行。
|
||||
|
||||
1. coordinator 输出中 A/B `started` 时间相差不超过 3 秒，`start_error` 为空。
|
||||
2. 2000 条发送全部得到明确 ACK；数据库按 `client_mid` 精确匹配，缺失、重复、字段不一致均为 0。
|
||||
3. 实时期望 4,000,000 次投递全部到达，重复、乱序和断连均为 0。
|
||||
4. JPush 设备查询不再随群消息数线性增长，日志中 `jpush: db error` 为 0。
|
||||
5. Pod 不重启，Event 中无 OOM、Unhealthy、Evicted；负载窗口内持续采集 CPU、内存和重启次数。
|
||||
6. `10.43.0.10:53`、MySQL、Redis、Kafka 域名解析错误均为 0。
|
||||
7. Kafka lag 最终归零，数据库最终消息数与 ACK 台账一致。
|
||||
|
||||
若业务正确性通过但 DNS 仍有拒绝请求，本轮仍判基础设施门禁失败，不继续用增加应用重试掩盖 DNS 容量问题。
|
||||
@@ -0,0 +1,191 @@
|
||||
# IM 单机 60% 容量压力测试方案
|
||||
|
||||
日期：2026-09-29
|
||||
|
||||
本方案首轮执行结果见 [IM-单机60%容量压力测试结果-20260929.md](./IM-单机60%容量压力测试结果-20260929.md)。
|
||||
|
||||
## 1. 目标与口径
|
||||
|
||||
本方案用一台物理压测机完成单聊、群聊和混合场景，容量按双机正式方案全局规模的 60% 计算。
|
||||
|
||||
单机不等于只运行一个 agent。为保留现有好友对和投递契约，同一台物理机上同时运行逻辑 agent A 和 agent B，每个 agent 承载 600 个连接，合计 1200 个 WebSocket。Coordinator 仍使用 `expected-agents=2`。
|
||||
|
||||
| 项目 | 双机正式方案 | 单机 60% 方案 |
|
||||
|---|---:|---:|
|
||||
| 物理压测机 | 2 | 1 |
|
||||
| 逻辑 agent | 2 | 2 |
|
||||
| 每个 agent 连接 | 1000 | 600 |
|
||||
| WebSocket 总连接 | 2000 | 1200 |
|
||||
| 好友对 | 1000 | 600 |
|
||||
| 群成员 | 2000 | 1200 |
|
||||
|
||||
## 2. 场景容量
|
||||
|
||||
### 2.1 单聊
|
||||
|
||||
- 600 对好友，A/B 各 600 人。
|
||||
- 每个用户发送 10 条，每秒 1 条。
|
||||
- 总发送尝试 12000 条，是双机 20000 条的 60%。
|
||||
- 每条 ACK 消息应在配对用户的 WebSocket 到达一次。
|
||||
|
||||
### 2.2 群聊
|
||||
|
||||
- 1200 个在线群成员。
|
||||
- A/B 各取前 300 个用户作为发送者，合计 600 个发送者。
|
||||
- 每人发送 2 条，全局每 100ms 一条，合计 1200 条群消息，耗时约 120 秒。
|
||||
- 全部发送被 ACK 时，期望实时投递 `1200 x 1200 = 1,440,000` 次。
|
||||
|
||||
### 2.3 单聊和群聊混合
|
||||
|
||||
- 前 180 对好友互发，每人 5 条，共 1800 条单聊。
|
||||
- A/B 各 150 个群发送者，每人 2 条，共 600 条群消息。
|
||||
- 群消息期望实时投递 `600 x 1200 = 720,000` 次。
|
||||
- 单聊和群聊同时开始，不分两轮单独执行。
|
||||
|
||||
工具按 `users_per_machine` 等比计算发送者数量。`users_per_machine=1000` 时保持原双机参数，设为 600 时自动使用上述单机参数。
|
||||
|
||||
## 3. 文件隔离
|
||||
|
||||
单机方案不复用双机的运行文件，避免把 1200 人结果和 2000 人结果混在一起。
|
||||
|
||||
```text
|
||||
loadtest/im-pressure/config.single-machine.json
|
||||
loadtest/im-pressure/manifest.single-machine.json
|
||||
loadtest/im-pressure/results/<RUN_ID>/
|
||||
```
|
||||
|
||||
从模板生成本地配置：
|
||||
|
||||
```bash
|
||||
cp loadtest/im-pressure/config.single-machine.example.json \
|
||||
loadtest/im-pressure/config.single-machine.json
|
||||
chmod 600 loadtest/im-pressure/config.single-machine.json
|
||||
```
|
||||
|
||||
修改租户、HTTP/WS 地址和注册手机前缀。A/B endpoint 可以填同一个公网入口；若需确认跨 Pod 投递，应分别填写两个定向入口。
|
||||
|
||||
## 4. 一次性数据准备
|
||||
|
||||
```bash
|
||||
source loadtest/im-pressure/.env.test
|
||||
go build -o loadtest/im-pressure/im-pressure ./loadtest/im-pressure
|
||||
ulimit -n 100000
|
||||
|
||||
./loadtest/im-pressure/im-pressure prepare \
|
||||
-config loadtest/im-pressure/config.single-machine.json \
|
||||
-manifest loadtest/im-pressure/manifest.single-machine.json
|
||||
|
||||
./loadtest/im-pressure/im-pressure verify \
|
||||
-config loadtest/im-pressure/config.single-machine.json \
|
||||
-manifest loadtest/im-pressure/manifest.single-machine.json
|
||||
```
|
||||
|
||||
`verify` 必须输出 1200 个账号、600 对好友和 1200 个群成员。单机 manifest 可以复用已注册的 `lta0001..0600` 和 `ltb0001..0600` 账号，但必须使用独立群 Topic。
|
||||
|
||||
## 5. 正式运行
|
||||
|
||||
以群聊为例：
|
||||
|
||||
```bash
|
||||
export RUN_ID='single60-group-r1-YYYYMMDD-01'
|
||||
export SCENARIO='group'
|
||||
mkdir -p "loadtest/im-pressure/results/${RUN_ID}"
|
||||
```
|
||||
|
||||
终端 1，启动 coordinator：
|
||||
|
||||
```bash
|
||||
./loadtest/im-pressure/im-pressure coordinator \
|
||||
-listen :8090 \
|
||||
-run-id "${RUN_ID}" \
|
||||
-scenario "${SCENARIO}" \
|
||||
-expected-agents 2 \
|
||||
-output "loadtest/im-pressure/results/${RUN_ID}/coordinator.json"
|
||||
```
|
||||
|
||||
终端 2，启动逻辑 agent A：
|
||||
|
||||
```bash
|
||||
./loadtest/im-pressure/im-pressure agent \
|
||||
-config loadtest/im-pressure/config.single-machine.json \
|
||||
-manifest loadtest/im-pressure/manifest.single-machine.json \
|
||||
-machine a \
|
||||
-coordinator http://127.0.0.1:8090 \
|
||||
-metrics-listen :9105 \
|
||||
-output "loadtest/im-pressure/results/${RUN_ID}/agent-a-summary.json"
|
||||
```
|
||||
|
||||
终端 3，启动逻辑 agent B：
|
||||
|
||||
```bash
|
||||
./loadtest/im-pressure/im-pressure agent \
|
||||
-config loadtest/im-pressure/config.single-machine.json \
|
||||
-manifest loadtest/im-pressure/manifest.single-machine.json \
|
||||
-machine b \
|
||||
-coordinator http://127.0.0.1:8090 \
|
||||
-metrics-listen :9106 \
|
||||
-output "loadtest/im-pressure/results/${RUN_ID}/agent-b-summary.json"
|
||||
```
|
||||
|
||||
两个 agent 都报告 `600/600 READY` 并稳定通过 30 秒屏障后才会开始。任一 agent 未满连接时 coordinator 会撤销开始时间。
|
||||
|
||||
固定 drain 结束后，agent 还会等待入站投递连续静默后再生成实时 summary。默认 `delivery_quiet_period=10s`、`delivery_quiet_timeout=3m`；接收计数增长或回调队列不为空都会重置静默窗口，避免公网晚到帧被误判为实时丢失。
|
||||
|
||||
## 6. 指标采集
|
||||
|
||||
终端 4 在起压前至少 2 分钟启动：
|
||||
|
||||
```bash
|
||||
./loadtest/im-pressure/im-pressure collect \
|
||||
-duration 20m -interval 5s \
|
||||
-coordinator http://127.0.0.1:8090 \
|
||||
-coordinator-output "loadtest/im-pressure/results/${RUN_ID}/coordinator.json" \
|
||||
-url agent-a=http://127.0.0.1:9105/metrics \
|
||||
-url agent-b=http://127.0.0.1:9106/metrics \
|
||||
-url node=http://127.0.0.1:9100/metrics \
|
||||
-output "loadtest/im-pressure/results/${RUN_ID}/metrics.jsonl"
|
||||
```
|
||||
|
||||
同时采集 Redis、Kafka、MySQL、Pod CPU/内存/网络、Kafka consumer lag、MySQL 连接池等指标。如本机没有 `node-exporter :9100`，删除 `node` URL，但不影响 agent 指标采集。
|
||||
|
||||
发版后必须在服务端 `/metrics` 确认 `im_ws_heartbeat_seconds{setting="ping_period"}=15`、`im_ws_heartbeat_seconds{setting="pong_wait"}=90`，并采集 `im_ws_heartbeat_total{event="pong_received"}` 和 `im_ws_disconnect_total{reason="timeout"}` 的分 Pod 增量。
|
||||
|
||||
混合场景还必须采集 `im_ctrl_ack_delivery_duration_seconds{stage="enqueue"}` 和 `im_ctrl_ack_delivery_duration_seconds{stage="write"}`。agent 端同时采集 `im_load_agent_publish_ctrl_observed_total`、`im_load_agent_publish_ctrl_over_20s_total`、`im_load_agent_publish_ctrl_late_total` 和 `im_load_agent_publish_ctrl_unresolved`，用于区分及时 ACK、超过 20 秒的 ACK、超过命令超时后晚到的 ACK 与最终未到 ACK。
|
||||
|
||||
## 7. 报告和独立落库核验
|
||||
|
||||
```bash
|
||||
./loadtest/im-pressure/im-pressure merge \
|
||||
-a "loadtest/im-pressure/results/${RUN_ID}/agent-a-summary.json" \
|
||||
-b "loadtest/im-pressure/results/${RUN_ID}/agent-b-summary.json" \
|
||||
-output "loadtest/im-pressure/results/${RUN_ID}/report.json"
|
||||
|
||||
export MYSQL_DSN='readonly:password@tcp(mysql-host:3306)/meeting_im?parseTime=true'
|
||||
export TENANT='006' # 必须与 config.single-machine.json 中的 tenant 一致
|
||||
./loadtest/im-pressure/im-pressure verify-db \
|
||||
-coordinator "loadtest/im-pressure/results/${RUN_ID}/coordinator.json" \
|
||||
-tenant "${TENANT}" \
|
||||
-dsn-env MYSQL_DSN \
|
||||
-registry-db hotgo \
|
||||
-output "loadtest/im-pressure/results/${RUN_ID}/db-verification.json"
|
||||
```
|
||||
|
||||
## 8. 验收门禁
|
||||
|
||||
- READY 必须是 `1200/1200`，统计窗口内断连为 0。
|
||||
- publish 明确拒绝和结果未知均为 0，成功率至少 99.99%。
|
||||
- `publish_ctrl_unresolved` 必须为 0；`publish_ctrl_over_20s` 和 `publish_ctrl_late` 作为定位指标单独报告，不与消息丢失混为一项。
|
||||
- ACK P95 和投递 P95 保留记录用于趋势对比，不作为当前公网单机容量测试的阻断门禁。
|
||||
- 实时缺失、重复、乱序、未知本轮消息均为 0。
|
||||
- ACK 台账和无 ACK 但已观测台账的历史回查命中率必须 100%，独立 MySQL 核验丢失、错配、重复均为 0。
|
||||
- 实时发送、历史回查和渐进关闭三个阶段分别统计 Topic/Proxy/Session 队列满，不能只看整轮总数。
|
||||
|
||||
## 9. 单机特有风险
|
||||
|
||||
1. 两个 agent 共用一台机器的 CPU、内存、FD、NAT 和出口带宽，测到的是“单机压测端 + 服务端”综合上限，不能代替双机最终验收。
|
||||
2. 正式前做 120/600/1200 连接三档阶梯，观察 agent CPU、`go_goroutines`、FD、接收速率和 Ping/Pong 断连，确认压测端本身没有先饱和。当前工具已将投递观测按 `client_mid` 分片、接收账号顺序状态独立加锁，使用原子延迟直方图，并将数据统计回调从 WebSocket socket 读循环解耦；仍需用指标证明本机有余量。
|
||||
3. 两个 agent 必须使用不同指标端口，不能同时绑定 `:9105`。
|
||||
4. 压测域名如被 VPN/TUN 接管，起压前必须确认所有当前 IP 的路由都指向物理网卡。
|
||||
5. 单机 60% 通过后只能进入双机 100% 验收，不能直接得出 2000 连接容量通过的结论。
|
||||
|
||||
断连时同时查看 agent summary 中的 `disconnect_reasons` 和 `disconnect_samples`。前者按 timeout、abnormal close、connection reset 等原因分类计数，后者最多保留 100 条带账号与时间的原始错误样本，用于区分压测端读循环阻塞、网络中断和服务端主动关闭。
|
||||
@@ -0,0 +1,949 @@
|
||||
# IM 单机 60% 容量压力测试结果
|
||||
|
||||
日期：2026-09-29
|
||||
|
||||
## 1. 结论
|
||||
|
||||
本轮使用一台物理压测机同时运行逻辑 agent A/B，每侧 600 个 WebSocket，合计 1200 连接。60% 容量的正确性和性能总验收未通过。
|
||||
|
||||
- 单聊实时和持久化正确性通过；ACK P95 和投递 P95 仅记录，不再作为当前公网单机压测的阻断条件。
|
||||
- 群聊在 1200 条群消息、144 万设计投递量下出现 ACK 超时、连接重置和分钟级投递延迟。
|
||||
- 混合场景明显越过容量拐点，737 个连接断开，仅 44.83% 发送得到 ACK。
|
||||
- 独立 MySQL 核验确认：三个场景的所有 ACK 消息均精确落库，0 错配、0 重复；所有“无 ACK 但 WS 已观测”消息也均已落库。
|
||||
- 压测工具本轮峰值 RSS 不超过 127 MB，每个 agent 峰值 FD 614，未显示本机内存或 FD 先饱和。
|
||||
- 混合实时阶段的群 Topic 所在 `im-server-0` 出现 broadcast queue full，服务端 Topic 处理能力是已证实的瓶颈之一。
|
||||
|
||||
## 2. 环境与口径
|
||||
|
||||
- 租户：`allrich`
|
||||
- HTTP/WS 入口：`api.118im.com`
|
||||
- 物理压测机：1 台
|
||||
- 逻辑 agent：A/B 各2个中的一个，每侧 600 连接
|
||||
- 账号：1200，好友对：600，群成员：1200
|
||||
- API 域名当前 3 个 IP 均经 `en1` 和局域网网关直连，未走 VPN/TUN
|
||||
- IM Server：3 Pod，无重启；每 Pod request `1 CPU/512Mi`，limit `2 CPU/1Gi`
|
||||
- 独立群与 manifest：`loadtest/im-pressure/manifest.single-machine.json`
|
||||
|
||||
准备阶段最终验证为 `accounts=1200 pairs=600 group_members=1200`。群批量邀请曾出现入口 502 和幂等任务长时间处理中，工具通过按实际成员补齐缺失用户完成恢复。
|
||||
|
||||
## 3. 结果汇总
|
||||
|
||||
| 场景 | Run ID | ACK | 实时投递 | 断连 | ACK P95 | 投递 P95 | 独立 DB 核验 | 结果 |
|
||||
|---|---|---:|---:|---:|---:|---:|---:|---|
|
||||
| 单聊 | `s60-p2p-r2-0929` | 12000/12000 | 12000/12000 | 0 | 1341.555ms | 2000ms | 12000 ACK 全部精确命中 | 正确性通过 |
|
||||
| 群聊 | `s60-group-r1-0929` | 1145/1200 | 1350508/1374000 ACK 台账 | 47 | 11207.859ms | 94621.882ms | 1145 ACK + 39 无 ACK 落库 | 未通过 |
|
||||
| 混合 | `s60-mixed-r1-0929` | 1076/2400 | 234829/263657 ACK 台账 | 737 | 17491.634ms | 101226.392ms | 1076 ACK + 1017 无 ACK 落库 | 未通过 |
|
||||
|
||||
表中群聊和混合的“期望投递”只按有 ACK 的 ledger 计算。发送结果未知的消息不能在没有 seq 时直接加入期望投递分母，因此该分母小于设计投递量。
|
||||
|
||||
## 4. 分场景记录
|
||||
|
||||
### 4.1 单聊
|
||||
|
||||
R1 的历史查询将 `limit=10` 错误地按目标消息数计算，没有计入 P2P Topic 中双方交错的 seq，造成 4619 条假缺失。修复后 R2 结果：
|
||||
|
||||
- 12000 条全部 ACK，无结果未知。
|
||||
- 12000 次对端投递全部到达，无重复、乱序、断连。
|
||||
- 12000 条历史回查全部命中，持久化缺失 0。
|
||||
- ACK P95 1341.555ms，同机投递 P95 落在 2000ms 桶；两项作为公网链路观测数据，不影响正确性验收。
|
||||
|
||||
### 4.2 群聊
|
||||
|
||||
- 发送 1200 条，1145 条 ACK，43 条 ACK 超时，12 条 socket 结果未知，无明确拒绝。
|
||||
- 47 个连接断开：22 个 abnormal close，25 个 connection reset by peer。
|
||||
- ACK 台账下实时缺失 23492 次，这些消息最终均可从历史查到。
|
||||
- 55 条结果未知中，39 条被接收端实时观测，且全部被独立 SQL 确认落库；另 16 条既无 ACK、无 WS 观测，也无数据库记录。
|
||||
- 断连发生时间为 `03:40:10Z` 至 `03:41:15Z`，与群实时发送阶段重合。
|
||||
|
||||
### 4.3 混合
|
||||
|
||||
- 设计发送为 1800 条单聊和 600 条群聊，合计 2400 条。
|
||||
- 1076 条 ACK，1324 条结果未知，其中 1036 条 ACK 超时、288 条 socket 失败，无明确拒绝。
|
||||
- 737 个连接断开：69 个 abnormal close，668 个 connection reset by peer。
|
||||
- ACK 台账下实时缺失 28828 次，其中 28633 次对应消息最终已查到持久化记录。
|
||||
- 历史验证 ledger 含 2002 条，命中 1331 条，缺失 671 条全部为 P2P。独立 SQL 确认所有 1076 条 ACK 消息和全部 926 条 WS 已观测的无 ACK 消息均已落库，因此这 671 条是 verifier 连接掉线和 3 分钟总预算造成的验证不完整，不是数据库丢失。
|
||||
- 1324 条结果未知中，1017 条实际落库：926 条被 WS 观测，另有 91 条未被 WS 观测但已落库。剩余 307 条无 ACK、无 WS 观测、无数据库记录。
|
||||
|
||||
### 4.4 独立 MySQL 核验
|
||||
|
||||
核验租户 ID 为 8，消息表为 `im_008_messages` 和 `im_008_group_messages_202609`。
|
||||
|
||||
| 场景 | ACK expected | 精确命中 | ACK 缺失 | 错配 | 重复 | 无 ACK 但落库 |
|
||||
|---|---:|---:|---:|---:|---:|---:|
|
||||
| 单聊 | 12000 | 12000 | 0 | 0 | 0 | 0 |
|
||||
| 群聊 | 1145 | 1145 | 0 | 0 | 0 | 39 |
|
||||
| 混合 | 1076 | 1076 | 0 | 0 | 0 | 1017 |
|
||||
|
||||
`verify-db` 对无 ACK 但落库的行按 unexpected 报告，所以群聊和混合命令会以非零状态结束；这不代表数据错误，而是发送结果未知但服务端已持久化的重要诊断证据。
|
||||
|
||||
## 5. 服务端观测
|
||||
|
||||
- `im-server-0/1/2` 全部 Running，本轮重启数均为 0。
|
||||
- 混合实时阶段 `03:53:00Z..03:53:02Z`，`im-server-0` 记录 55 次 `topic's broadcast queue is full`。
|
||||
- 混合渐进关闭末尾 `04:03:45Z`，`im-server-0` 再记录 375 次同类队列满。当前 60 秒 close ramp 仍未完全避免关闭风暴。
|
||||
- 同一观测窗口的 `03:45:13Z`，`im-server-2` 在 `serveIMFriendRequestAction` (`server/im_http.go:6256`) 出现一次 nil pointer panic。该端点不属于群实时发送路径，不将其归因为本轮断连，但需单独修复。
|
||||
- 当前 kubeconfig 无权读取 metrics-server，未能取得负载窗口的 Pod CPU/内存历史值。
|
||||
|
||||
## 6. 压测工具修复
|
||||
|
||||
1. 新增单机 60% 配置和执行方案，A/B 各 600 连接。
|
||||
2. 群聊和混合发送者按 `users_per_machine` 比例缩放，1000 用户的原参数保持不变。
|
||||
3. 投递观测按 `client_mid` 分为 64 片，每个接收账号独立维护 seq，延迟直方图改为原子计数，移除 WS 读循环中的全局锁争用。
|
||||
4. summary/report 新增 `disconnect_reasons` 和最多 100 条 `disconnect_samples`。
|
||||
5. 群创建后立即 checkpoint manifest；批量邀请失败后重读实际成员，仅邀请缺失用户。
|
||||
6. P2P 历史查询 limit 按 seq 区间扩展，上限 200，修复双方消息交错造成的假缺失。
|
||||
|
||||
## 7. 已实施改进
|
||||
|
||||
1. 好友申请接受和自动接受路径增加 store 空结果保护。底层异常返回 `nil, nil` 时记录租户、申请、操作人和状态并返回 HTTP 500，不再触发进程 panic。
|
||||
2. Hub 向已满的 Topic client queue 投递失败时，立即向原发送连接返回 `503 service unavailable`，并记录 topic、tenant、队列长度和容量。客户端可据此重试，压测也不再把这类请求归为等待超时的未知结果。
|
||||
3. 群消息权限检查改为复用群设置缓存，并为个人禁言增加 Redis 正向和负向缓存；新增禁言和解除禁言会同步更新缓存，到期禁言会自动淘汰。此改动减少群 Topic 串行处理路径上的 MySQL 查询。
|
||||
4. `message_publish_stage_duration_seconds` 新增 `group/permission_check` 和 `*/realtime_fanout` 阶段，下轮可区分权限查询、持久化和实时扩散耗时。
|
||||
5. A/B 两个逻辑 agent 按账号顺序交错关闭连接。`close_ramp=60s` 时每个 agent 的本地关闭过程约 120 秒，A/B 相差半个时隙，整体约每 100ms 关闭一个连接，避免两侧同时下线形成双倍尖峰。
|
||||
6. 已增加空结果防 panic、Topic queue 满返回 503、双 agent 关闭时隙的自动化测试。
|
||||
|
||||
## 8. 复测顺序
|
||||
|
||||
1. 发版后先跑单机群聊 60%，验收实时发送阶段和关闭阶段 `topic_client` enqueue failed 均为 0；若仍过载，发送端必须收到 503，不能再出现对应的纯 ACK timeout。
|
||||
2. 对比 `permission_check`、`save`、`realtime_fanout` 三阶段的 P50/P95/P99，确认群权限缓存是否消除了数据库检查瓶颈。
|
||||
3. 群聊通过后再跑单机混合 60%，要求 ACK、历史和独立 SQL 三方账本一致，连接断开数为 0。
|
||||
4. 补齐 Prometheus/metrics-server 权限并同步保存 Pod CPU、内存、网络、Kafka lag、Redis、MySQL 指标；在此之前不进行双机 100% 最终验收。
|
||||
|
||||
## 9. 发版后群聊 R2
|
||||
|
||||
Run ID：`s60-group-r2-0929`。
|
||||
|
||||
本轮开始前 3 个 Pod 均为 Running、0 重启，账号和群成员核验为 `accounts=1200 pairs=600 group_members=1200`。测试完整运行到 A/B 历史验证完成，并执行了错峰关闭和独立 MySQL 核验。
|
||||
|
||||
| 指标 | 结果 |
|
||||
|---|---:|
|
||||
| 发送尝试 / ACK | 1200 / 871 |
|
||||
| 结果未知 | 329（222 timeout、107 socket） |
|
||||
| 断连 | 360（315 reset、44 abnormal close、1 timeout） |
|
||||
| ACK P95 | 16630.441ms |
|
||||
| 投递 P95 | 101823.156ms |
|
||||
| ACK 台账期望 / 实时收到 | 1045200 / 889777 |
|
||||
| 实时缺失 | 155423，最终全部查到持久化 |
|
||||
| 历史回查 | 1080 / 1080 |
|
||||
| DB：ACK 精确命中 | 871 / 871 |
|
||||
| DB：无 ACK 但落库 | 209 |
|
||||
| DB：缺失 / 错配 / 重复 | 0 / 0 / 0 |
|
||||
|
||||
329 条结果未知中，209 条已被 WS 观测且独立 SQL 确认落库，剩余 120 条数据库中不存在。断连样本主要是公网对端 `connection reset by peer`，且样本连接均指向 `64.118.158.131:443`；本机到域名全部当前 IP 均使用 `en1` 静态直连，没有经过 VPN/TUN。
|
||||
|
||||
服务端指标显示群 Topic 所在 `im-server-0` 的 1080 条 `save` 全部低于 100ms，Kafka consumer lag 为 0；但 `topic_server` 队列累计出现 393 次 full。压测客户端 CPU、RSS 不构成瓶颈。
|
||||
|
||||
本轮不能作为提交 `75bb448` 的有效回归：Pod 中运行的二进制不包含该提交新增的 `permission_check`、`realtime_fanout` 阶段字符串和好友申请空结果保护日志，Prometheus 也没有相应时序，说明实际部署镜像不是该提交构建结果。必须正确部署 `75bb448` 或后续提交后重跑。
|
||||
|
||||
此外，本轮发现 agent 只在正式开始前维护连接，统计窗口内断线后不会恢复，导致历史核验依赖剩余连接。后续已改为持续维护到 A/B 历史验证均完成，并将重连并发限制为每 agent 16，避免恢复风暴；每次计划发送前重新取得当前连接。
|
||||
|
||||
## 10. 产物
|
||||
|
||||
- 方案：`docs/test/IM-单机60%容量压力测试方案.md`
|
||||
- 单聊 R2：`loadtest/im-pressure/results/s60-p2p-r2-0929/`
|
||||
- 群聊 R1：`loadtest/im-pressure/results/s60-group-r1-0929/`
|
||||
- 混合 R1：`loadtest/im-pressure/results/s60-mixed-r1-0929/`
|
||||
- 群聊 R2：`loadtest/im-pressure/results/s60-group-r2-0929/`
|
||||
|
||||
结果目录包含 coordinator、A/B summary、合并 report 和 agent 指标 JSONL。
|
||||
|
||||
## 12. 精确成员规模诊断复测
|
||||
|
||||
为区分“在线连接数”和“群成员总数”的影响，新增 300、600 人独立群复测。每个测试群在起压前均按用户 ID 集合核验，要求实际成员与目标账号完全相等。
|
||||
|
||||
| 场景 | Run ID | 在线/群成员 | ACK | 实时投递 | 历史 | DB 精确命中 | ACK P95 | 投递 P95 |
|
||||
|---|---|---:|---:|---:|---:|---:|---:|---:|
|
||||
| 原 1200 人群、仅 300 人在线 | `s300-group-r1b-0929` | 300/1200 | 45/300 | 13500/13500（按 ACK 台账） | 206/206 | 45 ACK + 210 结果未知全部落库 | 18994ms | 89489ms |
|
||||
| 精确 300 人群 | `s300m-group-r1-0930` | 300/300 | 300/300 | 90000/90000 | 300/300 | 300/300 | 235.177ms | 500ms 桶内 |
|
||||
| 精确 600 人群 | `s600m-group-r1-0930` | 600/600 | 600/600 | 360000/360000 | 600/600 | 600/600 | 235.177ms | 500ms 桶内 |
|
||||
| 精确 900 人群 | `s900m-group-r1-0930` | 900/900 | 890/900 | 800555/801000（按 ACK 台账） | 899/899 | 890 ACK + 9 结果未知落库 | >10s，最大19.272s | >10s，最大55.279s |
|
||||
|
||||
三轮均未出现 WS 发送队列满、WS 写失败或 Kafka lag。精确 300、600 人群在实时阶段也没有新增 `session_topic_publish`、`topic_client`、`topic_server` 满队列，说明 WS 写链路在 600 在线规模内不是瓶颈。
|
||||
|
||||
原 1200 人群仅 300 人在线时有约 900 个离线成员。群消息处理会在 Topic 串行路径中为每个离线成员逐个调用 Redis presence 查询；上一条消息 ACK 后的推送准备持续占用 Topic goroutine，使后续消息长时间等待，最终表现为 ACK 超时和分钟级投递延迟。精确成员群消除这部分离线成员后，同等 300 在线规模恢复正常，验证了该路径是主要根因。
|
||||
|
||||
600 人实时发送通过后，240 秒错峰下线仍使群主 Pod 的 `topic_server full` 从 36 增至 128，净增 92；其他两类内部队列无增量。该问题来自最后会话离线时的 presence 扩散，与消息保存、Kafka 消费和 WS 实时投递无关，仍需在服务端合并或限流离线状态广播。
|
||||
|
||||
精确 900 人群出现容量拐点：900 次发送中 890 次 ACK，9 次 ACK 超时、1 次 socket 失败；7 条连接在实时阶段超时断开并自动恢复。按 ACK 台账计算的 445 次实时缺口全部可从历史补回，890 条 ACK 全部精确落库；9 条结果未知实际落库，1 条 socket 失败未落库。因此本轮没有持久化丢失，但实时正确性和延迟门禁未通过。
|
||||
|
||||
900 人实时阶段三个内部队列的 full 计数均无增量，Kafka lag 为 0。成功处理的 899 条消息中，`permission_check` P95 不超过 10ms、`save` P95 不超过 50ms、`kafka_produce` P95 不超过 25ms、`realtime_fanout` P95 不超过 5ms；抽样覆盖后半段 575468 次 WS 投递，发送队列等待 P95 不超过 10ms、socket write P95 不超过 0.5ms，且没有发送队列满或 text write 失败。
|
||||
|
||||
现有 `realtime_fanout` 指标在 `pushForData` 之前结束。`pushForData` 位于 Topic 串行处理路径中，每条群消息遍历全部成员、构造推送收件人并调用 `sendPush`；即使成员全部在线，900 人规模下这段未计时工作仍可让后续发布在 `topic_client` 中排队。队列未达到容量上限并不代表没有排队，这与 ACK 延迟逐步增长到 19 秒、投递延迟增长到 55 秒的现象一致。下一步应先补充 `push_prepare`、`push_dispatch` 和 Topic client queue wait 指标，再将群推送准备移出 Topic 串行热路径或改为异步批处理。
|
||||
|
||||
900 人在 240 秒错峰下线后，群主 Pod 的 `topic_server full` 从 0 增至 379，说明 presence 离线扩散问题随群规模明显放大。压测机两个 agent 的峰值 CPU 分别约 39.5% 和 38.5% 单核，峰值 RSS 各约 76MB、FD 各 460，不是本轮瓶颈。
|
||||
|
||||
新增产物：
|
||||
|
||||
- 300 人精确群：`loadtest/im-pressure/results/s300-group-members-setup-0929/`
|
||||
- 300 人复测：`loadtest/im-pressure/results/s300m-group-r1-0930/`
|
||||
- 600 人精确群：`loadtest/im-pressure/results/s600-group-members-setup-0930/`
|
||||
- 600 人复测：`loadtest/im-pressure/results/s600m-group-r1-0930/`
|
||||
- 900 人精确群：`loadtest/im-pressure/results/s900-group-members-setup-0930/`
|
||||
- 900 人复测：`loadtest/im-pressure/results/s900m-group-r1-0930/`
|
||||
|
||||
## 13. 容量拐点修复（待发版复测）
|
||||
|
||||
针对精确 900 人群暴露的 Topic 串行积压和批量下线 presence 风暴，已完成以下代码改造：
|
||||
|
||||
1. `pushForData` 在 Topic goroutine 内只复制成员推送快照，不再逐成员同步查询 Redis，也不再执行跨 Pod 收件人拆分和 user updater 投递。
|
||||
2. 新增总容量 4096 的有界异步推送队列和 4 个固定分片 worker。同一 `tenant + topic` 固定进入同一分片，保证同一群的推送和未读更新顺序；不同群可并行处理。
|
||||
3. 离线成员 presence 使用一次 Redis pipeline 批量读取，单条消息的 Redis 网络往返由成员数 N 次降为 1 次。
|
||||
4. 群离线 presence 遇到 Topic server queue 满时不再直接丢弃。服务端按 `tenant + topic + source user` 合并为最新状态，并在 Topic 队列恢复后重投；待处理集合上限 16384。
|
||||
5. 新增 Topic client queue wait、push prepare、push dispatch queue、push dispatch duration、push recipient count、deferred presence 当前深度等指标。
|
||||
|
||||
新增重点指标：
|
||||
|
||||
- `im_topic_queue_wait_duration_seconds`
|
||||
- `im_message_publish_stage_duration_seconds{stage="push_prepare"}`
|
||||
- `im_push_dispatch_queue_enqueue_total`
|
||||
- `im_push_dispatch_queue_depth`
|
||||
- `im_push_dispatch_queue_wait_duration_seconds`
|
||||
- `im_push_dispatch_duration_seconds`
|
||||
- `im_push_recipient_count`
|
||||
- `im_internal_queue_current{queue="topic_presence_pending"}`
|
||||
- `im_internal_queue_enqueue_total{queue="topic_presence_deferred"}`
|
||||
|
||||
发版后先复测精确 600 人群，确认基线无回归；再复测精确 900 人群。900 人验收要求：900/900 ACK、810000/810000 实时投递、0 断连、0 持久化缺失，`topic_client/topic_server/push_dispatch` 无 full，Topic queue wait P95 不超过 500ms，ACK P95 不超过 500ms，投递 P95 不超过 1s。渐进下线结束后 deferred presence 必须回落到 0，且不得出现 `topic_server full`。
|
||||
|
||||
## 14. 异步推送发版后精确 600 人回归
|
||||
|
||||
Run ID：`s600m-pushfix-r1-0930`。测试使用精确 600 人群 `grpJZjWbOek928`，A/B 各建立 300 条连接；两端均完整运行至 5 分钟落库沉淀、历史回查和 240 秒交错关闭结束，没有提前终止连接。
|
||||
|
||||
| 指标 | 结果 |
|
||||
|---|---:|
|
||||
| 就绪连接 | 600 / 600 |
|
||||
| 发送尝试 / 收到 ACK | 600 / 523 |
|
||||
| 发送结果未知 | 77（71 timeout、6 socket） |
|
||||
| 服务端实际接收并落库 | 593 / 600 |
|
||||
| ACK P95 | 17522.536ms |
|
||||
| ACK 台账实时投递 | 308745 / 313800 |
|
||||
| 实时到达率 | 98.3891% |
|
||||
| 实时缺口 | 5055，全部可从历史补回 |
|
||||
| 历史回查 | 593 / 593 |
|
||||
| 无 ACK 但历史可查 | 70 / 70 |
|
||||
| MySQL 精确命中 | 593 / 593 |
|
||||
| MySQL 缺失 / 错配 / 重复 / 额外 | 0 / 0 / 0 / 0 |
|
||||
| 断连 | 34（21 reset、13 abnormal close） |
|
||||
| 重启 | 0 |
|
||||
|
||||
异步推送改造本身没有形成服务端队列瓶颈。群主 Pod 处理的 600 次发布 Topic queue wait 全部不超过 50ms；593 次异步推送均成功入队，`push_dispatch` 没有 full，队列等待 P95 不超过 0.5ms，dispatch P95 不超过 10ms、最大不超过 25ms，deferred presence 最终为 0。Kafka lag 仍为 0。
|
||||
|
||||
本轮 77 条 ACK 未知中，70 条已被对端 WS 观察并精确落库，剩余 7 条 socket 失败未进入服务端。由此可确认本轮失败发生在客户端连接和 ACK 返回链路，不是消息持久化、Kafka 消费、Topic 排队或异步推送丢弃。
|
||||
|
||||
测试期间 3 个 IM Pod 持续收到培养服务文件确认请求，并在 `store.Files.LinkAttachments` 中对 `cvp...` 业务保留键调用 `types.GetTopicCat`，约每 5 秒产生一轮 HTTP panic 和完整堆栈。该接口允许业务保留键，不能按 Tinode Topic 强制分类；持续 panic 日志和并发异常请求污染了本轮环境，因此本轮不能作为异步推送改造的有效 600 人基线。
|
||||
|
||||
已完成两项修复：
|
||||
|
||||
1. 附件关联仅对 `usr...` 用户引用执行用户 Topic 转换，`cvp...` 等业务保留键直接按普通关联键落库，消除确定性 panic。
|
||||
2. `verify-db` 将双机共同观察到的无 ACK 消息与 ACK ledger 合并去重。本轮修正后的独立 SQL 结果为 `expected=593, found_exact=593`，不再把 70 条已落库的 ACK 超时消息误报为 unexpected。
|
||||
|
||||
下一轮必须在上述修复发版后重新执行相同的精确 600 人场景。验收前先确认 3 个 Pod 不再出现 `invalid topic type for name 'cvp...'`；随后要求 600/600 ACK、360000/360000 实时投递、MySQL 600/600 精确命中、0 断连，并保持 Topic、push dispatch、Kafka 和数据库指标无积压。
|
||||
|
||||
新增产物：
|
||||
|
||||
- `loadtest/im-pressure/results/s600m-pushfix-r1-0930/report.json`
|
||||
- `loadtest/im-pressure/results/s600m-pushfix-r1-0930/db-verification.json`
|
||||
- `loadtest/im-pressure/results/s600m-pushfix-r1-0930/server-metrics/`
|
||||
|
||||
## 15. Panic 修复后精确 600 人连续复测
|
||||
|
||||
部署提交 `06e08a6` 后，3 个 Pod 使用同一镜像摘要，均为 Ready、零重启。连续观测和三轮压测期间不再出现 `invalid topic type for name 'cvp...'` 或对应 HTTP panic，确认培养文件关联修复生效。
|
||||
|
||||
为区分代码回归与公网瞬态，使用同一配置和精确 600 人群连续执行 R2、R3、R4。每轮 A/B 各 300 连接，保持到 5 分钟沉淀、历史验证和 240 秒交错关闭全部完成。
|
||||
|
||||
| 指标 | R2 | R3 | R4 |
|
||||
|---|---:|---:|---:|
|
||||
| ACK | 600 / 600 | 600 / 600 | 600 / 600 |
|
||||
| 实时投递 | 359400 / 360000 | 360000 / 360000 | 360000 / 360000 |
|
||||
| 断连 | 1 次 timeout | 0 | 0 |
|
||||
| 历史回查 | 600 / 600 | 600 / 600 | 600 / 600 |
|
||||
| MySQL 精确命中 | 600 / 600 | 600 / 600 | 600 / 600 |
|
||||
| MySQL 缺失 / 错配 / 重复 | 0 / 0 / 0 | 0 / 0 / 0 | 0 / 0 / 0 |
|
||||
| ACK P95 | 314.530ms | 1555.109ms | 6228.311ms |
|
||||
| 投递 P95 | 500ms 桶内 | 2000ms 桶内 | 10000ms 桶内 |
|
||||
|
||||
R2 唯一异常是 `lta0259` 到 `216.236.31.189:443` 的一次客户端读超时。该连接在发送阶段开始后自动恢复，因此该账号错过 600 条实时消息；600 条均可从历史补回，服务端和 MySQL 没有丢失。R3、R4 均达到消息正确性全通过：零断连、零实时缺失、零重复、零乱序、零持久化缺失。
|
||||
|
||||
R3、R4 的延迟门禁没有通过，但服务端分段指标未出现同步恶化：
|
||||
|
||||
- Topic queue wait P95 不超过 0.5ms。
|
||||
- 消息保存 P95 不超过 25ms。
|
||||
- push dispatch P95 不超过 10ms，队列无 full，完成后深度为 0。
|
||||
- WS send queue wait P95 约 3ms，socket write 超过 99% 不超过 0.5ms。
|
||||
- deferred presence 当前深度为 0。
|
||||
- 600 条历史和数据库记录全部精确命中，Kafka 未出现消费积压。
|
||||
|
||||
ACK 延迟按 3 个拨号 IP 分组后同步上升，不集中于单一入口。R2 三个 IP 的 ACK P95 为 302–372ms，R3 为 1536–1643ms，R4 为 5999–6559ms，说明延迟发生在服务端 socket write 之后的共享公网、负载均衡或压测机接收链路。
|
||||
|
||||
压测结束后的网络基线同样异常：
|
||||
|
||||
| 入口 IP | TCP connect 平均 | HTTPS TTFB 平均 | ICMP 丢包 | ICMP RTT |
|
||||
|---|---:|---:|---:|---:|
|
||||
| `216.236.31.189` | 179ms | 623ms | 10% | 平均 256ms，最大 511ms |
|
||||
| `216.236.31.191` | 178ms | 543ms | 10% | 平均 154ms |
|
||||
| `64.118.158.131` | 178ms | 588ms | 0% | 平均 168ms |
|
||||
|
||||
三条静态路由均经 `en1` 和网关 `192.168.1.1` 直连，没有经过 VPN/TUN。当前公网 RTT、丢包和 HTTPS TTFB 已高于或接近 500ms ACK 门禁，TCP 重传会进一步放大 WebSocket 队头等待。因此本组结果可证明精确 600 人下服务端消息正确性和持久化能力通过，但不能证明公网端到端延迟 SLA 通过。
|
||||
|
||||
后续在进入精确 900 人测试前，应先满足以下任一条件：
|
||||
|
||||
1. 使用与测试集群同地域、低丢包的压测机，并保留当前 500ms/1s 延迟门禁。
|
||||
2. 取得 Ingress/LB 的 upstream、出站带宽、连接 reset 和重传指标，将公网延迟与服务端处理延迟分开验收。
|
||||
3. 双机正式压测时分别记录本机到 3 个入口的 RTT、丢包和 HTTPS TTFB；网络基线不合格时只验收正确性，不输出容量延迟结论。
|
||||
|
||||
新增产物：
|
||||
|
||||
- `loadtest/im-pressure/results/s600m-pushfix-r2-0930/`
|
||||
- `loadtest/im-pressure/results/s600m-pushfix-r3-0930/`
|
||||
- `loadtest/im-pressure/results/s600m-pushfix-r4-0930/`
|
||||
|
||||
## 16. 异步推送发版后精确 900 人复测
|
||||
|
||||
Run ID：`s900m-pushfix-r2-0930`，精确群成员 900 人，A/B 各 450 条 WebSocket 连接。两端均完整运行至 5 分钟沉淀、历史回查和 240 秒渐进关闭结束，没有提前终止。
|
||||
|
||||
| 指标 | 结果 |
|
||||
|---|---:|
|
||||
| 就绪连接 | 900 / 900 |
|
||||
| 发送尝试 / 收到 ACK | 900 / 590 |
|
||||
| 发送结果未知 | 310（186 条已被对端 WS 观察，124 条未入服） |
|
||||
| 服务端实际接收并落库 | 776 / 900 |
|
||||
| ACK P95 | 17075.685ms |
|
||||
| ACK 台账实时投递 | 381424 / 531000 |
|
||||
| ACK 台账实时到达率 | 71.8313% |
|
||||
| 实时缺口 | 149576，全部可从历史补回 |
|
||||
| 重复 / 乱序 | 0 / 0 |
|
||||
| 历史回查 | 776 / 776 |
|
||||
| MySQL 精确命中 | 776 / 776 |
|
||||
| MySQL 缺失 / 错配 / 重复 / 额外 | 0 / 0 / 0 / 0 |
|
||||
| 断连 | 410（322 reset、88 abnormal close） |
|
||||
| Pod 重启 | 0 |
|
||||
|
||||
本轮消息正确性结论很明确：590 条有 ACK，186 条虽然 ACK 超时或 socket 失败，但已被其他 WS 客户端观察到；这 776 条全部能从历史查到，且 MySQL 精确落库 776 条。服务端已接收消息的持久化丢失率为 0，无重复、无字段错配。余下 124 条在连接断开期间未进入服务端，不属于服务端落库后丢失。
|
||||
|
||||
服务端在 776 条实际入服消息上没有出现容量积压：
|
||||
|
||||
- Topic queue wait P95 不超过 10ms，95.7% 不超过 10ms。
|
||||
- push dispatch 776/776 成功入队，无 full；队列等待 P95 不超过 5ms，dispatch P95 不超过 25ms。
|
||||
- WS send queue 等待 P95 不超过 5ms，socket text write P95 不超过 0.5ms；1177345 次成功写入，16 次写入失败均是对端已关闭连接。
|
||||
- 3 个 Pod 的 Kafka consumer lag 和 oldest backlog age 始终为 0，无 Topic、push dispatch 或 WS send queue full。
|
||||
- 渐进关闭后 push dispatch 和 WS send queue 深度均为 0，`topic_presence_pending` 回落为 0。
|
||||
|
||||
压测期间客户端记录 410 次非预期断连，服务端指标同期新增 412 次 WS timeout，三个 Pod 分别为 135、133、144 次，并非单一 Pod 故障。客户端 reset 样本同样均匀分布在三个公网 IP。压测 agent A/B 峰值 CPU 约为 20.8%/22.0% 单核，峰值 RSS 约为 108/101MiB，FD 均为 460，本机进程没有出现 CPU、内存或 FD 容量耗尽。结合本章前已观测到的高 RTT、ICMP 丢包和 HTTPS TTFB，本轮只能判定“当前端到端公网路径下 900 长连接门禁失败”，具体故障点可能在压测机网络、NAT/ISP、Ingress/LB 或其间 TCP 链路，不能据此判定 IM 服务端在 900 人处理容量上失败。
|
||||
|
||||
下一次 900 人容量验收必须改为集群同地域压测机，或先取得 Ingress/LB 的 reset、TCP 重传、出站带宽和 upstream 时延指标。网络基线不合格时，继续从当前机器反复施压只会重复得到链路上限，无法给出服务端容量上限。
|
||||
|
||||
新增产物：
|
||||
|
||||
- `loadtest/im-pressure/results/s900m-pushfix-r2-0930/report.json`
|
||||
- `loadtest/im-pressure/results/s900m-pushfix-r2-0930/db-verification.json`
|
||||
- `loadtest/im-pressure/results/s900m-pushfix-r2-0930/server-metrics/`
|
||||
|
||||
## 17. 900 连接心跳容错与压测端解耦（待发版复测）
|
||||
|
||||
针对 R2 在约一个心跳超时窗口后出现的 410 次断连，已完成三项改造：
|
||||
|
||||
1. WebSocket Ping 周期和 Pong 等待时间从硬编码中拆出，支持 `ws_ping_period_seconds` 和 `ws_pong_wait_seconds` 配置。默认为 15 秒 Ping、90 秒 Pong 容错，不再复用 45 秒业务 idle timeout。
|
||||
2. 新增 `im_ws_heartbeat_seconds` 和 `im_ws_heartbeat_total{event="pong_received"}`，用于核对每个 Pod 的有效心跳配置和 Pong 接收数；测试环境 ArgoCD patch 显式设置 15/90 秒。
|
||||
3. 压测客户端为每条连接新增有序数据回调队列。Socket 读循环不再同步执行消息去重、顺序和延迟统计，避免统计锁竞争阻塞 ACK、Ping 和 Pong 读取。
|
||||
|
||||
发版后仍使用同一精确 900 人群复测。起压前先确认三个 Pod 的 heartbeat gauge 均为 15/90；验收时对比 Pong 增量、timeout 增量、agent 断连和入服/落库数。若仍然同时批量 timeout，下一步必须取得 Ingress/LB TCP 指标或改用同地域压测机，不再继续放大应用心跳窗口。
|
||||
|
||||
## 18. 90 秒心跳发版后精确 900 人复测
|
||||
|
||||
Run ID：`s900m-heart-r1-0930`。三个 Pod 发版后均为 Ready、零重启，指标确认有效心跳配置均为 `ping_period=15`、`pong_wait=90`。使用与上一轮相同的精确 900 人群，A/B 各 450 条连接，完整运行至历史回查和渐进关闭结束。
|
||||
|
||||
| 指标 | 本轮 | 改造前 R2 |
|
||||
|---|---:|---:|
|
||||
| 发送尝试 | 900 | 900 |
|
||||
| 收到 ACK | 854 | 590 |
|
||||
| ACK 超时 / socket 错误 | 45 / 0 | 310 条结果未知，其中 124 条未入服 |
|
||||
| 明确失败 | 1 条 `500 internal error` | 0 |
|
||||
| 服务端实际接收并落库 | 899 / 900 | 776 / 900 |
|
||||
| 断连 | 11 | 410 |
|
||||
| ACK P95 | 11475.912ms | 17075.685ms |
|
||||
| ACK 台账实时投递 | 765336 / 768600 | 381424 / 531000 |
|
||||
| ACK 台账实时到达率 | 99.5753% | 71.8313% |
|
||||
| 重复 / 乱序 | 0 / 0 | 0 / 0 |
|
||||
| 历史回查 | 899 / 899 | 776 / 776 |
|
||||
| MySQL 精确命中 | 899 / 899 | 776 / 776 |
|
||||
| MySQL 缺失 / 错配 / 重复 / 额外 | 0 / 0 / 0 / 0 | 0 / 0 / 0 / 0 |
|
||||
|
||||
心跳容错和压测端读循环解耦有明显效果：断连从 410 次降到 11 次，降幅约 97.3%；未入服数从 124 降到 1。负载期三 Pod 累计成功发送约 14554 次 Ping，已观察到约 14472 次 Pong。11 次断连仍在 90 秒窗口附近发生，客户端全部自动重连。当前心跳仅在 Pong 回调时续期；下一步应将有效的客户端 Ping 和业务入站帧同样视为连接活动，而不是继续扩大 Pong 窗口。
|
||||
|
||||
45 条 ACK 超时消息全部被对端 WS 观察，且 899 条实际入服消息全部历史可查、MySQL 精确命中，证明已入服消息仍为零持久化丢失。唯一未入服消息已定位到 `10:07:58` 群主 Pod 的 MySQL `invalid connection`：同时出现群禁言策略查询失败和消息保存失败，客户端收到 `500 internal error`。需要对失效 MySQL 连接进行识别、丢弃并安全重试。
|
||||
|
||||
服务端分段指标中，899 条成功消息的 permission check P95 不超过 25ms、save P95 不超过 50ms、Kafka produce P95 不超过 25ms、realtime fanout P95 不超过 5ms、push prepare 和 push queue wait P95 不超过 10ms、push dispatch P95 不超过 25ms，无任何内部队列 full，Kafka lag 为 0。但 Topic queue wait P95 为 5s，最后 124 条在连接 timeout 与 presence 处理同期等待 0.1–10s，使 ACK P95 仍未过门禁。
|
||||
|
||||
压测 agent 峰值 CPU 约为 28.8%/22.8% 单核，RSS 约 121MiB，FD 各 460，进程资源未饱和。但异步数据回调队列尚未暴露当前深度和等待时间，因此本轮 98 秒投递 P95 包含网络、socket 读取和压测统计队列三段时间，不能单独作为服务端投递 SLA。下一轮前需增加压测回调队列深度/等待指标，并减少 recorder 在同一 `client_mid` 上的锁竞争。
|
||||
|
||||
新增产物：
|
||||
|
||||
- `loadtest/im-pressure/results/s900m-heartbeat-r1-0930/report.json`
|
||||
- `loadtest/im-pressure/results/s900m-heartbeat-r1-0930/db-verification.json`
|
||||
- `loadtest/im-pressure/results/s900m-heartbeat-r1-0930/server-metrics/`
|
||||
|
||||
## 19. 900 人复测后第二轮修复
|
||||
|
||||
针对第 18 节剩余的 11 次 timeout、1 条 MySQL `invalid connection` 和压测端回调排队不可观测问题，已完成以下修复：
|
||||
|
||||
1. 服务端收到有效客户端 Ping 时立即续期读截止时间并回 Pong；成功读取业务帧时也续期读截止时间。连接活跃判定不再只依赖服务端 Ping 对应的 Pong，新增 `im_ws_heartbeat_total{event="ping_received"}` 观测客户端心跳。
|
||||
2. MySQL 只读查询遇到 `ErrInvalidConn` 时向 `database/sql` 返回 `ErrBadConn`，使连接池丢弃坏连接并在新连接上重试。个人消息仅在存在 `client_msg_id` 唯一键保护时重试一次；群消息由 `message_id` 和 `client_msg_id` 唯一键保护，整个保存操作可安全重试一次。新增 `im_db_invalid_connection_retry_total{operation,outcome}`。
|
||||
3. recorder 的群消息接收位图改为原子 CAS。同一 `client_mid` 只在首次创建 observation 时使用 shard 锁，后续数百个收件人不再串行争用同一把写锁。
|
||||
4. agent 新增回调队列当前深度、处理数、峰值、入队/完成数、累计等待和最大等待指标，并将快照写入 `summary.json` 的 `callback_queue`。生成实时到达率前会等待排队及正在处理的回调全部完成，避免统计尚未落入 recorder 时产生假丢失。
|
||||
|
||||
下一轮仍使用精确 900 人群复测。除原有正确性门禁外，重点检查：
|
||||
|
||||
- `im_ws_heartbeat_total{event="ping_received"}` 有持续增量，服务端 timeout 和客户端断连进一步下降。
|
||||
- `im_db_invalid_connection_retry_total` 若出现，结果应为 `discard` 或 `recovered`，客户端不再收到对应 500。
|
||||
- `callback_queue.current` 和 `callback_queue.processing` 在结算时均为 0；结合 `maximum`、`wait_seconds_total / completed` 和 `wait_seconds_max` 区分压测端排队与服务端投递延迟。
|
||||
|
||||
## 20. 第二轮修复后精确 900 人复测
|
||||
|
||||
Run ID：`s900m-activity-r1-0930`，部署提交 `36129e3`。复用精确 900 人群 `grpQ5_hPdAz3cE`，A/B 各 450 条 WebSocket 连接。两个 agent 均通过 30 秒满连接屏障，并一直运行至实时投递、5 分钟沉淀、历史回查、MySQL 校验和 120 秒渐进关闭全部完成。
|
||||
|
||||
| 指标 | 结果 |
|
||||
|---|---:|
|
||||
| 就绪连接 | 900 / 900 |
|
||||
| 发送尝试 / 收到 ACK | 900 / 899 |
|
||||
| 结果未知 | 1 条 ACK 超时，socket 错误 0 |
|
||||
| 服务端实际接收并落库 | 900 / 900 |
|
||||
| 实际实时投递 | 810000 / 810000 |
|
||||
| 实时到达率 | 100% |
|
||||
| 重复 / 乱序 | 0 / 0 |
|
||||
| 压测期非预期断连 | 0 |
|
||||
| 服务端 WS timeout | 0 |
|
||||
| 历史回查 | 900 / 900 |
|
||||
| MySQL 精确命中 | 900 / 900 |
|
||||
| MySQL 缺失 / 错配 / 重复 / 额外 | 0 / 0 / 0 / 0 |
|
||||
| ACK P95 | 4748.423ms |
|
||||
| 投递 P95 | 10000ms 桶内 |
|
||||
|
||||
唯一 ACK 超时为 `s900m-activity-r1-0930-g-b-0121-01`。该消息虽然 20 秒内没有收到 ctrl ACK，但 A/B 均实时观察到，服务端分配 seq `3714`，历史可查且 MySQL 精确命中；因此是 ACK 返回链路超时，不是消息未入服或丢失。报告中的 `unexpected_messages=900` 正是该条没有进入 ACK ledger、但被 900 个连接实际收到的消息。
|
||||
|
||||
本轮连接活跃改造生效。三 Pod 均持续记录 `ping_received` 和 `pong_received`，负载阶段没有 WS timeout；两个 agent 全程 450/450 在线且断连为 0。与上一轮 11 次断连相比，客户端 Ping 和业务帧续期后已消除本轮心跳误断。
|
||||
|
||||
压测端回调队列指标证明 recorder 不再构成秒级延迟：
|
||||
|
||||
| agent | 入队 / 完成 | 峰值等待数 | 最大等待 | 平均等待 | 结算时等待 / 处理中 |
|
||||
|---|---:|---:|---:|---:|---:|
|
||||
| A | 405000 / 405000 | 106 | 11.353ms | 约 0.022ms | 0 / 0 |
|
||||
| B | 405000 / 405000 | 98 | 5.075ms | 约 0.020ms | 0 / 0 |
|
||||
|
||||
服务端内部处理也没有出现容量积压：
|
||||
|
||||
- Topic queue wait P95 不超过 50ms，全部 900 条不超过 250ms。
|
||||
- permission check P95 不超过 25ms，消息保存 P95 不超过 50ms，realtime fanout P95 不超过 5ms，push prepare P95 不超过 10ms。
|
||||
- WS send queue wait P95 不超过 25ms，socket text write P95 不超过 0.5ms。
|
||||
- push dispatch queue 无 full，Kafka consumer lag 和 oldest backlog age 均为 0。
|
||||
- 没有 `invalid connection`、消息保存失败、内部队列 full、panic 或 Pod 重启；关闭后 push 队列和 `topic_presence_pending` 均回落为 0。
|
||||
|
||||
因此本轮消息正确性和持久化门禁已经通过，但端到端延迟门禁仍未通过。测试结束后的三入口公网 RTT 为 153–189ms，HTTPS TTFB 为 0.50–1.04s；三条路由均经 `en1` 和网关 `192.168.1.1` 直连。服务端从 Topic 排队到 socket write 的 P95 远低于端到端 ACK/投递 P95，压测回调队列最大等待也只有 11ms，剩余延迟主要位于服务端 socket 写入之后的 Ingress/LB、公网 TCP、NAT 或压测机网络路径。后续服务端容量验收应在集群同地域压测机执行，并同时采集 Ingress/LB 出站带宽、TCP 重传、reset 和 upstream 指标。
|
||||
|
||||
新增产物：
|
||||
|
||||
- `loadtest/im-pressure/results/s900m-activity-r1-0930/report.json`
|
||||
- `loadtest/im-pressure/results/s900m-activity-r1-0930/db-verification.json`
|
||||
- `loadtest/im-pressure/results/s900m-activity-r1-0930/metrics.jsonl`
|
||||
- `loadtest/im-pressure/results/s900m-activity-r1-0930/server-metrics/`
|
||||
|
||||
## 21. 精确 900 人混合场景测试
|
||||
|
||||
混合场景使用 900 条 WebSocket、135 对单聊用户和 224 个群发送者。单聊双方各发 5 条，共 1350 条；群发送者各发 2 条，共 448 条；总发送尝试 1798 条。每个 agent 理论应观察 201600 次群投递和 1350 次单聊发送方回显/接收方投递，共 202950 次唯一回调。
|
||||
|
||||
### 21.1 R1：发现结算窗口过早
|
||||
|
||||
Run ID：`s900m-mixed-r1-1001`。
|
||||
|
||||
- 1798 条消息全部进入服务端，历史 1798/1798，MySQL 1798/1798，缺失、错配、重复和额外记录均为 0。
|
||||
- 收到 ACK 1411 条，387 条 ctrl ACK 超时；ACK P95 为 15756.928ms。
|
||||
- A/B 负载期均为 450 条连接且断连为 0。
|
||||
- 固定 60 秒 drain 结束时，A 的 summary 记录 202867/202950，误报 83 次实时缺口；summary 生成后指标继续增长至 202950/202950，说明这些帧是公网链路中的晚到帧，并未丢失。
|
||||
- B 为 202950/202950。回调队列最大等待分别约 98ms/92ms，结算时等待和处理中均为 0。
|
||||
- 发现 1 次单聊顺序回退：接收方 A-27 在同一 P2P Topic 上先收到 seq 245、后收到 seq 244。
|
||||
|
||||
本轮证明只等待“回调队列瞬时为空”不够：帧可能仍位于服务端 socket、Ingress/LB、TCP 或客户端 socket 缓冲区。压测程序已增加 `delivery_quiet_period` 和 `delivery_quiet_timeout`。固定 drain 后，接收计数和回调队列必须连续静默 10 秒才允许生成实时 summary；任意晚到帧都会重置静默窗口，最长等待 3 分钟。
|
||||
|
||||
同时修正实时统计口径：无 ACK 但已被任一 WebSocket 观察到的消息会与 ACK ledger 合并后参与全部收件人的实时到达率和“实时缺失但已落库”计算，不再被记为 `unexpected_messages`。该口径从后续轮次开始生效；R1/R2 原始报告仍保留当时输出，以上结论已按全量唯一回调和历史/MySQL 结果校正。
|
||||
|
||||
### 21.2 R2：静默结算修复验证
|
||||
|
||||
Run ID：`s900m-mixed-r2-1001`。本轮使用修复后的压测程序，静默等待在 A/B 均正常执行并完成后才生成 summary。
|
||||
|
||||
| 指标 | 结果 |
|
||||
|---|---:|
|
||||
| 就绪连接 | 900 / 900 |
|
||||
| 发送尝试 / ACK | 1798 / 1760 |
|
||||
| ACK 超时 / socket 错误 | 37 / 1 |
|
||||
| 压测期断连 | 14，全部为 timeout，A/B 各 7 |
|
||||
| A 唯一实时回调 | 202535 / 202950 |
|
||||
| B 唯一实时回调 | 202787 / 202950 |
|
||||
| 全量唯一实时回调 | 405322 / 405900，约 99.8576% |
|
||||
| ACK 台账实时缺口 | 566，全部已落库 |
|
||||
| 重复 / 乱序 | 0 / 0 |
|
||||
| 历史回查 | 1798 / 1798 |
|
||||
| MySQL 精确命中 | 1798 / 1798 |
|
||||
| MySQL 缺失 / 错配 / 重复 / 额外 | 0 / 0 / 0 / 0 |
|
||||
| ACK P95 | 14317.159ms |
|
||||
| 投递 P95 | 46622.373ms |
|
||||
|
||||
R2 的 A/B 各 7 条连接在负载期超时断开，随后全部自动恢复至 450/450。实时缺口与断连窗口一致，静默窗口结束后接收计数不再增长，因此不是 R1 那种过早结算。所有实时缺口均可由历史补回，MySQL 独立校验证明服务端持久化零丢失。
|
||||
|
||||
两个 agent 的回调队列最大等待约 10ms，结算时 current/processing 均为 0，压测 recorder 不是本轮瓶颈。服务端 1350 条 P2P 消息全部处理完成，P2P Topic queue P95 位于 250–500ms，relationship check P95 位于 100–250ms；Kafka lag 为 0，未出现 MySQL invalid connection、内部队列 full 或 Pod 重启。关闭后 push 队列和 deferred presence 均回落为 0。
|
||||
|
||||
混合场景在当前公网压测机上未通过实时和延迟门禁，但持久化门禁通过。下一步不应直接扩大到 1200 人，应先在集群同地域压测机重复 900 人混合场景；若同地域仍出现乱序或实时缺口，再分别处理跨 Pod P2P 顺序和 WS 投递路径。
|
||||
|
||||
新增产物：
|
||||
|
||||
- `loadtest/im-pressure/results/s900m-mixed-r1-1001/`
|
||||
- `loadtest/im-pressure/results/s900m-mixed-r2-1001/`
|
||||
|
||||
### 21.3 R3：零断连、零实时丢失，但仍有 ctrl ACK 超时
|
||||
|
||||
Run ID：`s900m-mixed-r3-1001`。本轮沿用 900 条 WebSocket、135 对单聊用户和 224 个群发送者，共发送 1350 条单聊和 448 条群消息。端到端 P95 延迟按最新口径仅作观测，不参与通过判定。
|
||||
|
||||
| 指标 | R3 | R2 |
|
||||
|---|---:|---:|
|
||||
| 就绪连接 | 900 / 900 | 900 / 900 |
|
||||
| 发送尝试 / ACK | 1798 / 1751 | 1798 / 1760 |
|
||||
| ctrl ACK 超时 / socket 错误 | 47 / 0 | 37 / 1 |
|
||||
| 压测期非预期断连 | 0 | 14 |
|
||||
| 两端唯一 WS 回调 | 405900 / 405900 | 405322 / 405900 |
|
||||
| 实时缺口 | 0 | 578 次底层回调，ACK 台账折算缺口 566 |
|
||||
| 重复 / 乱序 | 0 / 0 | 0 / 0 |
|
||||
| 历史回查 | 1798 / 1798 | 1798 / 1798 |
|
||||
| 无 ACK 但已观察消息的历史命中 | 47 / 47 | 38 / 38 |
|
||||
| MySQL 精确命中 | 1798 / 1798 | 1798 / 1798 |
|
||||
| MySQL 缺失 / 错配 / 重复 / 额外 | 0 / 0 / 0 / 0 | 0 / 0 / 0 / 0 |
|
||||
| ACK P95（仅观测） | 14020.142ms | 14317.159ms |
|
||||
| 投递 P95（仅观测） | 52784.346ms | 46622.373ms |
|
||||
|
||||
R3 的消息正确性明显好于 R2：A/B 各收到 202950/202950 个唯一回调，包括每端 201600 次群投递、675 次单聊对端投递和 675 次发送方回显。固定 drain 后又经过连续 10 秒静默窗口，计数未再变化，本轮没有过早结算。
|
||||
|
||||
47 条 ctrl ACK 超时中，单聊 26 条、群聊 21 条，A/B 分别为 22/25 条。它们全部被 WS 实时观察，47/47 历史可查，并被 MySQL 独立校验纳入 1798/1798 精确命中，所以不是消息未入服或持久化丢失。但客户端在 20 秒内未收到 ctrl ACK，仍是独立的发送协议问题，不属于已取消的 500ms 延迟门禁。因此本轮应判定为：
|
||||
|
||||
- 实时到达、重复、顺序、断连、历史和落库正确性通过。
|
||||
- 混合场景总验收仍未通过，唯一阻断为 47 条 `command_timeout=20s` 的 ctrl ACK 超时。
|
||||
- ACK P95 和投递 P95 保留在报告中，不影响上述判定。
|
||||
|
||||
服务端负载期没有 Topic/WS 发送队列 full，WS send queue 结算时均为 0，所有 Kafka consumer lag 为 0，三个 Pod 均为 Ready 且零重启。连接渐进关闭后，三 Pod 连接数从 518/373/550 回落至 7/2/7，与起压前 8/3/6 基本一致。Pod 2 增加了 1 次 `Cluster.Route` 和 1 次 `Cluster.UserCacheUpdate` 瞬时 error，均不足 1ms，最近日志无对应错误，且未影响本轮投递与落库。
|
||||
|
||||
压测端回调队列在结算时均为 0；A/B 队列峰值为 355/174，最大等待仅为 37.054ms/25.036ms，不是 20 秒 ACK 超时或 52 秒投递 P95 的原因。
|
||||
|
||||
新增产物：
|
||||
|
||||
- `loadtest/im-pressure/results/s900m-mixed-r3-1001/report.json`
|
||||
- `loadtest/im-pressure/results/s900m-mixed-r3-1001/db-verification.json`
|
||||
- `loadtest/im-pressure/results/s900m-mixed-r3-1001/metrics.jsonl`
|
||||
- `loadtest/im-pressure/results/s900m-mixed-r3-1001/server-metrics/`
|
||||
|
||||
### 21.4 R3 ctrl ACK 超时原因分析
|
||||
|
||||
#### 21.4.1 代码路径
|
||||
|
||||
服务端的消息发布顺序为：Topic 队列出队、权限/关系校验、持久化、生成 `NoErrAccepted` ctrl ACK 并调用 `queueOut`，然后才做当前消息的 Topic 广播、跨 Pod 实时 fanout 和离线推送准备。因此，当前消息的大规模 fanout 不会阻塞同一条消息的 ACK 生成。
|
||||
|
||||
但所有 ctrl 和 data 最终共用用户的同一个 WebSocket/TCP 有序字节流。ACK 可以在当前消息的 data 之前入队，但无法超过已经写入 socket 或正在 Ingress/LB/公网 TCP 中传输的早先 data 帧。混合场景中发送者同时也是 900 人群的收件人，因此每条发送连接在等待自己 ACK 时，前面可能已有数百条群 data 帧。
|
||||
|
||||
#### 21.4.2 服务端分段证据
|
||||
|
||||
按起压前与负载结束 Prometheus 快照计算本轮增量：
|
||||
|
||||
| 阶段 | 增量样本 | P95 |
|
||||
|---|---:|---:|
|
||||
| P2P Topic 队列等待 | 1350 | 100ms 桶内 |
|
||||
| P2P 关系校验 | 1350 | 100ms 桶内 |
|
||||
| P2P 保存 | 1350 | 50ms 桶内 |
|
||||
| 群 Topic 队列等待 | 448 | 25ms 桶内 |
|
||||
| 群权限校验 | 448 | 25ms 桶内 |
|
||||
| 群保存 | 448 | 50ms 桶内 |
|
||||
| Cluster TopicMaster RPC | 2495 | 3至5ms 桶内 |
|
||||
| Cluster proxy 队列深度 | 7988 | P95=1 |
|
||||
| WS 发送队列深度 | 817924 | P95=0 |
|
||||
| WS 发送队列等待 | 817924 | 10ms 桶内 |
|
||||
| WS text frame 写入 | 817946 | 0.5ms 桶内 |
|
||||
|
||||
本轮三 Pod 共成功写出约 81.8 万个文本帧、193.6MB。服务端应用层队列、保存、Cluster RPC 和 socket write 均远低于 20 秒；压测时间窗内无 queue full、write timeout、数据库错误、Pod 重启或 panic。因此没有证据支持“业务处理或服务端应用队列阻塞 20 秒”。
|
||||
|
||||
#### 21.4.3 超时分布证据
|
||||
|
||||
- 47 条超时全部发生在起压后 20至40 秒，正好是每条连接已经接收约 200至400 条群广播帧的区间。
|
||||
- 40/47 是同一用户的第 2 条消息，5 条是第 3 条，只有 2 条是第 1 条；超时不是随机散布在整个测试周期。
|
||||
- 三个入口 IP 分别有 16/16/15 条超时，排除单一 Ingress IP 热点。
|
||||
- 压测端回调队列最大等待只有 37ms，数据回调已与 socket 读循环解耦；超时不是 recorder 锁竞争或回调队列堵塞。
|
||||
- 测试后三个入口的当前 ICMP RTT 约为 167至197ms，HTTPS TTFB 约为 0.49至0.61 秒；路由均经 `en1` 直连，未经 VPN/TUN。
|
||||
|
||||
综合上述证据，当前根因判断为：**高置信度的 WebSocket/TCP 单流队头阻塞，延迟主要发生在服务端完成 socket write 之后的 Ingress/LB、公网 TCP、NAT 和压测机接收路径。**这个结论可以解释“服务端已快速保存且客户端最终收到全部 data，但部分 ctrl 在 20 秒窗口内未被读到”。
|
||||
|
||||
尚未完全证明的环节是：当前压测程序在 ctrl 超时后删除 wait entry，晚到 ctrl 会被静默忽略；服务端 WS 写指标也没有区分 ctrl 和 data。因此现有数据无法逐条证明 47 个 ACK 全部在 20 秒后到达，只能证明对应消息全部已被服务端接受、投递和持久化。
|
||||
|
||||
#### 21.4.4 处理建议
|
||||
|
||||
1. 压测端增加 late ctrl 台账：ctrl 超时后保留 ID 和发送时间 10 分钟，记录晚到 ACK 数、最终延迟和完全未到数。
|
||||
2. 服务端将 WS 出站指标按 `ctrl/data/pres/info` 分类，新增从客户端请求时间到 ctrl ACK 入队、实际 socket write 完成的直方图；跨 Pod 时再记录 master-to-proxy ctrl 延迟。
|
||||
3. 下一轮保持完全相同的 900 人混合负载，将 `command_timeout` 改为 90 秒，但单独统计超过 20 秒的 ACK。若 47 类消息都在 90 秒内到达，且服务端 ctrl write 延迟仍在毫秒级，则队头阻塞结论得到直接验证。
|
||||
4. 当前业务不对端到端延迟设阻断门禁，因此不建议现在改造独立控制连接或破坏 WebSocket 有序性。应先补齐上述观测，并将“消息已实时观测且历史/MySQL 命中”与“20 秒内收到 ctrl ACK”分开报告。
|
||||
5. 另有一项与 ACK 超时无关的独立缺陷：负载结束后 Pod 2 记录 `gob: type not registered for interface: []map[string]interface {}`，影响另一个群 Topic 的 Cluster Route。当前 cluster gob 已注册 `[]any` 和 `map[string]any`，但未注册 `[]map[string]any`，应单独补注册和编解码回归测试。
|
||||
|
||||
#### 21.4.5 观测改造完成情况与 R4 口径
|
||||
|
||||
上述观测缺口已补齐：
|
||||
|
||||
- 压测端会统计 `publish_ctrl_observed`、`publish_ctrl_over_20s`、`publish_ctrl_late`、`publish_ctrl_unresolved` 和 `publish_ctrl_p95_ms`，并保留最多 1000 条超过 20 秒或晚到的样本。
|
||||
- agent Prometheus 新增 `im_load_agent_publish_ctrl_observed_total`、`im_load_agent_publish_ctrl_over_20s_total`、`im_load_agent_publish_ctrl_late_total` 和 `im_load_agent_publish_ctrl_unresolved`。
|
||||
- 服务端 Prometheus 新增 `im_ctrl_ack_delivery_duration_seconds{stage="enqueue|write"}`，按 Pod 记录从请求进入到 ACK 进入 WS 发送队列、以及完成 socket write 的时间。
|
||||
- Cluster gob 已注册 `[]map[string]any`，并用真实系统消息结构补了编解码回归测试。
|
||||
|
||||
R4 保持 R3 的 900 连接混合负载，仅将 `command_timeout` 从 `20s` 改为 `90s`。验证时必须同时满足：`publish_ctrl_unresolved=0`、消息实时与历史/MySQL 零丢失，并对比服务端 `enqueue`/`write` P95 与客户端 ctrl P95。如服务端两阶段仍为毫秒级，但 `publish_ctrl_over_20s>0`，则可将超过 20 秒的部分定位在 socket write 之后的网络/TCP 接收路径。
|
||||
|
||||
## 22. 精确 900 人全量单聊测试
|
||||
|
||||
Run ID：`s900m-p2p-r1-1001`。900 条 WebSocket 连接组成 450 对好友，A/B 各承担 450 个用户。每个用户以 1 秒间隔发送 10 条消息，共尝试发送 9000 条。两个 agent 均完整运行至实时静默判定、5 分钟沉淀、历史回查、MySQL 校验和 120 秒渐进关闭结束，没有提前终止。
|
||||
|
||||
| 指标 | 结果 |
|
||||
|---|---:|
|
||||
| 就绪连接 | 900 / 900 |
|
||||
| 发送尝试 / ACK | 9000 / 9000 |
|
||||
| 明确拒绝 / 结果未知 | 0 / 0 |
|
||||
| 对端实时投递 | 9000 / 9000 |
|
||||
| 发送方回显 | 9000 / 9000 |
|
||||
| WS 唯一回调总数 | 18000 / 18000 |
|
||||
| 实时到达率 / 丢失率 | 100% / 0% |
|
||||
| 重复 / 乱序 / 非预期消息 | 0 / 0 / 0 |
|
||||
| 压测期非预期断连 | 0 |
|
||||
| 历史回查 | 9000 / 9000 |
|
||||
| MySQL 精确命中 | 9000 / 9000 |
|
||||
| MySQL 缺失 / 错配 / 重复 / 额外 | 0 / 0 / 0 / 0 |
|
||||
| ACK P95 / 最大值 | 854.385ms / 1324.17ms |
|
||||
| 实时投递 P95 / 最大值 | 1000ms 桶内 / 1324.255ms |
|
||||
| 服务端 Pod 重启 | 0 |
|
||||
|
||||
报告的 `expected_deliveries=9000` 只统计对端投递，不包含发送方自身回显。A/B 各收到 4500 次对端投递和 4500 次本地回显，因此底层唯一 WS 数据回调合计为 18000 次，不是重复消息。
|
||||
|
||||
本轮单聊正确性和持久化门禁完整通过：9000 条发送全部收到 ACK，所有收发双方均实时收到应有消息，历史接口与 `im_008_messages` 都精确命中 9000 条。没有发生 ACK 超时、socket 错误、重复、乱序、实时丢失或持久化丢失。
|
||||
|
||||
压测端的回调处理没有构成瓶颈：
|
||||
|
||||
| agent | 回调入队 / 完成 | 队列峰值 | 最大等待 | 结算时等待 / 处理中 |
|
||||
|---|---:|---:|---:|---:|
|
||||
| A | 9000 / 9000 | 7 | 16.317ms | 0 / 0 |
|
||||
| B | 9000 / 9000 | 9 | 13.004ms | 0 / 0 |
|
||||
|
||||
服务端运行状态稳定：
|
||||
|
||||
- 三个 Pod 均为 Ready、0 重启，单 Pod 资源配置为 `requests=1 CPU/512Mi`、`limits=2 CPU/1Gi`。
|
||||
- 负载结束时 WS 连接分布为 527/373/544；压测连接渐进关闭后回落至 1/0/2，剩余为环境常驻连接，未再出现压测 agent FD 不释放。
|
||||
- 三个 Pod 的 WS send queue 结算时均为 0；Kafka 的消息持久化、群消息持久化、管理命令、配置事件、缓存失效、死信和工作台 consumer lag 均为 0。
|
||||
- 关闭过程中的少量 `ping closed` 累计指标来自客户端主动渐退，本轮 agent 记录的负载期非预期断连为 0。
|
||||
|
||||
按最新验收口径，ACK P95 和投递 P95 仅用于趋势对比，不作为当前公网单机容量测试的阻断条件。重新合并后本轮 `passed=true`：单聊消息正确性、实时到达、顺序性、连接稳定性和落库均通过。ACK P95 854.385ms 和投递 P95 1000ms 仍保留在报告中，但不影响本轮通过结论。
|
||||
|
||||
全量单聊已通过，无需为 500ms 延迟门禁单独重复 R2。下一步可进入更高风险的群聊或混合场景复测，仍以零丢失、零重复、零乱序、零断连和全量落库为通过条件。
|
||||
|
||||
新增产物：
|
||||
|
||||
- `loadtest/im-pressure/results/s900m-p2p-r1-1001/report.json`
|
||||
- `loadtest/im-pressure/results/s900m-p2p-r1-1001/db-verification.json`
|
||||
- `loadtest/im-pressure/results/s900m-p2p-r1-1001/metrics.jsonl`
|
||||
- `loadtest/im-pressure/results/s900m-p2p-r1-1001/server-metrics/`
|
||||
|
||||
## 23. 900 人混合 R4 长 ACK 观测复测
|
||||
|
||||
Run ID：`s900m-mixed-r4-1001`。负载与 R3 相同：900 条 WebSocket、135 对单聊用户和 224 个群发送者，共发送 1350 条单聊和 448 条群消息。本轮仅将 `command_timeout` 改为 90 秒，并启用 ctrl ACK 晚到台账和服务端 `enqueue/write` 分段指标。
|
||||
|
||||
| 指标 | 结果 |
|
||||
|---|---:|
|
||||
| 就绪连接 | 900 / 900 |
|
||||
| 发送尝试 / ACK | 1798 / 1798 |
|
||||
| 明确拒绝 / 结果未知 | 0 / 0 |
|
||||
| 超过 20 秒的 ACK | 118 |
|
||||
| 90 秒超时 / 晚到 / 最终未到 | 0 / 0 / 0 |
|
||||
| ACK P95 / 最大值 | 23425.942ms / 68730.192ms |
|
||||
| 单聊对端实时投递 | 1350 / 1350 |
|
||||
| 群聊实时投递 | 403200 / 403200 |
|
||||
| 实时到达率 / 丢失率 | 100% / 0% |
|
||||
| 重复 / 乱序 / 断连 | 0 / 0 / 0 |
|
||||
| 历史回查 | 1798 / 1798 |
|
||||
| MySQL 精确命中 | 1798 / 1798 |
|
||||
| MySQL 缺失 / 错配 / 重复 / 额外 | 0 / 0 / 0 / 0 |
|
||||
| Pod Ready / 重启 | 3 / 3，0 重启 |
|
||||
|
||||
118 条慢 ACK 的精确延迟范围为 20.317 至 68.731 秒，中位数约 27.062 秒，慢样本 P95 约 39.465 秒。两个 agent 分别有 56 和 62 条超过 20 秒的 ACK，但全部在 90 秒内返回。R4 直接证明同负载下存在“20 秒后才到、但未丢失”的 ACK，强力支持 R3 的 47 条超时属于同类晚 ACK，而非服务端拒绝或消息丢失。
|
||||
|
||||
服务端三个 Pod 的 ctrl ACK 指标共有 1801 个自 Pod 启动后的样本，与本轮 1798 条发送只相差 3 条环境流量：
|
||||
|
||||
| Pod | enqueue 平均 | write 平均 | enqueue/write P95 |
|
||||
|---|---:|---:|---:|
|
||||
| im-server-0 | 49.672ms | 50.059ms | 均在 250ms 桶内 |
|
||||
| im-server-1 | 51.612ms | 52.176ms | 均在 250ms 桶内 |
|
||||
| im-server-2 | 48.051ms | 48.358ms | 均在 250ms 桶内 |
|
||||
|
||||
ACK 从 WS 队列入队到 socket write 完成只增加约 0.3至0.6ms，而客户端有 118 条超过 20 秒。因此可确认长延迟发生在服务端 socket write 完成之后的 Ingress/LB、公网 TCP/NAT 或压测机接收路径，不在 Topic、持久化、Cluster RPC 或 WS 应用发送队列内。
|
||||
|
||||
本轮底层共处理 405900 个匹配的 WS data 回调，A/B 的回调队列峰值为 120/149，最大等待仅 9.603/9.199ms，结算时全部排空。Kafka 各 consumer lag 均为 0，未出现 Topic、Cluster proxy 或 WS send queue full；渐进关闭后只有 4 次预期的 ping closed 写错误。
|
||||
|
||||
报告中 `ack_p95_ms=23425.942` 是基于 1798 条 ACK 原始时间戳的精确值。本轮二进制的 ctrl 直方图上界仅到 10 秒，使 `publish_ctrl_p95_ms` 在 `+Inf` 桶中退化为最大值 68731.141ms，该字段本轮不作为 P95 使用。直方图已增加 20、30、45、60、90 和 120 秒桶及回归测试，下轮会输出正确的长延迟分位桶。
|
||||
|
||||
服务端日志还发现两次与本轮 Topic 无关的 Cluster Route 失败：`gob: type not registered for interface: types.KVMap`。`types.KVMap` 是命名 map 类型，不会被 `map[string]any` 的 gob 注册覆盖。已补充 `types.KVMap` 和 `[]types.KVMap` 注册，并扩展 Cluster gob 回归测试；该修复需要再发版。
|
||||
|
||||
本轮 `report.json` 最终 `passed=true`。混合场景的消息正确性、实时到达、顺序、连接稳定性、历史回查和 MySQL 落库均通过；长 ACK 已被定位为 socket write 之后的端到端队头阻塞，不再阻断当前容量验收。
|
||||
|
||||
新增产物：
|
||||
|
||||
- `loadtest/im-pressure/results/s900m-mixed-r4-1001/report.json`
|
||||
- `loadtest/im-pressure/results/s900m-mixed-r4-1001/db-verification.json`
|
||||
- `loadtest/im-pressure/results/s900m-mixed-r4-1001/metrics.jsonl`
|
||||
- `loadtest/im-pressure/results/s900m-mixed-r4-1001/server-metrics/`
|
||||
- `loadtest/im-pressure/results/s900m-mixed-r4-1001/im-server-{0,1,2}.log`
|
||||
|
||||
## 24. 900 人混合 R5 顺序复测
|
||||
|
||||
Run ID：`s900m-mixed-r5-1001`。本轮在 R4 相同的 900 条 WebSocket、135 对单聊用户和 224 个群发送者负载下，验证长延迟直方图、Cluster gob 类型修复及消息顺序。
|
||||
|
||||
| 指标 | 结果 |
|
||||
|---|---:|
|
||||
| 就绪连接 | 900 / 900 |
|
||||
| 发送尝试 / 90 秒命令窗口内 ACK | 1798 / 1797 |
|
||||
| 晚到 2xx / 最终接受 | 1 / 1798 |
|
||||
| 明确拒绝 / 最终未知 | 0 / 0 |
|
||||
| 超过 20 秒的 ACK | 375 |
|
||||
| ctrl ACK P95 桶 / 精确 ACK P95 | 45s / 41746.854ms |
|
||||
| 单聊对端实时投递 | 1350 / 1350 |
|
||||
| 群聊实时投递 | 403200 / 403200 |
|
||||
| 实时总投递 / 丢失 | 404550 / 404550，0 丢失 |
|
||||
| 重复 / 乱序 / 断连 | 0 / 1 / 0 |
|
||||
| 历史回查 | 1798 / 1798 |
|
||||
| 无及时 ACK 消息历史命中 | 1 / 1 |
|
||||
| MySQL 精确命中 | 1798 / 1798 |
|
||||
| MySQL 缺失 / 错配 / 重复 / 额外 | 0 / 0 / 0 / 0 |
|
||||
| Pod Ready / 重启 | 3 / 3，0 重启 |
|
||||
|
||||
唯一超时消息 `s900m-mixed-r5-1001-g-a-0136-02` 在 95.028 秒收到 202，因此属于晚到成功，不是拒绝或丢失。压测报告已区分 `publish_accepted`（命令等待窗口内）与 `publish_final_accepted`（包含晚到 2xx）；用新口径重算后发送成功率为 100%，报告只因真实乱序而失败。长延迟直方图的 P95 正确落在 45 秒桶，不再退化为最大值。
|
||||
|
||||
本轮唯一乱序发生在 A 机 97 号连接的同一单聊 Topic：客户端先收到 seq=322，后收到 seq=321。两条消息由会话两端在几乎同一时刻发送，服务端均实时投递并精确落库，所以问题不在 MySQL、Kafka 或历史回查，而在 Topic master 到 proxy Pod 的实时转发顺序。
|
||||
|
||||
代码检查确认，同一个 multiplex session 每次入队都会调度 `clusterWriteLoop`；此前没有排他状态，多个 worker 可以同时消费同一个 `send` channel。随后使用 fire-and-forget RPC 时，先取出的 seq=321 与后取出的 seq=322 会在远端并发执行，导致 seq=322 先进入 proxy Topic。该竞态与本轮现象完全一致。
|
||||
|
||||
修复内容：
|
||||
|
||||
1. multiplex session 增加原子调度所有权，同一 session 同时只允许一个 Cluster 写任务排空；释放所有权时再次检查队列，覆盖入队与退出之间的竞态。
|
||||
2. Master 到 Proxy 的同一 Topic 响应改为逐条等待 RPC 完成，保证远端处理完 seq=N 后才发送 seq=N+1；不同 Topic 仍由 worker pool 并行处理。
|
||||
3. 新增本地 RPC 回归测试，故意延迟 seq=1 的处理并重复调度同一 session，验证接收顺序始终为 1、2；该测试在 `go test -race` 下通过。
|
||||
4. 压测工具新增晚到 2xx 最终接受计数，已解决“最终已 202 但报告仍算发送失败”的重复判错。
|
||||
|
||||
服务端三个 Pod 共记录 1798 条 ctrl ACK，分布为 574/611/613；enqueue/write 平均约 60至66ms，P95 均不超过 250ms。未出现 Topic、Cluster proxy、WS send queue full、Kafka 错误、WS text write 错误或 Pod 重启。A/B 回调队列峰值为 253/290，最大等待 10.703/5.929ms，结算时均已排空。完整 Pod 日志中未再出现 `types.KVMap` 或其他 gob 注册错误，证明 R4 后的 Cluster gob 修复已生效。
|
||||
|
||||
本轮 MySQL 校验覆盖 `im_008_group_messages_202610` 与 `im_008_messages`，1798 条业务消息全部精确命中。结论是持久化、实时到达、长 ACK 统计和 gob 修复均通过，当前唯一阻断项为已完成代码修复但尚待发版复验的跨 Pod 顺序竞态。
|
||||
|
||||
下一步发版后执行完全相同的 900 人混合 R6。R6 通过条件为：最终接受 1798/1798、实时投递 404550/404550、重复/乱序/断连均为 0、历史和 MySQL 均为 1798/1798；ACK 延迟继续观测但不作为阻断门禁。
|
||||
|
||||
新增产物：
|
||||
|
||||
- `loadtest/im-pressure/results/s900m-mixed-r5-1001/report.json`
|
||||
- `loadtest/im-pressure/results/s900m-mixed-r5-1001/db-verification.json`
|
||||
- `loadtest/im-pressure/results/s900m-mixed-r5-1001/metrics.jsonl`
|
||||
- `loadtest/im-pressure/results/s900m-mixed-r5-1001/server-metrics/`
|
||||
- `loadtest/im-pressure/results/s900m-mixed-r5-1001/im-server-{0,1,2}.log`
|
||||
|
||||
## 25. 900 人混合 R6/R7 顺序修复发版复验
|
||||
|
||||
### 25.1 R6：顺序修复通过，公网瞬时断连导致整轮失败
|
||||
|
||||
Run ID：`s900m-mixed-r6-1001`。本轮首次验证同步 Master-to-Proxy 顺序投递版本，跨 Pod 乱序已从 R5 的 1 次降为 0，但压测机在 15:42:13 至 15:42:18 UTC 同时对两个入口 IP 发生 TCP 写超时。
|
||||
|
||||
| 指标 | R6 |
|
||||
|---|---:|
|
||||
| 就绪连接 | 900 / 900 |
|
||||
| 发送尝试 / ACK | 1798 / 1785 |
|
||||
| socket 结果未知 | 13 |
|
||||
| 压测期断连 | 11，均为 write timeout |
|
||||
| 实时投递 | 401361 / 404542 |
|
||||
| 实时缺口 | 3181，全部历史可查 |
|
||||
| 重复 / 乱序 | 0 / 0 |
|
||||
| 历史回查 | 1790 / 1790 |
|
||||
| MySQL 精确命中 | 1790 / 1790 |
|
||||
| MySQL 缺失 / 错配 / 重复 / 额外 | 0 / 0 / 0 / 0 |
|
||||
|
||||
11 次断连分布在 `216.236.31.189` 和 `216.236.31.191` 两个入口，并集中于约 5 秒窗口；三个服务 Pod 均 Ready、零重启，负载期无 Topic/Cluster/WS 队列满、Kafka 错误或 gob 错误。13 条 socket 结果未知中有 5 条被对端观察并精确落库，另外 8 条未落库且未形成额外数据，符合请求在进入服务端前失败的特征。
|
||||
|
||||
R6 的顺序修复目标已通过，但整轮因断连、发送未知和实时缺口判定失败。三条目标 IP 路由复查均为 `en1` 直连，未经过 VPN/TUN。为排除一次性公网抖动，随即执行相同参数的 R7。
|
||||
|
||||
### 25.2 R7：消息业务门禁全部通过
|
||||
|
||||
Run ID：`s900m-mixed-r7-1001`。运行前对三个入口 IP 的 HTTPS 预检均返回 200，TLS 总耗时约 1.0 至 1.1 秒。
|
||||
|
||||
| 指标 | R7 |
|
||||
|---|---:|
|
||||
| 就绪连接 | 900 / 900 |
|
||||
| 发送尝试 / 最终接受 | 1798 / 1798 |
|
||||
| 明确拒绝 / 结果未知 | 0 / 0 |
|
||||
| 超过 20 秒 / 晚到 / 最终未到 ACK | 74 / 0 / 0 |
|
||||
| 精确 ACK P95 | 16795.372ms |
|
||||
| 单聊对端实时投递 | 1350 / 1350 |
|
||||
| 群聊实时投递 | 403200 / 403200 |
|
||||
| 实时总投递 | 404550 / 404550 |
|
||||
| WS 匹配回调总数（含单聊回显） | 405900 / 405900 |
|
||||
| 重复 / 乱序 / 断连 | 0 / 0 / 0 |
|
||||
| 历史回查 | 1798 / 1798 |
|
||||
| MySQL 精确命中 | 1798 / 1798 |
|
||||
| MySQL 缺失 / 错配 / 重复 / 额外 | 0 / 0 / 0 / 0 |
|
||||
| Pod Ready / 重启 | 3 / 3，0 重启 |
|
||||
|
||||
R7 的 `report.json` 为 `passed=true`。R5 暴露的同 Topic 跨 Pod 乱序在 R6 和 R7 连续两轮均为 0，证明单 session 排他 drain 与同步 RPC 顺序修复有效。压测端 A/B 回调队列峰值为 156/150，最大等待 13.281/25.038ms，结算时均为 0。
|
||||
|
||||
按 R7 起压前和负载结束快照计算增量：服务端新增 1798 条 ctrl ACK，enqueue/write 平均约 82.43/82.89ms，P95 均在 500ms 桶内；`Cluster.TopicProxy` 新增 7943 次调用，平均约 2.27ms、P95 在 10ms 桶内。负载期无新增 RPC error、内部队列失败、Kafka lag、queue full、gob 错误或 Pod 重启，说明同步 RPC 没有形成负载期瓶颈。关闭后服务端 active WS 回落为 0，压测进程和 FD 均完全释放。
|
||||
|
||||
### 25.3 关闭阶段 multiplex 队列竞态
|
||||
|
||||
R6 和 R7 均在最后一批连接关闭后记录 1 次 `s.queueOut: session's send queue full 8:grpQ5_hPdAz3cE-im-server-0`。该事件发生在业务发送、实时静默、历史回查和 summary 生成之后，没有影响上述消息结果，但违反“渐进关闭阶段也不得队列满”的严格运维门禁。
|
||||
|
||||
根因是 proxy stop/detach 后 multiplex worker 已退出并从 SessionStore 删除，而 Topic 的异步 unsubscribe 尚未完成，短时间内仍可能通过旧指针调用 `queueOut`。旧 session 未标记 terminating，消息进入无人消费的 send channel 并最终填满。修复已在本地完成：Cluster session 清理开始时先原子标记 terminating，并排空残留 channel；后续对旧指针的写入会被直接丢弃，不再进入失去 drain owner 的队列。新增关闭竞态测试和顺序测试均在普通模式连续 20 次、race 模式连续 10 次通过。
|
||||
|
||||
因此当前结论分为两层：
|
||||
|
||||
- 900 人混合消息业务门禁已经通过，跨 Pod 顺序修复有效。
|
||||
- 完整运维门禁还需发布关闭竞态补丁后做一次 R8，重点验证业务指标继续全绿且渐进关闭阶段 `queue full=0`。
|
||||
|
||||
新增产物：
|
||||
|
||||
- `loadtest/im-pressure/results/s900m-mixed-r6-1001/`
|
||||
- `loadtest/im-pressure/results/s900m-mixed-r7-1001/`
|
||||
|
||||
## 26. 900 人混合 R8 关闭竞态补丁复验
|
||||
|
||||
Run ID：`s900m-mixed-r8-1002`。本轮使用与 R7 完全相同的 900 条 WebSocket、135 对单聊用户和 224 个群发送者负载，重点验证关闭竞态补丁发布后，业务负载期和 120 秒渐进关闭全过程不再出现 `send queue full`。
|
||||
|
||||
| 指标 | R8 |
|
||||
|---|---:|
|
||||
| 就绪连接 | 900 / 900 |
|
||||
| 计划发送 / 实际尝试 | 1798 / 1798 |
|
||||
| 最终接受 / socket 结果未知 | 1796 / 2 |
|
||||
| 明确拒绝 / ACK 超过 20 秒 | 0 / 0 |
|
||||
| 精确 ACK P95 / 最大值 | 1041.578ms / 2055.915ms |
|
||||
| 单聊对端实时投递 | 1350 / 1350 |
|
||||
| 已接受群消息应投 / 实投 | 401400 / 400954 |
|
||||
| 已接受消息实时总投递 | 402750 / 402304 |
|
||||
| 实时缺口 | 446，全部可持久化回查 |
|
||||
| 重复 / 乱序 | 0 / 0 |
|
||||
| 压测期非预期断连 | 1，公网读超时后自动重连 |
|
||||
| 历史回查 | 1796 / 1796 |
|
||||
| MySQL 精确命中 | 1796 / 1796 |
|
||||
| MySQL 缺失 / 错配 / 重复 / 额外 | 0 / 0 / 0 / 0 |
|
||||
| 负载期 / 关闭期 queue full | 0 / 0 |
|
||||
| Pod Ready / 重启 | 3 / 3，0 重启 |
|
||||
|
||||
### 26.1 严格业务门禁结论
|
||||
|
||||
本轮 `report.json` 为 `passed=false`。失败由 A 机第 231 个连接的一次公网读超时引起：
|
||||
|
||||
```text
|
||||
read tcp 192.168.1.135:57717->216.236.31.191:443: read: operation timed out
|
||||
```
|
||||
|
||||
断连发生于 `2026-10-02T06:31:53.391465Z`，客户端约 1.5 秒后报告连接恢复。该用户的两条群发送 `s900m-mixed-r8-1002-g-a-0231-01/02` 分别得到 `websocket closed` 和 `websocket connection unavailable`，未进入服务端，MySQL 中也没有对应额外行。因此实际被服务端接受的业务消息为 1796 条，不能把这两条计为服务端持久化丢失。
|
||||
|
||||
同一连接在群消息阶段漏收 446 条已接受群消息，形成 446 次实时投递缺口；这些消息全部可通过历史回查获得。其余连接的群投递、全部 1350 次单聊对端投递、重复和顺序检查均正确。按严格验收标准，本轮仍因 `disconnects=1`、`publish_outcome_unknown=2` 和实时缺口判定失败，不能替代 R7 的业务全绿结论。
|
||||
|
||||
服务端三个 Pod 在超时窗口均保持 Ready、零重启、Kafka lag 为 0，也没有 Topic、Cluster、WS 队列满、RPC error、gob 或持久化错误。证据更符合压测机到单个公网入口的瞬时网络读超时，不支持服务端容量拐点或队列阻塞判断。
|
||||
|
||||
### 26.2 关闭竞态补丁结论
|
||||
|
||||
关闭竞态补丁的专项验收通过。两个 agent 均完成实时静默、5 分钟沉淀、历史回查、summary 生成和 120 秒渐进关闭，并以退出码 0 结束；没有提前停止进程。从负载开始到全部压测连接关闭后的完整 Pod 日志中：
|
||||
|
||||
- `s.queueOut: session's send queue full` 为 0；
|
||||
- Cluster P2M `queue_full` 增量为 0；
|
||||
- WS send queue `queue_full` 增量为 0，三个 Pod 结算队列深度均为 0；
|
||||
- 未出现旧 multiplex session 无消费者却继续积压消息的现象；
|
||||
- 三个 Pod 均保持 Ready、0 重启，Cluster peer 均为 2/2。
|
||||
|
||||
这证明“清理 Cluster session 前先标记 terminating 并 purge channel”的修复已经消除 R6/R7 关闭末尾出现的孤儿发送队列积压。关闭期日志仅有 1 次客户端渐退附近的 `close 1006 unexpected EOF`，没有引发队列堆积或 Pod 状态异常。
|
||||
|
||||
### 26.3 服务端与压测端观测
|
||||
|
||||
按起压前和负载结束快照计算增量：
|
||||
|
||||
| 指标 | 结果 |
|
||||
|---|---:|
|
||||
| `Cluster.TopicProxy` | 8227 次，平均 2.285ms，P95 在 10ms 桶内 |
|
||||
| ctrl ACK enqueue | 1798 次，平均 62.11ms，P95 在 250ms 桶内 |
|
||||
| ctrl ACK write | 1798 次，平均 62.43ms，P95 在 250ms 桶内 |
|
||||
| Cluster P2M queue full | 0 |
|
||||
| WS send queue full | 0 |
|
||||
| Kafka consumer lag | 全部为 0 |
|
||||
|
||||
A/B 回调队列峰值为 63/47，最大等待为 9.865/10.031ms，结算时等待和处理中均为 0。客户端回调处理没有形成瓶颈。MySQL 校验覆盖 `im_008_group_messages_202610` 与 `im_008_messages`，1796 条服务端已接受消息全部精确命中。
|
||||
|
||||
本轮结论应分开使用：**关闭竞态修复已通过专项复验；R8 整体业务门禁因一次公网断连失败。**如需获得“修复版本下业务与关闭运维门禁同时全绿”的单轮证据，应保持参数不变再执行 R9；无需因本轮结果继续修改服务端队列代码。
|
||||
|
||||
新增产物：
|
||||
|
||||
- `loadtest/im-pressure/results/s900m-mixed-r8-1002/report.json`
|
||||
- `loadtest/im-pressure/results/s900m-mixed-r8-1002/db-verification.json`
|
||||
- `loadtest/im-pressure/results/s900m-mixed-r8-1002/metrics.jsonl`
|
||||
- `loadtest/im-pressure/results/s900m-mixed-r8-1002/server-metrics/`
|
||||
- `loadtest/im-pressure/results/s900m-mixed-r8-1002/im-server-{0,1,2}.log`
|
||||
|
||||
## 27. 900 人混合 R9 全量复验
|
||||
|
||||
Run ID：`s900m-mixed-r9-1002`。本轮保持 R8 的配置、账号、消息量和验收口径不变。起压前三个公网入口均返回 200，HTTPS 总耗时约 0.48 至 0.56 秒；三个 Pod 使用相同镜像，均为 Ready、0 重启。
|
||||
|
||||
| 指标 | R9 |
|
||||
|---|---:|
|
||||
| 就绪连接 | 900 / 900 |
|
||||
| 发送尝试 / 最终接受 | 1798 / 1798 |
|
||||
| 明确拒绝 / 结果未知 | 0 / 0 |
|
||||
| 超过 20 秒的 ACK | 10 |
|
||||
| 精确 ACK P95 | 11417.014ms |
|
||||
| 单聊对端实时投递 | 1350 / 1350 |
|
||||
| 群聊实时投递 | 403200 / 403200 |
|
||||
| 实时总投递 | 404550 / 404550 |
|
||||
| WS 匹配回调总数（含单聊回显） | 405900 / 405900 |
|
||||
| 实时到达率 / 丢失率 | 100% / 0% |
|
||||
| 重复 / 乱序 / 断连 | 0 / 0 / 0 |
|
||||
| 历史回查 | 1798 / 1798 |
|
||||
| MySQL 精确命中 | 1798 / 1798 |
|
||||
| MySQL 缺失 / 错配 / 重复 / 额外 | 0 / 0 / 0 / 0 |
|
||||
| 负载期 queue full | 0 |
|
||||
| 渐进关闭期 queue full | 1 |
|
||||
| Pod Ready / 重启 | 3 / 3，0 重启 |
|
||||
|
||||
### 27.1 业务门禁通过
|
||||
|
||||
本轮 `report.json` 为 `passed=true`。1798 条发送全部得到最终 ACK，404550 次对端投递全部实时到达；没有发送未知、实时丢失、重复、乱序或非预期断连。A/B 各收到 202950 次匹配回调，其中包含 675 次单聊发送方回显，因此底层回调总数为 405900。
|
||||
|
||||
历史接口和 MySQL 均精确命中 1798/1798。数据库校验覆盖 `im_008_group_messages_202610` 与 `im_008_messages`，缺失、错配、重复和额外行均为 0。Kafka consumer lag 全部为 0，三个 Pod 全程 Ready、0 重启。
|
||||
|
||||
A/B 回调队列峰值为 218/210，结算时均完全排空。按起压前和负载结束快照计算，服务端 `Cluster.TopicProxy` 新增 7857 次调用，平均 1.752ms、P95 在 5ms 桶内；ctrl ACK enqueue/write 各 1798 次，平均 72.513/72.812ms，P95 均在 500ms 桶内。负载期没有 Topic、Cluster、WS 队列满、RPC 失败、gob 或持久化错误。
|
||||
|
||||
因此 R9 已取得修复版本下完整的消息业务全绿证据，并再次验证跨 Pod 顺序修复有效。10 条超过 20 秒的 ACK 均在 90 秒命令窗口内返回，继续按既定口径只做趋势观测。
|
||||
|
||||
### 27.2 关闭期仍有一个低概率队列满
|
||||
|
||||
在最后一批连接退出时，Pod `im-server-1` 于 `2026-10-02T07:23:16Z` 记录：
|
||||
|
||||
```text
|
||||
s.queueOut: session's send queue full 8:grpQ5_hPdAz3cE-im-server-2
|
||||
topic[grpQ5_hPdAz3cE]: connection stuck, detaching - 8:grpQ5_hPdAz3cE-im-server-2
|
||||
cluster: session proxy closed 8:grpQ5_hPdAz3cE-im-server-2
|
||||
```
|
||||
|
||||
该事件发生在 summary 已生成、1798 条历史核验完成之后，没有影响消息正确性和持久化。结算时三个 Pod 的 WS send queue 与 Cluster P2M queue 深度均为 0，Kafka lag 为 0。
|
||||
|
||||
R9 证据修正了 R6/R7 时的部分根因判断。此次队列满发生时，`grp...-im-server-2` multiplex session 尚未终止；日志顺序是先队列满、Topic 判定连接卡住并 detach，最后 session 才关闭。因此 R8 发布的 `terminating + purgeChannels` 补丁只能防止“已进入终止流程的旧 session 被继续写入”，不能处理仍存活 multiplex session 在大量连接关闭时的瞬时响应突发。
|
||||
|
||||
当前 multiplex `send` channel 容量为 `sendQueueLimit + 32 = 160`。Cluster 写任务采用排他调度和同步 Master-to-Proxy RPC；正常负载期吞吐稳定，但关闭阶段多个 unsubscribe/presence 响应可在 drain worker 获得调度前快速填满 160 个槽位。R8 未复现、R9 复现 1 次，符合低概率调度竞态，而不是持续消费能力不足：R9 关闭阶段 `Cluster.TopicProxy` 仍完成 3195 次调用，平均约 0.9ms、P95 在 3ms 桶内。
|
||||
|
||||
后续修复不应只继续增加 terminating 判断。建议为 MULTIPLEX 出站队列增加一个很短的有界背压重试：首次非阻塞入队失败时确保 drain 已调度，并等待最多约 250ms 获取队列槽位；超时后才维持现有的 `queue full -> detach` 保护。这样可以吸收调度抖动和关闭突发，同时避免永久阻塞 Topic goroutine。需要增加“队列已满但 worker 很快释放槽位”与“worker 永久不消费时按时失败”两类回归测试，发版后再执行 R10 专项复验。
|
||||
|
||||
上述修复已在 R9 后完成：MULTIPLEX 出站队列首次满时会确保 Cluster drain 已调度，并在 250ms 有界窗口内等待一个槽位；只有持续无消费超过窗口才沿用原来的失败和 detach 保护。新增的 `clusterQueueMu` 将 terminating 检查与入队、以及终止标记与 channel 清理串行化，防止等待期间 session 恰好终止后消息重新进入已清空队列。普通 WebSocket、gRPC 和长轮询队列仍保持原有非阻塞语义，不受此次改动影响。
|
||||
|
||||
回归测试覆盖短暂队列满后恢复、持续不消费按时失败、等待期间 session 终止不残留消息，以及原有跨 Pod 顺序与终止拒绝写入场景。针对性测试普通模式连续 20 次、race 模式连续 10 次通过，`go test ./server ./loadtest/im-pressure` 全部通过。该修复尚待发版并执行 R10 复验。
|
||||
|
||||
本轮最终结论：**消息业务门禁通过；完整运维门禁仍因关闭期 1 次 multiplex queue full 未通过。**
|
||||
|
||||
新增产物：
|
||||
|
||||
- `loadtest/im-pressure/results/s900m-mixed-r9-1002/report.json`
|
||||
- `loadtest/im-pressure/results/s900m-mixed-r9-1002/db-verification.json`
|
||||
- `loadtest/im-pressure/results/s900m-mixed-r9-1002/metrics.jsonl`
|
||||
- `loadtest/im-pressure/results/s900m-mixed-r9-1002/server-metrics/`
|
||||
- `loadtest/im-pressure/results/s900m-mixed-r9-1002/im-server-{0,1,2}.log`
|
||||
@@ -128,6 +128,12 @@ WebSocket Upgrade 阶段由 `server/hdl_websock.go` 先调用 `checkAPIKey`。
|
||||
|
||||
客户端只使用稳定的 `reason` 分支和本地化文案，不解析 `message`。ID、Topic、Seq 游标在客户端均按字符串处理；时间统一为 UTC RFC3339 毫秒。分页统一使用 `limit` 与不透明 `cursor`，返回 `items`、`next_cursor`、`has_more`。
|
||||
|
||||
资源不存在类错误统一语义（2026-09-29 修订）：
|
||||
|
||||
- 好友申请 `accept/reject` 在申请不存在时统一返回 `404/20004 RESOURCE_NOT_FOUND`，两个入口语义一致。
|
||||
- 群类接口（`DELETE /v1/im/groups/{topic}`、`POST /v1/im/groups/{topic}/leave`、`POST /v1/im/groups/{topic}/qrcodes`、`PUT /v1/im/groups/{topic}/members/me/profile`、`GET/PATCH /v1/im/groups/{topic}/settings` 等经群成员校验的路径）在群不存在时返回 `404/30004 RESOURCE_NOT_FOUND`；群已解散时返回 `410/30004 GROUP_DISSOLVED`；群存在但当前用户非成员仍是 `403/30005 PERMISSION_DENIED`。
|
||||
- 服务端对不存在的资源不再返回 5xx 或网关错误。
|
||||
|
||||
### 0.5 WebSocket 建连、登录和恢复顺序
|
||||
|
||||
#### 0.5.1 现有服务端可直接执行的冷启动流程
|
||||
@@ -210,6 +216,8 @@ WebSocket Upgrade 阶段由 `server/hdl_websock.go` 先调用 `checkAPIKey`。
|
||||
{"sub":{"id":"sub-topic-1","topic":"grp...","get":{"what":"desc sub data del","data":{"since":125,"limit":50},"del":{"since":1,"limit":50}}}}
|
||||
```
|
||||
|
||||
**向上翻页（加载更早历史）最终参数约定（后端已确认，2026-09-29）**：复用同一 `get data`，最终参数为 `before`（排他上界，返回 `seq < before`），**没有 REST cursor 语义**。聊天页滚动到顶部时发 `data:{"before":<当前已加载最小 seq>,"limit":50}`（`since` 可省略，默认 0）。服务端返回该锚点之前**最接近 `before` 的 `limit` 条**，`{data}` 按 `seq` 从大到小下发（首条即锚点前最新一条）；收到 `ctrl 204 no content`（`params.what=="data"`）表示已到会话顶部。`data` 参数全集：`since`（含下界）、`before`（排他上界）、`limit`、`ranges`。P2P 与群聊（含热缓存 fallback）语义一致。REST `GET /v1/im/conversations/{topic}/messages` 当前为空占位（见 4.21.1），不得用于历史分页。
|
||||
|
||||
订阅成功后才允许 `{pub}`、`note read/recv` 等会话操作。断线恢复必须重新执行 `WebSocket Upgrade -> hi -> token login -> sub me -> 恢复 Topic`，同一连接内不得跳过 `hi` 或切换企业码。
|
||||
|
||||
#### 0.5.2 Long Polling 兼容流程：先取得的是 sid
|
||||
@@ -901,7 +909,7 @@ X-MeetingIM-APIKey: <预置的普通 API Key>
|
||||
|
||||
**状态：新增。**
|
||||
|
||||
- 敏感操作先调用 `POST /v1/im/security/challenges`；开启青少年模式设置独立密码时 Body 为 `{"scene":"set_independent_password","channel":"sms"}`，关闭青少年模式时 Body 为 `{"scene":"close_youth_mode","channel":"sms"}`。手机号由登录身份读取，客户端不提交目标手机号。
|
||||
- 敏感操作先调用 `POST /v1/im/security/challenges`；开启青少年模式设置独立密码时 Body 为 `{"scene":"set_independent_password","channel":"sms"}`，关闭青少年模式时 Body 为 `{"scene":"close_youth_mode","channel":"sms"}`；群主转让使用 `{"scene":"group_owner_transfer","channel":"sms"}`；解散群聊使用 `{"scene":"group_dissolve","channel":"sms"}`；注销账号使用 `{"scene":"delete_account","channel":"sms"}`。手机号由登录身份读取，客户端不提交目标手机号。
|
||||
- 发码/校验：服务端返回 `challenge_id` 和脱敏手机号；提交 `POST /v1/im/security/challenges/{id}/verify`，Body `{"code":"..."}`。
|
||||
- 重复发码：同一用户、同一手机号、同一敏感场景存在未过期 challenge 且仍在 `resend_after_seconds` 冷却期内时，服务端返回 HTTP `429 VERIFICATION_RATE_LIMITED`，`details` 带 `challenge_id` 和 `retry_after_seconds`。客户端应复用该 `challenge_id` 继续倒计时，不要当成接口不可达或重新登录。
|
||||
- 小时限流：超过一小时创建次数上限时同样返回 HTTP `429 VERIFICATION_RATE_LIMITED`，`details.retry_after_seconds` 可能大于 60 秒且不一定带 `challenge_id`。客户端只展示倒计时，倒计时结束后允许用户重新获取验证码。
|
||||
@@ -927,7 +935,7 @@ X-MeetingIM-APIKey: <预置的普通 API Key>
|
||||
|
||||
- 页面只调用 `GET /v1/im/security/realname`。`verified` 时返回如 `masked_name:"张*"`、`masked_id_number:"110***********1234"`、`verified_at`。
|
||||
- 前端只读展示，不缓存原始证件信息，不提供直接修改接口。
|
||||
- 更正信息使用独立申诉 `POST /v1/im/security/realname/appeals`，需二次身份验证；不复用本页保存按钮。
|
||||
- 更正信息使用独立申诉 `POST /v1/im/security/realname/appeals`，Body `{"reason":"...","attachments":[]}`（`reason` 必填、≤500 字；`attachments` 最多 9 个 file_id），Bearer 即可，当前无需二次身份验证；不复用本页保存按钮。
|
||||
|
||||
### 1.10 实名认证（安卓）.png
|
||||
|
||||
@@ -975,7 +983,7 @@ X-MeetingIM-APIKey: <预置的普通 API Key>
|
||||
| `POST /v1/im/auth/password-reset/challenges/{challenge_id}/verify` | `{"code":"123456"}` | `{"reset_token":"reset_token_123","expires_in_seconds":300}` |
|
||||
| `POST /v1/im/auth/password-reset/confirm` | `{"reset_token":"reset_token_123","new_password":"NewPass123"}` | `{"status":"ok"}` |
|
||||
| `POST /admin/v1/tenant/users/{user_id}/reset-password` 管理后台 | `{"new_password":"AdminReset123","reason":"用户无手机号无法自助找回"}` | `{"user_id":"usr...","status":"reset","auth_version":2}` |
|
||||
| `POST /v1/im/auth/logout` 待实现 | `{}` | `{"status":"ok"}` |
|
||||
| `POST /v1/im/auth/logout` | `{}`（可选 `device_id`；或 Header `X-Device-ID`/`X-IM-Device-ID`） | `{"status":"ok","device_id":"...","push_unbound":true}`；当前 Token 写入 Redis 撤销集合（TTL=剩余有效期），仅当前设备失效，不影响其他设备；带 `device_id` 时同步解绑该设备推送 |
|
||||
| `GET /v1/im/app/releases/latest?platform=ios\|android\|web¤t_version=1.0.0&channel=stable` | Query only | `{"platform":"web","channel":"stable","latest_version":"1.0.0","min_supported_version":"1.0.0","update_type":"none","package_url":"","store_url":"","release_notes":"当前已是最新版本。","remind_after_seconds":0}` |
|
||||
|
||||
## 2. 消息首页、全局导航与系统通知
|
||||
@@ -1389,7 +1397,7 @@ X-MeetingIM-APIKey: <预置的普通 API Key>
|
||||
| `POST /v1/im/conversations/p2p` | `{"peer_user_id":"usr..."}` | `{"topic":"p2p...","relationship":"friend","can_send":true}` |
|
||||
| `GET /v1/im/conversations/{topic}/settings` | Path topic | `{"pinned":false,"mute_notifications":false,"message_retention":"default","blocked":false,"remark":"","version":1,"permissions":{"can_update":true}}` |
|
||||
| `PATCH /v1/im/conversations/{topic}/settings` | `{"pinned":true,"mute_notifications":false,"version":1}` | `{"pinned":true,"mute_notifications":false,"message_retention":"default","blocked":false,"remark":"","version":2}` |
|
||||
| `GET /v1/im/conversations/{topic}/messages` | Query `?since=12&limit=20` | `{"topic":"p2p...","items":[{"seq":13,"from":"usr...","ts":"2026-08-20T09:00:00Z","head":{"x-im-type":"text"},"content":{"text":"你好"}}],"next_cursor":""}` |
|
||||
| `GET /v1/im/conversations/{topic}/messages` | **服务端当前为空占位**：GET 恒返回空列表，勿用 `?since=&limit=` 调它；历史消息（首屏与向上翻页）一律走 WS `get data` 的 `since`/`before`/`limit`（向上翻页见 0.5.1 约定） | `{"topic":"p2p...","items":[],"next_cursor":"","has_more":false}` |
|
||||
| `PUT /v1/im/topics/{topic}/draft` 待实现 | `{"content":"未发送草稿","version":1}` | `{"topic":"grp...","content":"未发送草稿","version":2,"updated_at":"2026-08-20T09:00:00Z"}` |
|
||||
| `POST /v1/im/topics/{topic}/messages/{seq}/recall` | 单条撤回；`topic` 可传 `p2p...` 或 `grp...`；Body `{}`，可选 `{"version":1,"reason":"mistake"}` | `{"topic":"grp...","seq":12,"status":"recalled","version":2,"updated_at":"2026-08-20T09:00:00Z"}` |
|
||||
| `POST /v1/im/topics/{topic}/messages/recall` | 多选批量撤回；`{"items":[{"seq":12,"version":1},{"seq":13}],"reason":"mistake"}`；`items` 1～50 条，`version` 可选 | `{"topic":"grp...","items":[{"topic":"grp...","seq":12,"status":"recalled","version":2,"updated_at":"2026-08-20T09:00:00Z"},{"seq":13,"status":"failed","http":409,"code":30006,"reason":"VERSION_CONFLICT","message":"message version conflict"}]}` |
|
||||
@@ -1647,6 +1655,7 @@ GET /v1/im/topics/{topic}/message-mutations?since_version={localMutationVersion}
|
||||
- 列表：`GET /v1/im/friend-requests?direction=inbound&status=all&cursor=`，返回 `request_id,applicant,message,source,status,created_at,handled_at`。
|
||||
- 同意：`POST /v1/im/friend-requests/{request_id}/accept`；拒绝：`POST /v1/im/friend-requests/{request_id}/reject`，均带幂等键和可选 `version`。
|
||||
- 成功返回最终 `status` 和 P2P `topic`。重复审批返回原结果；并发冲突返回 `REQUEST_ALREADY_HANDLED` 并附当前状态，前端刷新该项。
|
||||
- 申请不存在时 `accept/reject` 统一返回 `404/20004 RESOURCE_NOT_FOUND`；通用语义见 0.4。
|
||||
- 主动下行：创建申请时被申请人收到 `info.what="im", event="friend.request.created"`；通过/拒绝后申请人与处理人本人都收到 `friend.request.accepted/rejected`。前端收到后刷新新朋友列表、好友列表、会话 context 和 `bootstrap.entry_badges`。当前系统通知列表只给被申请人/申请人写记录，处理人本人主要靠实时业务事件刷新。
|
||||
|
||||
### 5.3 添加好友.png
|
||||
@@ -1901,6 +1910,7 @@ GET /v1/im/topics/{topic}/message-mutations?since_version={localMutationVersion}
|
||||
- `GET /v1/im/groups/{topic}/settings` 返回 `my_role,permissions,profile,announcement,member_count,pinned,mute_notifications` 和群治理开关。
|
||||
- 个人置顶/免打扰通过 `PATCH /v1/im/groups/{topic}/member-settings`；群级设置必须使用 6.17，不混在个人设置里。
|
||||
- 退出：`POST /v1/im/groups/{topic}/leave`；群主必须先转让或解散。清记录、举报分别见 5.14、11.3。
|
||||
- 群不存在返回 `404/30004 RESOURCE_NOT_FOUND`；群已解散返回 `410/30004 GROUP_DISSOLVED`；非群成员返回 `403/30005 PERMISSION_DENIED`。通用语义见 0.4。
|
||||
|
||||
### 6.11 群资料.png
|
||||
|
||||
@@ -1930,6 +1940,7 @@ GET /v1/im/topics/{topic}/message-mutations?since_version={localMutationVersion}
|
||||
|
||||
- `PUT /v1/im/groups/{topic}/members/me/profile`，Body `{"nickname":"...","version":n}`。
|
||||
- 服务端校验成员资格、长度和敏感词，返回 `nickname,version,updated_at` 并推送成员资料变化。
|
||||
- 群不存在返回 `404/30004 RESOURCE_NOT_FOUND`；群已解散返回 `410/30004 GROUP_DISSOLVED`；非群成员返回 `403/30005 PERMISSION_DENIED`。通用语义见 0.4。
|
||||
- 该昵称仅作用当前群，不能修改全局用户昵称或其他群资料。
|
||||
|
||||
### 6.13.1 群成员列表拉取与缓存策略
|
||||
@@ -1997,6 +2008,7 @@ GET /v1/im/topics/{topic}/message-mutations?since_version={localMutationVersion}
|
||||
**状态：新增。**
|
||||
|
||||
- `POST /v1/im/groups/{topic}/qrcodes`，Body `{"expires_in_seconds":86400}`，返回签名 `qr_token,qr_payload,expires_at,group,group_snapshot`；`group` 与 `group_snapshot` 内容一致，前端优先使用 `group`。
|
||||
- 群不存在返回 `404/30004 RESOURCE_NOT_FOUND`；群已解散返回 `410/30004 GROUP_DISSOLVED`；非群主/管理员返回 `403/30005 PERMISSION_DENIED`。通用语义见 0.4。
|
||||
- 扫码调用 `POST /v1/im/qrcodes/resolve`，`purpose:"join_group"`；随后 `POST /v1/im/groups/{topic}/join-requests` 并携带 `qr_token`。
|
||||
- 群解散、二维码过期或管理员撤销后必须失效；二维码不得直接承载内部群权限。
|
||||
|
||||
@@ -2009,6 +2021,7 @@ GET /v1/im/topics/{topic}/message-mutations?since_version={localMutationVersion}
|
||||
- 查询/更新：`GET/PATCH /v1/im/groups/{topic}/governance`，字段为 `join_approval,mute_all,member_upload_files,member_private_chat,member_invite` 和 `version`。
|
||||
- 每个字段返回 `editable`；PATCH 仅群主或被授权管理员可用，记录操作者、旧值、新值和 Request ID。
|
||||
- 解散：`DELETE /v1/im/groups/{topic}`，需二次 `X-Action-Token` 和幂等键；成功后撤销 Topic、二维码、邀请和文件新增权限。
|
||||
- 群不存在返回 `404/30004 RESOURCE_NOT_FOUND`；已解散重复解散返回 `410/30004 GROUP_DISSOLVED`；非群主返回 `403/30005 PERMISSION_DENIED`。通用语义见 0.4。
|
||||
|
||||
### 6.18 设置群管理员.png
|
||||
|
||||
@@ -2915,7 +2928,7 @@ GET /v1/im/topics/{topic}/message-mutations?since_version={localMutationVersion}
|
||||
**状态：菜单纯客户端；数据接口部分新增。**
|
||||
|
||||
- 菜单来自 `bootstrap.settings_menu`，安全、通知、隐私、通用、关于按权限展示。系统通知权限状态由 iOS API 读取，不由服务端伪造。
|
||||
- 退出：`POST /v1/im/auth/logout` 撤销当前登录 Token 对应的设备会话，成功或 Token 已失效后关闭 WS、清安全存储和本租户用户缓存。
|
||||
- 退出：`POST /v1/im/auth/logout` 将当前登录 Token 的 SHA-256 摘要写入 Redis 撤销集合，TTL 与 Token 剩余有效期一致，不影响同账号其他设备。成功后关闭 WS、清安全存储和本租户用户缓存；请求须带 `Authorization: Bearer <token>` 和 `Idempotency-Key`。
|
||||
- 多设备退出使用 `DELETE /v1/im/security/sessions/{session_id}`，需二次验证。
|
||||
|
||||
### 10.8 设置（安卓）.png
|
||||
@@ -3083,9 +3096,29 @@ GET /v1/im/topics/{topic}/message-mutations?since_version={localMutationVersion}
|
||||
| `POST /v1/im/security/challenges/{id}/verify` 关闭青少年模式 | `{"code":"123456"}` | `{"action_token":"close_token","action":"close_youth_mode","expires_at":"2026-08-20T09:05:00Z"}` |
|
||||
| `GET /v1/im/security/realname` | Query only | `{"status":"unverified","can_submit":true}` |
|
||||
| `POST /v1/im/security/realname/submissions` | `{"name":"张三","id_type":"cn_id","id_number":"110101199001011234"}` | `{"submission_id":"rn_1","status":"pending","can_submit":false}` |
|
||||
| `POST /v1/im/security/realname/appeals` 待实现 | `{"reason":"证件变更","description":"需要更正实名信息","action_token":"action_token"}` | `{"appeal_id":"rna_1","status":"pending","submitted_at":"2026-08-20T09:00:00Z"}` |
|
||||
| `POST /v1/im/security/realname/appeals` | `{"reason":"证件变更需要更正实名信息","attachments":["file_1"]}` | `{"appeal_id":"rna_1","reason":"...","attachments":["file_1"],"status":"pending"}`；当前无需二次身份验证（如后续启用，将复用 1.7 的 action_token 机制并在此处更新） |
|
||||
| `PUT /v1/im/devices/{device_id}/push-token` | `{"provider":"jpush","token":"registration_id","platform":"android","app_version":"1.0.0","push_env":"test","sdk_version":"5.0.0","notification_permission":"granted"}` | `{"device_id":"device_1","provider":"jpush","platform":"android","app_version":"1.0.0","updated_at":"2026-08-20T09:00:00Z"}` |
|
||||
| `DELETE /v1/im/security/sessions/{session_id}` 待实现 | Header `X-Action-Token: action_token` | `{"session_id":"sess_1","revoked":true}` |
|
||||
| `POST /v1/im/me/password` | `{"old_password":"OldPass123","new_password":"NewPass123"}` | `{"status":"ok"}`；旧密码错误返回 HTTP `401`：`{"code":30001,"reason":"AUTH_REQUIRED","message":"invalid credentials"}`；新密码不满足策略返回 HTTP `400`：`{"code":30040,"reason":"PASSWORD_POLICY_VIOLATION"}` |
|
||||
| `POST /v1/im/security/challenges` 注销账号发码 | Header `Idempotency-Key`；Body `{"scene":"delete_account","channel":"sms"}` | `{"challenge_id":"sec_chg_9","phone_mask":"+861****000","expires_in_seconds":300,"resend_after_seconds":60}` |
|
||||
| `POST /v1/im/security/challenges/{id}/verify` 注销账号验证 | `{"code":"123456"}` | `{"action_token":"del_token","action":"delete_account","expires_at":"2026-08-20T09:05:00Z"}` |
|
||||
| `POST /v1/im/me/deletion` | Header `X-Action-Token: del_token`；Body `{}` | `{"status":"ok"}` |
|
||||
|
||||
#### 10.22.2 账号安全：修改密码与注销账号
|
||||
|
||||
**修改密码 `POST /v1/im/me/password`（已实现）：**
|
||||
|
||||
- 已登录（Bearer）用户提交 `old_password`、`new_password`；两者必填，缺一返回 `400 MALFORMED_REQUEST`。
|
||||
- 服务端先用旧密码走 `basic` 认证校验，失败返回 HTTP `401`：`{"code":30001,"reason":"AUTH_REQUIRED","message":"invalid credentials"}`，不泄露账号状态。
|
||||
- 新密码不满足密码策略返回 HTTP `400`：`{"code":30040,"reason":"PASSWORD_POLICY_VIOLATION"}`。
|
||||
- 成功后服务端递增 `auth_version` 并撤销该用户全部在线 Session（广播 `password_changed`）；包括当前请求在内的所有旧 Token 全部失效，客户端收到成功响应后必须清理本租户凭证并跳转登录页。
|
||||
|
||||
**注销账号 `POST /v1/im/me/deletion`（已实现）：**
|
||||
|
||||
- 两步挑战复用 1.7 敏感操作基建：`POST /v1/im/security/challenges`（Body `{"scene":"delete_account","channel":"sms"}`，发码到账号绑定手机号）→ `POST /v1/im/security/challenges/{id}/verify`（返回一次性 `action_token`）→ `POST /v1/im/me/deletion`（Header `X-Action-Token` 携带）。
|
||||
- 服务端配置 `permanent_accounts=true` 时拒绝注销：HTTP `403` `{"code":30040,"reason":"FORBIDDEN","message":"account deletion is disabled"}`。
|
||||
- 执行成功为**软删除**：撤销全部认证记录（含登录密码）、踢出所有在线 Session（`reason=user_deleted`）、退出并停用其拥有的群与 P2P 会话、向关注该用户的好友下发 `gone` 在线状态；账号不再可登录。客户端收到成功响应后清理本地数据并返回企业入口。
|
||||
- 挑战阶段限流与错误码同 1.7（`VERIFICATION_RATE_LIMITED`、`CHALLENGE_EXPIRED`、`CODE_INVALID`、`TOO_MANY_ATTEMPTS`）；action token 缺失/无效返回 HTTP `403` `{"code":30042,"reason":"ACTION_TOKEN_INVALID"}`。
|
||||
|
||||
## 11. 反诈安全、举报与意见反馈
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
/im-pressure
|
||||
/config.json
|
||||
/config.single-machine.json
|
||||
/config.local*.json
|
||||
/.env.test
|
||||
/manifest*.json
|
||||
|
||||
@@ -2,6 +2,9 @@
|
||||
|
||||
该工具按配置指定的企业租户运行：两台压测机各承载 1000 个 WebSocket 用户，共 2000 个用户、1000 对好友和一个 2000 人群。当前测试环境使用 `allrich`，本轮实时参数见 [MACHINE-B.md](./MACHINE-B.md) 和 [allrich 双机压测执行记录](../../docs/test/IM-allrich双机压测执行记录-20260927.md)。
|
||||
|
||||
只使用一台物理压测机时，请使用 [单机 60% 容量压测方案](../../docs/test/IM-单机60%容量压力测试方案.md) 和 [config.single-machine.example.json](./config.single-machine.example.json)。单机依然运行 A/B 两个逻辑 agent，用于保留好友对、跨逻辑分片投递和合并报告的完整契约。
|
||||
首轮实测数据与瓶颈分析见 [单机 60% 容量压测结果](../../docs/test/IM-单机60%容量压力测试结果-20260929.md)。
|
||||
|
||||
机器 B 上的 AI 或操作人请直接阅读 [MACHINE-B.md](./MACHINE-B.md)。该文件不重复讲方案，只给出机器 B 从核验到启动、观察和回传结果的完整命令。
|
||||
|
||||
## 两台机器的固定分工
|
||||
@@ -43,7 +46,7 @@
|
||||
- 两台机器必须使用 NTP/chrony 同步时间。报告会分别输出同机时钟延迟和跨机器单向延迟；跨机器出现负延迟样本时，后者仅用于诊断，不参与门禁。
|
||||
- agent 默认每 20 秒发送 WebSocket Ping，覆盖等待另一台机器、发压、drain 和历史核验阶段；`ws_ping_interval` 必须小于公网入口的空闲连接超时。
|
||||
- 正式开始前 agent 会持续检查实际存活连接；连接断开后自动重新登录并订阅 P2P、群 Topic。初次连接不齐或等待另一台机器期间掉线不会退出。
|
||||
- coordinator 使用两阶段屏障：两边实时连接数都达到配置值后进入验证倒计时；倒计时内任一方掉线就撤销屏障。两边再次确认满连接后才提交最终开始时间。
|
||||
- coordinator 使用三阶段屏障：两边实时连接数都达到配置值后进入验证倒计时；倒计时内任一方掉线就撤销屏障；提交最终开始时间后，两边必须在该时间前后 3 秒内再次到达起跑屏障，且双方都到达后才允许发送。任一方迟到会让两边共同终止本轮，避免生成错时压测结果。
|
||||
- 压测机如保持 OpenVPN 等全局 TUN，仅在另一套代理中设置 `DIRECT` 无法绕过它。起压前必须确认 `http_base` 和 WS 域名的每个当前 IP 都走物理网卡；需保留 VPN 访问内网时，可为这些 IP 配置经局域网关的 `/32` 主机路由。VPN 重连、DNS 变更或机器重启后需重新核验。
|
||||
|
||||
## 统计口径
|
||||
@@ -194,8 +197,8 @@ mkdir -p "loadtest/im-pressure/results/${RUN_ID}"
|
||||
两个 agent 都建立 1000 条连接后，协调器下发统一的 UTC `start_at`。场景可替换为：
|
||||
|
||||
- `p2p`：1000 对用户两侧同时发送，默认每人 10 条、间隔 1 秒。
|
||||
- `group`：每台机器 500 个发送者，每人 2 条；全局每 100ms 一条，共 2000 条。
|
||||
- `mixed`：300 对私聊用户每人 5 条，同时每台 250 个群发送者各发 2 条。
|
||||
- `group`：`users_per_machine=1000` 时每个 agent 有 500 个发送者，每人 2 条；全局每 100ms 一条，共 2000 条。其他容量按每个 agent 用户数的 50% 选发送者。
|
||||
- `mixed`：`users_per_machine=1000` 时使用 300 对私聊用户和每个 agent 250 个群发送者；其他容量分别按 30% 和 25% 同比计算。
|
||||
|
||||
## 机器 A：回收并合并报告
|
||||
|
||||
@@ -215,7 +218,7 @@ scp "${MACHINE_B_SSH}:${MACHINE_B_REPO}/loadtest/im-pressure/results/${RUN_ID}/a
|
||||
-output "loadtest/im-pressure/results/${RUN_ID}/report.json"
|
||||
```
|
||||
|
||||
`report.json` 给出 ACK、实时投递、重复、乱序、断连和历史持久化核验结果。当前正确性门禁要求上述错误计数全部为 0。
|
||||
`report.json` 给出 ACK、实时投递、重复、乱序、断连和历史持久化核验结果。断连会额外输出 `disconnect_reasons` 分类计数和最多 100 条 `disconnect_samples` 原始错误样本。当前正确性门禁要求上述错误计数全部为 0。
|
||||
|
||||
每个 agent 在 `:9105/metrics` 暴露连接、发送、接收、重复、乱序、断连以及 Go/process 指标；端口冲突时可通过 `-metrics-listen` 修改，传空字符串可关闭。
|
||||
|
||||
|
||||
+273
-41
@@ -26,7 +26,9 @@ type agentRun struct {
|
||||
clients map[int]*WSClient
|
||||
clientsMu sync.RWMutex
|
||||
recorder *deliveryRecorder
|
||||
callbackQueue *callbackQueueMetrics
|
||||
ack *DurationHistogram
|
||||
ctrlLatency *DurationHistogram
|
||||
attempted atomic.Int64
|
||||
ready atomic.Int64
|
||||
publish403 atomic.Int64
|
||||
@@ -35,10 +37,15 @@ type agentRun struct {
|
||||
publishTimeout atomic.Int64
|
||||
publishSocket atomic.Int64
|
||||
publishOther atomic.Int64
|
||||
publishCtrlObserved atomic.Int64
|
||||
publishCtrlOver20 atomic.Int64
|
||||
publishLateCtrl atomic.Int64
|
||||
publishLateAccepted atomic.Int64
|
||||
|
||||
mu sync.Mutex
|
||||
accepted []AcceptedMessage
|
||||
errors []string
|
||||
ctrlSamples []CtrlObservationSample
|
||||
}
|
||||
|
||||
func validScenario(scenario string) bool {
|
||||
@@ -115,9 +122,7 @@ func runAgent(ctx context.Context, cfg *Config, manifest *Manifest, machine, coo
|
||||
if len(accounts) != cfg.UsersPerMachine {
|
||||
return fmt.Errorf("machine %s accounts=%d expected=%d", machine, len(accounts), cfg.UsersPerMachine)
|
||||
}
|
||||
for i := range accounts {
|
||||
accounts[i].Endpoint = cfg.Endpoints[machine][(accounts[i].Index-1)%len(cfg.Endpoints[machine])]
|
||||
}
|
||||
assignAgentEndpoints(cfg, machine, accounts)
|
||||
if !cfg.SkipP2PPreflight {
|
||||
if err = preflightP2P(ctx, cfg, manifest, accounts, apiKey); err != nil {
|
||||
return err
|
||||
@@ -125,7 +130,9 @@ func runAgent(ctx context.Context, cfg *Config, manifest *Manifest, machine, coo
|
||||
}
|
||||
coord := newCoordinatorClient(coordinatorURL)
|
||||
placeholderRunID := "pending"
|
||||
run := &agentRun{cfg: cfg, machine: machine, clients: make(map[int]*WSClient), ack: NewDurationHistogram()}
|
||||
run := &agentRun{cfg: cfg, machine: machine, clients: make(map[int]*WSClient), ack: NewDurationHistogram(),
|
||||
ctrlLatency: NewDurationHistogram(),
|
||||
callbackQueue: &callbackQueueMetrics{}}
|
||||
run.recorder = newDeliveryRecorder(placeholderRunID, machine, len(accounts))
|
||||
stopMetrics, err := startAgentMetrics(metricsListen, run)
|
||||
if err != nil {
|
||||
@@ -154,31 +161,34 @@ func runAgent(ctx context.Context, cfg *Config, manifest *Manifest, machine, coo
|
||||
defer close(maintenanceDone)
|
||||
run.maintainConnections(maintenanceCtx, accounts, manifest.GroupTopic, apiKey, password)
|
||||
}()
|
||||
var stopMaintenanceOnce sync.Once
|
||||
stopConnectionMaintenance := func() {
|
||||
stopMaintenanceOnce.Do(func() {
|
||||
stopMaintenance()
|
||||
<-maintenanceDone
|
||||
})
|
||||
}
|
||||
defer stopConnectionMaintenance()
|
||||
start, err := coord.ready(ctx, func() AgentReady {
|
||||
return AgentReady{
|
||||
Machine: machine, ReadyConnections: run.aliveConnections(), ExpectedConnections: len(accounts),
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
stopMaintenance()
|
||||
<-maintenanceDone
|
||||
return err
|
||||
}
|
||||
if !runIDPattern.MatchString(start.RunID) || !validScenario(start.Scenario) {
|
||||
stopMaintenance()
|
||||
<-maintenanceDone
|
||||
return fmt.Errorf("invalid coordinator start: %+v", start)
|
||||
}
|
||||
run.runID = start.RunID
|
||||
run.scenario = start.Scenario
|
||||
run.recorder.setRunID(start.RunID)
|
||||
if err = waitUntil(ctx, start.StartAt); err != nil {
|
||||
stopMaintenance()
|
||||
<-maintenanceDone
|
||||
return err
|
||||
}
|
||||
stopMaintenance()
|
||||
<-maintenanceDone
|
||||
if err = coord.started(ctx, machine, start.StartAt); err != nil {
|
||||
return fmt.Errorf("coordinated start rejected: %w", err)
|
||||
}
|
||||
if alive := run.aliveConnections(); alive != len(accounts) {
|
||||
return fmt.Errorf("connections alive at scheduled start=%d expected=%d", alive, len(accounts))
|
||||
}
|
||||
@@ -206,6 +216,15 @@ func runAgent(ctx context.Context, cfg *Config, manifest *Manifest, machine, coo
|
||||
if err = waitContext(ctx, drain); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = waitForCallbackQueue(ctx, run.callbackQueue, 30*time.Second); err != nil {
|
||||
return err
|
||||
}
|
||||
quietPeriod, _ := configDuration(cfg.DeliveryQuiet)
|
||||
quietTimeout, _ := configDuration(cfg.DeliveryTimeout)
|
||||
log.Printf("agent=%s waiting for %s of delivery quiescence", machine, quietPeriod)
|
||||
if err = waitForDeliveryQuiescence(ctx, run.recorder, run.callbackQueue, quietPeriod, quietTimeout); err != nil {
|
||||
return err
|
||||
}
|
||||
observedLocal := run.recorder.unackedObservedMessages(ledger, start.Scenario)
|
||||
if err = coord.submitObserved(ctx, machine, observedLocal); err != nil {
|
||||
return err
|
||||
@@ -223,11 +242,15 @@ func runAgent(ctx context.Context, cfg *Config, manifest *Manifest, machine, coo
|
||||
PublishRejected: run.explicitRejected(), PublishOutcomeUnknown: run.outcomeUnknown(),
|
||||
PublishErrors: run.publishErrorCounts(),
|
||||
AckLatency: run.ack.Snapshot(), AcceptedLocal: local,
|
||||
CallbackQueue: run.callbackQueue.snapshot(),
|
||||
}
|
||||
run.mu.Lock()
|
||||
base.Errors = append(base.Errors, run.errors...)
|
||||
run.mu.Unlock()
|
||||
summary := run.recorder.summarize(base, ledger)
|
||||
// Include messages observed over WebSocket without a timely ctrl ACK. They
|
||||
// are confirmed server-side publishes and must participate in realtime
|
||||
// delivery accounting instead of being reported as unexpected traffic.
|
||||
summary := run.recorder.summarize(base, verificationLedger)
|
||||
historySettle, _ := configDuration(cfg.HistorySettle)
|
||||
log.Printf("agent=%s waiting %s before history verification", machine, historySettle)
|
||||
if err = waitContext(ctx, historySettle); err != nil {
|
||||
@@ -241,13 +264,16 @@ func runAgent(ctx context.Context, cfg *Config, manifest *Manifest, machine, coo
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Both agents have completed history verification. Connection maintenance can
|
||||
// now stop without making the other agent verify through stale sockets.
|
||||
stopConnectionMaintenance()
|
||||
summary.HistoryChecked = int64(history.checked)
|
||||
summary.HistoryFound = int64(history.found)
|
||||
summary.HistoryUnackedChecked = int64(history.unackedChecked)
|
||||
summary.HistoryUnackedFound = int64(history.unackedFound)
|
||||
persistenceMissing := history.missing
|
||||
summary.PersistenceLost = int64(len(persistenceMissing))
|
||||
summary.RealtimeLostStored = run.recorder.countRealtimeLostStored(ledger, storedMessageIDs)
|
||||
summary.RealtimeLostStored = run.recorder.countRealtimeLostStored(verificationLedger, storedMessageIDs)
|
||||
if len(persistenceMissing) > 10000 {
|
||||
summary.PersistenceMissing = append([]string(nil), persistenceMissing[:10000]...)
|
||||
summary.Errors = append(summary.Errors, "persistence missing details truncated at 10000")
|
||||
@@ -255,6 +281,7 @@ func runAgent(ctx context.Context, cfg *Config, manifest *Manifest, machine, coo
|
||||
summary.PersistenceMissing = persistenceMissing
|
||||
}
|
||||
summary.FinishedAt = time.Now().UTC()
|
||||
run.applyCtrlObservations(&summary)
|
||||
if err = writeAgentSummary(output, summary); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -267,6 +294,68 @@ func runAgent(ctx context.Context, cfg *Config, manifest *Manifest, machine, coo
|
||||
return nil
|
||||
}
|
||||
|
||||
func waitForCallbackQueue(ctx context.Context, queue *callbackQueueMetrics, timeout time.Duration) error {
|
||||
deadline := time.NewTimer(timeout)
|
||||
defer deadline.Stop()
|
||||
ticker := time.NewTicker(10 * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
for !queue.isDrained() {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-deadline.C:
|
||||
return fmt.Errorf("data callback queue did not drain within %s: current=%d processing=%d",
|
||||
timeout, queue.current.Load(), queue.processing.Load())
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func waitForDeliveryQuiescence(ctx context.Context, recorder *deliveryRecorder, queue *callbackQueueMetrics,
|
||||
quietPeriod, timeout time.Duration) error {
|
||||
deadline := time.NewTimer(timeout)
|
||||
defer deadline.Stop()
|
||||
ticker := time.NewTicker(100 * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
lastReceived := recorder.received.Load()
|
||||
stableSince := time.Now()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-deadline.C:
|
||||
return fmt.Errorf("deliveries did not become quiescent within %s: received=%d queued=%d processing=%d",
|
||||
timeout, recorder.received.Load(), queue.current.Load(), queue.processing.Load())
|
||||
case now := <-ticker.C:
|
||||
received := recorder.received.Load()
|
||||
if received != lastReceived || !queue.isDrained() {
|
||||
lastReceived = received
|
||||
stableSince = now
|
||||
continue
|
||||
}
|
||||
if now.Sub(stableSince) >= quietPeriod {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func assignAgentEndpoints(cfg *Config, machine string, accounts []AccountRecord) {
|
||||
endpoints := cfg.Endpoints[machine]
|
||||
dialIPs := cfg.DialIPs[machine]
|
||||
machineOffset := 0
|
||||
if machine == "b" {
|
||||
machineOffset = 1
|
||||
}
|
||||
for i := range accounts {
|
||||
accounts[i].Endpoint = endpoints[(accounts[i].Index-1)%len(endpoints)]
|
||||
if len(dialIPs) > 0 {
|
||||
slot := (accounts[i].Index-1)*2 + machineOffset
|
||||
accounts[i].DialIP = strings.TrimSpace(dialIPs[slot%len(dialIPs)])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type historyQuery struct {
|
||||
client *WSClient
|
||||
topic string
|
||||
@@ -458,8 +547,18 @@ func newHistoryQuery(client *WSClient, topic string, messages []AcceptedMessage,
|
||||
for _, message := range messages {
|
||||
expected = append(expected, message.ClientMid)
|
||||
}
|
||||
return historyQuery{client: client, topic: topic, since: messages[0].Seq,
|
||||
before: messages[len(messages)-1].Seq + 1, limit: limit, expected: expected}
|
||||
since := messages[0].Seq
|
||||
before := messages[len(messages)-1].Seq + 1
|
||||
// P2P topics interleave both senders. The requested range must leave room
|
||||
// for non-target messages between the expected sequence numbers.
|
||||
if span := before - since; span > int64(limit) {
|
||||
limit = int(span)
|
||||
}
|
||||
if limit > 200 {
|
||||
limit = 200
|
||||
}
|
||||
return historyQuery{client: client, topic: topic, since: since,
|
||||
before: before, limit: limit, expected: expected}
|
||||
}
|
||||
|
||||
func (r *agentRun) waitHistoryPage(ctx context.Context, expected []string, timeout time.Duration) {
|
||||
@@ -591,29 +690,57 @@ func (r *agentRun) maintainConnections(ctx context.Context, accounts []AccountRe
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
jobs := make(chan AccountRecord)
|
||||
var wg sync.WaitGroup
|
||||
for index, account := range byIndex {
|
||||
current := r.client(index)
|
||||
if current != nil && current.alive() {
|
||||
workers := 16
|
||||
if workers > len(accounts) {
|
||||
workers = len(accounts)
|
||||
}
|
||||
for range workers {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for account := range jobs {
|
||||
index := account.Index
|
||||
previous := r.client(index)
|
||||
if previous != nil && previous.alive() {
|
||||
continue
|
||||
}
|
||||
wg.Add(1)
|
||||
go func(index int, account AccountRecord, previous *WSClient) {
|
||||
defer wg.Done()
|
||||
if previous != nil {
|
||||
previous.Close()
|
||||
}
|
||||
client, err := r.connectAccountWithRetry(ctx, account, groupTopic, apiKey, password)
|
||||
if err != nil {
|
||||
if ctx.Err() == nil {
|
||||
log.Printf("account=%s pre-start reconnect failed: %v", account.Username, err)
|
||||
log.Printf("account=%s reconnect failed: %v", account.Username, err)
|
||||
}
|
||||
return
|
||||
continue
|
||||
}
|
||||
r.setClient(index, client)
|
||||
log.Printf("account=%s pre-start connection restored", account.Username)
|
||||
}(index, account, current)
|
||||
log.Printf("account=%s connection restored", account.Username)
|
||||
}
|
||||
}()
|
||||
}
|
||||
indexes := make([]int, 0, len(byIndex))
|
||||
for index := range byIndex {
|
||||
indexes = append(indexes, index)
|
||||
}
|
||||
sort.Ints(indexes)
|
||||
for _, index := range indexes {
|
||||
account := byIndex[index]
|
||||
current := r.client(index)
|
||||
if current != nil && current.alive() {
|
||||
continue
|
||||
}
|
||||
select {
|
||||
case jobs <- account:
|
||||
case <-ctx.Done():
|
||||
close(jobs)
|
||||
wg.Wait()
|
||||
return
|
||||
}
|
||||
}
|
||||
close(jobs)
|
||||
wg.Wait()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
@@ -626,7 +753,8 @@ func (r *agentRun) maintainConnections(ctx context.Context, accounts []AccountRe
|
||||
func (r *agentRun) connectAccountWithRetry(ctx context.Context, account AccountRecord, groupTopic, apiKey, password string) (*WSClient, error) {
|
||||
var lastErr error
|
||||
for attempt := 1; attempt <= 6; attempt++ {
|
||||
client, err := connectWSClient(ctx, r.cfg, account, apiKey, password, r.recorder.onData, r.recorder.onDisconnect)
|
||||
client, err := connectWSClient(ctx, r.cfg, account, apiKey, password, r.recorder.onData, r.recorder.onDisconnect,
|
||||
r.observeCtrl, r.callbackQueue)
|
||||
if err == nil {
|
||||
if err = client.Subscribe(ctx, account.P2PTopic); err == nil {
|
||||
err = client.Subscribe(ctx, groupTopic)
|
||||
@@ -659,13 +787,24 @@ func (r *agentRun) execute(ctx context.Context, start time.Time, groupTopic stri
|
||||
case "p2p":
|
||||
return r.scheduleP2P(ctx, start, r.cfg.UsersPerMachine, r.cfg.P2PMessages, mustDuration(r.cfg.P2PInterval))
|
||||
case "group":
|
||||
return r.scheduleGroup(ctx, start, groupTopic, 1, 500, 2)
|
||||
return r.scheduleGroup(ctx, start, groupTopic, 1, scaledUsers(r.cfg.UsersPerMachine, 1, 2), 2)
|
||||
case "mixed":
|
||||
p2pUsers := scaledUsers(r.cfg.UsersPerMachine, 3, 10)
|
||||
groupSenders := scaledUsers(r.cfg.UsersPerMachine, 1, 4)
|
||||
if p2pUsers+groupSenders > r.cfg.UsersPerMachine {
|
||||
groupSenders = r.cfg.UsersPerMachine - p2pUsers
|
||||
}
|
||||
if groupSenders <= 0 {
|
||||
return fmt.Errorf("users_per_machine=%d is too small for mixed scenario", r.cfg.UsersPerMachine)
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
errCh := make(chan error, 2)
|
||||
wg.Add(2)
|
||||
go func() { defer wg.Done(); errCh <- r.scheduleP2P(ctx, start, 300, 5, 20*time.Second) }()
|
||||
go func() { defer wg.Done(); errCh <- r.scheduleGroup(ctx, start, groupTopic, 301, 550, 2) }()
|
||||
go func() { defer wg.Done(); errCh <- r.scheduleP2P(ctx, start, p2pUsers, 5, 20*time.Second) }()
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
errCh <- r.scheduleGroup(ctx, start, groupTopic, p2pUsers+1, p2pUsers+groupSenders, 2)
|
||||
}()
|
||||
wg.Wait()
|
||||
close(errCh)
|
||||
for err := range errCh {
|
||||
@@ -679,15 +818,25 @@ func (r *agentRun) execute(ctx context.Context, start time.Time, groupTopic stri
|
||||
}
|
||||
}
|
||||
|
||||
func scaledUsers(total, numerator, denominator int) int {
|
||||
count := total * numerator / denominator
|
||||
if count < 1 {
|
||||
return 1
|
||||
}
|
||||
if count > total {
|
||||
return total
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
func (r *agentRun) scheduleP2P(ctx context.Context, start time.Time, users, messages int, interval time.Duration) error {
|
||||
var wg sync.WaitGroup
|
||||
for index := 1; index <= users; index++ {
|
||||
client := r.client(index)
|
||||
if client == nil {
|
||||
if r.client(index) == nil {
|
||||
return fmt.Errorf("missing client %d", index)
|
||||
}
|
||||
wg.Add(1)
|
||||
go func(index int, client *WSClient) {
|
||||
go func(index int) {
|
||||
defer wg.Done()
|
||||
jitter := time.Duration((index*37)%200) * time.Millisecond
|
||||
for messageIndex := 1; messageIndex <= messages; messageIndex++ {
|
||||
@@ -696,6 +845,11 @@ func (r *agentRun) scheduleP2P(ctx context.Context, start time.Time, users, mess
|
||||
return
|
||||
}
|
||||
mid := fmt.Sprintf("%s-p-%s-%04d-%02d", r.runID, r.machine, index, messageIndex)
|
||||
client := r.client(index)
|
||||
if client == nil || !client.alive() {
|
||||
r.recordUnavailablePublish(mid)
|
||||
continue
|
||||
}
|
||||
accepted, err := r.publish(ctx, client, client.account.P2PTopic, mid, messageIndex)
|
||||
if err == nil {
|
||||
accepted.PeerIndex = index
|
||||
@@ -707,7 +861,7 @@ func (r *agentRun) scheduleP2P(ctx context.Context, start time.Time, users, mess
|
||||
r.addAccepted(accepted)
|
||||
}
|
||||
}
|
||||
}(index, client)
|
||||
}(index)
|
||||
}
|
||||
wg.Wait()
|
||||
return ctx.Err()
|
||||
@@ -725,10 +879,9 @@ func (r *agentRun) scheduleGroup(ctx context.Context, start time.Time, topic str
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
for index := first; index <= last; index++ {
|
||||
client := r.client(index)
|
||||
position := index - first
|
||||
wg.Add(1)
|
||||
go func(index, position int, client *WSClient) {
|
||||
go func(index, position int) {
|
||||
defer wg.Done()
|
||||
for round := 1; round <= rounds; round++ {
|
||||
target := start.Add(time.Duration(round-1)*roundDuration + time.Duration(position)*200*time.Millisecond + offset)
|
||||
@@ -736,17 +889,32 @@ func (r *agentRun) scheduleGroup(ctx context.Context, start time.Time, topic str
|
||||
return
|
||||
}
|
||||
mid := fmt.Sprintf("%s-g-%s-%04d-%02d", r.runID, r.machine, index, round)
|
||||
client := r.client(index)
|
||||
if client == nil || !client.alive() {
|
||||
r.recordUnavailablePublish(mid)
|
||||
continue
|
||||
}
|
||||
accepted, err := r.publish(ctx, client, topic, mid, round)
|
||||
if err == nil {
|
||||
r.addAccepted(accepted)
|
||||
}
|
||||
}
|
||||
}(index, position, client)
|
||||
}(index, position)
|
||||
}
|
||||
wg.Wait()
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
func (r *agentRun) recordUnavailablePublish(mid string) {
|
||||
r.attempted.Add(1)
|
||||
r.publishSocket.Add(1)
|
||||
r.mu.Lock()
|
||||
if len(r.errors) < 1000 {
|
||||
r.errors = append(r.errors, fmt.Sprintf("publish %s: websocket connection unavailable", mid))
|
||||
}
|
||||
r.mu.Unlock()
|
||||
}
|
||||
|
||||
func (r *agentRun) publish(ctx context.Context, client *WSClient, topic, mid string, messageIndex int) (AcceptedMessage, error) {
|
||||
r.attempted.Add(1)
|
||||
accepted, err := client.Publish(ctx, topic, mid, r.runID, r.scenario, messageIndex)
|
||||
@@ -794,7 +962,57 @@ func (r *agentRun) explicitRejected() int64 {
|
||||
}
|
||||
|
||||
func (r *agentRun) outcomeUnknown() int64 {
|
||||
return r.publishTimeout.Load() + r.publishSocket.Load()
|
||||
return r.unresolvedCtrl() + r.publishSocket.Load()
|
||||
}
|
||||
|
||||
func (r *agentRun) unresolvedCtrl() int64 {
|
||||
unresolved := r.publishTimeout.Load() - r.publishLateCtrl.Load()
|
||||
if unresolved < 0 {
|
||||
return 0
|
||||
}
|
||||
return unresolved
|
||||
}
|
||||
|
||||
func (r *agentRun) observeCtrl(account AccountRecord, observation ctrlObservation) {
|
||||
if observation.Kind != "pub" {
|
||||
return
|
||||
}
|
||||
r.publishCtrlObserved.Add(1)
|
||||
r.ctrlLatency.Observe(observation.Latency)
|
||||
if observation.Latency > 20*time.Second {
|
||||
r.publishCtrlOver20.Add(1)
|
||||
}
|
||||
if observation.Late {
|
||||
r.publishLateCtrl.Add(1)
|
||||
if observation.Code >= 200 && observation.Code < 300 {
|
||||
r.publishLateAccepted.Add(1)
|
||||
}
|
||||
}
|
||||
if observation.Late || observation.Latency > 20*time.Second {
|
||||
r.mu.Lock()
|
||||
if len(r.ctrlSamples) < 1000 {
|
||||
r.ctrlSamples = append(r.ctrlSamples, CtrlObservationSample{
|
||||
Machine: account.Machine, Index: account.Index, ID: observation.ID, Code: observation.Code,
|
||||
SentAt: observation.SentAt, ReceivedAt: observation.ReceivedAt,
|
||||
LatencyMS: float64(observation.Latency) / float64(time.Millisecond), Late: observation.Late,
|
||||
})
|
||||
}
|
||||
r.mu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
func (r *agentRun) applyCtrlObservations(summary *AgentSummary) {
|
||||
summary.CtrlTrackingEnabled = true
|
||||
summary.PublishCtrlObserved = r.publishCtrlObserved.Load()
|
||||
summary.PublishCtrlOver20 = r.publishCtrlOver20.Load()
|
||||
summary.PublishLateCtrl = r.publishLateCtrl.Load()
|
||||
summary.PublishLateAccepted = r.publishLateAccepted.Load()
|
||||
summary.PublishCtrlUnresolved = r.unresolvedCtrl()
|
||||
summary.PublishOutcomeUnknown = r.outcomeUnknown()
|
||||
summary.PublishCtrlLatency = r.ctrlLatency.Snapshot()
|
||||
r.mu.Lock()
|
||||
summary.CtrlObservationSamples = append([]CtrlObservationSample(nil), r.ctrlSamples...)
|
||||
r.mu.Unlock()
|
||||
}
|
||||
|
||||
func (r *agentRun) addAccepted(message AcceptedMessage) {
|
||||
@@ -858,9 +1076,10 @@ func (r *agentRun) drainDuration() (time.Duration, error) {
|
||||
|
||||
func (r *agentRun) close() {
|
||||
clients := r.clientsSnapshot()
|
||||
var interval time.Duration
|
||||
if r.cfg != nil && r.cfg.CloseRamp != "" && len(clients) > 1 {
|
||||
interval = mustDuration(r.cfg.CloseRamp) / time.Duration(len(clients)-1)
|
||||
sort.Slice(clients, func(i, j int) bool { return clients[i].account.Index < clients[j].account.Index })
|
||||
interval, initialDelay := closeSchedule(r.cfg, r.machine, len(clients))
|
||||
if initialDelay > 0 {
|
||||
time.Sleep(initialDelay)
|
||||
}
|
||||
for index, client := range clients {
|
||||
client.Close()
|
||||
@@ -870,6 +1089,19 @@ func (r *agentRun) close() {
|
||||
}
|
||||
}
|
||||
|
||||
func closeSchedule(cfg *Config, machine string, clients int) (interval, initialDelay time.Duration) {
|
||||
if cfg == nil || cfg.CloseRamp == "" || clients <= 1 {
|
||||
return 0, 0
|
||||
}
|
||||
// The two logical agents share one global ramp. Interleaving their slots avoids
|
||||
// doubling the disconnect rate when both agents begin cleanup at the same time.
|
||||
interval = 2 * mustDuration(cfg.CloseRamp) / time.Duration(clients-1)
|
||||
if machine == "b" {
|
||||
initialDelay = interval / 2
|
||||
}
|
||||
return interval, initialDelay
|
||||
}
|
||||
|
||||
func waitUntil(ctx context.Context, target time.Time) error {
|
||||
delay := time.Until(target)
|
||||
if delay <= 0 {
|
||||
|
||||
@@ -2,7 +2,9 @@ package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestHistoryQueriesForMessagesGroupsRangesByTopic(t *testing.T) {
|
||||
@@ -51,6 +53,40 @@ func TestClassifyPublishErrorTreatsClosedConnectionAsUnknownSocketError(t *testi
|
||||
}
|
||||
}
|
||||
|
||||
func TestScenarioUsersScaleWithConfiguredCapacity(t *testing.T) {
|
||||
tests := []struct {
|
||||
users int
|
||||
group int
|
||||
mixedP2P int
|
||||
mixedGroup int
|
||||
}{
|
||||
{users: 1000, group: 500, mixedP2P: 300, mixedGroup: 250},
|
||||
{users: 600, group: 300, mixedP2P: 180, mixedGroup: 150},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
if got := scaledUsers(tc.users, 1, 2); got != tc.group {
|
||||
t.Fatalf("users=%d group=%d want=%d", tc.users, got, tc.group)
|
||||
}
|
||||
if got := scaledUsers(tc.users, 3, 10); got != tc.mixedP2P {
|
||||
t.Fatalf("users=%d mixed p2p=%d want=%d", tc.users, got, tc.mixedP2P)
|
||||
}
|
||||
if got := scaledUsers(tc.users, 1, 4); got != tc.mixedGroup {
|
||||
t.Fatalf("users=%d mixed group=%d want=%d", tc.users, got, tc.mixedGroup)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewHistoryQueryIncludesInterleavedSequenceRange(t *testing.T) {
|
||||
messages := make([]AcceptedMessage, 10)
|
||||
for i := range messages {
|
||||
messages[i] = AcceptedMessage{ClientMid: fmt.Sprintf("mid-%d", i), Seq: int64(100 + i*2)}
|
||||
}
|
||||
query := newHistoryQuery(nil, "p2p-one", messages, 10)
|
||||
if query.since != 100 || query.before != 119 || query.limit != 19 {
|
||||
t.Fatalf("history query range=%d-%d limit=%d want=100-119/19", query.since, query.before, query.limit)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeMessageLedgersPrefersPeerObservationAndAck(t *testing.T) {
|
||||
senderEcho := AcceptedMessage{ClientMid: "mid-1", SenderMachine: "a", PeerMachine: "a", Seq: 10}
|
||||
peerDelivery := AcceptedMessage{ClientMid: "mid-1", SenderMachine: "a", PeerMachine: "b", PeerIndex: 7, Seq: 10}
|
||||
@@ -65,3 +101,15 @@ func TestMergeMessageLedgersPrefersPeerObservationAndAck(t *testing.T) {
|
||||
t.Fatalf("ACK ledger was not authoritative: %+v", merged)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCloseScheduleInterleavesLogicalAgents(t *testing.T) {
|
||||
cfg := &Config{CloseRamp: "60s"}
|
||||
intervalA, delayA := closeSchedule(cfg, "a", 601)
|
||||
intervalB, delayB := closeSchedule(cfg, "b", 601)
|
||||
if intervalA != 200*time.Millisecond || intervalB != intervalA {
|
||||
t.Fatalf("intervals: a=%s b=%s, want 200ms", intervalA, intervalB)
|
||||
}
|
||||
if delayA != 0 || delayB != 100*time.Millisecond {
|
||||
t.Fatalf("delays: a=%s b=%s, want 0/100ms", delayA, delayB)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -46,6 +46,10 @@ func startAgentMetrics(address string, run *agentRun) (func(), error) {
|
||||
prometheus.NewCounterFunc(prometheus.CounterOpts{Name: "im_load_agent_publish_502_total", Help: "Publish commands rejected with 502.", ConstLabels: labels}, func() float64 { return float64(run.publish502.Load()) }),
|
||||
prometheus.NewCounterFunc(prometheus.CounterOpts{Name: "im_load_agent_publish_503_total", Help: "Publish commands rejected with 503.", ConstLabels: labels}, func() float64 { return float64(run.publish503.Load()) }),
|
||||
prometheus.NewCounterFunc(prometheus.CounterOpts{Name: "im_load_agent_publish_ctrl_timeout_total", Help: "Publish commands with control-response timeout.", ConstLabels: labels}, func() float64 { return float64(run.publishTimeout.Load()) }),
|
||||
prometheus.NewCounterFunc(prometheus.CounterOpts{Name: "im_load_agent_publish_ctrl_observed_total", Help: "Publish control responses observed, including responses received after command timeout.", ConstLabels: labels}, func() float64 { return float64(run.publishCtrlObserved.Load()) }),
|
||||
prometheus.NewCounterFunc(prometheus.CounterOpts{Name: "im_load_agent_publish_ctrl_over_20s_total", Help: "Publish control responses taking longer than 20 seconds.", ConstLabels: labels}, func() float64 { return float64(run.publishCtrlOver20.Load()) }),
|
||||
prometheus.NewCounterFunc(prometheus.CounterOpts{Name: "im_load_agent_publish_ctrl_late_total", Help: "Publish control responses received after command timeout.", ConstLabels: labels}, func() float64 { return float64(run.publishLateCtrl.Load()) }),
|
||||
prometheus.NewGaugeFunc(prometheus.GaugeOpts{Name: "im_load_agent_publish_ctrl_unresolved", Help: "Publish control responses still unresolved after accounting for late responses.", ConstLabels: labels}, func() float64 { return float64(run.unresolvedCtrl()) }),
|
||||
prometheus.NewCounterFunc(prometheus.CounterOpts{Name: "im_load_agent_publish_socket_error_total", Help: "Publish commands with socket errors.", ConstLabels: labels}, func() float64 { return float64(run.publishSocket.Load()) }),
|
||||
prometheus.NewCounterFunc(prometheus.CounterOpts{Name: "im_load_agent_received_total", Help: "Unique matching data deliveries received.", ConstLabels: labels}, func() float64 {
|
||||
received, _, _, _ := run.recorder.metricCounts()
|
||||
@@ -63,6 +67,27 @@ func startAgentMetrics(address string, run *agentRun) (func(), error) {
|
||||
_, _, _, disconnects := run.recorder.metricCounts()
|
||||
return float64(disconnects)
|
||||
}),
|
||||
prometheus.NewGaugeFunc(prometheus.GaugeOpts{Name: "im_load_agent_callback_queue_current", Help: "Data callbacks currently waiting to start.", ConstLabels: labels}, func() float64 {
|
||||
return float64(run.callbackQueue.current.Load())
|
||||
}),
|
||||
prometheus.NewGaugeFunc(prometheus.GaugeOpts{Name: "im_load_agent_callback_queue_max", Help: "Peak data callbacks waiting to start.", ConstLabels: labels}, func() float64 {
|
||||
return float64(run.callbackQueue.maximum.Load())
|
||||
}),
|
||||
prometheus.NewGaugeFunc(prometheus.GaugeOpts{Name: "im_load_agent_callback_processing", Help: "Data callbacks currently processing.", ConstLabels: labels}, func() float64 {
|
||||
return float64(run.callbackQueue.processing.Load())
|
||||
}),
|
||||
prometheus.NewCounterFunc(prometheus.CounterOpts{Name: "im_load_agent_callback_queue_enqueued_total", Help: "Data callbacks enqueued.", ConstLabels: labels}, func() float64 {
|
||||
return float64(run.callbackQueue.enqueued.Load())
|
||||
}),
|
||||
prometheus.NewCounterFunc(prometheus.CounterOpts{Name: "im_load_agent_callback_queue_completed_total", Help: "Data callbacks which completed processing.", ConstLabels: labels}, func() float64 {
|
||||
return float64(run.callbackQueue.completed.Load())
|
||||
}),
|
||||
prometheus.NewCounterFunc(prometheus.CounterOpts{Name: "im_load_agent_callback_queue_wait_seconds_total", Help: "Cumulative time spent waiting to start data callbacks.", ConstLabels: labels}, func() float64 {
|
||||
return float64(run.callbackQueue.waitNanos.Load()) / float64(time.Second)
|
||||
}),
|
||||
prometheus.NewGaugeFunc(prometheus.GaugeOpts{Name: "im_load_agent_callback_queue_wait_seconds_max", Help: "Maximum time spent waiting to start a data callback.", ConstLabels: labels}, func() float64 {
|
||||
return float64(run.callbackQueue.maxWait.Load()) / float64(time.Second)
|
||||
}),
|
||||
)
|
||||
mux := http.NewServeMux()
|
||||
mux.Handle("/metrics", promhttp.HandlerFor(registry, promhttp.HandlerOpts{}))
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
type callbackQueueMetrics struct {
|
||||
current atomic.Int64
|
||||
processing atomic.Int64
|
||||
maximum atomic.Int64
|
||||
enqueued atomic.Int64
|
||||
completed atomic.Int64
|
||||
waitNanos atomic.Int64
|
||||
maxWait atomic.Int64
|
||||
}
|
||||
|
||||
func (m *callbackQueueMetrics) enqueue() {
|
||||
if m == nil {
|
||||
return
|
||||
}
|
||||
current := m.current.Add(1)
|
||||
m.enqueued.Add(1)
|
||||
updateAtomicMax(&m.maximum, current)
|
||||
}
|
||||
|
||||
func (m *callbackQueueMetrics) startCallback(queuedAt time.Time) {
|
||||
if m == nil {
|
||||
return
|
||||
}
|
||||
wait := time.Since(queuedAt).Nanoseconds()
|
||||
if wait < 0 {
|
||||
wait = 0
|
||||
}
|
||||
m.current.Add(-1)
|
||||
m.processing.Add(1)
|
||||
m.waitNanos.Add(wait)
|
||||
updateAtomicMax(&m.maxWait, wait)
|
||||
}
|
||||
|
||||
func (m *callbackQueueMetrics) finishCallback() {
|
||||
if m == nil {
|
||||
return
|
||||
}
|
||||
m.processing.Add(-1)
|
||||
m.completed.Add(1)
|
||||
}
|
||||
|
||||
func (m *callbackQueueMetrics) snapshot() CallbackQueueSnapshot {
|
||||
if m == nil {
|
||||
return CallbackQueueSnapshot{}
|
||||
}
|
||||
return CallbackQueueSnapshot{
|
||||
Current: m.current.Load(), Processing: m.processing.Load(), Maximum: m.maximum.Load(), Enqueued: m.enqueued.Load(),
|
||||
Completed: m.completed.Load(), WaitSecondsTotal: float64(m.waitNanos.Load()) / float64(time.Second),
|
||||
WaitSecondsMax: float64(m.maxWait.Load()) / float64(time.Second),
|
||||
}
|
||||
}
|
||||
|
||||
func (m *callbackQueueMetrics) isDrained() bool {
|
||||
return m == nil || (m.current.Load() == 0 && m.processing.Load() == 0)
|
||||
}
|
||||
|
||||
func updateAtomicMax(target *atomic.Int64, value int64) {
|
||||
for current := target.Load(); value > current; current = target.Load() {
|
||||
if target.CompareAndSwap(current, value) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestCallbackQueueMetricsTrackWaitingAndProcessing(t *testing.T) {
|
||||
metrics := &callbackQueueMetrics{}
|
||||
metrics.enqueue()
|
||||
metrics.enqueue()
|
||||
metrics.startCallback(time.Now().Add(-10 * time.Millisecond))
|
||||
|
||||
snapshot := metrics.snapshot()
|
||||
if snapshot.Current != 1 || snapshot.Processing != 1 || snapshot.Maximum != 2 || snapshot.Enqueued != 2 {
|
||||
t.Fatalf("unexpected active queue snapshot: %+v", snapshot)
|
||||
}
|
||||
if snapshot.WaitSecondsTotal <= 0 || snapshot.WaitSecondsMax <= 0 || metrics.isDrained() {
|
||||
t.Fatalf("wait or drain state not tracked: %+v", snapshot)
|
||||
}
|
||||
|
||||
metrics.finishCallback()
|
||||
metrics.startCallback(time.Now())
|
||||
metrics.finishCallback()
|
||||
if snapshot = metrics.snapshot(); snapshot.Current != 0 || snapshot.Processing != 0 || snapshot.Completed != 2 || !metrics.isDrained() {
|
||||
t.Fatalf("unexpected drained queue snapshot: %+v", snapshot)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWaitForCallbackQueueWaitsForProcessingCallback(t *testing.T) {
|
||||
metrics := &callbackQueueMetrics{}
|
||||
metrics.enqueue()
|
||||
metrics.startCallback(time.Now())
|
||||
go func() {
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
metrics.finishCallback()
|
||||
}()
|
||||
|
||||
started := time.Now()
|
||||
if err := waitForCallbackQueue(context.Background(), metrics, time.Second); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if time.Since(started) < 15*time.Millisecond {
|
||||
t.Fatal("wait returned before the callback finished")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWaitForDeliveryQuiescenceResetsForLateDelivery(t *testing.T) {
|
||||
recorder := newDeliveryRecorder("run", "a", 1)
|
||||
queue := &callbackQueueMetrics{}
|
||||
go func() {
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
recorder.received.Add(1)
|
||||
}()
|
||||
|
||||
started := time.Now()
|
||||
if err := waitForDeliveryQuiescence(context.Background(), recorder, queue, 150*time.Millisecond, time.Second); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if elapsed := time.Since(started); elapsed < 250*time.Millisecond {
|
||||
t.Fatalf("quiet period was not reset after late delivery: elapsed=%s", elapsed)
|
||||
}
|
||||
}
|
||||
@@ -27,5 +27,7 @@
|
||||
"p2p_messages_per_user": 10,
|
||||
"p2p_interval": "1s",
|
||||
"drain_p2p": "30s",
|
||||
"drain_group": "60s"
|
||||
"drain_group": "60s",
|
||||
"delivery_quiet_period": "10s",
|
||||
"delivery_quiet_timeout": "3m"
|
||||
}
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
{
|
||||
"tenant": "006",
|
||||
"http_base": "https://im-api.example.com",
|
||||
"api_key_env": "IM_API_KEY",
|
||||
"password_env": "IM_LOAD_PASSWORD",
|
||||
"verification_code_env": "IM_VERIFICATION_CODE",
|
||||
"registration_phone_prefix": "",
|
||||
"users_per_machine": 600,
|
||||
"endpoints": {
|
||||
"a": ["wss://im-api.example.com/v0/channels"],
|
||||
"b": ["wss://im-api.example.com/v0/channels"]
|
||||
},
|
||||
"dial_ips": {
|
||||
"a": [],
|
||||
"b": []
|
||||
},
|
||||
"insecure_tls": false,
|
||||
"prepare_workers": 20,
|
||||
"connect_ramp": "30s",
|
||||
"command_timeout": "90s",
|
||||
"ws_ping_interval": "20s",
|
||||
"preflight_workers": 32,
|
||||
"skip_p2p_preflight": false,
|
||||
"history_workers": 12,
|
||||
"history_page_size": 10,
|
||||
"history_query_interval": "500ms",
|
||||
"history_page_wait": "3s",
|
||||
"history_settle": "5m",
|
||||
"history_timeout": "3m",
|
||||
"close_ramp": "60s",
|
||||
"p2p_messages_per_user": 10,
|
||||
"p2p_interval": "1s",
|
||||
"drain_p2p": "30s",
|
||||
"drain_group": "60s",
|
||||
"delivery_quiet_period": "10s",
|
||||
"delivery_quiet_timeout": "3m"
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
@@ -29,6 +30,20 @@ type AgentReady struct {
|
||||
Armed bool `json:"armed"`
|
||||
}
|
||||
|
||||
type AgentStarted struct {
|
||||
Machine string `json:"machine"`
|
||||
ScheduledAt time.Time `json:"scheduled_at"`
|
||||
}
|
||||
|
||||
type StartGate struct {
|
||||
Proceed bool `json:"proceed"`
|
||||
Error string `json:"error,omitempty"`
|
||||
ArrivedAgents int `json:"arrived_agents"`
|
||||
ExpectedAgents int `json:"expected_agents"`
|
||||
}
|
||||
|
||||
const maxAgentStartSkew = 3 * time.Second
|
||||
|
||||
type coordinatorState struct {
|
||||
mu sync.Mutex
|
||||
start StartInfo
|
||||
@@ -37,6 +52,8 @@ type coordinatorState struct {
|
||||
expectedAgents int
|
||||
ready map[string]AgentReady
|
||||
armed map[string]bool
|
||||
started map[string]time.Time
|
||||
startError string
|
||||
ledgers map[string][]AcceptedMessage
|
||||
observed map[string][]AcceptedMessage
|
||||
history map[string][]string
|
||||
@@ -60,11 +77,13 @@ func runCoordinator(ctx context.Context, listen, runID, scenario, output string,
|
||||
expectedAgents: expectedAgents,
|
||||
ready: make(map[string]AgentReady), ledgers: make(map[string][]AcceptedMessage),
|
||||
observed: make(map[string][]AcceptedMessage), history: make(map[string][]string),
|
||||
armed: make(map[string]bool), summaries: make(map[string]AgentSummary), complete: make(chan struct{}),
|
||||
armed: make(map[string]bool), started: make(map[string]time.Time),
|
||||
summaries: make(map[string]AgentSummary), complete: make(chan struct{}),
|
||||
}
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/v1/ready", state.handleReady)
|
||||
mux.HandleFunc("/v1/start", state.handleStart)
|
||||
mux.HandleFunc("/v1/started", state.handleStarted)
|
||||
mux.HandleFunc("/v1/ledger", state.handleLedger)
|
||||
mux.HandleFunc("/v1/observed", state.handleObserved)
|
||||
mux.HandleFunc("/v1/history", state.handleHistory)
|
||||
@@ -210,11 +229,15 @@ func (s *coordinatorState) handleReady(w http.ResponseWriter, r *http.Request) {
|
||||
s.start.StartAt = time.Time{}
|
||||
s.start.Committed = false
|
||||
clear(s.armed)
|
||||
clear(s.started)
|
||||
s.startError = ""
|
||||
}
|
||||
if allStable && s.start.StartAt.IsZero() {
|
||||
s.start.StartAt = now.Add(s.startDelay)
|
||||
s.start.Committed = false
|
||||
clear(s.armed)
|
||||
clear(s.started)
|
||||
s.startError = ""
|
||||
log.Printf("all agents ready; validation_at=%s", s.start.StartAt.Format(time.RFC3339Nano))
|
||||
}
|
||||
if allStable && !s.start.Committed && ready.Armed && !now.Before(s.start.StartAt) {
|
||||
@@ -234,6 +257,43 @@ func (s *coordinatorState) handleReady(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusOK, start)
|
||||
}
|
||||
|
||||
func (s *coordinatorState) handleStarted(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPost {
|
||||
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
var arrival AgentStarted
|
||||
if !decodeLimitedJSON(w, r, &arrival) || (arrival.Machine != "a" && arrival.Machine != "b") || arrival.ScheduledAt.IsZero() {
|
||||
http.Error(w, "invalid start arrival", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if !s.start.Committed || !arrival.ScheduledAt.Equal(s.start.StartAt) {
|
||||
writeJSON(w, http.StatusOK, StartGate{Error: "start schedule changed", ExpectedAgents: s.expectedAgents})
|
||||
return
|
||||
}
|
||||
if s.started == nil {
|
||||
s.started = make(map[string]time.Time)
|
||||
}
|
||||
if _, exists := s.started[arrival.Machine]; !exists {
|
||||
now := time.Now().UTC()
|
||||
s.started[arrival.Machine] = now
|
||||
skew := now.Sub(s.start.StartAt)
|
||||
if skew < -maxAgentStartSkew || skew > maxAgentStartSkew {
|
||||
s.startError = fmt.Sprintf("agent %s reached start barrier %s from schedule (limit %s)",
|
||||
arrival.Machine, skew.Round(time.Millisecond), maxAgentStartSkew)
|
||||
log.Printf("start barrier rejected: %s", s.startError)
|
||||
}
|
||||
}
|
||||
gate := StartGate{
|
||||
Proceed: len(s.started) == s.expectedAgents && s.startError == "", Error: s.startError,
|
||||
ArrivedAgents: len(s.started), ExpectedAgents: s.expectedAgents,
|
||||
}
|
||||
writeJSON(w, http.StatusOK, gate)
|
||||
}
|
||||
|
||||
func (s *coordinatorState) handleStart(w http.ResponseWriter, r *http.Request) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
@@ -311,7 +371,8 @@ func (s *coordinatorState) handleStatus(w http.ResponseWriter, _ *http.Request)
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
writeJSON(w, http.StatusOK, map[string]any{
|
||||
"start": s.start, "ready": s.ready, "armed": s.armed,
|
||||
"start": s.start, "ready": s.ready, "armed": s.armed, "started": s.started,
|
||||
"start_error": s.startError,
|
||||
"ledger_agents": len(s.ledgers), "observed_agents": len(s.observed),
|
||||
"history_agents": len(s.history), "summary_agents": len(s.summaries),
|
||||
})
|
||||
@@ -322,12 +383,15 @@ func (s *coordinatorState) writeOutput(path string) error {
|
||||
data := struct {
|
||||
Start StartInfo `json:"start"`
|
||||
Ready map[string]AgentReady `json:"ready"`
|
||||
Started map[string]time.Time `json:"started"`
|
||||
StartError string `json:"start_error,omitempty"`
|
||||
Ledgers map[string][]AcceptedMessage `json:"ledgers"`
|
||||
Observed map[string][]AcceptedMessage `json:"observed"`
|
||||
History map[string][]string `json:"history"`
|
||||
Summaries map[string]AgentSummary `json:"summaries"`
|
||||
}{
|
||||
Start: s.start, Ready: s.ready, Ledgers: s.ledgers, Observed: s.observed,
|
||||
Start: s.start, Ready: s.ready, Started: s.started, StartError: s.startError,
|
||||
Ledgers: s.ledgers, Observed: s.observed,
|
||||
History: s.history, Summaries: s.summaries,
|
||||
}
|
||||
s.mu.Unlock()
|
||||
@@ -403,6 +467,25 @@ func (c *coordinatorClient) submitLedger(ctx context.Context, machine string, me
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *coordinatorClient) started(ctx context.Context, machine string, scheduledAt time.Time) error {
|
||||
arrival := AgentStarted{Machine: machine, ScheduledAt: scheduledAt}
|
||||
for {
|
||||
var gate StartGate
|
||||
if _, err := c.request(ctx, http.MethodPost, "/v1/started", arrival, &gate); err != nil {
|
||||
return err
|
||||
}
|
||||
if gate.Error != "" {
|
||||
return errors.New(gate.Error)
|
||||
}
|
||||
if gate.Proceed {
|
||||
return nil
|
||||
}
|
||||
if err := waitContext(ctx, 100*time.Millisecond); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *coordinatorClient) ledger(ctx context.Context) ([]AcceptedMessage, error) {
|
||||
for {
|
||||
var messages []AcceptedMessage
|
||||
|
||||
@@ -15,6 +15,7 @@ func TestCoordinatorResetsBarrierWhenConnectionCountDrops(t *testing.T) {
|
||||
start: StartInfo{RunID: "run-reset", Scenario: "group"}, startDelay: time.Minute,
|
||||
commitDelay: time.Second, expectedAgents: 2, ready: make(map[string]AgentReady),
|
||||
armed: make(map[string]bool), ledgers: make(map[string][]AcceptedMessage),
|
||||
started: make(map[string]time.Time),
|
||||
summaries: make(map[string]AgentSummary), complete: make(chan struct{}),
|
||||
}
|
||||
postReady := func(ready AgentReady) StartInfo {
|
||||
@@ -52,6 +53,7 @@ func TestCoordinatorClientWaitsForBothAgentsAndMergesLedger(t *testing.T) {
|
||||
commitDelay: 50 * time.Millisecond, expectedAgents: 2,
|
||||
ready: make(map[string]AgentReady), ledgers: make(map[string][]AcceptedMessage),
|
||||
armed: make(map[string]bool), summaries: make(map[string]AgentSummary), complete: make(chan struct{}),
|
||||
started: make(map[string]time.Time),
|
||||
}
|
||||
server := httptest.NewServer(testCoordinatorMux(state))
|
||||
defer server.Close()
|
||||
@@ -116,6 +118,7 @@ func testCoordinatorMux(state *coordinatorState) *http.ServeMux {
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/v1/ready", state.handleReady)
|
||||
mux.HandleFunc("/v1/start", state.handleStart)
|
||||
mux.HandleFunc("/v1/started", state.handleStarted)
|
||||
mux.HandleFunc("/v1/ledger", state.handleLedger)
|
||||
mux.HandleFunc("/v1/observed", state.handleObserved)
|
||||
mux.HandleFunc("/v1/history", state.handleHistory)
|
||||
@@ -123,3 +126,50 @@ func testCoordinatorMux(state *coordinatorState) *http.ServeMux {
|
||||
mux.HandleFunc("/v1/status", state.handleStatus)
|
||||
return mux
|
||||
}
|
||||
|
||||
func TestCoordinatorRejectsLateAgentForBothMachines(t *testing.T) {
|
||||
startAt := time.Now().Add(-maxAgentStartSkew - time.Second).UTC()
|
||||
state := &coordinatorState{
|
||||
start: StartInfo{RunID: "run-late", Scenario: "group", StartAt: startAt, Committed: true},
|
||||
expectedAgents: 2, started: make(map[string]time.Time),
|
||||
}
|
||||
server := httptest.NewServer(testCoordinatorMux(state))
|
||||
defer server.Close()
|
||||
client := newCoordinatorClient(server.URL)
|
||||
if err := client.started(context.Background(), "b", startAt); err == nil {
|
||||
t.Fatal("late agent must be rejected")
|
||||
}
|
||||
if err := client.started(context.Background(), "a", startAt); err == nil {
|
||||
t.Fatal("the other agent must observe the same rejected run")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCoordinatorReleasesAgentsOnlyAfterBothReachStart(t *testing.T) {
|
||||
startAt := time.Now().UTC()
|
||||
state := &coordinatorState{
|
||||
start: StartInfo{RunID: "run-synced", Scenario: "group", StartAt: startAt, Committed: true},
|
||||
expectedAgents: 2, started: make(map[string]time.Time),
|
||||
}
|
||||
server := httptest.NewServer(testCoordinatorMux(state))
|
||||
defer server.Close()
|
||||
client := newCoordinatorClient(server.URL)
|
||||
aDone := make(chan error, 1)
|
||||
go func() { aDone <- client.started(context.Background(), "a", startAt) }()
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
select {
|
||||
case err := <-aDone:
|
||||
t.Fatalf("machine A passed before B arrived: %v", err)
|
||||
default:
|
||||
}
|
||||
if err := client.started(context.Background(), "b", startAt); err != nil {
|
||||
t.Fatalf("machine B start: %v", err)
|
||||
}
|
||||
select {
|
||||
case err := <-aDone:
|
||||
if err != nil {
|
||||
t.Fatalf("machine A start: %v", err)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("machine A was not released")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -25,6 +25,7 @@ type dbVerifyResult struct {
|
||||
Unexpected int `json:"unexpected_rows"`
|
||||
Tables []string `json:"tables"`
|
||||
MissingMIDs []string `json:"missing_client_mids,omitempty"`
|
||||
UnexpectedMIDs []string `json:"unexpected_client_mids,omitempty"`
|
||||
}
|
||||
|
||||
type persistedIdentity struct {
|
||||
@@ -42,6 +43,7 @@ func runDBVerify(ctx context.Context, coordinatorPath, output, tenantCode, dsnEn
|
||||
var coordinator struct {
|
||||
Start StartInfo `json:"start"`
|
||||
Ledgers map[string][]AcceptedMessage `json:"ledgers"`
|
||||
Observed map[string][]AcceptedMessage `json:"observed"`
|
||||
}
|
||||
if err = json.Unmarshal(raw, &coordinator); err != nil {
|
||||
return fmt.Errorf("decode coordinator: %w", err)
|
||||
@@ -76,19 +78,13 @@ func runDBVerify(ctx context.Context, coordinatorPath, output, tenantCode, dsnEn
|
||||
return err
|
||||
}
|
||||
|
||||
expected := make(map[string]AcceptedMessage)
|
||||
expected := expectedDBMessages(coordinator.Start.RunID, coordinator.Ledgers, coordinator.Observed)
|
||||
months := make(map[string]struct{})
|
||||
for _, ledger := range coordinator.Ledgers {
|
||||
for _, message := range ledger {
|
||||
if message.RunID != coordinator.Start.RunID {
|
||||
continue
|
||||
}
|
||||
expected[message.ClientMid] = message
|
||||
for _, message := range expected {
|
||||
if message.Kind == "group" {
|
||||
months[message.SentAt.UTC().Format("200601")] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
tables := []string{fmt.Sprintf("im_%03d_messages", tenantID)}
|
||||
for month := range months {
|
||||
tables = append(tables, fmt.Sprintf("im_%03d_group_messages_%s", tenantID, month))
|
||||
@@ -145,9 +141,11 @@ func runDBVerify(ctx context.Context, coordinatorPath, output, tenantCode, dsnEn
|
||||
for mid := range actual {
|
||||
if _, ok := expected[mid]; !ok {
|
||||
result.Unexpected++
|
||||
result.UnexpectedMIDs = append(result.UnexpectedMIDs, mid)
|
||||
}
|
||||
}
|
||||
sort.Strings(result.MissingMIDs)
|
||||
sort.Strings(result.UnexpectedMIDs)
|
||||
resultRaw, _ := json.MarshalIndent(result, "", " ")
|
||||
if output == "" || output == "-" {
|
||||
if _, err = fmt.Fprintln(os.Stdout, string(resultRaw)); err != nil {
|
||||
@@ -164,3 +162,20 @@ func runDBVerify(ctx context.Context, coordinatorPath, output, tenantCode, dsnEn
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func expectedDBMessages(runID string, ledgers, observed map[string][]AcceptedMessage) map[string]AcceptedMessage {
|
||||
all := make([]AcceptedMessage, 0)
|
||||
for _, messages := range observed {
|
||||
all = append(all, messages...)
|
||||
}
|
||||
for _, messages := range ledgers {
|
||||
all = append(all, messages...)
|
||||
}
|
||||
expected := make(map[string]AcceptedMessage, len(all))
|
||||
for _, message := range mergeMessageLedgers(nil, all) {
|
||||
if message.RunID == runID && message.ClientMid != "" {
|
||||
expected[message.ClientMid] = message
|
||||
}
|
||||
}
|
||||
return expected
|
||||
}
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
package main
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestExpectedDBMessagesIncludesObservedWithoutAck(t *testing.T) {
|
||||
ledgers := map[string][]AcceptedMessage{
|
||||
"a": {{RunID: "run-1", ClientMid: "acked", Topic: "grp-one", Seq: 1}},
|
||||
}
|
||||
observed := map[string][]AcceptedMessage{
|
||||
"a": {
|
||||
{RunID: "run-1", ClientMid: "unacked", Topic: "grp-one", Seq: 2},
|
||||
{RunID: "other", ClientMid: "foreign", Topic: "grp-one", Seq: 3},
|
||||
},
|
||||
"b": {{RunID: "run-1", ClientMid: "unacked", Topic: "grp-one", Seq: 2}},
|
||||
}
|
||||
|
||||
expected := expectedDBMessages("run-1", ledgers, observed)
|
||||
if len(expected) != 2 {
|
||||
t.Fatalf("expected messages=%d want=2: %+v", len(expected), expected)
|
||||
}
|
||||
if expected["acked"].Seq != 1 || expected["unacked"].Seq != 2 {
|
||||
t.Fatalf("unexpected merged messages: %+v", expected)
|
||||
}
|
||||
}
|
||||
@@ -2,7 +2,7 @@ package main
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -10,14 +10,14 @@ var latencyBounds = []time.Duration{
|
||||
time.Millisecond, 3 * time.Millisecond, 5 * time.Millisecond, 10 * time.Millisecond,
|
||||
25 * time.Millisecond, 50 * time.Millisecond, 100 * time.Millisecond, 200 * time.Millisecond,
|
||||
500 * time.Millisecond, time.Second, 2 * time.Second, 5 * time.Second, 10 * time.Second,
|
||||
20 * time.Second, 30 * time.Second, 45 * time.Second, 60 * time.Second, 90 * time.Second, 120 * time.Second,
|
||||
}
|
||||
|
||||
type DurationHistogram struct {
|
||||
mu sync.Mutex
|
||||
count int64
|
||||
sum time.Duration
|
||||
max time.Duration
|
||||
bucket []int64
|
||||
count atomic.Int64
|
||||
sum atomic.Int64
|
||||
max atomic.Int64
|
||||
bucket []atomic.Int64
|
||||
}
|
||||
|
||||
type HistogramSnapshot struct {
|
||||
@@ -28,37 +28,36 @@ type HistogramSnapshot struct {
|
||||
}
|
||||
|
||||
func NewDurationHistogram() *DurationHistogram {
|
||||
return &DurationHistogram{bucket: make([]int64, len(latencyBounds)+1)}
|
||||
return &DurationHistogram{bucket: make([]atomic.Int64, len(latencyBounds)+1)}
|
||||
}
|
||||
|
||||
func (h *DurationHistogram) Observe(value time.Duration) {
|
||||
if value < 0 {
|
||||
return
|
||||
}
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
h.count++
|
||||
h.sum += value
|
||||
if value > h.max {
|
||||
h.max = value
|
||||
nanos := int64(value)
|
||||
h.count.Add(1)
|
||||
h.sum.Add(nanos)
|
||||
for previous := h.max.Load(); nanos > previous; previous = h.max.Load() {
|
||||
if h.max.CompareAndSwap(previous, nanos) {
|
||||
break
|
||||
}
|
||||
}
|
||||
idx := sort.Search(len(latencyBounds), func(i int) bool { return value <= latencyBounds[i] })
|
||||
h.bucket[idx]++
|
||||
h.bucket[idx].Add(1)
|
||||
}
|
||||
|
||||
func (h *DurationHistogram) Snapshot() HistogramSnapshot {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
buckets := make(map[string]int64, len(h.bucket))
|
||||
for i, count := range h.bucket {
|
||||
for i := range h.bucket {
|
||||
label := "+Inf"
|
||||
if i < len(latencyBounds) {
|
||||
label = latencyBounds[i].String()
|
||||
}
|
||||
buckets[label] = count
|
||||
buckets[label] = h.bucket[i].Load()
|
||||
}
|
||||
return HistogramSnapshot{
|
||||
Count: h.count, SumMS: float64(h.sum) / float64(time.Millisecond),
|
||||
MaxMS: float64(h.max) / float64(time.Millisecond), Buckets: buckets,
|
||||
Count: h.count.Load(), SumMS: float64(h.sum.Load()) / float64(time.Millisecond),
|
||||
MaxMS: float64(h.max.Load()) / float64(time.Millisecond), Buckets: buckets,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -269,6 +269,11 @@ type groupMembersResponse struct {
|
||||
}
|
||||
|
||||
func (a *apiClient) countGroupMembers(ctx context.Context, account AccountRecord, topic string) (int, error) {
|
||||
members, err := a.groupMemberIDs(ctx, account, topic)
|
||||
return len(members), err
|
||||
}
|
||||
|
||||
func (a *apiClient) groupMemberIDs(ctx context.Context, account AccountRecord, topic string) (map[string]bool, error) {
|
||||
cursor := ""
|
||||
seen := make(map[string]bool)
|
||||
for {
|
||||
@@ -278,7 +283,7 @@ func (a *apiClient) countGroupMembers(ctx context.Context, account AccountRecord
|
||||
}
|
||||
var page groupMembersResponse
|
||||
if err := a.do(ctx, http.MethodGet, path, account.Token, "", nil, &page); err != nil {
|
||||
return 0, err
|
||||
return nil, err
|
||||
}
|
||||
for _, item := range page.Items {
|
||||
seen[item.UserID] = true
|
||||
@@ -287,11 +292,11 @@ func (a *apiClient) countGroupMembers(ctx context.Context, account AccountRecord
|
||||
break
|
||||
}
|
||||
if page.NextCursor == cursor {
|
||||
return 0, fmt.Errorf("group member cursor did not advance: %s", cursor)
|
||||
return nil, fmt.Errorf("group member cursor did not advance: %s", cursor)
|
||||
}
|
||||
cursor = page.NextCursor
|
||||
}
|
||||
return len(seen), nil
|
||||
return seen, nil
|
||||
}
|
||||
|
||||
func queryInt(value string, fallback int) int {
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -18,10 +19,18 @@ type CombinedReport struct {
|
||||
ReadyConnections int `json:"ready_connections"`
|
||||
PublishAttempted int64 `json:"publish_attempted"`
|
||||
PublishAccepted int64 `json:"publish_accepted"`
|
||||
PublishLateAccepted int64 `json:"publish_late_accepted"`
|
||||
PublishFinalAccepted int64 `json:"publish_final_accepted"`
|
||||
PublishRejected int64 `json:"publish_rejected"`
|
||||
PublishExplicitReject int64 `json:"publish_explicitly_rejected"`
|
||||
PublishOutcomeUnknown int64 `json:"publish_outcome_unknown"`
|
||||
PublishErrors map[string]int64 `json:"publish_errors,omitempty"`
|
||||
PublishCtrlObserved int64 `json:"publish_ctrl_observed"`
|
||||
PublishCtrlOver20 int64 `json:"publish_ctrl_over_20s"`
|
||||
PublishLateCtrl int64 `json:"publish_ctrl_late"`
|
||||
PublishCtrlUnresolved int64 `json:"publish_ctrl_unresolved"`
|
||||
PublishCtrlP95MS float64 `json:"publish_ctrl_p95_ms"`
|
||||
CtrlObservationSamples []CtrlObservationSample `json:"ctrl_observation_samples,omitempty"`
|
||||
PublishSuccessRate float64 `json:"publish_success_rate"`
|
||||
AckP95MS float64 `json:"ack_p95_ms"`
|
||||
DeliveryP95MS float64 `json:"delivery_p95_ms"`
|
||||
@@ -38,6 +47,8 @@ type CombinedReport struct {
|
||||
OutOfOrder int64 `json:"out_of_order"`
|
||||
OrderingViolations []OrderingViolation `json:"ordering_violations,omitempty"`
|
||||
Disconnects int64 `json:"disconnects"`
|
||||
DisconnectReasons map[string]int64 `json:"disconnect_reasons,omitempty"`
|
||||
DisconnectSamples []DisconnectSample `json:"disconnect_samples,omitempty"`
|
||||
UnexpectedMessages int64 `json:"unexpected_messages"`
|
||||
HistoryChecked int64 `json:"history_checked"`
|
||||
HistoryFound int64 `json:"history_found"`
|
||||
@@ -62,7 +73,7 @@ func runMerge(aPath, bPath, output string) error {
|
||||
return fmt.Errorf("agent summaries do not belong to the same run")
|
||||
}
|
||||
report := CombinedReport{RunID: a.RunID, Scenario: a.Scenario, GeneratedAt: time.Now().UTC(), Agents: []AgentSummary{a, b}, Passed: true,
|
||||
PublishErrors: make(map[string]int64)}
|
||||
PublishErrors: make(map[string]int64), DisconnectReasons: make(map[string]int64)}
|
||||
for _, summary := range report.Agents {
|
||||
expectedConnections := summary.ExpectedConnections
|
||||
if expectedConnections == 0 {
|
||||
@@ -72,16 +83,39 @@ func runMerge(aPath, bPath, output string) error {
|
||||
report.ReadyConnections += summary.ReadyConnections
|
||||
report.PublishAttempted += summary.PublishAttempted
|
||||
report.PublishAccepted += summary.PublishAccepted
|
||||
lateAccepted := summary.PublishLateAccepted
|
||||
// Backfill summaries generated before publish_ctrl_late_accepted was added.
|
||||
if summary.CtrlTrackingEnabled && lateAccepted == 0 {
|
||||
for _, sample := range summary.CtrlObservationSamples {
|
||||
if sample.Late && sample.Code >= 200 && sample.Code < 300 {
|
||||
lateAccepted++
|
||||
}
|
||||
}
|
||||
}
|
||||
report.PublishLateAccepted += lateAccepted
|
||||
report.PublishFinalAccepted += summary.PublishAccepted + lateAccepted
|
||||
for category, count := range summary.PublishErrors {
|
||||
report.PublishErrors[category] += count
|
||||
}
|
||||
unknown := summary.PublishOutcomeUnknown
|
||||
if unknown == 0 {
|
||||
if summary.CtrlTrackingEnabled {
|
||||
unknown = summary.PublishCtrlUnresolved + summary.PublishErrors["socket"]
|
||||
} else if unknown == 0 {
|
||||
unknown = summary.PublishErrors["ctrl_timeout"] + summary.PublishErrors["socket"]
|
||||
}
|
||||
explicit := summary.PublishErrors["403"] + summary.PublishErrors["502"] +
|
||||
summary.PublishErrors["503"] + summary.PublishErrors["other"]
|
||||
report.PublishOutcomeUnknown += unknown
|
||||
report.PublishCtrlObserved += summary.PublishCtrlObserved
|
||||
report.PublishCtrlOver20 += summary.PublishCtrlOver20
|
||||
report.PublishLateCtrl += summary.PublishLateCtrl
|
||||
report.PublishCtrlUnresolved += summary.PublishCtrlUnresolved
|
||||
if remaining := 1000 - len(report.CtrlObservationSamples); remaining > 0 {
|
||||
if remaining > len(summary.CtrlObservationSamples) {
|
||||
remaining = len(summary.CtrlObservationSamples)
|
||||
}
|
||||
report.CtrlObservationSamples = append(report.CtrlObservationSamples, summary.CtrlObservationSamples[:remaining]...)
|
||||
}
|
||||
report.PublishExplicitReject += explicit
|
||||
report.PublishRejected += explicit
|
||||
report.ExpectedDeliveries += summary.ExpectedPeer + summary.ExpectedGroup
|
||||
@@ -92,6 +126,15 @@ func runMerge(aPath, bPath, output string) error {
|
||||
report.OutOfOrder += summary.OutOfOrder
|
||||
report.OrderingViolations = append(report.OrderingViolations, summary.OrderingViolations...)
|
||||
report.Disconnects += summary.Disconnects
|
||||
for category, count := range summary.DisconnectReasons {
|
||||
report.DisconnectReasons[category] += count
|
||||
}
|
||||
if remaining := 100 - len(report.DisconnectSamples); remaining > 0 {
|
||||
if remaining > len(summary.DisconnectSamples) {
|
||||
remaining = len(summary.DisconnectSamples)
|
||||
}
|
||||
report.DisconnectSamples = append(report.DisconnectSamples, summary.DisconnectSamples[:remaining]...)
|
||||
}
|
||||
report.UnexpectedMessages += summary.UnexpectedMessages
|
||||
report.HistoryChecked += summary.HistoryChecked
|
||||
report.HistoryFound += summary.HistoryFound
|
||||
@@ -100,14 +143,23 @@ func runMerge(aPath, bPath, output string) error {
|
||||
report.PersistenceLost += summary.PersistenceLost
|
||||
report.CrossClockSkewSamples += summary.CrossClockSkewSamples
|
||||
report.Missing = append(report.Missing, summary.Missing...)
|
||||
if len(summary.Errors) > 0 {
|
||||
report.FailureReasons = append(report.FailureReasons, fmt.Sprintf("agent %s errors=%d", summary.Machine, len(summary.Errors)))
|
||||
actionableErrors := len(summary.Errors)
|
||||
lateTimeouts := int(lateAccepted)
|
||||
for _, message := range summary.Errors {
|
||||
if lateTimeouts > 0 && strings.Contains(message, "ctrl timeout") {
|
||||
actionableErrors--
|
||||
lateTimeouts--
|
||||
}
|
||||
}
|
||||
if actionableErrors > 0 {
|
||||
report.FailureReasons = append(report.FailureReasons, fmt.Sprintf("agent %s unresolved errors=%d", summary.Machine, actionableErrors))
|
||||
}
|
||||
}
|
||||
if report.PublishAttempted > 0 {
|
||||
report.PublishSuccessRate = float64(report.PublishAccepted) / float64(report.PublishAttempted)
|
||||
report.PublishSuccessRate = float64(report.PublishFinalAccepted) / float64(report.PublishAttempted)
|
||||
}
|
||||
report.AckP95MS = mergedAckP95(report.Agents)
|
||||
report.PublishCtrlP95MS = mergedHistogramP95(report.Agents, func(summary AgentSummary) HistogramSnapshot { return summary.PublishCtrlLatency })
|
||||
report.DeliveryLocalP95MS = mergedHistogramP95(report.Agents, func(summary AgentSummary) HistogramSnapshot { return summary.DeliveryLatencyLocal })
|
||||
report.DeliveryCrossP95MS = mergedHistogramP95(report.Agents, func(summary AgentSummary) HistogramSnapshot { return summary.DeliveryLatencyCross })
|
||||
// delivery_p95_ms remains the compatibility field, but now uses only same-clock
|
||||
@@ -136,13 +188,6 @@ func runMerge(aPath, bPath, output string) error {
|
||||
report.FailureReasons = append(report.FailureReasons,
|
||||
fmt.Sprintf("deliveries observed without ACK ledger=%d", report.UnexpectedMessages))
|
||||
}
|
||||
if report.AckP95MS > 500 {
|
||||
report.FailureReasons = append(report.FailureReasons, fmt.Sprintf("ack p95=%.3fms expected<=500ms", report.AckP95MS))
|
||||
}
|
||||
if report.DeliveryP95MS > 1000 {
|
||||
report.FailureReasons = append(report.FailureReasons,
|
||||
fmt.Sprintf("delivery p95=%.0fms expected<=1000ms", report.DeliveryP95MS))
|
||||
}
|
||||
if report.PublishRejected > 0 || report.RealtimeLost > 0 || report.PersistenceLost > 0 || report.Duplicates > 0 || report.OutOfOrder > 0 || report.Disconnects > 0 || report.UnexpectedMessages > 0 {
|
||||
report.FailureReasons = append(report.FailureReasons, "message correctness gate failed")
|
||||
}
|
||||
|
||||
@@ -13,12 +13,18 @@ func TestRunMergeComputesArrivalAndPersistenceGate(t *testing.T) {
|
||||
a := AgentSummary{RunID: "run1", Scenario: "p2p", Machine: "a", ReadyConnections: 1000,
|
||||
ExpectedConnections: 1000,
|
||||
PublishAttempted: 10, PublishAccepted: 10, ExpectedPeer: 10, ReceivedPeer: 9, UnexpectedMessages: 42,
|
||||
CtrlTrackingEnabled: true, PublishCtrlObserved: 9, PublishCtrlOver20: 2, PublishLateCtrl: 2,
|
||||
PublishCtrlUnresolved: 1,
|
||||
PublishErrors: map[string]int64{"ctrl_timeout": 3, "socket": 2, "503": 1},
|
||||
RealtimeLost: 1, HistoryChecked: 10, HistoryFound: 10}
|
||||
RealtimeLost: 1, HistoryChecked: 10, HistoryFound: 10, Disconnects: 1,
|
||||
DisconnectReasons: map[string]int64{"timeout": 1},
|
||||
DisconnectSamples: []DisconnectSample{{ReceiverMachine: "a", ReceiverIndex: 3, Category: "timeout"}}}
|
||||
b := AgentSummary{RunID: "run1", Scenario: "p2p", Machine: "b", ReadyConnections: 1000,
|
||||
ExpectedConnections: 1000,
|
||||
PublishAttempted: 10, PublishAccepted: 10, ExpectedPeer: 10, ReceivedPeer: 10,
|
||||
HistoryChecked: 10, HistoryFound: 9, PersistenceLost: 1, OutOfOrder: 1,
|
||||
CtrlTrackingEnabled: true, PublishCtrlObserved: 10,
|
||||
Disconnects: 2, DisconnectReasons: map[string]int64{"timeout": 1, "connection_reset": 1},
|
||||
OrderingViolations: []OrderingViolation{{ReceiverMachine: "b", ReceiverIndex: 7, PreviousSeq: 12, CurrentSeq: 11}}}
|
||||
aPath, bPath, reportPath := filepath.Join(dir, "a.json"), filepath.Join(dir, "b.json"), filepath.Join(dir, "report.json")
|
||||
writeSummaryForTest(t, aPath, a)
|
||||
@@ -40,13 +46,22 @@ func TestRunMergeComputesArrivalAndPersistenceGate(t *testing.T) {
|
||||
if report.Passed || report.PersistenceLost != 1 {
|
||||
t.Fatalf("correctness gate did not fail: %+v", report)
|
||||
}
|
||||
if report.UnexpectedMessages != 42 || report.PublishOutcomeUnknown != 5 ||
|
||||
if report.UnexpectedMessages != 42 || report.PublishOutcomeUnknown != 3 ||
|
||||
report.PublishExplicitReject != 1 || report.PublishRejected != 1 {
|
||||
t.Fatalf("ambiguous publish outcomes were not merged: %+v", report)
|
||||
}
|
||||
if report.PublishCtrlObserved != 19 || report.PublishCtrlOver20 != 2 || report.PublishLateCtrl != 2 ||
|
||||
report.PublishCtrlUnresolved != 1 {
|
||||
t.Fatalf("ctrl observations were not merged: %+v", report)
|
||||
}
|
||||
if len(report.OrderingViolations) != 1 || report.OrderingViolations[0].ReceiverIndex != 7 {
|
||||
t.Fatalf("ordering violations were not merged: %+v", report.OrderingViolations)
|
||||
}
|
||||
if report.Disconnects != 3 || report.DisconnectReasons["timeout"] != 2 ||
|
||||
report.DisconnectReasons["connection_reset"] != 1 || len(report.DisconnectSamples) != 1 {
|
||||
t.Fatalf("disconnect diagnostics were not merged: count=%d reasons=%v samples=%+v",
|
||||
report.Disconnects, report.DisconnectReasons, report.DisconnectSamples)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergedHistogramP95(t *testing.T) {
|
||||
@@ -59,6 +74,23 @@ func TestMergedHistogramP95(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergedHistogramP95DoesNotUseMaximumForLongTail(t *testing.T) {
|
||||
histogram := NewDurationHistogram()
|
||||
for i := 0; i < 94; i++ {
|
||||
histogram.Observe(5 * time.Second)
|
||||
}
|
||||
for i := 0; i < 5; i++ {
|
||||
histogram.Observe(25 * time.Second)
|
||||
}
|
||||
histogram.Observe(70 * time.Second)
|
||||
agents := []AgentSummary{{PublishCtrlLatency: histogram.Snapshot()}}
|
||||
if got := mergedHistogramP95(agents, func(summary AgentSummary) HistogramSnapshot {
|
||||
return summary.PublishCtrlLatency
|
||||
}); got != 30000 {
|
||||
t.Fatalf("p95=%v want=30000", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergedAckP95UsesExactAcceptedTimestamps(t *testing.T) {
|
||||
base := time.Date(2026, 9, 27, 8, 0, 0, 0, time.UTC)
|
||||
agents := []AgentSummary{{
|
||||
@@ -85,7 +117,7 @@ func TestMergedAckP95FallsBackWhenRawSamplesAreIncomplete(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunMergeUsesSameClockDeliveryLatencyForGate(t *testing.T) {
|
||||
func TestRunMergeReportsSameClockDeliveryLatency(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
local := HistogramSnapshot{Count: 10, MaxMS: 200, Buckets: map[string]int64{"200ms": 10}}
|
||||
cross := HistogramSnapshot{Count: 10, MaxMS: 5000, Buckets: map[string]int64{"5s": 10}}
|
||||
@@ -112,7 +144,44 @@ func TestRunMergeUsesSameClockDeliveryLatencyForGate(t *testing.T) {
|
||||
t.Fatalf("unexpected delivery percentiles: %+v", report)
|
||||
}
|
||||
if report.CrossClockSkewSamples != 6 || !report.Passed {
|
||||
t.Fatalf("cross-machine clock skew must not fail the same-clock latency gate: %+v", report)
|
||||
t.Fatalf("cross-machine clock skew must not fail correctness acceptance: %+v", report)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunMergeTreatsLateSuccessAsFinalAcceptance(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
late := AgentSummary{
|
||||
RunID: "run1", Scenario: "group", Machine: "a", ExpectedConnections: 1, ReadyConnections: 1,
|
||||
PublishAttempted: 1, PublishAccepted: 0, CtrlTrackingEnabled: true,
|
||||
PublishCtrlObserved: 1, PublishLateCtrl: 1, PublishCtrlUnresolved: 0,
|
||||
PublishErrors: map[string]int64{"ctrl_timeout": 1},
|
||||
CtrlObservationSamples: []CtrlObservationSample{{Code: 202, Late: true}},
|
||||
Errors: []string{"publish message-1: ctrl timeout id=pub-a-1"},
|
||||
}
|
||||
timely := AgentSummary{
|
||||
RunID: "run1", Scenario: "group", Machine: "b", ExpectedConnections: 1, ReadyConnections: 1,
|
||||
PublishAttempted: 1, PublishAccepted: 1, CtrlTrackingEnabled: true,
|
||||
PublishCtrlObserved: 1, PublishErrors: map[string]int64{},
|
||||
}
|
||||
aPath, bPath, reportPath := filepath.Join(dir, "a.json"), filepath.Join(dir, "b.json"), filepath.Join(dir, "report.json")
|
||||
writeSummaryForTest(t, aPath, late)
|
||||
writeSummaryForTest(t, bPath, timely)
|
||||
if err := runMerge(aPath, bPath, reportPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var report CombinedReport
|
||||
raw, err := os.ReadFile(reportPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = json.Unmarshal(raw, &report); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if report.PublishAccepted != 1 || report.PublishLateAccepted != 1 || report.PublishFinalAccepted != 2 || report.PublishSuccessRate != 1 {
|
||||
t.Fatalf("late acceptance was not included in final publish outcome: %+v", report)
|
||||
}
|
||||
if !report.Passed || len(report.FailureReasons) != 0 {
|
||||
t.Fatalf("resolved late ACK must not fail the report: %+v", report.FailureReasons)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
@@ -21,6 +22,7 @@ type Config struct {
|
||||
PasswordEnv string `json:"password_env"`
|
||||
UsersPerMachine int `json:"users_per_machine"`
|
||||
Endpoints map[string][]string `json:"endpoints"`
|
||||
DialIPs map[string][]string `json:"dial_ips,omitempty"`
|
||||
InsecureTLS bool `json:"insecure_tls"`
|
||||
PrepareWorkers int `json:"prepare_workers"`
|
||||
ConnectRamp string `json:"connect_ramp"`
|
||||
@@ -30,6 +32,8 @@ type Config struct {
|
||||
P2PInterval string `json:"p2p_interval"`
|
||||
DrainP2P string `json:"drain_p2p"`
|
||||
DrainGroup string `json:"drain_group"`
|
||||
DeliveryQuiet string `json:"delivery_quiet_period"`
|
||||
DeliveryTimeout string `json:"delivery_quiet_timeout"`
|
||||
PhonePrefix string `json:"registration_phone_prefix"`
|
||||
VerifyCodeEnv string `json:"verification_code_env"`
|
||||
HistoryWorkers int `json:"history_workers"`
|
||||
@@ -85,6 +89,18 @@ func loadConfig(path string) (*Config, error) {
|
||||
if cfg.DrainGroup == "" {
|
||||
cfg.DrainGroup = "60s"
|
||||
}
|
||||
if cfg.DeliveryQuiet == "" {
|
||||
cfg.DeliveryQuiet = "10s"
|
||||
}
|
||||
if _, err = configDuration(cfg.DeliveryQuiet); err != nil {
|
||||
return nil, fmt.Errorf("delivery_quiet_period: %w", err)
|
||||
}
|
||||
if cfg.DeliveryTimeout == "" {
|
||||
cfg.DeliveryTimeout = "3m"
|
||||
}
|
||||
if _, err = configDuration(cfg.DeliveryTimeout); err != nil {
|
||||
return nil, fmt.Errorf("delivery_quiet_timeout: %w", err)
|
||||
}
|
||||
if cfg.HistoryWorkers <= 0 {
|
||||
cfg.HistoryWorkers = 12
|
||||
}
|
||||
@@ -140,6 +156,11 @@ func loadConfig(path string) (*Config, error) {
|
||||
if len(cfg.Endpoints[machine]) == 0 {
|
||||
return nil, fmt.Errorf("at least one endpoint is required for machine %s", machine)
|
||||
}
|
||||
for _, ip := range cfg.DialIPs[machine] {
|
||||
if net.ParseIP(strings.TrimSpace(ip)) == nil {
|
||||
return nil, fmt.Errorf("invalid dial IP %q for machine %s", ip, machine)
|
||||
}
|
||||
}
|
||||
}
|
||||
if cfg.PhonePrefix != "" {
|
||||
if len(cfg.PhonePrefix) != 7 || strings.Trim(cfg.PhonePrefix, "0123456789") != "" {
|
||||
@@ -214,6 +235,7 @@ type AccountRecord struct {
|
||||
TokenExpires time.Time `json:"token_expires,omitempty"`
|
||||
P2PTopic string `json:"p2p_topic,omitempty"`
|
||||
Endpoint string `json:"endpoint,omitempty"`
|
||||
DialIP string `json:"-"`
|
||||
}
|
||||
|
||||
func newManifest(cfg *Config) *Manifest {
|
||||
@@ -398,6 +420,14 @@ type AgentSummary struct {
|
||||
PublishRejected int64 `json:"publish_rejected"`
|
||||
PublishOutcomeUnknown int64 `json:"publish_outcome_unknown"`
|
||||
PublishErrors map[string]int64 `json:"publish_errors,omitempty"`
|
||||
CtrlTrackingEnabled bool `json:"ctrl_tracking_enabled"`
|
||||
PublishCtrlObserved int64 `json:"publish_ctrl_observed"`
|
||||
PublishCtrlOver20 int64 `json:"publish_ctrl_over_20s"`
|
||||
PublishLateCtrl int64 `json:"publish_ctrl_late"`
|
||||
PublishLateAccepted int64 `json:"publish_ctrl_late_accepted"`
|
||||
PublishCtrlUnresolved int64 `json:"publish_ctrl_unresolved"`
|
||||
PublishCtrlLatency HistogramSnapshot `json:"publish_ctrl_latency"`
|
||||
CtrlObservationSamples []CtrlObservationSample `json:"ctrl_observation_samples,omitempty"`
|
||||
ExpectedPeer int64 `json:"expected_peer"`
|
||||
ReceivedPeer int64 `json:"received_peer"`
|
||||
ExpectedEcho int64 `json:"expected_echo"`
|
||||
@@ -410,6 +440,8 @@ type AgentSummary struct {
|
||||
OutOfOrder int64 `json:"out_of_order"`
|
||||
OrderingViolations []OrderingViolation `json:"ordering_violations,omitempty"`
|
||||
Disconnects int64 `json:"disconnects"`
|
||||
DisconnectReasons map[string]int64 `json:"disconnect_reasons,omitempty"`
|
||||
DisconnectSamples []DisconnectSample `json:"disconnect_samples,omitempty"`
|
||||
UnexpectedMessages int64 `json:"unexpected_messages"`
|
||||
HistoryChecked int64 `json:"history_checked"`
|
||||
HistoryFound int64 `json:"history_found"`
|
||||
@@ -421,12 +453,42 @@ type AgentSummary struct {
|
||||
DeliveryLatencyLocal HistogramSnapshot `json:"delivery_latency_local"`
|
||||
DeliveryLatencyCross HistogramSnapshot `json:"delivery_latency_cross_machine"`
|
||||
CrossClockSkewSamples int64 `json:"cross_machine_negative_latency_samples"`
|
||||
CallbackQueue CallbackQueueSnapshot `json:"callback_queue"`
|
||||
Errors []string `json:"errors,omitempty"`
|
||||
Missing []MissingDelivery `json:"missing,omitempty"`
|
||||
PersistenceMissing []string `json:"persistence_missing,omitempty"`
|
||||
AcceptedLocal []AcceptedMessage `json:"accepted_local,omitempty"`
|
||||
}
|
||||
|
||||
type CtrlObservationSample struct {
|
||||
Machine string `json:"machine"`
|
||||
Index int `json:"index"`
|
||||
ID string `json:"id"`
|
||||
Code int `json:"code"`
|
||||
SentAt time.Time `json:"sent_at"`
|
||||
ReceivedAt time.Time `json:"received_at"`
|
||||
LatencyMS float64 `json:"latency_ms"`
|
||||
Late bool `json:"late"`
|
||||
}
|
||||
|
||||
type CallbackQueueSnapshot struct {
|
||||
Current int64 `json:"current"`
|
||||
Processing int64 `json:"processing"`
|
||||
Maximum int64 `json:"maximum"`
|
||||
Enqueued int64 `json:"enqueued"`
|
||||
Completed int64 `json:"completed"`
|
||||
WaitSecondsTotal float64 `json:"wait_seconds_total"`
|
||||
WaitSecondsMax float64 `json:"wait_seconds_max"`
|
||||
}
|
||||
|
||||
type DisconnectSample struct {
|
||||
ReceiverMachine string `json:"receiver_machine"`
|
||||
ReceiverIndex int `json:"receiver_index"`
|
||||
Category string `json:"category"`
|
||||
Error string `json:"error"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
type OrderingViolation struct {
|
||||
ReceiverMachine string `json:"receiver_machine"`
|
||||
ReceiverIndex int `json:"receiver_index"`
|
||||
|
||||
@@ -3,6 +3,7 @@ package main
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
@@ -24,6 +25,46 @@ func TestLoadConfigAppliesBoundedHistoryDefaults(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssignAgentEndpointsDistributesDialIPsAcrossMachines(t *testing.T) {
|
||||
cfg := &Config{
|
||||
Endpoints: map[string][]string{"a": {"wss://api.example.test/ws"}, "b": {"wss://api.example.test/ws"}},
|
||||
DialIPs: map[string][]string{"a": {"192.0.2.1", "192.0.2.2", "192.0.2.3"}, "b": {"192.0.2.1", "192.0.2.2", "192.0.2.3"}},
|
||||
}
|
||||
a := []AccountRecord{{Index: 1}, {Index: 2}, {Index: 3}}
|
||||
b := []AccountRecord{{Index: 1}, {Index: 2}, {Index: 3}}
|
||||
assignAgentEndpoints(cfg, "a", a)
|
||||
assignAgentEndpoints(cfg, "b", b)
|
||||
got := []string{a[0].DialIP, b[0].DialIP, a[1].DialIP, b[1].DialIP, a[2].DialIP, b[2].DialIP}
|
||||
want := []string{"192.0.2.1", "192.0.2.2", "192.0.2.3", "192.0.2.1", "192.0.2.2", "192.0.2.3"}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("unexpected dial IP distribution: got=%v want=%v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadConfigRejectsInvalidDialIP(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "config.json")
|
||||
raw := []byte(`{"http_base":"https://example.test","endpoints":{"a":["wss://a"],"b":["wss://b"]},"dial_ips":{"a":["not-an-ip"]}}`)
|
||||
if err := os.WriteFile(path, raw, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := loadConfig(path); err == nil {
|
||||
t.Fatal("invalid dial IP was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSingleMachineExampleUsesSixtyPercentCapacity(t *testing.T) {
|
||||
cfg, err := loadConfig("config.single-machine.example.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if cfg.UsersPerMachine != 600 || cfg.CloseRamp != "60s" {
|
||||
t.Fatalf("unexpected single-machine capacity: users=%d close_ramp=%s", cfg.UsersPerMachine, cfg.CloseRamp)
|
||||
}
|
||||
if len(cfg.Endpoints["a"]) != 1 || len(cfg.Endpoints["b"]) != 1 {
|
||||
t.Fatalf("single-machine endpoints must define both logical agents: %+v", cfg.Endpoints)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewManifestDistributesUsersAndEndpoints(t *testing.T) {
|
||||
cfg := &Config{Tenant: "006", UsersPerMachine: 3, Endpoints: map[string][]string{
|
||||
"a": {"ws://a1", "ws://a2"}, "b": {"ws://b1"},
|
||||
|
||||
@@ -49,7 +49,7 @@ func runPrepare(ctx context.Context, cfg *Config, manifestPath string) error {
|
||||
if err = saveManifest(manifestPath, manifest); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = prepareGroup(ctx, cfg, manifest, api); err != nil {
|
||||
if err = prepareGroup(ctx, cfg, manifest, api, manifestPath); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = saveManifest(manifestPath, manifest); err != nil {
|
||||
@@ -184,7 +184,7 @@ func prepareFriendships(ctx context.Context, cfg *Config, manifest *Manifest, ap
|
||||
return err
|
||||
}
|
||||
|
||||
func prepareGroup(ctx context.Context, cfg *Config, manifest *Manifest, api *apiClient) error {
|
||||
func prepareGroup(ctx context.Context, cfg *Config, manifest *Manifest, api *apiClient, manifestPath string) error {
|
||||
if err := parallelAccounts(ctx, manifest.Accounts, cfg.PrepareWorkers, func(account AccountRecord) error {
|
||||
return retryPrepareOperation(ctx, "privacy-"+account.Username, func() error {
|
||||
return api.disableGroupInviteVerification(ctx, account)
|
||||
@@ -200,6 +200,9 @@ func prepareGroup(ctx context.Context, cfg *Config, manifest *Manifest, api *api
|
||||
return err
|
||||
}
|
||||
manifest.GroupTopic = group.Topic
|
||||
if err = saveManifest(manifestPath, manifest); err != nil {
|
||||
return fmt.Errorf("checkpoint created group: %w", err)
|
||||
}
|
||||
log.Printf("created load-test group topic_hash=%s", topicHash(group.Topic))
|
||||
}
|
||||
memberIDs := make([]string, 0, len(manifest.Accounts)-1)
|
||||
@@ -214,20 +217,9 @@ func prepareGroup(ctx context.Context, cfg *Config, manifest *Manifest, api *api
|
||||
if end > len(memberIDs) {
|
||||
end = len(memberIDs)
|
||||
}
|
||||
var response *groupInvitationResponse
|
||||
err := retryPrepareOperation(ctx, fmt.Sprintf("group-invite-%03d", batch), func() error {
|
||||
var inviteErr error
|
||||
response, inviteErr = api.inviteGroup(ctx, owner, manifest.GroupTopic, batch, memberIDs[offset:end])
|
||||
return inviteErr
|
||||
})
|
||||
if err != nil {
|
||||
if err := ensureGroupMembers(ctx, api, owner, manifest.GroupTopic, batch, memberIDs[offset:end]); err != nil {
|
||||
return fmt.Errorf("invite group batch %d: %w", batch, err)
|
||||
}
|
||||
for _, item := range response.Items {
|
||||
if item.Status != "joined" && item.Status != "already_member" {
|
||||
return fmt.Errorf("invite user %s status=%s reason=%s", item.UserID, item.Status, item.Reason)
|
||||
}
|
||||
}
|
||||
}
|
||||
count, err := api.countGroupMembers(ctx, owner, manifest.GroupTopic)
|
||||
if err != nil {
|
||||
@@ -240,9 +232,60 @@ func prepareGroup(ctx context.Context, cfg *Config, manifest *Manifest, api *api
|
||||
return nil
|
||||
}
|
||||
|
||||
func retryPrepareOperation(ctx context.Context, label string, operation func() error) error {
|
||||
func ensureGroupMembers(ctx context.Context, api *apiClient, owner AccountRecord, topic string, batch int, desired []string) error {
|
||||
var lastErr error
|
||||
for attempt := 1; attempt <= 6; attempt++ {
|
||||
for attempt := 1; attempt <= 12; attempt++ {
|
||||
members, err := api.groupMemberIDs(ctx, owner, topic)
|
||||
if err != nil {
|
||||
lastErr = fmt.Errorf("list current members: %w", err)
|
||||
} else {
|
||||
missing := make([]string, 0, len(desired))
|
||||
for _, userID := range desired {
|
||||
if !members[userID] {
|
||||
missing = append(missing, userID)
|
||||
}
|
||||
}
|
||||
if len(missing) == 0 {
|
||||
return nil
|
||||
}
|
||||
response, inviteErr := api.inviteGroup(ctx, owner, topic, batch, missing)
|
||||
if inviteErr != nil {
|
||||
lastErr = inviteErr
|
||||
} else {
|
||||
lastErr = nil
|
||||
for _, item := range response.Items {
|
||||
if item.Status != "joined" && item.Status != "already_member" {
|
||||
lastErr = fmt.Errorf("invite user %s status=%s reason=%s", item.UserID, item.Status, item.Reason)
|
||||
break
|
||||
}
|
||||
}
|
||||
if lastErr == nil {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
if attempt == 12 {
|
||||
break
|
||||
}
|
||||
delay := time.Duration(attempt*5) * time.Second
|
||||
log.Printf("operation=group-invite-%03d attempt=%d failed; retry_in=%s: %v", batch, attempt, delay, lastErr)
|
||||
if err := waitContext(ctx, delay); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("missing members not recovered after retries: %w", lastErr)
|
||||
}
|
||||
|
||||
func retryPrepareOperation(ctx context.Context, label string, operation func() error) error {
|
||||
return retryOperation(ctx, label, 6, operation)
|
||||
}
|
||||
|
||||
func retryOperation(ctx context.Context, label string, attempts int, operation func() error) error {
|
||||
var lastErr error
|
||||
for attempt := 1; attempt <= attempts; attempt++ {
|
||||
if err := operation(); err == nil {
|
||||
return nil
|
||||
} else {
|
||||
@@ -251,6 +294,9 @@ func retryPrepareOperation(ctx context.Context, label string, operation func() e
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
if attempt == attempts {
|
||||
break
|
||||
}
|
||||
delay := time.Duration(attempt*5) * time.Second
|
||||
log.Printf("operation=%s attempt=%d failed; retry_in=%s: %v", label, attempt, delay, lastErr)
|
||||
if err := waitContext(ctx, delay); err != nil {
|
||||
|
||||
@@ -2,8 +2,11 @@ package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/bits"
|
||||
"net"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -11,7 +14,19 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
const observationShardCount = 64
|
||||
|
||||
type observation struct {
|
||||
topic string
|
||||
seq int64
|
||||
senderMachine string
|
||||
senderIndex int
|
||||
scenario string
|
||||
sentAt int64
|
||||
bits []atomic.Uint64
|
||||
}
|
||||
|
||||
type observationView struct {
|
||||
topic string
|
||||
seq int64
|
||||
senderMachine string
|
||||
@@ -21,6 +36,17 @@ type observation struct {
|
||||
bits []uint64
|
||||
}
|
||||
|
||||
type observationShard struct {
|
||||
mu sync.RWMutex
|
||||
observed map[string]*observation
|
||||
history map[string]bool
|
||||
}
|
||||
|
||||
type sequenceState struct {
|
||||
mu sync.Mutex
|
||||
last map[string]int64
|
||||
}
|
||||
|
||||
type deliveryRecorder struct {
|
||||
machine string
|
||||
localUsers int
|
||||
@@ -32,26 +58,35 @@ type deliveryRecorder struct {
|
||||
runID atomic.Value
|
||||
historyMode atomic.Bool
|
||||
|
||||
mu sync.Mutex
|
||||
observed map[string]*observation
|
||||
lastSeq []map[string]int64
|
||||
duplicates int64
|
||||
outOfOrder int64
|
||||
shards []observationShard
|
||||
sequences []sequenceState
|
||||
duplicates atomic.Int64
|
||||
outOfOrder atomic.Int64
|
||||
malformed atomic.Int64
|
||||
crossClockSkewSamples atomic.Int64
|
||||
|
||||
orderingMu sync.Mutex
|
||||
ordering []OrderingViolation
|
||||
malformed int64
|
||||
crossClockSkewSamples int64
|
||||
history map[string]bool
|
||||
|
||||
disconnectMu sync.Mutex
|
||||
disconnectReasons map[string]int64
|
||||
disconnectSamples []DisconnectSample
|
||||
}
|
||||
|
||||
func newDeliveryRecorder(runID, machine string, localUsers int) *deliveryRecorder {
|
||||
lastSeq := make([]map[string]int64, localUsers)
|
||||
for i := range lastSeq {
|
||||
lastSeq[i] = make(map[string]int64)
|
||||
shards := make([]observationShard, observationShardCount)
|
||||
for i := range shards {
|
||||
shards[i].observed = make(map[string]*observation)
|
||||
shards[i].history = make(map[string]bool)
|
||||
}
|
||||
sequences := make([]sequenceState, localUsers)
|
||||
for i := range sequences {
|
||||
sequences[i].last = make(map[string]int64)
|
||||
}
|
||||
recorder := &deliveryRecorder{
|
||||
machine: machine, localUsers: localUsers, latency: NewDurationHistogram(),
|
||||
localLatency: NewDurationHistogram(), crossLatency: NewDurationHistogram(),
|
||||
observed: make(map[string]*observation), lastSeq: lastSeq, history: make(map[string]bool),
|
||||
shards: shards, sequences: sequences, disconnectReasons: make(map[string]int64),
|
||||
}
|
||||
recorder.runID.Store(runID)
|
||||
return recorder
|
||||
@@ -59,7 +94,13 @@ func newDeliveryRecorder(runID, machine string, localUsers int) *deliveryRecorde
|
||||
|
||||
func (r *deliveryRecorder) setRunID(runID string) { r.runID.Store(runID) }
|
||||
|
||||
func (r *deliveryRecorder) resetDisconnects() { r.disconnects.Store(0) }
|
||||
func (r *deliveryRecorder) resetDisconnects() {
|
||||
r.disconnectMu.Lock()
|
||||
r.disconnects.Store(0)
|
||||
r.disconnectReasons = make(map[string]int64)
|
||||
r.disconnectSamples = nil
|
||||
r.disconnectMu.Unlock()
|
||||
}
|
||||
|
||||
func (r *deliveryRecorder) onData(account AccountRecord, data dataPacket) {
|
||||
runID, _ := r.runID.Load().(string)
|
||||
@@ -75,40 +116,53 @@ func (r *deliveryRecorder) onData(account AccountRecord, data dataPacket) {
|
||||
SentAt int64 `json:"sent_at_unix_nano"`
|
||||
}
|
||||
if json.Unmarshal(data.Content, &content) != nil || content.RunID != runID {
|
||||
r.mu.Lock()
|
||||
r.malformed++
|
||||
r.mu.Unlock()
|
||||
r.malformed.Add(1)
|
||||
return
|
||||
}
|
||||
if r.historyMode.Load() {
|
||||
r.mu.Lock()
|
||||
r.history[mid] = true
|
||||
r.mu.Unlock()
|
||||
shard := r.shard(mid)
|
||||
shard.mu.Lock()
|
||||
shard.history[mid] = true
|
||||
shard.mu.Unlock()
|
||||
return
|
||||
}
|
||||
index := account.Index - 1
|
||||
if index < 0 || index >= r.localUsers {
|
||||
return
|
||||
}
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
obs := r.observed[mid]
|
||||
shard := r.shard(mid)
|
||||
shard.mu.RLock()
|
||||
obs := shard.observed[mid]
|
||||
shard.mu.RUnlock()
|
||||
if obs == nil {
|
||||
shard.mu.Lock()
|
||||
obs = shard.observed[mid]
|
||||
if obs == nil {
|
||||
obs = &observation{topic: data.Topic, seq: data.Seq, senderMachine: content.SenderMachine,
|
||||
senderIndex: content.SenderIndex, scenario: content.Scenario, sentAt: content.SentAt,
|
||||
bits: make([]uint64, (r.localUsers+63)/64)}
|
||||
r.observed[mid] = obs
|
||||
bits: make([]atomic.Uint64, (r.localUsers+63)/64)}
|
||||
shard.observed[mid] = obs
|
||||
}
|
||||
shard.mu.Unlock()
|
||||
}
|
||||
word, bit := index/64, uint(index%64)
|
||||
mask := uint64(1) << bit
|
||||
if obs.bits[word]&mask != 0 {
|
||||
r.duplicates++
|
||||
if !markObservationBit(obs.bits, word, mask) {
|
||||
r.duplicates.Add(1)
|
||||
return
|
||||
}
|
||||
obs.bits[word] |= mask
|
||||
r.received.Add(1)
|
||||
if previous := r.lastSeq[index][data.Topic]; previous > 0 && data.Seq < previous {
|
||||
r.outOfOrder++
|
||||
sequence := &r.sequences[index]
|
||||
sequence.mu.Lock()
|
||||
previous := sequence.last[data.Topic]
|
||||
violation := previous > 0 && data.Seq < previous
|
||||
if data.Seq > previous {
|
||||
sequence.last[data.Topic] = data.Seq
|
||||
}
|
||||
sequence.mu.Unlock()
|
||||
if violation {
|
||||
r.outOfOrder.Add(1)
|
||||
r.orderingMu.Lock()
|
||||
if len(r.ordering) < 100 {
|
||||
r.ordering = append(r.ordering, OrderingViolation{
|
||||
ReceiverMachine: r.machine,
|
||||
@@ -119,9 +173,7 @@ func (r *deliveryRecorder) onData(account AccountRecord, data dataPacket) {
|
||||
ClientMid: mid,
|
||||
})
|
||||
}
|
||||
}
|
||||
if data.Seq > r.lastSeq[index][data.Topic] {
|
||||
r.lastSeq[index][data.Topic] = data.Seq
|
||||
r.orderingMu.Unlock()
|
||||
}
|
||||
if content.SentAt > 0 {
|
||||
delay := time.Since(time.Unix(0, content.SentAt))
|
||||
@@ -133,7 +185,7 @@ func (r *deliveryRecorder) onData(account AccountRecord, data dataPacket) {
|
||||
if senderMachine == r.machine {
|
||||
r.localLatency.Observe(delay)
|
||||
} else if delay < 0 {
|
||||
r.crossClockSkewSamples++
|
||||
r.crossClockSkewSamples.Add(1)
|
||||
} else {
|
||||
r.crossLatency.Observe(delay)
|
||||
}
|
||||
@@ -150,26 +202,22 @@ func senderMachineFromClientMid(runID, mid string) string {
|
||||
}
|
||||
|
||||
func (r *deliveryRecorder) metricCounts() (received, duplicates, outOfOrder, disconnects int64) {
|
||||
r.mu.Lock()
|
||||
duplicates = r.duplicates
|
||||
outOfOrder = r.outOfOrder
|
||||
r.mu.Unlock()
|
||||
return r.received.Load(), duplicates, outOfOrder, r.disconnects.Load()
|
||||
return r.received.Load(), r.duplicates.Load(), r.outOfOrder.Load(), r.disconnects.Load()
|
||||
}
|
||||
|
||||
func (r *deliveryRecorder) beginHistory() { r.historyMode.Store(true) }
|
||||
|
||||
func (r *deliveryRecorder) historyFound(mid string) bool {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
return r.history[mid]
|
||||
shard := r.shard(mid)
|
||||
shard.mu.RLock()
|
||||
found := shard.history[mid]
|
||||
shard.mu.RUnlock()
|
||||
return found
|
||||
}
|
||||
|
||||
func (r *deliveryRecorder) historyFoundAll(messageIDs []string) bool {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
for _, mid := range messageIDs {
|
||||
if !r.history[mid] {
|
||||
if !r.historyFound(mid) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
@@ -182,10 +230,11 @@ func (r *deliveryRecorder) unackedObservedMessages(ledger []AcceptedMessage, sce
|
||||
accepted[message.ClientMid] = true
|
||||
}
|
||||
runID, _ := r.runID.Load().(string)
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
messages := make([]AcceptedMessage, 0)
|
||||
for mid, obs := range r.observed {
|
||||
for i := range r.shards {
|
||||
shard := &r.shards[i]
|
||||
shard.mu.RLock()
|
||||
for mid, obs := range shard.observed {
|
||||
if accepted[mid] || obs.seq <= 0 || obs.topic == "" {
|
||||
continue
|
||||
}
|
||||
@@ -199,12 +248,14 @@ func (r *deliveryRecorder) unackedObservedMessages(ledger []AcceptedMessage, sce
|
||||
}
|
||||
message := AcceptedMessage{RunID: runID, Scenario: observedScenario, Kind: kind,
|
||||
ClientMid: mid, Topic: obs.topic, Seq: obs.seq, SenderMachine: obs.senderMachine,
|
||||
SenderIndex: obs.senderIndex, PeerMachine: r.machine, PeerIndex: firstObservedIndex(obs.bits)}
|
||||
SenderIndex: obs.senderIndex, PeerMachine: r.machine, PeerIndex: firstObservedAtomicIndex(obs.bits)}
|
||||
if obs.sentAt > 0 {
|
||||
message.SentAt = time.Unix(0, obs.sentAt).UTC()
|
||||
}
|
||||
messages = append(messages, message)
|
||||
}
|
||||
shard.mu.RUnlock()
|
||||
}
|
||||
sort.Slice(messages, func(i, j int) bool { return messages[i].ClientMid < messages[j].ClientMid })
|
||||
return messages
|
||||
}
|
||||
@@ -218,34 +269,109 @@ func firstObservedIndex(words []uint64) int {
|
||||
return 0
|
||||
}
|
||||
|
||||
func (r *deliveryRecorder) onDisconnect(_ AccountRecord, _ error) {
|
||||
func firstObservedAtomicIndex(words []atomic.Uint64) int {
|
||||
for wordIndex := range words {
|
||||
if word := words[wordIndex].Load(); word != 0 {
|
||||
return wordIndex*64 + bits.TrailingZeros64(word) + 1
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func markObservationBit(words []atomic.Uint64, word int, mask uint64) bool {
|
||||
for {
|
||||
current := words[word].Load()
|
||||
if current&mask != 0 {
|
||||
return false
|
||||
}
|
||||
if words[word].CompareAndSwap(current, current|mask) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (r *deliveryRecorder) onDisconnect(account AccountRecord, err error) {
|
||||
category := disconnectCategory(err)
|
||||
r.disconnectMu.Lock()
|
||||
r.disconnects.Add(1)
|
||||
r.disconnectReasons[category]++
|
||||
if len(r.disconnectSamples) < 100 {
|
||||
message := ""
|
||||
if err != nil {
|
||||
message = err.Error()
|
||||
if len(message) > 300 {
|
||||
message = message[:300]
|
||||
}
|
||||
}
|
||||
r.disconnectSamples = append(r.disconnectSamples, DisconnectSample{
|
||||
ReceiverMachine: r.machine, ReceiverIndex: account.Index, Category: category,
|
||||
Error: message, At: time.Now().UTC(),
|
||||
})
|
||||
}
|
||||
r.disconnectMu.Unlock()
|
||||
}
|
||||
|
||||
func disconnectCategory(err error) string {
|
||||
if err == nil {
|
||||
return "unknown"
|
||||
}
|
||||
if errors.Is(err, io.EOF) {
|
||||
return "eof"
|
||||
}
|
||||
if errors.Is(err, net.ErrClosed) {
|
||||
return "closed_connection"
|
||||
}
|
||||
var netErr net.Error
|
||||
if errors.As(err, &netErr) && netErr.Timeout() {
|
||||
return "timeout"
|
||||
}
|
||||
message := strings.ToLower(err.Error())
|
||||
switch {
|
||||
case strings.Contains(message, "close 1006") || strings.Contains(message, "abnormal closure"):
|
||||
return "abnormal_close"
|
||||
case strings.Contains(message, "close 1000") || strings.Contains(message, "normal closure"):
|
||||
return "normal_close"
|
||||
case strings.Contains(message, "connection reset"):
|
||||
return "connection_reset"
|
||||
case strings.Contains(message, "broken pipe"):
|
||||
return "broken_pipe"
|
||||
case strings.Contains(message, "timeout") || strings.Contains(message, "deadline exceeded"):
|
||||
return "timeout"
|
||||
case strings.Contains(message, "use of closed network connection"):
|
||||
return "closed_connection"
|
||||
default:
|
||||
return "other"
|
||||
}
|
||||
}
|
||||
|
||||
func (r *deliveryRecorder) summarize(base AgentSummary, ledger []AcceptedMessage) AgentSummary {
|
||||
base.DeliveryLatency = r.latency.Snapshot()
|
||||
base.DeliveryLatencyLocal = r.localLatency.Snapshot()
|
||||
base.DeliveryLatencyCross = r.crossLatency.Snapshot()
|
||||
r.disconnectMu.Lock()
|
||||
base.Disconnects = r.disconnects.Load()
|
||||
base.DisconnectReasons = cloneCountMap(r.disconnectReasons)
|
||||
base.DisconnectSamples = append([]DisconnectSample(nil), r.disconnectSamples...)
|
||||
r.disconnectMu.Unlock()
|
||||
runID, _ := r.runID.Load().(string)
|
||||
accepted := make(map[string]AcceptedMessage, len(ledger))
|
||||
const missingLimit = 100000
|
||||
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
base.Duplicates = r.duplicates
|
||||
base.OutOfOrder = r.outOfOrder
|
||||
base.CrossClockSkewSamples = r.crossClockSkewSamples
|
||||
base.Duplicates = r.duplicates.Load()
|
||||
base.OutOfOrder = r.outOfOrder.Load()
|
||||
base.CrossClockSkewSamples = r.crossClockSkewSamples.Load()
|
||||
r.orderingMu.Lock()
|
||||
base.OrderingViolations = append([]OrderingViolation(nil), r.ordering...)
|
||||
if r.malformed > 0 {
|
||||
base.Errors = append(base.Errors, fmt.Sprintf("malformed run messages=%d", r.malformed))
|
||||
r.orderingMu.Unlock()
|
||||
if malformed := r.malformed.Load(); malformed > 0 {
|
||||
base.Errors = append(base.Errors, fmt.Sprintf("malformed run messages=%d", malformed))
|
||||
}
|
||||
for _, message := range ledger {
|
||||
if message.RunID != runID {
|
||||
continue
|
||||
}
|
||||
accepted[message.ClientMid] = message
|
||||
obs := r.observed[message.ClientMid]
|
||||
obs := r.observationSnapshot(message.ClientMid)
|
||||
switch message.Kind {
|
||||
case "group":
|
||||
base.ExpectedGroup += int64(r.localUsers)
|
||||
@@ -273,11 +399,16 @@ func (r *deliveryRecorder) summarize(base AgentSummary, ledger []AcceptedMessage
|
||||
}
|
||||
}
|
||||
}
|
||||
for mid, obs := range r.observed {
|
||||
for i := range r.shards {
|
||||
shard := &r.shards[i]
|
||||
shard.mu.RLock()
|
||||
for mid, obs := range shard.observed {
|
||||
if _, ok := accepted[mid]; !ok {
|
||||
base.UnexpectedMessages += bitsetCount(obs.bits)
|
||||
base.UnexpectedMessages += atomicBitsetCount(obs.bits)
|
||||
}
|
||||
}
|
||||
shard.mu.RUnlock()
|
||||
}
|
||||
base.RealtimeLost = (base.ExpectedPeer - base.ReceivedPeer) + (base.ExpectedGroup - base.ReceivedGroup)
|
||||
if len(base.Missing) == missingLimit {
|
||||
base.Errors = append(base.Errors, "missing delivery details truncated at 100000")
|
||||
@@ -291,14 +422,12 @@ func (r *deliveryRecorder) countRealtimeLostStored(ledger []AcceptedMessage, sto
|
||||
for _, mid := range storedMessageIDs {
|
||||
stored[mid] = true
|
||||
}
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
var count int64
|
||||
for _, message := range ledger {
|
||||
if message.RunID != runID || !stored[message.ClientMid] {
|
||||
continue
|
||||
}
|
||||
obs := r.observed[message.ClientMid]
|
||||
obs := r.observationSnapshot(message.ClientMid)
|
||||
switch message.Kind {
|
||||
case "group":
|
||||
for index := 1; index <= r.localUsers; index++ {
|
||||
@@ -315,7 +444,44 @@ func (r *deliveryRecorder) countRealtimeLostStored(ledger []AcceptedMessage, sto
|
||||
return count
|
||||
}
|
||||
|
||||
func observationHas(obs *observation, receiverIndex int) bool {
|
||||
func (r *deliveryRecorder) shard(mid string) *observationShard {
|
||||
var hash uint32 = 2166136261
|
||||
for i := 0; i < len(mid); i++ {
|
||||
hash ^= uint32(mid[i])
|
||||
hash *= 16777619
|
||||
}
|
||||
return &r.shards[hash%uint32(len(r.shards))]
|
||||
}
|
||||
|
||||
func (r *deliveryRecorder) observationSnapshot(mid string) *observationView {
|
||||
shard := r.shard(mid)
|
||||
shard.mu.RLock()
|
||||
obs := shard.observed[mid]
|
||||
shard.mu.RUnlock()
|
||||
if obs == nil {
|
||||
return nil
|
||||
}
|
||||
view := &observationView{topic: obs.topic, seq: obs.seq, senderMachine: obs.senderMachine,
|
||||
senderIndex: obs.senderIndex, scenario: obs.scenario, sentAt: obs.sentAt,
|
||||
bits: make([]uint64, len(obs.bits))}
|
||||
for i := range obs.bits {
|
||||
view.bits[i] = obs.bits[i].Load()
|
||||
}
|
||||
return view
|
||||
}
|
||||
|
||||
func cloneCountMap(source map[string]int64) map[string]int64 {
|
||||
if len(source) == 0 {
|
||||
return nil
|
||||
}
|
||||
clone := make(map[string]int64, len(source))
|
||||
for key, value := range source {
|
||||
clone[key] = value
|
||||
}
|
||||
return clone
|
||||
}
|
||||
|
||||
func observationHas(obs *observationView, receiverIndex int) bool {
|
||||
if obs == nil || receiverIndex <= 0 {
|
||||
return false
|
||||
}
|
||||
@@ -324,6 +490,14 @@ func observationHas(obs *observation, receiverIndex int) bool {
|
||||
return word < len(obs.bits) && obs.bits[word]&(uint64(1)<<bit) != 0
|
||||
}
|
||||
|
||||
func atomicBitsetCount(words []atomic.Uint64) int64 {
|
||||
var count int64
|
||||
for i := range words {
|
||||
count += int64(bits.OnesCount64(words[i].Load()))
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
func bitsetCount(words []uint64) int64 {
|
||||
var count int64
|
||||
for _, word := range words {
|
||||
|
||||
@@ -2,6 +2,9 @@ package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
@@ -111,6 +114,65 @@ func TestDeliveryRecorderSeparatesLocalAndCrossMachineLatency(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeliveryRecorderConcurrentGroupFanout(t *testing.T) {
|
||||
const users = 32
|
||||
const messages = 64
|
||||
recorder := newDeliveryRecorder("run1", "a", users)
|
||||
ledger := make([]AcceptedMessage, messages)
|
||||
for messageIndex := range ledger {
|
||||
ledger[messageIndex] = AcceptedMessage{
|
||||
RunID: "run1", Kind: "group", Topic: "grp-one", Seq: 1,
|
||||
ClientMid: fmt.Sprintf("run1-g-b-%04d-01", messageIndex+1), SenderMachine: "b",
|
||||
}
|
||||
}
|
||||
|
||||
var writers sync.WaitGroup
|
||||
for receiver := 1; receiver <= users; receiver++ {
|
||||
for _, message := range ledger {
|
||||
writers.Add(1)
|
||||
go func(receiver int, message AcceptedMessage) {
|
||||
defer writers.Done()
|
||||
recorder.onData(AccountRecord{Index: receiver}, testData(message, time.Now().Add(-time.Millisecond)))
|
||||
}(receiver, message)
|
||||
}
|
||||
}
|
||||
writers.Wait()
|
||||
|
||||
summary := recorder.summarize(AgentSummary{}, ledger)
|
||||
want := int64(users * messages)
|
||||
if summary.ExpectedGroup != want || summary.ReceivedGroup != want || summary.RealtimeLost != 0 {
|
||||
t.Fatalf("group expected/received/lost = %d/%d/%d, want %d/%d/0",
|
||||
summary.ExpectedGroup, summary.ReceivedGroup, summary.RealtimeLost, want, want)
|
||||
}
|
||||
if summary.Duplicates != 0 || summary.DeliveryLatency.Count != want {
|
||||
t.Fatalf("duplicates=%d latency samples=%d want=0/%d", summary.Duplicates, summary.DeliveryLatency.Count, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeliveryRecorderDisconnectDiagnostics(t *testing.T) {
|
||||
recorder := newDeliveryRecorder("run1", "a", 1)
|
||||
recorder.onDisconnect(AccountRecord{Index: 1}, errors.New("websocket: close 1006 (abnormal closure): unexpected EOF"))
|
||||
recorder.onDisconnect(AccountRecord{Index: 1}, errors.New("read tcp: connection reset by peer"))
|
||||
|
||||
summary := recorder.summarize(AgentSummary{}, nil)
|
||||
if summary.Disconnects != 2 || summary.DisconnectReasons["abnormal_close"] != 1 ||
|
||||
summary.DisconnectReasons["connection_reset"] != 1 || len(summary.DisconnectSamples) != 2 {
|
||||
t.Fatalf("unexpected disconnect diagnostics: count=%d reasons=%v samples=%+v",
|
||||
summary.Disconnects, summary.DisconnectReasons, summary.DisconnectSamples)
|
||||
}
|
||||
recorder.resetDisconnects()
|
||||
summary = recorder.summarize(AgentSummary{}, nil)
|
||||
if summary.Disconnects != 0 || len(summary.DisconnectReasons) != 0 || len(summary.DisconnectSamples) != 0 {
|
||||
t.Fatalf("disconnect diagnostics were not reset: %+v", summary)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitsetCountCountsEachBitOnce(t *testing.T) {
|
||||
if got := bitsetCount([]uint64{0b1011, 0b100000}); got != 4 {
|
||||
t.Fatalf("bitset count=%d want=4", got)
|
||||
}
|
||||
}
|
||||
|
||||
func testData(message AcceptedMessage, sentAt time.Time) dataPacket {
|
||||
content, _ := json.Marshal(map[string]any{"run_id": message.RunID, "scenario": message.Scenario,
|
||||
"sender_machine": message.SenderMachine, "sender_index": message.SenderIndex,
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
@@ -35,11 +36,42 @@ type dataPacket struct {
|
||||
Content json.RawMessage `json:"content"`
|
||||
}
|
||||
|
||||
type queuedDataPacket struct {
|
||||
packet dataPacket
|
||||
queuedAt time.Time
|
||||
}
|
||||
|
||||
type ctrlWait struct {
|
||||
ch chan ctrlPacket
|
||||
kind string
|
||||
sentAt time.Time
|
||||
}
|
||||
|
||||
type timedOutCtrl struct {
|
||||
kind string
|
||||
sentAt time.Time
|
||||
}
|
||||
|
||||
type ctrlObservation struct {
|
||||
ID string
|
||||
Kind string
|
||||
Code int
|
||||
SentAt time.Time
|
||||
ReceivedAt time.Time
|
||||
Latency time.Duration
|
||||
Late bool
|
||||
}
|
||||
|
||||
type ctrlError struct {
|
||||
Code int
|
||||
Text string
|
||||
}
|
||||
|
||||
const (
|
||||
dataCallbackQueueSize = 512
|
||||
lateCtrlRetention = 10 * time.Minute
|
||||
)
|
||||
|
||||
func (e *ctrlError) Error() string {
|
||||
return fmt.Sprintf("ctrl code=%d text=%s", e.Code, e.Text)
|
||||
}
|
||||
@@ -50,17 +82,22 @@ type WSClient struct {
|
||||
commandTimeout time.Duration
|
||||
onData func(AccountRecord, dataPacket)
|
||||
onDisconnect func(AccountRecord, error)
|
||||
onCtrlObserved func(AccountRecord, ctrlObservation)
|
||||
callbackQueue *callbackQueueMetrics
|
||||
|
||||
sendMu sync.Mutex
|
||||
mu sync.Mutex
|
||||
seq uint64
|
||||
waits map[string]chan ctrlPacket
|
||||
waits map[string]ctrlWait
|
||||
timedOutCtrls map[string]timedOutCtrl
|
||||
done chan struct{}
|
||||
data chan queuedDataPacket
|
||||
closed atomic.Bool
|
||||
}
|
||||
|
||||
func connectWSClient(ctx context.Context, cfg *Config, account AccountRecord, apiKey, password string,
|
||||
onData func(AccountRecord, dataPacket), onDisconnect func(AccountRecord, error)) (*WSClient, error) {
|
||||
onData func(AccountRecord, dataPacket), onDisconnect func(AccountRecord, error),
|
||||
onCtrlObserved func(AccountRecord, ctrlObservation), callbackQueue *callbackQueueMetrics) (*WSClient, error) {
|
||||
timeout, err := configDuration(cfg.CommandTimeout)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -70,6 +107,17 @@ func connectWSClient(ctx context.Context, cfg *Config, account AccountRecord, ap
|
||||
return nil, err
|
||||
}
|
||||
dialer := *websocket.DefaultDialer
|
||||
if account.DialIP != "" {
|
||||
targetHost := endpointHostname(endpoint)
|
||||
var netDialer net.Dialer
|
||||
dialer.NetDialContext = func(ctx context.Context, network, address string) (net.Conn, error) {
|
||||
host, port, splitErr := net.SplitHostPort(address)
|
||||
if splitErr == nil && strings.EqualFold(host, targetHost) {
|
||||
address = net.JoinHostPort(account.DialIP, port)
|
||||
}
|
||||
return netDialer.DialContext(ctx, network, address)
|
||||
}
|
||||
}
|
||||
if cfg.InsecureTLS {
|
||||
dialer.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} // #nosec G402: explicit load-test option.
|
||||
}
|
||||
@@ -82,8 +130,11 @@ func connectWSClient(ctx context.Context, cfg *Config, account AccountRecord, ap
|
||||
}
|
||||
client := &WSClient{
|
||||
account: account, conn: conn, commandTimeout: timeout, onData: onData, onDisconnect: onDisconnect,
|
||||
waits: make(map[string]chan ctrlPacket), done: make(chan struct{}),
|
||||
onCtrlObserved: onCtrlObserved,
|
||||
callbackQueue: callbackQueue,
|
||||
waits: make(map[string]ctrlWait), timedOutCtrls: make(map[string]timedOutCtrl), done: make(chan struct{}),
|
||||
}
|
||||
client.startDataProcessor()
|
||||
go client.readLoop()
|
||||
if _, err = client.command(ctx, "hi", map[string]any{"hi": map[string]any{
|
||||
"id": client.nextID("hi"), "ver": "0.25", "ua": "im-pressure/1.0",
|
||||
@@ -125,6 +176,14 @@ func connectWSClient(ctx context.Context, cfg *Config, account AccountRecord, ap
|
||||
return client, nil
|
||||
}
|
||||
|
||||
func endpointHostname(raw string) string {
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return u.Hostname()
|
||||
}
|
||||
|
||||
func connectForPrepare(ctx context.Context, cfg *Config, account AccountRecord, api *apiClient, password, verificationCode string) (*WSClient, bool, error) {
|
||||
timeout, err := configDuration(cfg.CommandTimeout)
|
||||
if err != nil {
|
||||
@@ -142,7 +201,8 @@ func connectForPrepare(ctx context.Context, cfg *Config, account AccountRecord,
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
client := &WSClient{account: account, conn: conn, commandTimeout: timeout, waits: make(map[string]chan ctrlPacket), done: make(chan struct{})}
|
||||
client := &WSClient{account: account, conn: conn, commandTimeout: timeout,
|
||||
waits: make(map[string]ctrlWait), timedOutCtrls: make(map[string]timedOutCtrl), done: make(chan struct{})}
|
||||
go client.readLoop()
|
||||
if _, err = client.command(ctx, "hi", map[string]any{"hi": map[string]any{
|
||||
"id": client.nextID("hi"), "ver": "0.25", "ua": "im-pressure-prepare/1.0", "tenant": cfg.Tenant,
|
||||
@@ -285,14 +345,15 @@ func ctrlInt64(raw json.RawMessage) (int64, error) {
|
||||
return strconv.ParseInt(text, 10, 64)
|
||||
}
|
||||
|
||||
func (c *WSClient) command(ctx context.Context, _ string, message map[string]any) (ctrlPacket, error) {
|
||||
func (c *WSClient) command(ctx context.Context, kind string, message map[string]any) (ctrlPacket, error) {
|
||||
id := nestedMessageID(message)
|
||||
if id == "" {
|
||||
return ctrlPacket{}, errors.New("command id is required")
|
||||
}
|
||||
wait := make(chan ctrlPacket, 1)
|
||||
sentAt := time.Now()
|
||||
c.mu.Lock()
|
||||
c.waits[id] = wait
|
||||
c.waits[id] = ctrlWait{ch: wait, kind: kind, sentAt: sentAt}
|
||||
c.mu.Unlock()
|
||||
c.sendMu.Lock()
|
||||
err := c.conn.WriteJSON(message)
|
||||
@@ -310,7 +371,33 @@ func (c *WSClient) command(ctx context.Context, _ string, message map[string]any
|
||||
}
|
||||
return ctrl, nil
|
||||
case <-timer.C:
|
||||
c.removeWait(id)
|
||||
c.mu.Lock()
|
||||
entry, pending := c.waits[id]
|
||||
if pending {
|
||||
delete(c.waits, id)
|
||||
if entry.kind == "pub" {
|
||||
c.timedOutCtrls[id] = timedOutCtrl{kind: entry.kind, sentAt: entry.sentAt}
|
||||
}
|
||||
}
|
||||
c.mu.Unlock()
|
||||
if !pending {
|
||||
// The read loop won the timer race and placed the response in the
|
||||
// buffered channel before removing the waiter.
|
||||
ctrl := <-wait
|
||||
if ctrl.Code < 200 || ctrl.Code >= 300 {
|
||||
return ctrl, &ctrlError{Code: ctrl.Code, Text: ctrl.Text}
|
||||
}
|
||||
return ctrl, nil
|
||||
}
|
||||
if entry.kind == "pub" {
|
||||
time.AfterFunc(lateCtrlRetention, func() {
|
||||
c.mu.Lock()
|
||||
if current, ok := c.timedOutCtrls[id]; ok && current.sentAt.Equal(sentAt) {
|
||||
delete(c.timedOutCtrls, id)
|
||||
}
|
||||
c.mu.Unlock()
|
||||
})
|
||||
}
|
||||
return ctrlPacket{}, fmt.Errorf("ctrl timeout id=%s", id)
|
||||
case <-ctx.Done():
|
||||
c.removeWait(id)
|
||||
@@ -346,6 +433,9 @@ func (c *WSClient) removeWait(id string) {
|
||||
func (c *WSClient) readLoop() {
|
||||
defer func() {
|
||||
_ = c.conn.Close()
|
||||
if c.data != nil {
|
||||
close(c.data)
|
||||
}
|
||||
close(c.done)
|
||||
}()
|
||||
for {
|
||||
@@ -360,11 +450,23 @@ func (c *WSClient) readLoop() {
|
||||
var ctrl ctrlPacket
|
||||
if json.Unmarshal(raw, &ctrl) == nil && ctrl.ID != "" {
|
||||
c.mu.Lock()
|
||||
wait := c.waits[ctrl.ID]
|
||||
wait, pending := c.waits[ctrl.ID]
|
||||
late, timedOut := c.timedOutCtrls[ctrl.ID]
|
||||
receivedAt := time.Now()
|
||||
if pending {
|
||||
wait.ch <- ctrl
|
||||
delete(c.waits, ctrl.ID)
|
||||
} else if timedOut {
|
||||
delete(c.timedOutCtrls, ctrl.ID)
|
||||
}
|
||||
c.mu.Unlock()
|
||||
if wait != nil {
|
||||
wait <- ctrl
|
||||
if c.onCtrlObserved != nil && (pending && wait.kind == "pub" || timedOut && late.kind == "pub") {
|
||||
kind, sentAt, isLate := wait.kind, wait.sentAt, false
|
||||
if timedOut {
|
||||
kind, sentAt, isLate = late.kind, late.sentAt, true
|
||||
}
|
||||
c.onCtrlObserved(c.account, ctrlObservation{ID: ctrl.ID, Kind: kind, Code: ctrl.Code,
|
||||
SentAt: sentAt, ReceivedAt: receivedAt, Latency: receivedAt.Sub(sentAt), Late: isLate})
|
||||
}
|
||||
}
|
||||
continue
|
||||
@@ -372,10 +474,30 @@ func (c *WSClient) readLoop() {
|
||||
if raw := envelope["data"]; len(raw) > 0 && c.onData != nil {
|
||||
var data dataPacket
|
||||
if json.Unmarshal(raw, &data) == nil {
|
||||
if c.data == nil {
|
||||
c.onData(c.account, data)
|
||||
} else {
|
||||
queued := queuedDataPacket{packet: data, queuedAt: time.Now()}
|
||||
c.callbackQueue.enqueue()
|
||||
c.data <- queued
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *WSClient) startDataProcessor() {
|
||||
if c.onData == nil || c.data != nil {
|
||||
return
|
||||
}
|
||||
c.data = make(chan queuedDataPacket, dataCallbackQueueSize)
|
||||
go func() {
|
||||
for data := range c.data {
|
||||
c.callbackQueue.startCallback(data.queuedAt)
|
||||
c.onData(c.account, data.packet)
|
||||
c.callbackQueue.finishCallback()
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func (c *WSClient) startKeepalive(interval time.Duration) {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
@@ -10,6 +11,49 @@ import (
|
||||
"github.com/gorilla/websocket"
|
||||
)
|
||||
|
||||
func TestWSClientObservesCtrlAfterCommandTimeout(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
conn, err := (&websocket.Upgrader{}).Upgrade(w, r, nil)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer conn.Close()
|
||||
var command map[string]any
|
||||
if err = conn.ReadJSON(&command); err != nil {
|
||||
return
|
||||
}
|
||||
time.Sleep(40 * time.Millisecond)
|
||||
_ = conn.WriteJSON(map[string]any{"ctrl": map[string]any{"id": "pub-a-0001", "code": 202, "text": "accepted"}})
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
conn, _, err := websocket.DefaultDialer.Dial("ws"+strings.TrimPrefix(server.URL, "http"), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
observed := make(chan ctrlObservation, 1)
|
||||
client := &WSClient{
|
||||
account: AccountRecord{Machine: "a", Index: 1}, conn: conn, commandTimeout: 10 * time.Millisecond,
|
||||
waits: make(map[string]ctrlWait), timedOutCtrls: make(map[string]timedOutCtrl), done: make(chan struct{}),
|
||||
onCtrlObserved: func(_ AccountRecord, observation ctrlObservation) { observed <- observation },
|
||||
}
|
||||
go client.readLoop()
|
||||
defer client.Close()
|
||||
|
||||
_, err = client.command(context.Background(), "pub", map[string]any{"pub": map[string]any{"id": "pub-a-0001"}})
|
||||
if err == nil || !strings.Contains(err.Error(), "ctrl timeout") {
|
||||
t.Fatalf("command error=%v, want ctrl timeout", err)
|
||||
}
|
||||
select {
|
||||
case observation := <-observed:
|
||||
if !observation.Late || observation.Kind != "pub" || observation.Code != 202 || observation.Latency < 10*time.Millisecond {
|
||||
t.Fatalf("unexpected observation: %+v", observation)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("late ctrl response was not observed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWSClientKeepaliveSendsPing(t *testing.T) {
|
||||
ping := make(chan struct{}, 1)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -37,7 +81,7 @@ func TestWSClientKeepaliveSendsPing(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client := &WSClient{conn: conn, waits: make(map[string]chan ctrlPacket), done: make(chan struct{})}
|
||||
client := &WSClient{conn: conn, waits: make(map[string]ctrlWait), timedOutCtrls: make(map[string]timedOutCtrl), done: make(chan struct{})}
|
||||
go client.readLoop()
|
||||
client.startKeepalive(10 * time.Millisecond)
|
||||
defer client.Close()
|
||||
@@ -70,7 +114,7 @@ func TestWSClientReadLoopClosesConnectionAfterDecodeError(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client := &WSClient{conn: conn, waits: make(map[string]chan ctrlPacket), done: make(chan struct{})}
|
||||
client := &WSClient{conn: conn, waits: make(map[string]ctrlWait), timedOutCtrls: make(map[string]timedOutCtrl), done: make(chan struct{})}
|
||||
go client.readLoop()
|
||||
|
||||
select {
|
||||
@@ -84,3 +128,58 @@ func TestWSClientReadLoopClosesConnectionAfterDecodeError(t *testing.T) {
|
||||
t.Fatal("read loop exited without closing the underlying connection")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWSClientDataCallbackDoesNotBlockPong(t *testing.T) {
|
||||
pong := make(chan struct{}, 1)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
conn, err := (&websocket.Upgrader{}).Upgrade(w, r, nil)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer conn.Close()
|
||||
conn.SetPongHandler(func(string) error {
|
||||
select {
|
||||
case pong <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err = conn.WriteJSON(map[string]any{"data": map[string]any{
|
||||
"topic": "grp-test", "seq": 1, "head": map[string]any{}, "content": map[string]any{},
|
||||
}}); err != nil {
|
||||
return
|
||||
}
|
||||
if err = conn.WriteControl(websocket.PingMessage, nil, time.Now().Add(time.Second)); err != nil {
|
||||
return
|
||||
}
|
||||
conn.SetReadDeadline(time.Now().Add(time.Second))
|
||||
for {
|
||||
if _, _, readErr := conn.ReadMessage(); readErr != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
conn, _, err := websocket.DefaultDialer.Dial("ws"+strings.TrimPrefix(server.URL, "http"), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
blocked := make(chan struct{})
|
||||
client := &WSClient{
|
||||
conn: conn, waits: make(map[string]ctrlWait), timedOutCtrls: make(map[string]timedOutCtrl), done: make(chan struct{}),
|
||||
onData: func(AccountRecord, dataPacket) { <-blocked },
|
||||
}
|
||||
client.startDataProcessor()
|
||||
go client.readLoop()
|
||||
defer func() {
|
||||
close(blocked)
|
||||
client.Close()
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-pong:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("pong was blocked by the data callback")
|
||||
}
|
||||
}
|
||||
|
||||
+26
-8
@@ -489,14 +489,16 @@ func clusterReqShard(msg *ClusterReq, count int) int {
|
||||
return int(h.Sum32() % uint32(count))
|
||||
}
|
||||
|
||||
// masterToProxyAsync forwards response from topic master to topic proxy
|
||||
// in a fire-and-forget manner.
|
||||
func (n *ClusterNode) masterToProxyAsync(msg *ClusterResp) error {
|
||||
var unused bool
|
||||
if c := n.callAsync("Cluster.TopicProxy", msg, &unused, nil); c.Error != nil {
|
||||
return c.Error
|
||||
// masterToProxy forwards responses synchronously. A multiplexing session drains
|
||||
// one topic in order, so waiting for each RPC prevents the RPC server from
|
||||
// concurrently enqueueing later sequence IDs before earlier ones.
|
||||
func (n *ClusterNode) masterToProxy(msg *ClusterResp) error {
|
||||
var rejected bool
|
||||
err := n.call("Cluster.TopicProxy", msg, &rejected)
|
||||
if err == nil && rejected {
|
||||
return errClusterOverloaded
|
||||
}
|
||||
return nil
|
||||
return err
|
||||
}
|
||||
|
||||
// route routes server message within the cluster.
|
||||
@@ -1215,7 +1217,10 @@ func clusterInit(configString json.RawMessage, self *string) int {
|
||||
|
||||
func registerClusterGobTypes() {
|
||||
gob.Register([]any{})
|
||||
gob.Register([]map[string]any{})
|
||||
gob.Register(map[string]any{})
|
||||
gob.Register(types.KVMap{})
|
||||
gob.Register([]types.KVMap{})
|
||||
gob.Register(map[string]int{})
|
||||
gob.Register(map[string]string{})
|
||||
gob.Register(MsgAccessMode{})
|
||||
@@ -1393,6 +1398,13 @@ func (sess *Session) clusterWriteLoop(forTopic string) {
|
||||
terminate := true
|
||||
defer func() {
|
||||
if terminate {
|
||||
// The topic may retain this multiplexing session briefly while its
|
||||
// unsubscribe request is processed. Reject writes to the stale session
|
||||
// instead of filling a send queue which no longer has a drain owner.
|
||||
sess.clusterQueueMu.Lock()
|
||||
atomic.StoreInt32(&sess.terminating, 1)
|
||||
sess.purgeChannels()
|
||||
sess.clusterQueueMu.Unlock()
|
||||
sess.closeRPC()
|
||||
globals.sessionStore.Delete(sess)
|
||||
sess.inflightReqs = nil
|
||||
@@ -1434,7 +1446,7 @@ func (sess *Session) clusterWriteLoop(forTopic string) {
|
||||
srvMsg.RcptTo = forTopic
|
||||
response.RcptTo = forTopic
|
||||
|
||||
if err := sess.clnode.masterToProxyAsync(response); err != nil {
|
||||
if err := sess.clnode.masterToProxy(response); err != nil {
|
||||
logs.Warn.Printf("cluster: response to proxy failed \"%s\": %s", sess.sid, err.Error())
|
||||
return
|
||||
}
|
||||
@@ -1451,6 +1463,12 @@ func (sess *Session) clusterWriteLoop(forTopic string) {
|
||||
case <-sess.detach:
|
||||
return
|
||||
default:
|
||||
sess.clusterWriteScheduled.Store(false)
|
||||
// An enqueue can race with releasing ownership. Either the enqueuer
|
||||
// schedules the next drain or this check does; the CAS prevents both.
|
||||
if len(sess.send) > 0 {
|
||||
sess.scheduleClusterWriteLoop()
|
||||
}
|
||||
terminate = false
|
||||
return
|
||||
}
|
||||
|
||||
@@ -28,6 +28,9 @@ func TestClusterGobRegistersSystemMessagePayloadTypes(t *testing.T) {
|
||||
Content: map[string]any{
|
||||
"text": "成员加入了群聊",
|
||||
"template": "group_member_joined",
|
||||
"items": []map[string]any{{"type": "member", "id": "usr123"}},
|
||||
"action": types.KVMap{"type": "group_profile", "target_id": "grpTest"},
|
||||
"rows": []types.KVMap{{"type": "member", "id": "usr123"}},
|
||||
"actors": []imSystemMessageActor{{
|
||||
UserID: "usr123",
|
||||
Click: map[string]any{"type": "user_profile", "target_id": "usr123"},
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"net"
|
||||
"net/rpc"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/tinode/chat/server/concurrency"
|
||||
"github.com/tinode/chat/server/store/types"
|
||||
)
|
||||
|
||||
type orderedTopicProxy struct {
|
||||
mu sync.Mutex
|
||||
received []int64
|
||||
done chan struct{}
|
||||
}
|
||||
|
||||
func (proxy *orderedTopicProxy) TopicProxy(msg *ClusterResp, rejected *bool) error {
|
||||
if msg.SrvMsg.Data.SeqId == 1 {
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
proxy.mu.Lock()
|
||||
proxy.received = append(proxy.received, msg.SrvMsg.Data.SeqId)
|
||||
if len(proxy.received) == 2 {
|
||||
close(proxy.done)
|
||||
}
|
||||
proxy.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestClusterWriteLoopPreservesTopicOrder(t *testing.T) {
|
||||
serverConn, clientConn := net.Pipe()
|
||||
rpcServer := rpc.NewServer()
|
||||
proxy := &orderedTopicProxy{done: make(chan struct{})}
|
||||
if err := rpcServer.RegisterName("Cluster", proxy); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
go rpcServer.ServeConn(serverConn)
|
||||
|
||||
pool := concurrency.NewGoRoutinePool(4)
|
||||
previousCluster := globals.cluster
|
||||
globals.cluster = &Cluster{proxyEventQueue: pool}
|
||||
t.Cleanup(func() {
|
||||
globals.cluster = previousCluster
|
||||
_ = clientConn.Close()
|
||||
_ = serverConn.Close()
|
||||
pool.Stop()
|
||||
})
|
||||
|
||||
sess := &Session{
|
||||
sid: "multiplex-order-test",
|
||||
proto: MULTIPLEX,
|
||||
proxiedTopic: "p2p-order-test",
|
||||
send: make(chan any, 2),
|
||||
clnode: &ClusterNode{endpoint: rpc.NewClient(clientConn), connected: true},
|
||||
}
|
||||
tenantID := types.TenantID(1)
|
||||
sess.send <- &ServerComMessage{TenantID: tenantID, Data: &MsgServerData{SeqId: 1}}
|
||||
sess.send <- &ServerComMessage{TenantID: tenantID, Data: &MsgServerData{SeqId: 2}}
|
||||
|
||||
// Scheduling twice used to start two concurrent drains for the same topic.
|
||||
sess.scheduleClusterWriteLoop()
|
||||
sess.scheduleClusterWriteLoop()
|
||||
|
||||
select {
|
||||
case <-proxy.done:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("timed out waiting for cluster responses")
|
||||
}
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for sess.clusterWriteScheduled.Load() && time.Now().Before(deadline) {
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
if sess.clusterWriteScheduled.Load() {
|
||||
t.Fatal("cluster write task did not release session ownership")
|
||||
}
|
||||
|
||||
proxy.mu.Lock()
|
||||
defer proxy.mu.Unlock()
|
||||
if len(proxy.received) != 2 || proxy.received[0] != 1 || proxy.received[1] != 2 {
|
||||
t.Fatalf("cluster responses arrived out of order: %v", proxy.received)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClusterWriteLoopRejectsWritesAfterTermination(t *testing.T) {
|
||||
previousSessionStore := globals.sessionStore
|
||||
globals.sessionStore = NewSessionStore(time.Minute)
|
||||
t.Cleanup(func() { globals.sessionStore = previousSessionStore })
|
||||
|
||||
sess := &Session{
|
||||
sid: "multiplex-termination-test",
|
||||
proto: MULTIPLEX,
|
||||
send: make(chan any, 1),
|
||||
stop: make(chan any, 1),
|
||||
detach: make(chan string, 1),
|
||||
subs: make(map[string]*Subscription),
|
||||
}
|
||||
sess.stop <- nil
|
||||
|
||||
sess.clusterWriteLoop("grp-termination-test")
|
||||
|
||||
if atomic.LoadInt32(&sess.terminating) == 0 {
|
||||
t.Fatal("terminated multiplexing session still accepts writes")
|
||||
}
|
||||
if !sess.queueOut(&ServerComMessage{TenantID: types.TenantID(1)}) {
|
||||
t.Fatal("write to terminated multiplexing session should be discarded")
|
||||
}
|
||||
if len(sess.send) != 0 {
|
||||
t.Fatalf("terminated multiplexing session retained %d outbound messages", len(sess.send))
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultiplexEnqueueWaitsForDrain(t *testing.T) {
|
||||
sess := &Session{proto: MULTIPLEX, send: make(chan any, 1)}
|
||||
sess.send <- "existing"
|
||||
|
||||
go func() {
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
<-sess.send
|
||||
}()
|
||||
|
||||
payload := &ServerComMessage{TenantID: types.TenantID(1)}
|
||||
if !sess.enqueueMultiplexOutbound(payload, 200*time.Millisecond) {
|
||||
t.Fatal("multiplex enqueue failed while the drain was making progress")
|
||||
}
|
||||
if got := <-sess.send; got != payload {
|
||||
t.Fatalf("unexpected queued payload: %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultiplexEnqueueTimesOutWithoutDrain(t *testing.T) {
|
||||
sess := &Session{proto: MULTIPLEX, send: make(chan any, 1)}
|
||||
sess.send <- "existing"
|
||||
|
||||
started := time.Now()
|
||||
if sess.enqueueMultiplexOutbound("next", 20*time.Millisecond) {
|
||||
t.Fatal("multiplex enqueue succeeded without queue capacity")
|
||||
}
|
||||
if elapsed := time.Since(started); elapsed < 15*time.Millisecond || elapsed > 200*time.Millisecond {
|
||||
t.Fatalf("multiplex enqueue timeout outside expected bounds: %s", elapsed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultiplexEnqueueStopsWhenSessionTerminates(t *testing.T) {
|
||||
sess := &Session{proto: MULTIPLEX, send: make(chan any, 1)}
|
||||
sess.send <- "existing"
|
||||
|
||||
go func() {
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
sess.clusterQueueMu.Lock()
|
||||
atomic.StoreInt32(&sess.terminating, 1)
|
||||
<-sess.send
|
||||
sess.clusterQueueMu.Unlock()
|
||||
}()
|
||||
|
||||
if !sess.enqueueMultiplexOutbound("next", 200*time.Millisecond) {
|
||||
t.Fatal("terminating multiplex session should discard the pending write")
|
||||
}
|
||||
if len(sess.send) != 0 {
|
||||
t.Fatalf("terminating multiplex session retained %d outbound messages", len(sess.send))
|
||||
}
|
||||
}
|
||||
@@ -382,6 +382,8 @@ type ClientComMessage struct {
|
||||
MetaWhat int `json:"-"`
|
||||
// Timestamp when this message was received by the server.
|
||||
Timestamp time.Time `json:"-"`
|
||||
// Timestamp when this message entered the current topic client queue.
|
||||
queuedAt time.Time
|
||||
// Tenant context inherited from the originating session.
|
||||
TenantID types.TenantID `json:"-"`
|
||||
|
||||
|
||||
@@ -406,6 +406,7 @@ type TenantBusinessAdapter interface {
|
||||
TenantGroupMemberGet(t.TenantID, string, t.Uid) (*t.GroupMember, error)
|
||||
TenantGroupMemberList(t.TenantID, string, t.GroupMemberListOptions) ([]t.GroupMember, string, error)
|
||||
TenantGroupMemberUpdate(t.TenantID, string, t.Uid, t.GroupMemberPatch) (*t.GroupMember, error)
|
||||
TenantUsersShareOpenPrivateGroup(t.TenantID, t.Uid, t.Uid) (bool, error)
|
||||
TenantGroupInvitationCreate(t.TenantID, *t.GroupInvitation) (*t.GroupInvitation, error)
|
||||
TenantGroupInvitationAccept(t.TenantID, string, t.Uid, any, time.Time) (*t.GroupInvitation, error)
|
||||
TenantGroupJoinRequestCreate(t.TenantID, *t.GroupJoinRequest) (*t.GroupJoinRequest, error)
|
||||
|
||||
@@ -4407,10 +4407,10 @@ func (a *adapter) FileStartUpload(fd *t.FileDef) error {
|
||||
user = 0
|
||||
}
|
||||
_, err := a.db.ExecContext(ctx,
|
||||
"INSERT INTO im_fileuploads(id,createdat,updatedat,userid,status,mimetype,size,etag,location) "+
|
||||
"VALUES(?,?,?,?,?,?,?,?,?)",
|
||||
"INSERT INTO im_fileuploads(id,createdat,updatedat,userid,status,mimetype,size,etag,location,original_name) "+
|
||||
"VALUES(?,?,?,?,?,?,?,?,?,?)",
|
||||
store.DecodeUid(fd.Uid()), fd.CreatedAt, fd.UpdatedAt, user,
|
||||
fd.Status, fd.MimeType, fd.Size, fd.ETag, fd.Location)
|
||||
fd.Status, fd.MimeType, fd.Size, fd.ETag, fd.Location, strings.TrimSpace(fd.OriginalName))
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -4432,8 +4432,8 @@ func (a *adapter) FileFinishUpload(fd *t.FileDef, success bool, size int64) (*t.
|
||||
|
||||
now := t.TimeNow()
|
||||
if success {
|
||||
_, err = tx.ExecContext(ctx, "UPDATE im_fileuploads SET updatedat=?,status=?,size=?,etag=?,location=? WHERE id=?",
|
||||
now, t.UploadCompleted, size, fd.ETag, fd.Location, store.DecodeUid(fd.Uid()))
|
||||
_, err = tx.ExecContext(ctx, "UPDATE im_fileuploads SET updatedat=?,status=?,size=?,etag=?,location=?,original_name=? WHERE id=?",
|
||||
now, t.UploadCompleted, size, fd.ETag, fd.Location, strings.TrimSpace(fd.OriginalName), store.DecodeUid(fd.Uid()))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -4471,11 +4471,11 @@ func (a *adapter) FileGet(fid string) (*t.FileDef, error) {
|
||||
defer cancel()
|
||||
}
|
||||
var fd t.FileDef
|
||||
err := a.db.GetContext(ctx, &fd, "SELECT id,createdat,updatedat,userid AS user,status,mimetype,size,IFNULL(etag,'') AS etag,location "+
|
||||
err := a.db.GetContext(ctx, &fd, "SELECT id,createdat,updatedat,userid AS user,status,mimetype,size,IFNULL(etag,'') AS etag,location,IFNULL(original_name,'') AS original_name "+
|
||||
"FROM im_fileuploads WHERE id=?", store.DecodeUid(id))
|
||||
if err == sql.ErrNoRows {
|
||||
storageName := id.String32()
|
||||
err = a.db.GetContext(ctx, &fd, "SELECT id,createdat,updatedat,userid AS user,status,mimetype,size,IFNULL(etag,'') AS etag,location "+
|
||||
err = a.db.GetContext(ctx, &fd, "SELECT id,createdat,updatedat,userid AS user,status,mimetype,size,IFNULL(etag,'') AS etag,location,IFNULL(original_name,'') AS original_name "+
|
||||
"FROM im_fileuploads WHERE location=? OR location LIKE ? ORDER BY createdat DESC LIMIT 1", storageName, "%/"+storageName)
|
||||
}
|
||||
if err == sql.ErrNoRows {
|
||||
@@ -4836,6 +4836,27 @@ func isDupe(err error) bool {
|
||||
return ok && myerr.Number == 1062
|
||||
}
|
||||
|
||||
func isInvalidConnection(err error) bool {
|
||||
return errors.Is(err, ms.ErrInvalidConn)
|
||||
}
|
||||
|
||||
// retryInvalidConnection retries only operations which are independently
|
||||
// idempotent. Callers must not use it for unconstrained writes.
|
||||
func retryInvalidConnection(operation string, fn func() error) error {
|
||||
err := fn()
|
||||
if !isInvalidConnection(err) {
|
||||
return err
|
||||
}
|
||||
immetrics.IncDBInvalidConnectionRetry(operation, "attempt")
|
||||
err = fn()
|
||||
if err == nil || isDupe(err) {
|
||||
immetrics.IncDBInvalidConnectionRetry(operation, "recovered")
|
||||
} else {
|
||||
immetrics.IncDBInvalidConnectionRetry(operation, "failed")
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func isMissingTable(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
|
||||
@@ -4,6 +4,7 @@ package mysql
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -150,3 +151,44 @@ func TestTenantShardSQL(t *testing.T) {
|
||||
t.Fatalf("tenant shard query rewrote index name: %s", query)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSharedDBGroupFileLinkIncludesTenantID(t *testing.T) {
|
||||
source, err := os.ReadFile("tenant_groups.go")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body := sourceBetween(t, string(source), "func (a *adapter) TenantGroupFileCreate", "\nfunc (a *adapter) enrichGroupFileFromUpload")
|
||||
for _, required := range []string{
|
||||
"INSERT INTO im_filemsglinks(tenant_id,createdat,fileid,topic)",
|
||||
"SELECT ?,?,?,? WHERE NOT EXISTS",
|
||||
"SELECT 1 FROM im_filemsglinks WHERE tenant_id=? AND fileid=? AND topic=?)",
|
||||
} {
|
||||
if !strings.Contains(body, required) {
|
||||
t.Fatalf("TenantGroupFileCreate shared-db file link must include tenant_id; missing %q", required)
|
||||
}
|
||||
}
|
||||
if strings.Contains(body, "INSERT INTO im_filemsglinks(createdat,fileid,topic)") {
|
||||
t.Fatal("TenantGroupFileCreate must not rely on im_filemsglinks.tenant_id default")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTenantUsersShareOpenPrivateGroupChecksBothMembersAndSetting(t *testing.T) {
|
||||
source, err := os.ReadFile("tenant_groups.go")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body := sourceBetween(t, string(source), "func (a *adapter) TenantUsersShareOpenPrivateGroup", "\nfunc (a *adapter) TenantGroupMemberUpdate")
|
||||
for _, required := range []string{
|
||||
"INNER JOIN im_group_members AS gm2",
|
||||
"gm2.topic=gm1.topic",
|
||||
"INNER JOIN im_group_settings AS gs",
|
||||
"gs.member_private_chat=1",
|
||||
"gm1.member_status=?",
|
||||
"gm2.member_status=?",
|
||||
"gs.group_status=?",
|
||||
} {
|
||||
if !strings.Contains(body, required) {
|
||||
t.Fatalf("TenantUsersShareOpenPrivateGroup must check active shared group and setting; missing %q", required)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
//go:build mysql
|
||||
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql/driver"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
ms "github.com/go-sql-driver/mysql"
|
||||
)
|
||||
|
||||
func TestRetryInvalidConnectionRetriesOnce(t *testing.T) {
|
||||
attempts := 0
|
||||
err := retryInvalidConnection("test", func() error {
|
||||
attempts++
|
||||
if attempts == 1 {
|
||||
return ms.ErrInvalidConn
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil || attempts != 2 {
|
||||
t.Fatalf("err=%v attempts=%d", err, attempts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetryInvalidConnectionDoesNotRetryOtherErrors(t *testing.T) {
|
||||
want := errors.New("write failed")
|
||||
attempts := 0
|
||||
err := retryInvalidConnection("test", func() error {
|
||||
attempts++
|
||||
return want
|
||||
})
|
||||
if !errors.Is(err, want) || attempts != 1 {
|
||||
t.Fatalf("err=%v attempts=%d", err, attempts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInstrumentedQueryDiscardsInvalidConnection(t *testing.T) {
|
||||
conn := instrumentedConn{conn: invalidQueryConn{}}
|
||||
_, err := conn.QueryContext(context.Background(), "SELECT 1", nil)
|
||||
if !errors.Is(err, driver.ErrBadConn) {
|
||||
t.Fatalf("err=%v want=%v", err, driver.ErrBadConn)
|
||||
}
|
||||
}
|
||||
|
||||
type invalidQueryConn struct{}
|
||||
|
||||
func (invalidQueryConn) Prepare(string) (driver.Stmt, error) { return nil, driver.ErrSkip }
|
||||
func (invalidQueryConn) Close() error { return nil }
|
||||
func (invalidQueryConn) Begin() (driver.Tx, error) { return nil, driver.ErrSkip }
|
||||
func (invalidQueryConn) QueryContext(context.Context, string, []driver.NamedValue) (driver.Rows, error) {
|
||||
return nil, ms.ErrInvalidConn
|
||||
}
|
||||
@@ -113,6 +113,12 @@ func (c instrumentedConn) QueryContext(ctx context.Context, query string, args [
|
||||
started := time.Now()
|
||||
rows, err := queryer.QueryContext(ctx, query, args)
|
||||
immetrics.ObserveDB("mysql", "query", started, err)
|
||||
if isInvalidConnection(err) {
|
||||
// Queries are idempotent. Returning ErrBadConn makes database/sql discard
|
||||
// this connection and transparently retry on a fresh one.
|
||||
immetrics.IncDBInvalidConnectionRetry("query", "discard")
|
||||
return nil, driver.ErrBadConn
|
||||
}
|
||||
return rows, err
|
||||
}
|
||||
|
||||
@@ -211,6 +217,10 @@ func (s instrumentedStmt) QueryContext(ctx context.Context, args []driver.NamedV
|
||||
started := time.Now()
|
||||
rows, err := queryer.QueryContext(ctx, args)
|
||||
immetrics.ObserveDB("mysql", "query", started, err)
|
||||
if isInvalidConnection(err) {
|
||||
immetrics.IncDBInvalidConnectionRetry("prepared_query", "discard")
|
||||
return nil, driver.ErrBadConn
|
||||
}
|
||||
return rows, err
|
||||
}
|
||||
|
||||
|
||||
@@ -2233,21 +2233,29 @@ func (a *adapter) TenantMessageSave(id t.TenantID, msg *t.Message) error {
|
||||
if a.tenantScopedDB {
|
||||
return a.MessageSave(msg)
|
||||
}
|
||||
save := func() error {
|
||||
ctx, cancel := a.getContext()
|
||||
if cancel != nil {
|
||||
defer cancel()
|
||||
}
|
||||
clientMsgID := sql.NullString{String: msg.ClientMsgID, Valid: msg.ClientMsgID != ""}
|
||||
res, err := a.db.ExecContext(ctx,
|
||||
res, saveErr := a.db.ExecContext(ctx,
|
||||
fmt.Sprintf("INSERT INTO %s(tenant_id,createdat,updatedat,seqid,topic,`from`,client_msg_id,version,head,content,content_nonce,key_version) VALUES(?,?,?,?,?,?,?,?,?,?,?,?)", tenantMessagesTable(id)),
|
||||
id, msg.CreatedAt, msg.UpdatedAt, msg.SeqId, msg.Topic, decodeUID(t.ParseUid(msg.From)), clientMsgID, messageVersion(msg.Version), msg.Head, common.ToJSON(msg.Content), msg.ContentNonce, msg.KeyVersion)
|
||||
if isDupe(err) {
|
||||
return t.ErrDuplicate
|
||||
}
|
||||
if err == nil {
|
||||
if saveErr == nil {
|
||||
lastID, _ := res.LastInsertId()
|
||||
msg.SetUid(t.Uid(lastID))
|
||||
}
|
||||
return saveErr
|
||||
}
|
||||
if msg.ClientMsgID == "" {
|
||||
err = save()
|
||||
} else {
|
||||
err = retryInvalidConnection("tenant_message_save", save)
|
||||
}
|
||||
if isDupe(err) {
|
||||
return t.ErrDuplicate
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -2349,7 +2357,17 @@ func (a *adapter) TenantGroupMessageSave(id t.TenantID, msg *t.Message) error {
|
||||
if tenantAdp != a {
|
||||
return tenantAdp.TenantGroupMessageSave(id, msg)
|
||||
}
|
||||
if err = a.ensureCurrentGroupMessageShardWindow(); err != nil {
|
||||
err = retryInvalidConnection("tenant_group_message_save", func() error {
|
||||
return a.tenantGroupMessageSaveOnce(id, msg)
|
||||
})
|
||||
if isDupe(err) {
|
||||
return t.ErrDuplicate
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (a *adapter) tenantGroupMessageSaveOnce(id t.TenantID, msg *t.Message) error {
|
||||
if err := a.ensureCurrentGroupMessageShardWindow(); err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, cancel := a.getContext()
|
||||
@@ -2398,9 +2416,6 @@ func (a *adapter) TenantGroupMessageSave(id t.TenantID, msg *t.Message) error {
|
||||
id, messageID, msg.Topic, groupID, msg.SeqId, decodeUID(t.ParseUid(msg.From)), clientMsgID, messageHeadType(msg.Head), "normal",
|
||||
contentJSON, msg.Head, messageHeadJSON(msg.Head, "mentioned_user_ids"), messageHeadBool(msg.Head, "mention_all"),
|
||||
0, len(contentJSON), messageVersion(msg.Version), msg.CreatedAt, msg.UpdatedAt, msg.ContentNonce, msg.KeyVersion)
|
||||
if isDupe(err) {
|
||||
return t.ErrDuplicate
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -5200,8 +5215,8 @@ func (a *adapter) TenantFileStartUpload(id t.TenantID, fd *t.FileDef) error {
|
||||
defer cancel()
|
||||
}
|
||||
_, err = a.db.ExecContext(ctx,
|
||||
"INSERT INTO im_fileuploads(tenant_id,id,createdat,updatedat,userid,status,mimetype,size,etag,location) VALUES(?,?,?,?,?,?,?,?,?,?)",
|
||||
id, decodeUID(fd.Uid()), fd.CreatedAt, fd.UpdatedAt, user, fd.Status, fd.MimeType, fd.Size, fd.ETag, fd.Location)
|
||||
"INSERT INTO im_fileuploads(tenant_id,id,createdat,updatedat,userid,status,mimetype,size,etag,location,original_name) VALUES(?,?,?,?,?,?,?,?,?,?,?)",
|
||||
id, decodeUID(fd.Uid()), fd.CreatedAt, fd.UpdatedAt, user, fd.Status, fd.MimeType, fd.Size, fd.ETag, fd.Location, strings.TrimSpace(fd.OriginalName))
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -5232,8 +5247,8 @@ func (a *adapter) TenantFileFinishUpload(id t.TenantID, fd *t.FileDef, success b
|
||||
}()
|
||||
now := t.TimeNow()
|
||||
if success {
|
||||
_, err = tx.Exec("UPDATE im_fileuploads SET updatedat=?,status=?,size=?,etag=?,location=? WHERE tenant_id=? AND id=?",
|
||||
now, t.UploadCompleted, size, fd.ETag, fd.Location, id, decodeUID(fd.Uid()))
|
||||
_, err = tx.Exec("UPDATE im_fileuploads SET updatedat=?,status=?,size=?,etag=?,location=?,original_name=? WHERE tenant_id=? AND id=?",
|
||||
now, t.UploadCompleted, size, fd.ETag, fd.Location, strings.TrimSpace(fd.OriginalName), id, decodeUID(fd.Uid()))
|
||||
if err == nil {
|
||||
fd.UpdatedAt = now
|
||||
fd.Size = size
|
||||
@@ -5274,12 +5289,12 @@ func (a *adapter) TenantFileGet(id t.TenantID, fid string) (*t.FileDef, error) {
|
||||
}
|
||||
var fd t.FileDef
|
||||
err = a.db.GetContext(ctx, &fd,
|
||||
"SELECT tenant_id,id,createdat,updatedat,userid AS user,status,mimetype,size,IFNULL(etag,'') AS etag,location "+
|
||||
"SELECT tenant_id,id,createdat,updatedat,userid AS user,status,mimetype,size,IFNULL(etag,'') AS etag,location,IFNULL(original_name,'') AS original_name "+
|
||||
"FROM im_fileuploads WHERE tenant_id=? AND id=?", id, decodeUID(fileID))
|
||||
if err == sql.ErrNoRows {
|
||||
storageName := fileID.String32()
|
||||
err = a.db.GetContext(ctx, &fd,
|
||||
"SELECT tenant_id,id,createdat,updatedat,userid AS user,status,mimetype,size,IFNULL(etag,'') AS etag,location "+
|
||||
"SELECT tenant_id,id,createdat,updatedat,userid AS user,status,mimetype,size,IFNULL(etag,'') AS etag,location,IFNULL(original_name,'') AS original_name "+
|
||||
"FROM im_fileuploads WHERE tenant_id=? AND (location=? OR location LIKE ?) ORDER BY createdat DESC LIMIT 1",
|
||||
id, storageName, "%/"+storageName)
|
||||
}
|
||||
@@ -6899,8 +6914,12 @@ func (a *adapter) TenantFriendRequestResolve(id t.TenantID, requestID string, ac
|
||||
defer cancel()
|
||||
}
|
||||
req, err := a.friendRequestByRequestID(ctx, id, requestID)
|
||||
if err != nil || req == nil {
|
||||
return req, err
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if req == nil {
|
||||
// 好友申请不存在必须返回 ErrNotFound，否则上层拿到 (nil,nil) 会解引用 panic（网关 502）。
|
||||
return nil, t.ErrNotFound
|
||||
}
|
||||
if req.TargetID != actor {
|
||||
return nil, t.ErrPermissionDenied
|
||||
|
||||
@@ -394,6 +394,39 @@ func (a *adapter) TenantGroupMemberList(id t.TenantID, topic string, opts t.Grou
|
||||
return items, next, rows.Err()
|
||||
}
|
||||
|
||||
func (a *adapter) TenantUsersShareOpenPrivateGroup(id t.TenantID, userID, peerID t.Uid) (bool, error) {
|
||||
if tenantAdp, err := a.routeGroupTenant(id); err != nil {
|
||||
return false, err
|
||||
} else if tenantAdp != a {
|
||||
return tenantAdp.TenantUsersShareOpenPrivateGroup(id, userID, peerID)
|
||||
}
|
||||
if userID.IsZero() || peerID.IsZero() || userID == peerID {
|
||||
return false, t.ErrMalformed
|
||||
}
|
||||
ctx, cancel := a.getContext()
|
||||
if cancel != nil {
|
||||
defer cancel()
|
||||
}
|
||||
var exists int
|
||||
err := a.db.GetContext(ctx, &exists, tenantShardSQL(id, `
|
||||
SELECT 1
|
||||
FROM im_group_members AS gm1
|
||||
INNER JOIN im_group_members AS gm2
|
||||
ON gm2.tenant_id=gm1.tenant_id AND gm2.topic=gm1.topic
|
||||
INNER JOIN im_group_settings AS gs
|
||||
ON gs.tenant_id=gm1.tenant_id AND gs.topic=gm1.topic
|
||||
WHERE gm1.tenant_id=? AND gm1.user_id=? AND gm1.member_status=?
|
||||
AND gm2.user_id=? AND gm2.member_status=?
|
||||
AND gs.group_status=? AND gs.member_private_chat=1
|
||||
LIMIT 1`),
|
||||
id, decodeUID(userID), t.GroupMemberStatusActive,
|
||||
decodeUID(peerID), t.GroupMemberStatusActive, t.GroupStatusActive)
|
||||
if err == sql.ErrNoRows {
|
||||
return false, nil
|
||||
}
|
||||
return exists == 1, err
|
||||
}
|
||||
|
||||
func (a *adapter) TenantGroupMemberUpdate(id t.TenantID, topic string, uid t.Uid, patch t.GroupMemberPatch) (*t.GroupMember, error) {
|
||||
if tenantAdp, err := a.routeGroupTenant(id); err != nil {
|
||||
return nil, err
|
||||
@@ -1132,10 +1165,10 @@ func (a *adapter) TenantGroupFileCreate(id t.TenantID, file *t.GroupFile) (*t.Gr
|
||||
SELECT 1 FROM im_filemsglinks WHERE tenant_id=? AND fileid=? AND topic=?)`,
|
||||
id, now, decodeUID(fid), file.Topic, id, decodeUID(fid), file.Topic)
|
||||
} else {
|
||||
_, err = tx.ExecContext(ctx, `INSERT INTO im_filemsglinks(createdat,fileid,topic)
|
||||
SELECT ?,?,? WHERE NOT EXISTS (
|
||||
SELECT 1 FROM im_filemsglinks WHERE fileid=? AND topic=?)`,
|
||||
now, decodeUID(fid), file.Topic, decodeUID(fid), file.Topic)
|
||||
_, err = tx.ExecContext(ctx, `INSERT INTO im_filemsglinks(tenant_id,createdat,fileid,topic)
|
||||
SELECT ?,?,?,? WHERE NOT EXISTS (
|
||||
SELECT 1 FROM im_filemsglinks WHERE tenant_id=? AND fileid=? AND topic=?)`,
|
||||
id, now, decodeUID(fid), file.Topic, id, decodeUID(fid), file.Topic)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
+126
-26
@@ -11,26 +11,52 @@ package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/websocket"
|
||||
"github.com/tinode/chat/server/logs"
|
||||
immetrics "github.com/tinode/chat/server/metrics"
|
||||
"github.com/tinode/chat/server/store/types"
|
||||
)
|
||||
|
||||
const (
|
||||
// Time allowed to write a message to the peer.
|
||||
writeWait = 10 * time.Second
|
||||
|
||||
// Time allowed to read the next pong message from the peer.
|
||||
pongWait = idleSessionTimeout
|
||||
|
||||
// Send pings to peer with this period. Must be less than pongWait.
|
||||
pingPeriod = 15 * time.Second
|
||||
// Heartbeat defaults tolerate transient mobile and public-network stalls while
|
||||
// still detecting dead connections in a bounded amount of time.
|
||||
defaultWSPongWait = 90 * time.Second
|
||||
defaultWSPingPeriod = 15 * time.Second
|
||||
)
|
||||
|
||||
var (
|
||||
pongWait = defaultWSPongWait
|
||||
pingPeriod = defaultWSPingPeriod
|
||||
)
|
||||
|
||||
func configureWSHeartbeat(pingSeconds, pongSeconds int) error {
|
||||
ping := defaultWSPingPeriod
|
||||
pong := defaultWSPongWait
|
||||
if pingSeconds > 0 {
|
||||
ping = time.Duration(pingSeconds) * time.Second
|
||||
}
|
||||
if pongSeconds > 0 {
|
||||
pong = time.Duration(pongSeconds) * time.Second
|
||||
}
|
||||
if ping >= pong {
|
||||
return errors.New("websocket ping period must be shorter than pong wait")
|
||||
}
|
||||
pingPeriod = ping
|
||||
pongWait = pong
|
||||
immetrics.SetWSHeartbeatConfig(ping, pong)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (sess *Session) closeWS() {
|
||||
if sess.proto == WEBSOCK {
|
||||
sess.ws.Close()
|
||||
@@ -44,41 +70,55 @@ func (sess *Session) readLoop() {
|
||||
}()
|
||||
|
||||
sess.ws.SetReadLimit(globals.maxMessageSize)
|
||||
sess.ws.SetReadDeadline(time.Now().Add(pongWait))
|
||||
sess.ws.SetPongHandler(func(string) error {
|
||||
refreshReadDeadline := func(event, presenceSource string) {
|
||||
atomic.StoreInt64(&sess.lastAction, types.TimeNow().UnixNano())
|
||||
sess.refreshPresence("ws.pong")
|
||||
sess.ws.SetReadDeadline(time.Now().Add(pongWait))
|
||||
immetrics.IncWSHeartbeat(event)
|
||||
if presenceSource != "" {
|
||||
sess.refreshPresence(presenceSource)
|
||||
}
|
||||
_ = sess.ws.SetReadDeadline(time.Now().Add(pongWait))
|
||||
}
|
||||
_ = sess.ws.SetReadDeadline(time.Now().Add(pongWait))
|
||||
sess.ws.SetPongHandler(func(string) error {
|
||||
refreshReadDeadline("pong_received", "ws.pong")
|
||||
return nil
|
||||
})
|
||||
sess.ws.SetPingHandler(func(appData string) error {
|
||||
refreshReadDeadline("ping_received", "")
|
||||
return sess.ws.WriteControl(websocket.PongMessage, []byte(appData), time.Now().Add(writeWait))
|
||||
})
|
||||
|
||||
for {
|
||||
// Read a ClientComMessage
|
||||
_, raw, err := sess.ws.ReadMessage()
|
||||
if err != nil {
|
||||
if websocket.IsUnexpectedCloseError(err, websocket.CloseGoingAway, websocket.CloseAbnormalClosure,
|
||||
websocket.CloseNormalClosure) {
|
||||
logs.Err.Println("ws: readLoop", sess.sid, err)
|
||||
reason := classifyWSError(err)
|
||||
immetrics.IncWSDisconnect(reason)
|
||||
if websocket.IsUnexpectedCloseError(err, websocket.CloseGoingAway, websocket.CloseNormalClosure) {
|
||||
logs.Err.Printf("ws: readLoop sid=%s uid=%s reason=%s err=%v", sess.sid, sess.uid.UserId(), reason, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
// Any successfully read client frame proves that the connection is alive.
|
||||
// This prevents a busy client from timing out merely because its Pong was
|
||||
// delayed behind application traffic.
|
||||
atomic.StoreInt64(&sess.lastAction, types.TimeNow().UnixNano())
|
||||
_ = sess.ws.SetReadDeadline(time.Now().Add(pongWait))
|
||||
statsInc("IncomingMessagesWebsockTotal", 1)
|
||||
sess.dispatchRaw(raw)
|
||||
}
|
||||
}
|
||||
|
||||
func (sess *Session) sendMessage(msg any) bool {
|
||||
if len(sess.send) > sendQueueLimit {
|
||||
logs.Err.Println("ws: outbound queue limit exceeded", sess.sid)
|
||||
if depth := len(sess.send); depth > sendQueueLimit {
|
||||
immetrics.ObserveWSWrite("text", "rejected", "queue_limit", time.Now(), 0)
|
||||
logs.Err.Printf("ws: outbound queue limit exceeded sid=%s uid=%s depth=%d capacity=%d",
|
||||
sess.sid, sess.uid.UserId(), depth, cap(sess.send))
|
||||
return false
|
||||
}
|
||||
|
||||
statsInc("OutgoingMessagesWebsockTotal", 1)
|
||||
if err := wsWrite(sess.ws, websocket.TextMessage, msg); err != nil {
|
||||
if websocket.IsUnexpectedCloseError(err, websocket.CloseGoingAway, websocket.CloseAbnormalClosure,
|
||||
websocket.CloseNormalClosure) {
|
||||
logs.Err.Println("ws: writeLoop", sess.sid, err)
|
||||
}
|
||||
if err := sess.writeWSFrame(websocket.TextMessage, msg); err != nil {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
@@ -100,6 +140,7 @@ func (sess *Session) writeLoop() {
|
||||
// Channel closed.
|
||||
return
|
||||
}
|
||||
sess.observeOutboundDequeue(false)
|
||||
switch v := msg.(type) {
|
||||
case []*ServerComMessage: // batch of unserialized messages
|
||||
for _, msg := range v {
|
||||
@@ -107,12 +148,18 @@ func (sess *Session) writeLoop() {
|
||||
if !sess.sendMessage(w) {
|
||||
return
|
||||
}
|
||||
if isAcceptedCtrl(msg) {
|
||||
immetrics.ObserveCtrlACKDelivery("write", msg.Timestamp)
|
||||
}
|
||||
}
|
||||
case *ServerComMessage: // single unserialized message
|
||||
w := sess.serializeAndUpdateStats(v)
|
||||
if !sess.sendMessage(w) {
|
||||
return
|
||||
}
|
||||
if isAcceptedCtrl(v) {
|
||||
immetrics.ObserveCtrlACKDelivery("write", v.Timestamp)
|
||||
}
|
||||
default: // serialized message
|
||||
if !sess.sendMessage(v) {
|
||||
return
|
||||
@@ -128,7 +175,7 @@ func (sess *Session) writeLoop() {
|
||||
case msg := <-sess.stop:
|
||||
// Shutdown requested, don't care if the message is delivered
|
||||
if msg != nil {
|
||||
wsWrite(sess.ws, websocket.TextMessage, msg)
|
||||
sess.writeWSFrame(websocket.TextMessage, msg)
|
||||
}
|
||||
return
|
||||
|
||||
@@ -136,17 +183,70 @@ func (sess *Session) writeLoop() {
|
||||
sess.delSub(topic)
|
||||
|
||||
case <-ticker.C:
|
||||
if err := wsWrite(sess.ws, websocket.PingMessage, nil); err != nil {
|
||||
if websocket.IsUnexpectedCloseError(err, websocket.CloseGoingAway, websocket.CloseAbnormalClosure,
|
||||
websocket.CloseNormalClosure) {
|
||||
logs.Err.Println("ws: writeLoop ping", sess.sid, err)
|
||||
}
|
||||
if err := sess.writeWSFrame(websocket.PingMessage, nil); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (sess *Session) writeWSFrame(mt int, msg any) error {
|
||||
frame := "text"
|
||||
if mt == websocket.PingMessage {
|
||||
frame = "ping"
|
||||
}
|
||||
payloadBytes := 0
|
||||
if bits, ok := msg.([]byte); ok {
|
||||
payloadBytes = len(bits)
|
||||
}
|
||||
started := time.Now()
|
||||
err := wsWrite(sess.ws, mt, msg)
|
||||
if err == nil {
|
||||
immetrics.ObserveWSWrite(frame, "ok", "none", started, payloadBytes)
|
||||
return nil
|
||||
}
|
||||
reason := classifyWSError(err)
|
||||
immetrics.ObserveWSWrite(frame, "error", reason, started, payloadBytes)
|
||||
logs.Err.Printf("ws: writeLoop sid=%s uid=%s frame=%s queue=%d capacity=%d reason=%s err=%v",
|
||||
sess.sid, sess.uid.UserId(), frame, len(sess.send), cap(sess.send), reason, err)
|
||||
return err
|
||||
}
|
||||
|
||||
func classifyWSError(err error) string {
|
||||
if err == nil {
|
||||
return "none"
|
||||
}
|
||||
var closeErr *websocket.CloseError
|
||||
if errors.As(err, &closeErr) {
|
||||
switch closeErr.Code {
|
||||
case websocket.CloseNormalClosure:
|
||||
return "close_normal"
|
||||
case websocket.CloseGoingAway:
|
||||
return "close_going_away"
|
||||
case websocket.CloseAbnormalClosure:
|
||||
return "close_abnormal"
|
||||
default:
|
||||
return "close_other"
|
||||
}
|
||||
}
|
||||
var netErr net.Error
|
||||
if errors.As(err, &netErr) && netErr.Timeout() {
|
||||
return "timeout"
|
||||
}
|
||||
switch {
|
||||
case errors.Is(err, syscall.ECONNRESET):
|
||||
return "connection_reset"
|
||||
case errors.Is(err, syscall.EPIPE):
|
||||
return "broken_pipe"
|
||||
case errors.Is(err, net.ErrClosed):
|
||||
return "closed"
|
||||
case errors.Is(err, io.EOF):
|
||||
return "eof"
|
||||
default:
|
||||
return "other"
|
||||
}
|
||||
}
|
||||
|
||||
// Writes a message with the given message type (mt) and payload.
|
||||
func wsWrite(ws *websocket.Conn, mt int, msg any) error {
|
||||
var bits []byte
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestConfigureWSHeartbeat(t *testing.T) {
|
||||
originalPing, originalPong := pingPeriod, pongWait
|
||||
t.Cleanup(func() {
|
||||
pingPeriod, pongWait = originalPing, originalPong
|
||||
})
|
||||
|
||||
if err := configureWSHeartbeat(20, 120); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if pingPeriod != 20*time.Second || pongWait != 120*time.Second {
|
||||
t.Fatalf("heartbeat ping=%s pong=%s", pingPeriod, pongWait)
|
||||
}
|
||||
if err := configureWSHeartbeat(90, 90); err == nil {
|
||||
t.Fatal("expected invalid heartbeat configuration to fail")
|
||||
}
|
||||
if err := configureWSHeartbeat(0, 0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if pingPeriod != defaultWSPingPeriod || pongWait != defaultWSPongWait {
|
||||
t.Fatalf("default heartbeat ping=%s pong=%s", pingPeriod, pongWait)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"os"
|
||||
"syscall"
|
||||
"testing"
|
||||
|
||||
"github.com/gorilla/websocket"
|
||||
)
|
||||
|
||||
func TestClassifyWSError(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
err error
|
||||
want string
|
||||
}{
|
||||
{name: "none", want: "none"},
|
||||
{name: "normal close", err: &websocket.CloseError{Code: websocket.CloseNormalClosure}, want: "close_normal"},
|
||||
{name: "abnormal close", err: &websocket.CloseError{Code: websocket.CloseAbnormalClosure}, want: "close_abnormal"},
|
||||
{name: "timeout", err: &net.OpError{Op: "write", Err: os.ErrDeadlineExceeded}, want: "timeout"},
|
||||
{name: "reset", err: fmt.Errorf("write failed: %w", syscall.ECONNRESET), want: "connection_reset"},
|
||||
{name: "broken pipe", err: fmt.Errorf("write failed: %w", syscall.EPIPE), want: "broken_pipe"},
|
||||
{name: "closed", err: net.ErrClosed, want: "closed"},
|
||||
{name: "eof", err: io.EOF, want: "eof"},
|
||||
{name: "other", err: fmt.Errorf("unknown"), want: "other"},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := classifyWSError(tc.err); got != tc.want {
|
||||
t.Fatalf("classifyWSError()=%q want=%q", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestWebSocketOutboundQueueTiming(t *testing.T) {
|
||||
sess := &Session{proto: WEBSOCK, send: make(chan any, 2)}
|
||||
payload := []byte("message")
|
||||
if !sess.queueOutBytes(payload) {
|
||||
t.Fatal("queueOutBytes rejected payload")
|
||||
}
|
||||
|
||||
queued := <-sess.send
|
||||
sess.observeOutboundDequeue(false)
|
||||
if string(queued.([]byte)) != string(payload) {
|
||||
t.Fatalf("payload=%q want=%q", queued, payload)
|
||||
}
|
||||
}
|
||||
+81
-1
@@ -57,6 +57,12 @@ type userStatusReq struct {
|
||||
state types.ObjState
|
||||
}
|
||||
|
||||
type deferredPresenceKey struct {
|
||||
tenantID types.TenantID
|
||||
topic string
|
||||
source string
|
||||
}
|
||||
|
||||
// Hub is the core structure which holds topics.
|
||||
type Hub struct {
|
||||
|
||||
@@ -74,6 +80,9 @@ type Hub struct {
|
||||
|
||||
// Channel for routing server-generated messages, buffered at 4096
|
||||
routeSrv chan *ServerComMessage
|
||||
// Presence updates which could not immediately enter a topic queue.
|
||||
deferredPresence chan *ServerComMessage
|
||||
deferredPresenceStop chan struct{}
|
||||
|
||||
// subscribe session to topic, possibly creating a new topic, buffered at 256
|
||||
join chan *ClientComMessage
|
||||
@@ -123,6 +132,8 @@ func newHub() *Hub {
|
||||
// TODO: verify if these channels have to be buffered.
|
||||
routeCli: make(chan *ClientComMessage, 4096),
|
||||
routeSrv: make(chan *ServerComMessage, 4096),
|
||||
deferredPresence: make(chan *ServerComMessage, 8192),
|
||||
deferredPresenceStop: make(chan struct{}),
|
||||
join: make(chan *ClientComMessage, 256),
|
||||
unreg: make(chan *topicUnreg, 256),
|
||||
rehash: make(chan bool),
|
||||
@@ -154,11 +165,70 @@ func newHub() *Hub {
|
||||
statsRegisterHistogram("RequestLatency", requestLatencyDistribution)
|
||||
statsRegisterHistogram("OutgoingMessageSize", outgoingMessageSizeDistribution)
|
||||
|
||||
go h.runDeferredPresence()
|
||||
go h.run()
|
||||
|
||||
return h
|
||||
}
|
||||
|
||||
func (h *Hub) deferPresence(msg *ServerComMessage) bool {
|
||||
select {
|
||||
case h.deferredPresence <- msg:
|
||||
immetrics.ObserveInternalQueueEnqueue("topic_presence_deferred", len(h.deferredPresence), cap(h.deferredPresence), true)
|
||||
return true
|
||||
default:
|
||||
immetrics.ObserveInternalQueueEnqueue("topic_presence_deferred", len(h.deferredPresence), cap(h.deferredPresence), false)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (h *Hub) runDeferredPresence() {
|
||||
const maxPending = 16384
|
||||
ticker := time.NewTicker(5 * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
pending := make(map[deferredPresenceKey]*ServerComMessage)
|
||||
|
||||
for {
|
||||
select {
|
||||
case msg := <-h.deferredPresence:
|
||||
if msg == nil || msg.Pres == nil {
|
||||
continue
|
||||
}
|
||||
key := deferredPresenceKey{tenantID: msg.TenantID, topic: msg.RcptTo, source: msg.Pres.Src}
|
||||
if _, exists := pending[key]; exists || len(pending) < maxPending {
|
||||
pending[key] = msg
|
||||
} else {
|
||||
immetrics.ObserveInternalQueueEnqueue("topic_presence_pending", len(pending), maxPending, false)
|
||||
logs.Err.Printf("hub: deferred presence queue full topic=%s tenant=%d source=%s",
|
||||
msg.RcptTo, msg.TenantID, msg.Pres.Src)
|
||||
}
|
||||
immetrics.SetInternalQueueCurrent("topic_presence_pending", len(pending))
|
||||
case <-ticker.C:
|
||||
budget := 64
|
||||
for key, msg := range pending {
|
||||
dst := h.topicGet(types.TopicKey{TenantID: key.tenantID, Topic: key.topic})
|
||||
if dst == nil || dst.serverMsg == nil {
|
||||
delete(pending, key)
|
||||
continue
|
||||
}
|
||||
select {
|
||||
case dst.serverMsg <- msg:
|
||||
immetrics.ObserveInternalQueueEnqueue("topic_server", len(dst.serverMsg), cap(dst.serverMsg), true)
|
||||
delete(pending, key)
|
||||
budget--
|
||||
default:
|
||||
}
|
||||
if budget == 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
immetrics.SetInternalQueueCurrent("topic_presence_pending", len(pending))
|
||||
case <-h.deferredPresenceStop:
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (h *Hub) run() {
|
||||
for {
|
||||
select {
|
||||
@@ -246,12 +316,17 @@ func (h *Hub) run() {
|
||||
if dst := h.topicGet(key); dst != nil {
|
||||
// Everything is OK, sending packet to known topic
|
||||
if dst.clientMsg != nil {
|
||||
msg.queuedAt = time.Now()
|
||||
select {
|
||||
case dst.clientMsg <- msg:
|
||||
immetrics.ObserveInternalQueueEnqueue("topic_client", len(dst.clientMsg), cap(dst.clientMsg), true)
|
||||
default:
|
||||
immetrics.ObserveInternalQueueEnqueue("topic_client", len(dst.clientMsg), cap(dst.clientMsg), false)
|
||||
logs.Err.Println("hub: topic's broadcast queue is full", dst.name)
|
||||
if msg.sess != nil {
|
||||
msg.sess.queueOut(ErrServiceUnavailableReply(msg, types.TimeNow()))
|
||||
}
|
||||
logs.Err.Printf("hub: topic's broadcast queue is full topic=%s tenant=%d len=%d cap=%d",
|
||||
dst.name, dst.tenantID, len(dst.clientMsg), cap(dst.clientMsg))
|
||||
}
|
||||
} else {
|
||||
logs.Warn.Println("hub: invalid topic category for broadcast", dst.name)
|
||||
@@ -280,9 +355,11 @@ func (h *Hub) run() {
|
||||
case dst.serverMsg <- msg:
|
||||
immetrics.ObserveInternalQueueEnqueue("topic_server", len(dst.serverMsg), cap(dst.serverMsg), true)
|
||||
default:
|
||||
if msg.Pres == nil || !h.deferPresence(msg) {
|
||||
immetrics.ObserveInternalQueueEnqueue("topic_server", len(dst.serverMsg), cap(dst.serverMsg), false)
|
||||
logs.Err.Println("hub: topic's broadcast queue is full", dst.name)
|
||||
}
|
||||
}
|
||||
} else if (strings.HasPrefix(msg.RcptTo, "usr") || strings.HasPrefix(msg.RcptTo, "grp")) &&
|
||||
globals.cluster.isRemoteTopic(key) {
|
||||
// It is a remote topic.
|
||||
@@ -353,6 +430,9 @@ func (h *Hub) run() {
|
||||
}
|
||||
|
||||
logs.Info.Printf("Hub shutdown completed with %d topics", topicCount)
|
||||
if h.deferredPresenceStop != nil {
|
||||
close(h.deferredPresenceStop)
|
||||
}
|
||||
|
||||
// let the main goroutine know we are done with the cleanup
|
||||
hubdone <- true
|
||||
|
||||
@@ -102,3 +102,48 @@ func TestHubRejectsUnroutedClientMessageInsteadOfAcceptingWithoutSeq(t *testing.
|
||||
t.Fatal("timeout waiting for unrouted message response")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHubReturnsServiceUnavailableWhenTopicClientQueueIsFull(t *testing.T) {
|
||||
h := &Hub{
|
||||
topics: &sync.Map{},
|
||||
routeCli: make(chan *ClientComMessage, 1),
|
||||
shutdown: make(chan chan<- bool),
|
||||
routeSrv: make(chan *ServerComMessage),
|
||||
join: make(chan *ClientComMessage),
|
||||
meta: make(chan *ClientComMessage),
|
||||
unreg: make(chan *topicUnreg),
|
||||
userStatus: make(chan *userStatusReq),
|
||||
rehash: make(chan bool),
|
||||
}
|
||||
topic := &Topic{
|
||||
tenantID: 7, name: "grpBusy", clientMsg: make(chan *ClientComMessage, 1),
|
||||
exit: make(chan *shutDown, 1),
|
||||
}
|
||||
topic.clientMsg <- &ClientComMessage{Id: "occupy"}
|
||||
h.topicPut(types.TopicKey{TenantID: 7, Topic: topic.name}, topic)
|
||||
|
||||
go h.run()
|
||||
done := make(chan bool)
|
||||
defer func() {
|
||||
h.topicDel(types.TopicKey{TenantID: 7, Topic: topic.name})
|
||||
h.shutdown <- done
|
||||
<-done
|
||||
}()
|
||||
|
||||
sess := &Session{tenantID: 7, send: make(chan any, 1)}
|
||||
now := time.Now().UTC()
|
||||
h.routeCli <- &ClientComMessage{
|
||||
Id: "pub-overload", TenantID: 7, RcptTo: topic.name, Original: topic.name,
|
||||
Timestamp: now, sess: sess,
|
||||
}
|
||||
|
||||
select {
|
||||
case raw := <-sess.send:
|
||||
msg, ok := raw.(*ServerComMessage)
|
||||
if !ok || msg.Ctrl == nil || msg.Ctrl.Code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("unexpected response: %#v", raw)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("timeout waiting for overload response")
|
||||
}
|
||||
}
|
||||
|
||||
+195
-11
@@ -1704,6 +1704,155 @@ func serveIMFeedback(wrt http.ResponseWriter, req *http.Request) {
|
||||
writeIMSuccess(wrt, req, feedbackSubmissionPayload(feedback))
|
||||
}
|
||||
|
||||
type imMePasswordChangeRequest struct {
|
||||
OldPassword string `json:"old_password"`
|
||||
NewPassword string `json:"new_password"`
|
||||
}
|
||||
|
||||
// serveIMMePasswordChange lets an authenticated user change own password by
|
||||
// presenting the current one. All sessions are evicted afterwards.
|
||||
func serveIMMePasswordChange(wrt http.ResponseWriter, req *http.Request) {
|
||||
if req.Method != http.MethodPost {
|
||||
writeIMError(wrt, req, http.StatusMethodNotAllowed, 30002, imReasonMethodNotAllowed, "method not allowed", nil)
|
||||
return
|
||||
}
|
||||
principal, ok := authenticateIMHTTPBearer(wrt, req)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
var body imMePasswordChangeRequest
|
||||
if !decodeIMJSONBody(wrt, req, &body) {
|
||||
return
|
||||
}
|
||||
if strings.TrimSpace(body.OldPassword) == "" || strings.TrimSpace(body.NewPassword) == "" {
|
||||
writeIMError(wrt, req, http.StatusBadRequest, 30000, imReasonMalformedRequest, "malformed request", nil)
|
||||
return
|
||||
}
|
||||
basicAuth := store.Store.GetLogicalAuthHandler("basic")
|
||||
if basicAuth == nil || !basicAuth.IsInitialized() {
|
||||
writeIMError(wrt, req, http.StatusInternalServerError, 90000, imReasonInternal, "internal error", nil)
|
||||
return
|
||||
}
|
||||
login, authLevel, _, _, err := store.Users.GetAuthRecord(principal.TenantID, principal.Uid)
|
||||
if err != nil || login == "" {
|
||||
writeIMError(wrt, req, http.StatusInternalServerError, 90000, imReasonInternal, "internal error", nil)
|
||||
return
|
||||
}
|
||||
authCtx := auth.AuthContext{TenantID: principal.TenantID, RemoteAddr: req.RemoteAddr}
|
||||
if _, _, err = basicAuth.Authenticate(authCtx, []byte(login+":"+body.OldPassword)); err != nil {
|
||||
writeIMError(wrt, req, http.StatusUnauthorized, 30001, imReasonAuthRequired, "invalid credentials", nil)
|
||||
return
|
||||
}
|
||||
if _, err = basicAuth.UpdateRecord(authCtx, &auth.Rec{
|
||||
TenantID: principal.TenantID,
|
||||
Uid: principal.Uid,
|
||||
AuthLevel: authLevel,
|
||||
}, []byte(":"+body.NewPassword)); err != nil {
|
||||
if err == types.ErrPolicy {
|
||||
writeIMError(wrt, req, http.StatusBadRequest, 30040, imReasonPasswordPolicy, "password policy violation", nil)
|
||||
return
|
||||
}
|
||||
writeIMError(wrt, req, http.StatusInternalServerError, 90000, imReasonInternal, "internal error", nil)
|
||||
return
|
||||
}
|
||||
now := time.Now().UTC().Round(time.Millisecond)
|
||||
if store.Security != nil {
|
||||
if _, err = store.Security.BumpAuthVersion(principal.TenantID, principal.Uid, now); err != nil {
|
||||
logs.Warn.Println("im me password change: bump auth version failed", err)
|
||||
writeIMError(wrt, req, http.StatusInternalServerError, 90000, imReasonInternal, "internal error", nil)
|
||||
return
|
||||
}
|
||||
}
|
||||
if globals.sessionStore != nil {
|
||||
globals.sessionStore.EvictUser(principal.Uid, "")
|
||||
}
|
||||
publishRealtimeEvictUser(principal.TenantID, principal.Uid, "password_changed", "")
|
||||
writeIMSuccess(wrt, req, map[string]any{"status": "ok"})
|
||||
}
|
||||
|
||||
// serveIMMeDeletion deletes the authenticated user's account. Requires a
|
||||
// confirmed "delete_account" sensitive-action token obtained through
|
||||
// POST /v1/im/security/challenges + POST /v1/im/security/challenges/{id}.
|
||||
func serveIMMeDeletion(wrt http.ResponseWriter, req *http.Request) {
|
||||
if req.Method != http.MethodPost {
|
||||
writeIMError(wrt, req, http.StatusMethodNotAllowed, 30002, imReasonMethodNotAllowed, "method not allowed", nil)
|
||||
return
|
||||
}
|
||||
principal, ok := authenticateIMHTTPBearer(wrt, req)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if globals.permanentAccounts {
|
||||
writeIMError(wrt, req, http.StatusForbidden, 30040, imReasonForbidden, "account deletion is disabled", nil)
|
||||
return
|
||||
}
|
||||
if !consumeIMActionTokenHeader(wrt, req, principal, "delete_account") {
|
||||
return
|
||||
}
|
||||
tenantID, uid := principal.TenantID, principal.Uid
|
||||
if state, err := userGetState(tenantID, uid); err != nil || state == types.StateDeleted {
|
||||
logs.Warn.Println("im me deletion: user is missing or already deleted", err)
|
||||
writeIMError(wrt, req, http.StatusNotFound, 20004, imReasonResourceNotFound, "user not found", nil)
|
||||
return
|
||||
}
|
||||
|
||||
// Disable all authenticators.
|
||||
for _, name := range store.Store.GetAuthNames() {
|
||||
hdl := store.Store.GetLogicalAuthHandler(name)
|
||||
if !hdl.IsInitialized() {
|
||||
continue
|
||||
}
|
||||
if err := hdl.DelRecords(tenantID, uid); err != nil {
|
||||
// This could be completely benign, i.e. authenticator exists but not used.
|
||||
logs.Warn.Println("im me deletion: failed to delete auth record", uid.UserId(), name, err)
|
||||
if storeErr, ok := err.(types.StoreError); ok && storeErr == types.ErrUnsupported {
|
||||
writeIMError(wrt, req, http.StatusForbidden, 30040, imReasonForbidden, "account cannot be deleted", nil)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Terminate all sessions.
|
||||
if globals.sessionStore != nil {
|
||||
globals.sessionStore.EvictUser(uid, "", "user_deleted")
|
||||
}
|
||||
publishRealtimeEvictUser(tenantID, uid, "user_deleted", "")
|
||||
// Remove user from cache and announce to cluster that the user is deleted.
|
||||
usersRemoveUser(tenantID, uid)
|
||||
|
||||
// Stop topics where the user is the owner and p2p topics.
|
||||
done := make(chan bool)
|
||||
globals.hub.unreg <- &topicUnreg{tenantID: tenantID, forUser: uid, del: false, done: done}
|
||||
<-done
|
||||
|
||||
// Notify users of interest that the user is gone.
|
||||
if uoi, err := store.Users.GetSubs(tenantID, uid); err == nil {
|
||||
presUsersOfInterestOffline(tenantID, uid, uoi, "gone")
|
||||
} else {
|
||||
logs.Warn.Println("im me deletion: failed to send notifications to users", err)
|
||||
}
|
||||
// Notify subscribers of the group topics where the user was the owner.
|
||||
if ownTopics, err := store.Users.GetOwnTopics(tenantID, uid); err == nil {
|
||||
for _, topicName := range ownTopics {
|
||||
if subs, err := store.Topics.GetSubs(tenantID, topicName, nil); err == nil {
|
||||
presSubsOfflineOffline(tenantID, topicName, types.TopicCatGrp, subs, "gone", &presParams{}, "")
|
||||
} else {
|
||||
logs.Warn.Println("im me deletion: failed to notify topic subscribers", err, topicName)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
logs.Warn.Println("im me deletion: failed to send notifications to owned topics", err)
|
||||
}
|
||||
|
||||
// Delete user's records from the database (soft delete).
|
||||
if err := store.Users.Delete(tenantID, uid, false); err != nil {
|
||||
logs.Warn.Println("im me deletion: failed to delete user", err)
|
||||
writeIMError(wrt, req, http.StatusInternalServerError, 90000, imReasonInternal, "internal error", nil)
|
||||
return
|
||||
}
|
||||
writeIMSuccess(wrt, req, map[string]any{"status": "ok"})
|
||||
}
|
||||
|
||||
func serveIMSecurityChallenges(wrt http.ResponseWriter, req *http.Request) {
|
||||
if req.Method != http.MethodPost {
|
||||
writeIMError(wrt, req, http.StatusMethodNotAllowed, 30002, imReasonMethodNotAllowed, "method not allowed", nil)
|
||||
@@ -2488,6 +2637,10 @@ func serveIMSystemNotificationRead(wrt http.ResponseWriter, req *http.Request) {
|
||||
return
|
||||
}
|
||||
readAt = parsed.UTC().Round(time.Millisecond)
|
||||
if readAt.After(time.Now().UTC().Round(time.Millisecond)) {
|
||||
writeIMError(wrt, req, http.StatusBadRequest, 30000, imReasonMalformedRequest, "malformed request", map[string]any{"field": "read_at", "reason": "future_read_at"})
|
||||
return
|
||||
}
|
||||
}
|
||||
notification, err := store.Security.MarkSystemNotificationRead(principal.TenantID, principal.Uid, eventID, readAt)
|
||||
if err != nil {
|
||||
@@ -3873,15 +4026,6 @@ func serveIMFriendRequestCreate(wrt http.ResponseWriter, req *http.Request) {
|
||||
writeIMError(wrt, req, http.StatusForbidden, 30005, imReasonPermissionDenied, "group membership required", nil)
|
||||
return
|
||||
}
|
||||
settings, settingsErr := store.GroupBusiness.GetSettings(principal.TenantID, sourceTopic)
|
||||
if settingsErr != nil {
|
||||
writeIMRelationError(wrt, req, settingsErr)
|
||||
return
|
||||
}
|
||||
if settings == nil || !settings.MemberPrivateChat {
|
||||
writeIMError(wrt, req, http.StatusForbidden, 30005, imReasonPermissionDenied, "member private chat is disabled", nil)
|
||||
return
|
||||
}
|
||||
default:
|
||||
writeIMError(wrt, req, http.StatusBadRequest, 30000, imReasonMalformedRequest, "invalid friend request source", nil)
|
||||
return
|
||||
@@ -3930,6 +4074,12 @@ func serveIMFriendRequestCreate(wrt http.ResponseWriter, req *http.Request) {
|
||||
writeIMRelationError(wrt, req, acceptErr)
|
||||
return
|
||||
}
|
||||
if accepted == nil {
|
||||
logs.Err.Printf("friend request auto-accept returned an empty result: tenant=%d request=%s actor=%s",
|
||||
principal.TenantID, created.RequestID, target.String())
|
||||
writeIMError(wrt, req, http.StatusInternalServerError, 90000, imReasonInternal, "internal error", nil)
|
||||
return
|
||||
}
|
||||
created = accepted
|
||||
}
|
||||
if imReliableHTTPEventsEnabled(req) {
|
||||
@@ -4427,6 +4577,10 @@ func serveIMGroupDetail(wrt http.ResponseWriter, req *http.Request, topic string
|
||||
return
|
||||
}
|
||||
if selfSub == nil || !(selfSub.ModeGiven & selfSub.ModeWant).IsJoiner() || !(selfSub.ModeGiven & selfSub.ModeWant).IsReader() {
|
||||
// 群不存在（404）或已解散（410）要与"存在但无权限"（403）区分开。
|
||||
if selfSub == nil && writeIMGroupMissingIfGone(wrt, req, principal.TenantID, topic) {
|
||||
return
|
||||
}
|
||||
writeIMError(wrt, req, http.StatusForbidden, 30005, imReasonPermissionDenied, "topic permission required", nil)
|
||||
return
|
||||
}
|
||||
@@ -6238,6 +6392,12 @@ func serveIMFriendRequestAction(wrt http.ResponseWriter, req *http.Request) {
|
||||
writeIMRelationError(wrt, req, err)
|
||||
return
|
||||
}
|
||||
if updated == nil {
|
||||
logs.Err.Printf("friend request resolution returned an empty result: tenant=%d request=%s actor=%s status=%s",
|
||||
principal.TenantID, requestID, principal.Uid.String(), status)
|
||||
writeIMError(wrt, req, http.StatusInternalServerError, 90000, imReasonInternal, "internal error", nil)
|
||||
return
|
||||
}
|
||||
if status == types.FriendRequestAccepted && imReliableHTTPEventsEnabled(req) {
|
||||
topic := updated.ApplicantID.P2PName(updated.TargetID)
|
||||
publishIMConversationSystemNotice(principal.TenantID, topic,
|
||||
@@ -6838,6 +6998,8 @@ func defaultIMActionTarget(action, fallback string) string {
|
||||
return "youth_mode"
|
||||
case "group_owner_transfer", "group_dissolve":
|
||||
return action
|
||||
case "delete_account":
|
||||
return "account"
|
||||
default:
|
||||
return fallback
|
||||
}
|
||||
@@ -8512,7 +8674,7 @@ func currentUserAccountPayload(tenantID types.TenantID, uid types.Uid) (map[stri
|
||||
func validIMSecurityChallengeScene(scene string) bool {
|
||||
switch strings.TrimSpace(scene) {
|
||||
case "set_independent_password", "enable_youth_mode", "close_youth_mode", "reset_independent_password",
|
||||
"group_owner_transfer", "group_dissolve":
|
||||
"group_owner_transfer", "group_dissolve", "delete_account":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
@@ -9244,12 +9406,33 @@ func requireIMGroupMember(wrt http.ResponseWriter, req *http.Request, topic stri
|
||||
access = sub.ModeGiven & sub.ModeWant
|
||||
}
|
||||
if sub == nil || !access.IsJoiner() || (needRead && !access.IsReader()) {
|
||||
// 群不存在（404）或已解散（410）要与"存在但无权限"（403）区分开。
|
||||
if sub == nil && writeIMGroupMissingIfGone(wrt, req, principal.TenantID, topic) {
|
||||
return nil, nil, false
|
||||
}
|
||||
writeIMError(wrt, req, http.StatusForbidden, 30005, imReasonPermissionDenied, "topic permission required", nil)
|
||||
return nil, nil, false
|
||||
}
|
||||
return principal, sub, true
|
||||
}
|
||||
|
||||
// writeIMGroupMissingIfGone 在群不存在或已删除时写出 404/410 并返回 true；查询失败时返回 false 保持原错误路径。
|
||||
func writeIMGroupMissingIfGone(wrt http.ResponseWriter, req *http.Request, tenantID types.TenantID, topic string) bool {
|
||||
group, err := store.Topics.Get(tenantID, topic)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
if group == nil {
|
||||
writeIMError(wrt, req, http.StatusNotFound, 30004, imReasonResourceNotFound, "group not found", nil)
|
||||
return true
|
||||
}
|
||||
if group.State == types.StateDeleted {
|
||||
writeIMError(wrt, req, http.StatusGone, 30004, imReasonGroupDissolved, "group dissolved", map[string]any{"topic": topic})
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func authenticateTopicReader(wrt http.ResponseWriter, req *http.Request, topic string) (*imPrincipal, string, bool) {
|
||||
principal, ok := authenticateIMHTTPBearer(wrt, req)
|
||||
if !ok {
|
||||
@@ -10574,6 +10757,7 @@ func cleanStringList(values []string, limit int) []string {
|
||||
func isAllowedPublicUploadPurpose(purpose string) bool {
|
||||
switch strings.TrimSpace(purpose) {
|
||||
case types.PublicUploadPurposeEnterpriseLicense, types.PublicUploadPurposeTenantFeedback,
|
||||
types.PublicUploadPurposeFeedback,
|
||||
types.PublicUploadPurposeUserAvatar,
|
||||
types.PublicUploadPurposeGroupAvatar:
|
||||
return true
|
||||
@@ -10590,7 +10774,7 @@ func isAllowedPublicUploadMime(purpose, mimeType string) bool {
|
||||
strings.HasPrefix(mimeType, "image/png") ||
|
||||
strings.HasPrefix(mimeType, "image/webp") ||
|
||||
mimeType == "application/pdf"
|
||||
case types.PublicUploadPurposeTenantFeedback:
|
||||
case types.PublicUploadPurposeTenantFeedback, types.PublicUploadPurposeFeedback:
|
||||
return strings.HasPrefix(mimeType, "image/jpeg") ||
|
||||
strings.HasPrefix(mimeType, "image/png") ||
|
||||
strings.HasPrefix(mimeType, "image/webp")
|
||||
|
||||
+48
-10
@@ -158,6 +158,7 @@ type testRelationsStore struct {
|
||||
deleteTenant types.TenantID
|
||||
deleteUser types.Uid
|
||||
deletePeer types.Uid
|
||||
resolveNil bool
|
||||
}
|
||||
|
||||
type testFavoritesStore struct {
|
||||
@@ -962,6 +963,9 @@ func (s *testRelationsStore) CountPendingFriendRequests(tenantID types.TenantID,
|
||||
}
|
||||
|
||||
func (s *testRelationsStore) ResolveFriendRequest(_ types.TenantID, requestID string, actor types.Uid, status string, handledAt time.Time) (*types.FriendRequest, error) {
|
||||
if s.resolveNil {
|
||||
return nil, nil
|
||||
}
|
||||
return &types.FriendRequest{
|
||||
TenantID: 7,
|
||||
RequestID: requestID,
|
||||
@@ -975,6 +979,28 @@ func (s *testRelationsStore) ResolveFriendRequest(_ types.TenantID, requestID st
|
||||
}, nil
|
||||
}
|
||||
|
||||
func TestServeIMFriendRequestActionRejectsEmptyStoreResult(t *testing.T) {
|
||||
ctrl := gomock.NewController(t)
|
||||
defer ctrl.Finish()
|
||||
expectIMBearer(t, ctrl, "token-1", 7, 42)
|
||||
|
||||
oldRelations := store.Relations
|
||||
store.Relations = &testRelationsStore{resolveNil: true}
|
||||
t.Cleanup(func() { store.Relations = oldRelations })
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/im/friend-requests/frq-1/accept", nil)
|
||||
req.Header.Set("Authorization", "Bearer token-1")
|
||||
rec := httptest.NewRecorder()
|
||||
serveIMFriendRequestAction(rec, req)
|
||||
|
||||
if rec.Code != http.StatusInternalServerError {
|
||||
t.Fatalf("status: got %d, want %d; body=%s", rec.Code, http.StatusInternalServerError, rec.Body.String())
|
||||
}
|
||||
if response := decodeIMHTTPResponse(t, rec); response.Reason != imReasonInternal {
|
||||
t.Fatalf("reason: got %q, want %q", response.Reason, imReasonInternal)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *testRelationsStore) ListFriends(_ types.TenantID, _ types.Uid, _, _, _ string, _ int) ([]types.FriendListItem, string, error) {
|
||||
return s.friends, "", nil
|
||||
}
|
||||
@@ -3140,6 +3166,18 @@ func TestServeIMSystemNotificationsListAndRead(t *testing.T) {
|
||||
if readAgainData["id"] != "evt-2" || readAgainData["read_at"] != readAt.UTC().Format(time.RFC3339Nano) {
|
||||
t.Fatalf("read again response should keep persisted read_at: %#v", readAgainData)
|
||||
}
|
||||
|
||||
expectIMBearer(t, ctrl, "token-read-future", 7, 42)
|
||||
req = httptest.NewRequest(http.MethodPost, "/v1/im/system-notifications/evt-1/read", bytes.NewBufferString(`{"read_at":"2099-01-01T00:00:00Z"}`))
|
||||
req.Header.Set("Authorization", "Bearer token-read-future")
|
||||
req.Header.Set("Idempotency-Key", "read-future")
|
||||
rec = httptest.NewRecorder()
|
||||
|
||||
serveIMSystemNotificationRead(rec, req)
|
||||
|
||||
if rec.Code != http.StatusBadRequest {
|
||||
t.Fatalf("future read_at status: got %d, want %d; body=%s", rec.Code, http.StatusBadRequest, rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestServeIMMessageReceipts(t *testing.T) {
|
||||
@@ -7814,26 +7852,23 @@ func mustP2PNameForUser(t *testing.T, uid types.Uid, topic string) string {
|
||||
return name
|
||||
}
|
||||
|
||||
func TestServeIMFriendRequestCreateHonorsGroupPrivateChatSetting(t *testing.T) {
|
||||
func TestServeIMFriendRequestCreateAllowsGroupSourceWhenPrivateChatDisabled(t *testing.T) {
|
||||
ctrl := gomock.NewController(t)
|
||||
defer ctrl.Finish()
|
||||
expectIMBearer(t, ctrl, "token-1", 7, 1)
|
||||
mockStore := expectIMBearer(t, ctrl, "token-1", 7, 1)
|
||||
mockStore.EXPECT().GetUidString().Return("group-request-1")
|
||||
oldRelations := store.Relations
|
||||
oldProfiles := store.Profiles
|
||||
oldSubs := store.Subs
|
||||
oldGroups := store.GroupBusiness
|
||||
store.Relations = &testRelationsStore{context: &types.RelationshipContext{Relationship: types.RelationshipNone}}
|
||||
relations := &testRelationsStore{context: &types.RelationshipContext{Relationship: types.RelationshipNone}}
|
||||
store.Relations = relations
|
||||
store.Profiles = &testProfilesStore{}
|
||||
subs := mock_store.NewMockSubsPersistenceInterface(ctrl)
|
||||
store.Subs = subs
|
||||
store.GroupBusiness = &testGroupBusinessStore{settings: &types.GroupSettings{
|
||||
Topic: "grpAlpha", MemberPrivateChat: false, GroupStatus: types.GroupStatusActive,
|
||||
}}
|
||||
defer func() {
|
||||
store.Relations = oldRelations
|
||||
store.Profiles = oldProfiles
|
||||
store.Subs = oldSubs
|
||||
store.GroupBusiness = oldGroups
|
||||
}()
|
||||
memberMode := types.ModeCPublic
|
||||
for _, uid := range []types.Uid{1, 2} {
|
||||
@@ -7848,8 +7883,11 @@ func TestServeIMFriendRequestCreateHonorsGroupPrivateChatSetting(t *testing.T) {
|
||||
req.Header.Set("Idempotency-Key", "fr-group-1")
|
||||
rec := httptest.NewRecorder()
|
||||
serveIMFriendRequests(rec, req)
|
||||
if rec.Code != http.StatusForbidden {
|
||||
t.Fatalf("status: got %d, want %d; body=%s", rec.Code, http.StatusForbidden, rec.Body.String())
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status: got %d, want %d; body=%s", rec.Code, http.StatusOK, rec.Body.String())
|
||||
}
|
||||
if relations.created == nil || relations.created.Source != "group" || relations.created.SourceTopic != "grpAlpha" {
|
||||
t.Fatalf("unexpected group friend request: %#v", relations.created)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/golang/mock/gomock"
|
||||
"github.com/segmentio/kafka-go"
|
||||
"github.com/tinode/chat/server/store"
|
||||
"github.com/tinode/chat/server/store/mock_store"
|
||||
"github.com/tinode/chat/server/store/types"
|
||||
@@ -39,6 +40,9 @@ func TestNormalizeKafkaConfigUsesLowLatencyBatchTimeout(t *testing.T) {
|
||||
if cfg.MaxRetryCount != 3 {
|
||||
t.Fatalf("unexpected kafka max retry count: %d", cfg.MaxRetryCount)
|
||||
}
|
||||
if cfg.RequiredAcks != int(kafka.RequireAll) {
|
||||
t.Fatalf("unexpected kafka required acks: %d", cfg.RequiredAcks)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnqueueKafkaAuditEventPublishesCrossSystemEvent(t *testing.T) {
|
||||
|
||||
@@ -281,27 +281,27 @@ func persistKafkaMessageIndividually(ctx context.Context, item *kafkaMessagePers
|
||||
return store.HTTPInfrastructure.MarkKafkaConsumeFailed(item.tenantID, item.envelope.EventID, kafkaBus.config.ConsumerGroup,
|
||||
kafkaMessagePersistFailureReason(err), err.Error(), time.Now().UTC().Round(time.Millisecond))
|
||||
}
|
||||
markDead := func(failed *types.KafkaConsumeRecord, err error) error {
|
||||
immetrics.KafkaMessagePersistTotal.WithLabelValues(kind, "dead").Inc()
|
||||
immetrics.MessagePersistDuration.WithLabelValues(kind, "dead").Observe(time.Since(started).Seconds())
|
||||
reason := kafkaMessagePersistFailureReason(err)
|
||||
immetrics.KafkaDeadLettersTotal.WithLabelValues(kind, reason).Inc()
|
||||
return markKafkaDeadLetter(ctx, item.tenantID, item.envelope, failed, reason, err.Error())
|
||||
err := runKafkaMessageSingleRetries(ctx, kafkaBus.config.MessageRetries, operation, markFailed, waitKafkaRetry)
|
||||
if err != nil {
|
||||
// Message persistence failures are recoverable infrastructure failures. Returning
|
||||
// the error keeps the Kafka offset uncommitted so the outer consumer retries the
|
||||
// message until it is durable. Only malformed payloads are dead-lettered while
|
||||
// preparing the item.
|
||||
immetrics.KafkaMessagePersistTotal.WithLabelValues(kind, "retry_exhausted").Inc()
|
||||
immetrics.MessagePersistDuration.WithLabelValues(kind, "retry_exhausted").Observe(time.Since(started).Seconds())
|
||||
}
|
||||
return runKafkaMessageSingleRetries(ctx, kafkaBus.config.MessageRetries, operation, markFailed, markDead, waitKafkaRetry)
|
||||
return err
|
||||
}
|
||||
|
||||
func runKafkaMessageSingleRetries(ctx context.Context, retryCount int, operation func() error,
|
||||
markFailed func(error) (*types.KafkaConsumeRecord, error), markDead func(*types.KafkaConsumeRecord, error) error,
|
||||
wait kafkaRetryWaiter) error {
|
||||
markFailed func(error) (*types.KafkaConsumeRecord, error), wait kafkaRetryWaiter) error {
|
||||
var lastErr error
|
||||
var failed *types.KafkaConsumeRecord
|
||||
for attempt := 1; attempt <= retryCount+1; attempt++ {
|
||||
if lastErr = operation(); lastErr == nil {
|
||||
return nil
|
||||
}
|
||||
var err error
|
||||
if failed, err = markFailed(lastErr); err != nil {
|
||||
if _, err = markFailed(lastErr); err != nil {
|
||||
return err
|
||||
}
|
||||
if attempt <= retryCount {
|
||||
@@ -310,7 +310,7 @@ func runKafkaMessageSingleRetries(ctx context.Context, retryCount int, operation
|
||||
}
|
||||
}
|
||||
}
|
||||
return markDead(failed, lastErr)
|
||||
return fmt.Errorf("message persistence retries exhausted: %w", lastErr)
|
||||
}
|
||||
|
||||
func finishKafkaMessagePersist(ctx context.Context, item *kafkaMessagePersistItem) error {
|
||||
|
||||
@@ -182,7 +182,7 @@ func normalizeKafkaConfig(cfg *kafkaConfig) {
|
||||
cfg.BatchTimeoutMS = 10
|
||||
}
|
||||
if cfg.RequiredAcks == 0 {
|
||||
cfg.RequiredAcks = int(kafka.RequireOne)
|
||||
cfg.RequiredAcks = int(kafka.RequireAll)
|
||||
}
|
||||
if cfg.MaxRetryCount <= 0 {
|
||||
cfg.MaxRetryCount = 3
|
||||
|
||||
@@ -3,6 +3,7 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -47,6 +48,33 @@ func TestConsumeKafkaMessageBatchRetriesHandlerAndCommit(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestConsumeKafkaMessageBatchDoesNotCommitWhileHandlerFails(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
handlerCalls := 0
|
||||
commitCalls := 0
|
||||
|
||||
err := consumeKafkaMessageBatch(ctx, []kafka.Message{{Offset: 1}},
|
||||
func(context.Context, []kafka.Message) error {
|
||||
handlerCalls++
|
||||
return errors.New("database unavailable")
|
||||
},
|
||||
func(context.Context, ...kafka.Message) error {
|
||||
commitCalls++
|
||||
return nil
|
||||
},
|
||||
func(ctx context.Context, _ time.Duration) error {
|
||||
cancel()
|
||||
return ctx.Err()
|
||||
})
|
||||
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("consumeKafkaMessageBatch() error = %v, want context canceled", err)
|
||||
}
|
||||
if handlerCalls != 1 || commitCalls != 0 {
|
||||
t.Fatalf("calls handler=%d commit=%d, want 1/0", handlerCalls, commitCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConsumeKafkaMalformedMessageRetriesCommit(t *testing.T) {
|
||||
commitCalls := 0
|
||||
waitCalls := 0
|
||||
@@ -76,10 +104,9 @@ func TestConsumeKafkaMalformedMessageRetriesCommit(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunKafkaMessageSingleRetriesThenDeadLetters(t *testing.T) {
|
||||
func TestRunKafkaMessageSingleRetriesReturnsErrorWithoutTerminalCommit(t *testing.T) {
|
||||
operationCalls := 0
|
||||
markFailedCalls := 0
|
||||
markDeadCalls := 0
|
||||
waitCalls := 0
|
||||
|
||||
err := runKafkaMessageSingleRetries(context.Background(), 3,
|
||||
@@ -91,31 +118,23 @@ func TestRunKafkaMessageSingleRetriesThenDeadLetters(t *testing.T) {
|
||||
markFailedCalls++
|
||||
return &types.KafkaConsumeRecord{RetryCount: markFailedCalls}, nil
|
||||
},
|
||||
func(record *types.KafkaConsumeRecord, err error) error {
|
||||
markDeadCalls++
|
||||
if record == nil || record.RetryCount != 4 || err == nil {
|
||||
t.Fatalf("unexpected dead-letter input: record=%+v err=%v", record, err)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
func(context.Context, time.Duration) error {
|
||||
waitCalls++
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("runKafkaMessageSingleRetries() error = %v", err)
|
||||
if err == nil || !strings.Contains(err.Error(), "message persistence retries exhausted") {
|
||||
t.Fatalf("runKafkaMessageSingleRetries() error = %v, want retry exhaustion", err)
|
||||
}
|
||||
if operationCalls != 4 || markFailedCalls != 4 || waitCalls != 3 || markDeadCalls != 1 {
|
||||
t.Fatalf("calls operation=%d failed=%d wait=%d dead=%d, want 4/4/3/1",
|
||||
operationCalls, markFailedCalls, waitCalls, markDeadCalls)
|
||||
if operationCalls != 4 || markFailedCalls != 4 || waitCalls != 3 {
|
||||
t.Fatalf("calls operation=%d failed=%d wait=%d, want 4/4/3",
|
||||
operationCalls, markFailedCalls, waitCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunKafkaMessageSingleRetriesStopsOnSuccess(t *testing.T) {
|
||||
operationCalls := 0
|
||||
markFailedCalls := 0
|
||||
markDeadCalls := 0
|
||||
err := runKafkaMessageSingleRetries(context.Background(), 3,
|
||||
func() error {
|
||||
operationCalls++
|
||||
@@ -128,16 +147,12 @@ func TestRunKafkaMessageSingleRetriesStopsOnSuccess(t *testing.T) {
|
||||
markFailedCalls++
|
||||
return &types.KafkaConsumeRecord{RetryCount: markFailedCalls}, nil
|
||||
},
|
||||
func(*types.KafkaConsumeRecord, error) error {
|
||||
markDeadCalls++
|
||||
return nil
|
||||
},
|
||||
func(context.Context, time.Duration) error { return nil })
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("runKafkaMessageSingleRetries() error = %v", err)
|
||||
}
|
||||
if operationCalls != 3 || markFailedCalls != 2 || markDeadCalls != 0 {
|
||||
t.Fatalf("calls operation=%d failed=%d dead=%d, want 3/2/0", operationCalls, markFailedCalls, markDeadCalls)
|
||||
if operationCalls != 3 || markFailedCalls != 2 {
|
||||
t.Fatalf("calls operation=%d failed=%d, want 3/2", operationCalls, markFailedCalls)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -67,13 +67,14 @@ func executeKafkaWorkbenchEvent(event kafkaWorkbenchEvent) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
notificationEnabled := event.shouldCreateNotification()
|
||||
pushTargets := make(map[types.Uid]push.Recipient)
|
||||
for _, targetID := range event.TargetUserIDs {
|
||||
target := store.EncodeUid(targetID)
|
||||
if target.IsZero() {
|
||||
continue
|
||||
}
|
||||
if target != actor && event.Title != "" {
|
||||
if notificationEnabled && target != actor && event.Title != "" {
|
||||
action := event.notificationAction()
|
||||
if err = createIMSystemNotification(event.TenantID, target, stableIMEventID(event.EventID, "notification", target.UserId()),
|
||||
event.Action, event.Title, event.Summary, action); err != nil && err != types.ErrDuplicate {
|
||||
@@ -133,6 +134,15 @@ func (e kafkaWorkbenchEvent) realtimePayload() (map[string]any, error) {
|
||||
return payload, nil
|
||||
}
|
||||
|
||||
func (e kafkaWorkbenchEvent) shouldCreateNotification() bool {
|
||||
switch e.Action {
|
||||
case "workbench.ai.task.updated":
|
||||
return false
|
||||
default:
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
func (e kafkaWorkbenchEvent) notificationAction() types.KVMap {
|
||||
action := types.KVMap{
|
||||
"module": e.Module,
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/tinode/chat/server/store"
|
||||
)
|
||||
|
||||
func TestExecuteKafkaWorkbenchAITaskEventSkipsNotification(t *testing.T) {
|
||||
oldSecurity := store.Security
|
||||
security := &testUserSecurityStore{}
|
||||
store.Security = security
|
||||
t.Cleanup(func() {
|
||||
store.Security = oldSecurity
|
||||
})
|
||||
|
||||
event := kafkaWorkbenchEvent{
|
||||
EventID: "wbe_ai_1",
|
||||
TenantID: 7,
|
||||
ActorUserID: 1,
|
||||
Module: "ai",
|
||||
Action: "workbench.ai.task.updated",
|
||||
ResourceID: "aitask_1",
|
||||
TargetUserIDs: []int64{2},
|
||||
Title: "AI 任务完成",
|
||||
Summary: "表格分析已完成",
|
||||
Data: map[string]any{
|
||||
"task_id": "aitask_1",
|
||||
"feature": "smart_table",
|
||||
"status": "succeeded",
|
||||
"result_type": "record",
|
||||
"record_id": "airec_1",
|
||||
},
|
||||
}
|
||||
if event.shouldCreateNotification() {
|
||||
t.Fatal("ai task event must be realtime-only")
|
||||
}
|
||||
err := executeKafkaWorkbenchEvent(event)
|
||||
if err != nil {
|
||||
t.Fatalf("executeKafkaWorkbenchEvent returned error: %v", err)
|
||||
}
|
||||
if len(security.notifications) != 0 {
|
||||
t.Fatalf("ai task event should not create offline notification: %#v", security.notifications)
|
||||
}
|
||||
}
|
||||
@@ -14,17 +14,17 @@
|
||||
</head>
|
||||
<body>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">尊敬的笔墨讯使用户：</span></p>
|
||||
<p class="p2"><span class="s1">尊敬的讯工捷使用户：</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">欢迎您使用海口市枫群网络科技有限公司提供的“笔墨讯使”软件及相关服务！作为“笔墨讯使”的开发者（以下简称“我们”），我们深知个人信息对您的重要性，并会尽力保护您的个人信息安全可靠。我们致力于维持您对我们的信任，恪守以下原则，保护和处理您的个人信息：权责一致原则，目的明确原则，选择同意原则，最少必要原则，确保安全原则，主体参与原则，公开透明原则。同时我们承诺，我们将按业界成熟的安全解决方案，采取相应的安全保护措施来保护您的个人信息。</span></p>
|
||||
<p class="p2"><span class="s1">欢迎您使用南京赫顺名网络科技有限公司提供的“讯工捷使”软件及相关服务！作为“讯工捷使”的开发者（以下简称“我们”），我们深知个人信息对您的重要性，并会尽力保护您的个人信息安全可靠。我们致力于维持您对我们的信任，恪守以下原则，保护和处理您的个人信息：权责一致原则，目的明确原则，选择同意原则，最少必要原则，确保安全原则，主体参与原则，公开透明原则。同时我们承诺，我们将按业界成熟的安全解决方案，采取相应的安全保护措施来保护您的个人信息。</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">版本生效日期：2025年8月1日</span></p>
|
||||
<p class="p2"><span class="s1">版本生效日期：2026年10月8日</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">如您有任何疑问、意见或建议，您可通过以下联系方式与我们联系：</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">联系邮箱：fengqunwangluo2025@163.com</span></p>
|
||||
<p class="p2"><span class="s1">联系邮箱：support@heshunming.top</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">联系地址：海南省海口市美兰区海府街道海府路4号海口宾馆附楼第4层C区第7间</span></p>
|
||||
<p class="p2"><span class="s1">联系地址：江苏省南京市秦淮区光华路街道永智路6号云计算中心4号楼a栋303-93</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">【特别提示】</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
@@ -38,7 +38,7 @@
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">引言</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">笔墨讯使重视用户信息安全与隐私保护，竭尽全力通过合理有效的信息安全技术及管理流程，防止您的信息泄露、损毁、丢失；为您提供便利的信息管理选项，增强您对信息管理的便捷性；保护您的通信秘密，保障您的信息及通信安全。笔墨讯使严格遵守法律法规，仅收集必要的信息，并提升信息处理透明度，在产品或服务开发的各个环节，综合法律、产品、设计等多方因素，融入隐私保护的理念，为您提供更加安全、可靠的服务。</span></p>
|
||||
<p class="p2"><span class="s1">讯工捷使重视用户信息安全与隐私保护，竭尽全力通过合理有效的信息安全技术及管理流程，防止您的信息泄露、损毁、丢失；为您提供便利的信息管理选项，增强您对信息管理的便捷性；保护您的通信秘密，保障您的信息及通信安全。讯工捷使严格遵守法律法规，仅收集必要的信息，并提升信息处理透明度，在产品或服务开发的各个环节，综合法律、产品、设计等多方因素，融入隐私保护的理念，为您提供更加安全、可靠的服务。</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">我们制定并遵循本《隐私政策》（以下简称“本政策”），主要向您说明：</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
@@ -62,37 +62,53 @@
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">1.3 匿名化处理后的信息，即通过对个人信息的技术处理，使得个人信息主体无法被识别，且处理后不能被复原的信息，匿名化处理后的信息不属于个人信息。</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">1.4 其他无法识别特定自然人身份或者反映特定自然人活动情况的信息不属于个人信息。除上述定义外，本政策所用其他术语与《笔墨讯使软件服务协议》中的术语具有相同的涵义。如您有问题，请联系我们。</span></p>
|
||||
<p class="p2"><span class="s1">1.4 其他无法识别特定自然人身份或者反映特定自然人活动情况的信息不属于个人信息。除上述定义外，本政策所用其他术语与《讯工捷使软件服务协议》中的术语具有相同的涵义。如您有问题，请联系我们。</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">二、我们收集的信息</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">2.1 笔墨讯使软件功能分为核心功能和附加功能，我们将详细向您说明各业务功能所收集的个人信息以及收集方法、目的，拒绝提供的影响等。业务功能具体包括：</span></p>
|
||||
<p class="p2"><span class="s1">2.1 讯工捷使软件功能分为核心功能和附加功能，我们将详细向您说明各业务功能所收集的个人信息以及收集方法、目的，拒绝提供的影响等。业务功能具体包括：</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">2.1.1 核心功能，包括创建账号并完善资料、会话功能、发现功能。</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">2.1.2 附加功能，包括个人主页、身份认证。</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">2.2 您使用笔墨讯使软件核心功能前，需要创建账号并完善个人资料。通过用户主动提供的方式，我们会收集您的以下信息：</span></p>
|
||||
<p class="p2"><span class="s1">2.2 您使用讯工捷使软件核心功能前，需要创建账号并完善个人资料。通过用户主动提供的方式，我们会收集您的以下信息：</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">2.2.1 您需要向我们提供您的手机号码，并设置您在笔墨讯使的用户密码。手机号码确定后，暂时无法修改。如果您拒绝提供手机号码笔墨讯使将无法为您创建账号并提供服务。同时，为了保证软件正常运行，我们将会采集您的设备信息，具体包括： iOS系统：SDK版本、设备用户名、手机型号、系统型号、链接的无线MAC地址、设备随机序列号、IMSI、SIM卡国家、磁盘总空间、运行内存、电池电量、网络环境、运营商、系统版本、CPU型号、屏幕分辨率、IMEI、手机品牌、平台类型、设备序列号等。</span></p>
|
||||
<p class="p2"><span class="s1">2.2.1 您需要向我们提供您的手机号码，并设置您在讯工捷使的用户密码。手机号码确定后，暂时无法修改。如果您拒绝提供手机号码讯工捷使将无法为您创建账号并提供服务。同时，为了保证软件正常运行，我们将会采集您的设备信息，具体包括： iOS系统：SDK版本、设备用户名、手机型号、系统型号、链接的无线MAC地址、设备随机序列号、IMSI、SIM卡国家、磁盘总空间、运行内存、电池电量、网络环境、运营商、系统版本、CPU型号、屏幕分辨率、IMEI、手机品牌、平台类型、设备序列号等。</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">2.2.2 您使用会话功能进行语音通话、发送语音信息、拍摄及相册时；我们会调用您以下设备权限：设备麦克风权限、系统相册权限、系统摄像头权限。上述功能可能需要您在您的设备中向我们开启您的相机（摄像头）、相册（图片库）、麦克风以及通讯录的访问权限，以实现这些功能所涉及的信息的收集和使用。请您注意，您开启这些权限即代表您授权我们可以收集和使用这些个人信息来实现上述的功能，您关闭权限即代表您取消了这些授权，则我们将不再继续收集和使用您的这些个人信息，也无法为您提供上述与这些授权所对应的功能。您关闭权限的决定不会影响此前基于您的授权所进行的个人信息的处理。</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">2.3 为了提高您的社交属性，在社群中结实更多志同道合的朋友，我们在以下附加功能中可能会收集和使用您的个人信息。如果您不提供这些个人信息，您依然可以使用笔墨讯使软件的核心功能。</span></p>
|
||||
<p class="p2"><span class="s1">2.3 为了提高您的社交属性，在社群中结实更多志同道合的朋友，我们在以下附加功能中可能会收集和使用您的个人信息。如果您不提供这些个人信息，您依然可以使用讯工捷使软件的核心功能。</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">2.3.1 您可以选择向我们提供昵称、姓名、性别、年龄、所在地、行业、公司/机构、职业。您可以在个人主页资料中选择是否填写上述信息。如您提供上述信息，可以使其他用户对您有更全面的了解。您也可以在您的个人主页中将上述隐私事项设置为所有人/仅好友/仅自己可见。</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">2.4 您可以选择申请个人加V认证，申请加V认证您需要提供您的个人简介、昵称、并上传职位、技能证明材料，包含姓名、公司/组织信息、职位信息的加盖公章的截图材料、公开可查的官方个人信息页截图材料。成为笔墨讯使认证会员可以提升在社群的个人社交属性。</span></p>
|
||||
<p class="p2"><span class="s1">2.4 您可以选择申请个人加V认证，申请加V认证您需要提供您的个人简介、昵称、并上传职位、技能证明材料，包含姓名、公司/组织信息、职位信息的加盖公章的截图材料、公开可查的官方个人信息页截图材料。成为讯工捷使认证会员可以提升在社群的个人社交属性。</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">2.5 为了帮助您更好地使用我们的产品或服务，我们还会收集包括：您的好友列表、群列表信息、声纹特征值信息。为确保您使用我们服务时能与您认识的人进行联系，如您选择开启导入通讯录功能，我们可能对您联系人的姓名和电话号码进行加密，并仅收集加密后的信息。</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">2.6 我们可能会获得您在使用第三方合作伙伴服务时所产生或分享的信息。包括：您使用笔墨讯使帐户登录第三方合作伙伴服务时，或者您使用第三方合作伙伴服务账户登录笔墨讯使时，我们会获得您登录第三方合作伙伴服务的名称、登录时间或登录笔墨讯使的登录时间，方便您进行授权管理。请您仔细阅读第三方合作伙伴服务的用户协议或隐私政策。</span></p>
|
||||
<p class="p2"><span class="s1">2.6 我们可能会获得您在使用第三方合作伙伴服务时所产生或分享的信息。包括：您使用讯工捷使帐户登录第三方合作伙伴服务时，或者您使用第三方合作伙伴服务账户登录讯工捷使时，我们会获得您登录第三方合作伙伴服务的名称、登录时间或登录讯工捷使的登录时间，方便您进行授权管理。请您仔细阅读第三方合作伙伴服务的用户协议或隐私政策。</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">2.7 AI智能服务。为向您提供智能回复、内容生成、内容总结、智能客服、搜索推荐等AI辅助功能，我们可能会接入DeepSeek AI服务或我们部署、接入的其他人工智能模型。当您主动使用相关功能时，我们可能会处理您主动输入、选择提交或授权处理的文本、问题、上下文、反馈信息，以及为实现功能所必要的账号标识、设备信息、网络日志等信息。请您避免在使用AI功能时输入身份证件号码、银行卡号、精确住址、健康信息、未成年人信息、商业秘密或他人隐私等敏感信息；如特定场景确需处理敏感个人信息，我们将依法取得您的单独同意或满足法律法规规定的其他处理条件。</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">2.8 我们使用AI服务处理相关信息的目的包括：理解您的指令、生成或优化回复、提升交互体验、保障服务安全、排查故障以及改进产品质量。我们会采取合理措施限制处理范围，并要求AI服务提供方按照协议约定、法律法规及安全要求处理相关信息，不得将相关信息用于与提供服务无关的目的。AI生成内容可能存在不准确、不完整或不适合特定场景的情况，请您结合实际情况判断后使用。</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">我们的产品在应用运行期间需要收集您的设备唯一识别码（MAC地址及IMEI/IDFA、SIM 卡 IMSI 信息、OAID、ANDROID ID）以提供统计分析等服务，并通过应用启动数据及异常错误日志分析改进性能和用户体验，为用户提供更好的服务。</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">APP涉及嵌入第三方代码、插件传输个人信息的情形逐项列举如下：</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">DeepSeek AI服务</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">第三方主体：杭州深度求索人工智能基础技术研究有限公司及其关联公司</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">用途：提供智能回复、内容生成、内容总结、智能客服、搜索推荐等AI辅助能力</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">涉及个人信息：您主动输入、选择提交或授权处理的文本、问题、上下文、反馈信息，以及为实现功能所必要的账号标识、设备信息、网络日志等信息</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">收集方式：用户主动提交或在使用AI功能时通过接口传输</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">隐私政策：https://cdn.deepseek.com/policies/zh-CN/deepseek-privacy-policy.html</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">获取设备的传感器</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">用途：获取设备的传感器列表 为了检测、防止安卓模拟器等三方模拟器的使用</span></p>
|
||||
@@ -231,7 +247,7 @@
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">B. 为了提升我们的产品及服务质量或向您提供全新的产品及服务，我们会在关联公司内部共享您的相关信息。</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">C. 您通过笔墨讯使软件使用第三方服务时，笔墨讯使会按照“个人信息的收集”条款列明的共享内容与提供服务的第三方或合作伙伴（广告、分析服务类的授权合作伙伴、供应商、服务提供商和其他合作伙伴）共享您的个人信息。</span></p>
|
||||
<p class="p2"><span class="s1">C. 您通过讯工捷使软件使用第三方服务时，讯工捷使会按照“个人信息的收集”条款列明的共享内容与提供服务的第三方或合作伙伴（广告、分析服务类的授权合作伙伴、供应商、服务提供商和其他合作伙伴）共享您的个人信息。</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">D. 为维护用户合法权益，在协助处理与您有关的纠纷或争议时，我们可能向与您存在利害关系的第三方提供解决纠纷或争议所必需的信息。</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
@@ -260,7 +276,7 @@
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">F. 从合法公开披露的信息中收集个人信息的，如合法的新闻报道、政府信息公开等渠道。</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">4.5 如发生收购、兼并、重组等变更，我们会要求变更后的主体依然遵守本隐私政策约定，履行原有责任及义务。如变更后的主体需变更个人信息使用目的，我们会要求其事先获得您的明示同意。对内而言，在之前的收购、兼并活动中，对于被收购、被兼并对象管理的数据(包括用户数据)均采用与笔墨讯使自有数据相同的标准和要求进行处理和保护。</span></p>
|
||||
<p class="p2"><span class="s1">4.5 如发生收购、兼并、重组等变更，我们会要求变更后的主体依然遵守本隐私政策约定，履行原有责任及义务。如变更后的主体需变更个人信息使用目的，我们会要求其事先获得您的明示同意。对内而言，在之前的收购、兼并活动中，对于被收购、被兼并对象管理的数据(包括用户数据)均采用与讯工捷使自有数据相同的标准和要求进行处理和保护。</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">五、我们如何保护您的个人信息</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
@@ -294,7 +310,7 @@
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">E. 为执行相关服务协议或本政策、维护社会公共利益，为保护我们的客户、我们或我们的关联公司、其他用户或雇员的人身财产安全或其他合法权益所合理必需的用途。</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">6.3 请您注意，您在使用我们的服务期间，我们仅在法律法规规定、监管要求及为实现本条款声明的目的所必需的时限内保留您的个人信息，在您注销账号后，我们仅在法律法规规定、监管要求的期限内保留个人信息，且不会开展除存储和采取必要的安全保护措施之外的处理活动，但已经匿名化处理后的个人信息除外。当您成功注销笔墨讯使账户或超出保留期间后，我们将对您的个人信息进行删除或匿名化处理。</span></p>
|
||||
<p class="p2"><span class="s1">6.3 请您注意，您在使用我们的服务期间，我们仅在法律法规规定、监管要求及为实现本条款声明的目的所必需的时限内保留您的个人信息，在您注销账号后，我们仅在法律法规规定、监管要求的期限内保留个人信息，且不会开展除存储和采取必要的安全保护措施之外的处理活动，但已经匿名化处理后的个人信息除外。当您成功注销讯工捷使账户或超出保留期间后，我们将对您的个人信息进行删除或匿名化处理。</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">6.4 当我们的产品或服务发生停止运营的情形时，我们将采取例如，推送通知、公告等形式通知您，并在合理的期限内删除或匿名化处理您的个人信息。</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
@@ -310,7 +326,7 @@
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">您可以通过以下方式访问及管理您的信息：</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">8.1 查询、更正和补充您的信息 您有权查询、更正或补充您的信息。您可以通过以下方式自行进行： 登录笔墨讯使客户端，进入“我的”，点击用户名对个人资料、基本信息、认证信息、个人资料等进行查询、更正或补充；</span></p>
|
||||
<p class="p2"><span class="s1">8.1 查询、更正和补充您的信息 您有权查询、更正或补充您的信息。您可以通过以下方式自行进行： 登录讯工捷使客户端，进入“我的”，点击用户名对个人资料、基本信息、认证信息、个人资料等进行查询、更正或补充；</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">8.2 删除您的信息 您可以通过“8.1查询、更正和补充您的信息”中列明的方式删除您的部分信息。</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
@@ -329,13 +345,13 @@
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">8.3 改变您授权同意的范围</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">每个业务功能需要一些基本的个人信息才能得以完成（见本隐私政策“第一条 我们收集的信息”）。除此之外，您可以在笔墨讯使客户端“我的-设置-隐私设置”中设置或改变您的智能移动设备的设置等方式给予或收回您的授权同意的范围。 当您收回同意后，我们将不再处理相应的个人信息。但您收回同意的决定，不会影响此前基于您的授权而开展的个人信息处理。</span></p>
|
||||
<p class="p2"><span class="s1">每个业务功能需要一些基本的个人信息才能得以完成（见本隐私政策“第一条 我们收集的信息”）。除此之外，您可以在讯工捷使客户端“我的-设置-隐私设置”中设置或改变您的智能移动设备的设置等方式给予或收回您的授权同意的范围。 当您收回同意后，我们将不再处理相应的个人信息。但您收回同意的决定，不会影响此前基于您的授权而开展的个人信息处理。</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">8.4 注销您的账户</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">您可以通过以下方式申请注销您的账户：</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">登录笔墨讯使官网并选择联系客服寻求帮助，协助您申请注销您的账户。</span></p>
|
||||
<p class="p2"><span class="s1">登录讯工捷使官网并选择联系客服寻求帮助，协助您申请注销您的账户。</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">在您主动注销账户之后，我们将停止为您提供产品或服务，根据适用法律的要求删除您的个人信息，或使其匿名化处理。</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
@@ -343,7 +359,7 @@
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">在某些业务功能中，我们可能仅依据信息系统、算法等在内的非人工自动决策机制做出决定。</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">如果这些决定显著影响您的合法权益，您有权要求我们做出解释，我们也将在不侵害笔墨讯使商业秘密或其他用户权益、社会公共利益的前提下提供申诉方法。</span></p>
|
||||
<p class="p2"><span class="s1">如果这些决定显著影响您的合法权益，您有权要求我们做出解释，我们也将在不侵害讯工捷使商业秘密或其他用户权益、社会公共利益的前提下提供申诉方法。</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">8.6 响应您的上述请求</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
@@ -379,11 +395,11 @@
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">9.3 我们建议您查看此类社交网络或第三方服务提供商的隐私声明，以了解他们如何处理您的信息。对于您使用第三方服务时向其提供或与他人分享个人信息而产生的后果由您自行承担，并由相应责任方在法律规定范围内承担责任。</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">9.4 当您在笔墨讯使平台公开分享任何信息时，任何人都可能在使用笔墨讯使平台时查看或访问这些信息，如您在笔墨讯使平台上传的照片、评价等信息。您在使用我们服务时自愿共享的信息，可能会涉及您或他人的个人信息，请您谨慎选择并自行承担责任。</span></p>
|
||||
<p class="p2"><span class="s1">9.4 当您在讯工捷使平台公开分享任何信息时，任何人都可能在使用讯工捷使平台时查看或访问这些信息，如您在讯工捷使平台上传的照片、评价等信息。您在使用我们服务时自愿共享的信息，可能会涉及您或他人的个人信息，请您谨慎选择并自行承担责任。</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">十、未成年人保护</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">10.1 我们非常重视对未成年人个人信息的保护。根据相关法律法规的规定，若您是18周岁以下的未成年人，在使用笔墨讯使的服务前，应事先取得您的家长或法定监护人的同意。若您是未满14周岁的未成年人，您的监护人需要仔细阅读本政策并同意您依照本政策适用笔墨讯使平台。</span></p>
|
||||
<p class="p2"><span class="s1">10.1 我们非常重视对未成年人个人信息的保护。根据相关法律法规的规定，若您是18周岁以下的未成年人，在使用讯工捷使的服务前，应事先取得您的家长或法定监护人的同意。若您是未满14周岁的未成年人，您的监护人需要仔细阅读本政策并同意您依照本政策适用讯工捷使平台。</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">10.2 我们不会主动直接向未成年人收集其个人信息。对于经监护人同意而收集未成年人个人信息的情况，我们只会在受到法律允许、监护人同意或者保护未成年人所必要的情况下使用、共享、转让或披露此信息。</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
@@ -391,11 +407,11 @@
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">十一、适用范围</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">我们的所有服务均适用本政策。但某些服务有其特定的隐私指引/声明，该特定隐私指引/声明更具体地说明我们在该服务中如何处理您的信息。如本政策与特定服务的隐私指引/声明有不一致之处，请以该特定隐私指引/声明为准。 请您注意，本政策不适用由其他公司或个人提供的服务。例如，您通过使用笔墨讯使帐号登录其他公司或个人提供的服务。 您使用该等第三方服务须受其隐私政策（而非本政策）约束，您需要仔细阅读其政策内容。</span></p>
|
||||
<p class="p2"><span class="s1">我们的所有服务均适用本政策。但某些服务有其特定的隐私指引/声明，该特定隐私指引/声明更具体地说明我们在该服务中如何处理您的信息。如本政策与特定服务的隐私指引/声明有不一致之处，请以该特定隐私指引/声明为准。 请您注意，本政策不适用由其他公司或个人提供的服务。例如，您通过使用讯工捷使帐号登录其他公司或个人提供的服务。 您使用该等第三方服务须受其隐私政策（而非本政策）约束，您需要仔细阅读其政策内容。</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">十二、通知和修订</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">12.1 为给您提供更好的服务以及随着笔墨讯使业务的发展，本隐私政策也会随之更新。但未经您明确同意，我们不会削减您依据本隐私政策所应享有的权利。我们会通过在笔墨讯使官方网站、笔墨讯使移动端上发出更新版本并在生效前通过网站公告或以其他适当方式提醒您相关内容的更新，也请您访问及时了解最新的隐私政策。</span></p>
|
||||
<p class="p2"><span class="s1">12.1 为给您提供更好的服务以及随着讯工捷使业务的发展，本隐私政策也会随之更新。但未经您明确同意，我们不会削减您依据本隐私政策所应享有的权利。我们会通过在讯工捷使官方网站、讯工捷使移动端上发出更新版本并在生效前通过网站公告或以其他适当方式提醒您相关内容的更新，也请您访问及时了解最新的隐私政策。</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">12.2 对于重大变更，我们还会提供更为显著的通知（我们会通过包括但不限于邮件、短信或在浏览页面做特别提示等方式，说明隐私政策的具体变更内容）。</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
@@ -434,7 +450,7 @@
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">十四、如何联系我们</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">14.1 如您对本隐私政策或您个人信息的相关事宜有任何问题、意见或建议，请联系笔墨讯使客服（联系邮箱：【fengqunwangluo2025@163.com】）。</span></p>
|
||||
<p class="p2"><span class="s1">14.1 如您对本隐私政策或您个人信息的相关事宜有任何问题、意见或建议，请联系讯工捷使客服（联系邮箱：【support@heshunming.top】）。</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">14.2 一般情况下，我们将在15天内回复和处理您的疑问、意见或建议。需注意的是，我们可能不会回复与本政策或您的个人信息权利无关的问题。如果您对我们的回复不满意，特别是我们的个人信息处理行为损害了您的合法权益，您还可以向网信、电信、公安及工商等监管部门进行投诉或举报。或通过向我们所在地有管辖权的法院提起诉讼来寻求解决方案。</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
@@ -445,6 +461,6 @@
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">15.3 本政策的标题仅为方便阅读而设计，不影响本政策任何条款的含义或解释。</span></p>
|
||||
<p class="p1"><span class="s1"></span><br></p>
|
||||
<p class="p2"><span class="s1">开发者名称：【海口市枫群网络科技有限公司】</span></p>
|
||||
<p class="p2"><span class="s1">开发者名称：【南京赫顺名网络科技有限公司】</span></p>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
@@ -13,14 +13,14 @@
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<p class="p1"><span class="s1"><span class="Apple-converted-space"> </span>笔墨讯使软件服务协议</span></p>
|
||||
<p class="p1"><span class="s1"><span class="Apple-converted-space"> </span>讯工捷使软件服务协议</span></p>
|
||||
<p class="p1"><span class="s1"><span class="Apple-converted-space"> </span>前言</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">欢迎您使用【海口市枫群网络科技有限公司】公司（下称“我公司”或“我们”）为您提供的笔墨讯使的服务！</span></p>
|
||||
<p class="p1"><span class="s1">欢迎您使用【南京赫顺名网络科技有限公司】公司（下称“我公司”或“我们”）为您提供的讯工捷使的服务！</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">特别提示：</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">为使用笔墨讯使软件（以下简称“本软件”）及服务，您应当阅读并遵守《笔墨讯使软件服务协议》（以下简称“本协议”）。请您务必仔细阅读、充分理解本协议各条款内容，特别是豁免或者限制我公司责任的条款、对用户权利进行限制的条款、约定管辖与法律适用条款，以及开通或使用某项服务的单独协议。限制、免责条款或者其他涉及您重大权益的条款可能以加粗等形式提示您重点注意。如您对本协议有任何疑问，可通过文末联系方式联系我们。</span></p>
|
||||
<p class="p1"><span class="s1">为使用讯工捷使软件（以下简称“本软件”）及服务，您应当阅读并遵守《讯工捷使软件服务协议》（以下简称“本协议”）。请您务必仔细阅读、充分理解本协议各条款内容，特别是豁免或者限制我公司责任的条款、对用户权利进行限制的条款、约定管辖与法律适用条款，以及开通或使用某项服务的单独协议。限制、免责条款或者其他涉及您重大权益的条款可能以加粗等形式提示您重点注意。如您对本协议有任何疑问，可通过文末联系方式联系我们。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">除非您已阅读并接受本协议所有条款，否则您无权注册或登录使用本软件及相关服务。您点击“同意”，或者您注册或登录本软件的行为，或者其他明示或者默示方式表示接受本协议的行为即视为您已阅读并同意接受本协议的约束。本协议即在您与我公司之间产生法律效力，成为对双方均具有约束力的法律文件。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
@@ -34,20 +34,20 @@
|
||||
<p class="p1"><span class="s1">本协议内容同时包括我们可能不断发布的关于本服务的相关协议、业务规则、通知等内容。上述内容一经正式发布，即为本协议不可分割的组成部分，您同样应当遵守。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">1.3 相关定义</span></p>
|
||||
<p class="p1"><span class="s1">“我公司或我们”，是指笔墨讯使及相关服务可能存在的运营关联单位，即【海口市枫群网络科技有限公司】公司及其关联公司。</span></p>
|
||||
<p class="p1"><span class="s1">“我公司或我们”，是指讯工捷使及相关服务可能存在的运营关联单位，即【南京赫顺名网络科技有限公司】公司及其关联公司。</span></p>
|
||||
<p class="p1"><span class="s1">“用户”，是指使用本软件及相关服务的使用者，在本协议中更多地称为“您”。</span></p>
|
||||
<p class="p1"><span class="s1">“服务”，是指我们向您提供的笔墨讯使软件相关服务，具体以我们实际提供的为准。同时，我们有权对提供的软件服务进行增加、关停或其他调整。</span></p>
|
||||
<p class="p1"><span class="s1">“服务”，是指我们向您提供的讯工捷使软件相关服务，具体以我们实际提供的为准。同时，我们有权对提供的软件服务进行增加、关停或其他调整。</span></p>
|
||||
<p class="p1"><span class="s1">“账号”，指用户在使用服务时可能需要注册的账号。用户可在本软件注册并获得的账号，作为登录并使用本软件的凭证。</span></p>
|
||||
<p class="p1"><span class="s1"><span class="Apple-converted-space"> </span>二、关于本服务</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">2.1 本服务的内容</span></p>
|
||||
<p class="p1"><span class="s1">本服务内容是指我们向用户提供的跨平台的通讯工具（以下简称“笔墨讯使”），支持单人、多人参与，在发送语音短信、视频、图片和文字等即时通讯服务的基础上，同时为用户提供包括但不限于网络媒体、互联网增值、互动娱乐、电子商务、电子钱包、开放平台、与其他软件或硬件信息互通等功能或内容的软件许可及服务（以下简称“本服务”）和广告等服务（以下简称“本服务”）。</span></p>
|
||||
<p class="p1"><span class="s1">本服务内容是指我们向用户提供的跨平台的通讯工具（以下简称“讯工捷使”），支持单人、多人参与，在发送语音短信、视频、图片和文字等即时通讯服务的基础上，同时为用户提供包括但不限于网络媒体、互联网增值、互动娱乐、电子商务、电子钱包、开放平台、与其他软件或硬件信息互通等功能或内容的软件许可及服务（以下简称“本服务”）和广告等服务（以下简称“本服务”）。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">2.2 本服务的形式</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">2.2.1 您使用本服务需要下载笔墨讯使客户端软件，对于这些软件，我们给予您一项个人的、不可转让及非排他性的许可。您仅可为访问或使用本服务的目的而使用这些软件及服务。</span></p>
|
||||
<p class="p1"><span class="s1">2.2.1 您使用本服务需要下载讯工捷使客户端软件，对于这些软件，我们给予您一项个人的、不可转让及非排他性的许可。您仅可为访问或使用本服务的目的而使用这些软件及服务。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">2.2.2 本服务中笔墨讯使客户端软件可能会提供包括但不限于iOS、Android、Windows、Mac等多个应用版本，用户必须选择与所安装终端设备相匹配的软件版本。</span></p>
|
||||
<p class="p1"><span class="s1">2.2.2 本服务中讯工捷使客户端软件可能会提供包括但不限于iOS、Android、Windows、Mac等多个应用版本，用户必须选择与所安装终端设备相匹配的软件版本。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">2.3 本服务许可的范围</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
@@ -56,7 +56,7 @@
|
||||
<p class="p1"><span class="s1">2.3.2 本条及本协议其他条款未明确授权的其他一切权利仍由我们保留，您在行使这些权利时须另外取得我们的书面许可。我们如果未行使前述任何权利，并不构成对该权利的放弃。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">2.4 按现状提供服务</span></p>
|
||||
<p class="p1"><span class="s1">您理解并同意，笔墨讯使的服务是按照现有技术和条件所能达到的现状提供的。笔墨讯使会尽最大努力向您提供服务，确保服务的连续性和安全性；但笔墨讯使不能随时预见和防范法律、技术以及其他风险，包括但不限于不可抗力、病毒、木马、黑客攻击、系统不稳定、第三方服务缺陷、政府行为等原因可能导致的服务中断、数据丢失以及其他的损失和风险。</span></p>
|
||||
<p class="p1"><span class="s1">您理解并同意，讯工捷使的服务是按照现有技术和条件所能达到的现状提供的。讯工捷使会尽最大努力向您提供服务，确保服务的连续性和安全性；但讯工捷使不能随时预见和防范法律、技术以及其他风险，包括但不限于不可抗力、病毒、木马、黑客攻击、系统不稳定、第三方服务缺陷、政府行为等原因可能导致的服务中断、数据丢失以及其他的损失和风险。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">三、软件的获取与使用</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
@@ -70,7 +70,7 @@
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">3.5 为提供更加优质、安全的服务，在本软件安装时我们可能会推荐您安装其他软件，您可以选择安装或不安装。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">3.6 为了改善用户体验、保证服务的安全性及产品功能的一致性，我们可能会不时对笔墨讯使软件进行更新。我们有权不经过向您特别通知而对软件进行更新，或者对软件的部分功能效果进行改变或限制。您应当将相关软件更新到最新版本，否则我们并不保证其能正常使用。</span></p>
|
||||
<p class="p1"><span class="s1">3.6 为了改善用户体验、保证服务的安全性及产品功能的一致性，我们可能会不时对讯工捷使软件进行更新。我们有权不经过向您特别通知而对软件进行更新，或者对软件的部分功能效果进行改变或限制。您应当将相关软件更新到最新版本，否则我们并不保证其能正常使用。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">3.7 如果您不再需要使用本软件或者需要安装新版软件，可以自行卸载。如果您愿意帮助我们改进产品服务，请告知卸载的原因。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
@@ -78,7 +78,7 @@
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">4.1 保护用户个人信息是我们的一项基本原则，我们将按照本协议的规定收集、使用、存储您的个人信息，并会采取合理的措施保护您的个人信息。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">4.2 您在注册账号或使用本服务的过程中，需要提供一些必要的信息，例如：为向您提供账号注册服务或进行用户身份识别，需要您填写手机号码、身份证号码等；笔墨讯使某些功能可能需要收集您的个人信息包括：昵称、头像、手机号、身份证号、加入群组、好友关系等。若国家法律法规或政策有特殊规定的，您需要提供真实的身份信息。若您提供的信息不完整，则无法使用本服务或在使用过程中受到限制。您了解并同意，您有义务保持您提供信息的真实性、有效性及完整性。</span></p>
|
||||
<p class="p1"><span class="s1">4.2 您在注册账号或使用本服务的过程中，需要提供一些必要的信息，例如：为向您提供账号注册服务或进行用户身份识别，需要您填写手机号码、身份证号码等；讯工捷使某些功能可能需要收集您的个人信息包括：昵称、头像、手机号、身份证号、加入群组、好友关系等。若国家法律法规或政策有特殊规定的，您需要提供真实的身份信息。若您提供的信息不完整，则无法使用本服务或在使用过程中受到限制。您了解并同意，您有义务保持您提供信息的真实性、有效性及完整性。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">4.3 一般情况下，您可随时查看、修改自己提交的信息，但出于安全性和身份识别（如号码申诉服务）的考虑，您可能无法修改注册时提供的初始注册信息及其他验证信息。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
@@ -105,13 +105,13 @@
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">5.1 账号使用规范</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">5.1.1 您在使用本服务前需要注册一个笔墨讯使账号。本软件为您提供注册及登录通道，笔墨讯使账号通过手机号码进行注册，您可按照页面提示填写信息、阅读并同意本协议，在完成全部注册登录程序后，成为本软件用户。另外我们有权根据用户需求或产品需要对账号注册的方式进行变更，关于您使用账号的具体规则，请遵守相关账号使用协议以及我们为此发布的专项规则。</span></p>
|
||||
<p class="p1"><span class="s1">5.1.1 您在使用本服务前需要注册一个讯工捷使账号。本软件为您提供注册及登录通道，讯工捷使账号通过手机号码进行注册，您可按照页面提示填写信息、阅读并同意本协议，在完成全部注册登录程序后，成为本软件用户。另外我们有权根据用户需求或产品需要对账号注册的方式进行变更，关于您使用账号的具体规则，请遵守相关账号使用协议以及我们为此发布的专项规则。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">5.1.2 笔墨讯使账号的所有权归我们公司所有，用户完成申请注册手续后，仅获得笔墨讯使账号的使用权，且该使用权仅属于初始申请注册人。同时，初始申请注册人不得赠与、借用、出租、转让或售卖笔墨讯使账号或者以其他方式许可非初始申请注册人使用笔墨讯使账号。非初始申请注册人不得通过受赠、继承、承租、受让或者其他任何方式使用笔墨讯使账号。如我们判断您账号的使用可能违反法律法规、危及您的账号安全及/或本软件信息安全的，我们可拒绝提供相应服务或终止本协议。我们有权视情况封禁或注销、回收该账号，由此带来的包括但不限于内容、虚拟财产清空等损失由您本人及该账号使用者自行承担。</span></p>
|
||||
<p class="p1"><span class="s1">5.1.2 讯工捷使账号的所有权归我们公司所有，用户完成申请注册手续后，仅获得讯工捷使账号的使用权，且该使用权仅属于初始申请注册人。同时，初始申请注册人不得赠与、借用、出租、转让或售卖讯工捷使账号或者以其他方式许可非初始申请注册人使用讯工捷使账号。非初始申请注册人不得通过受赠、继承、承租、受让或者其他任何方式使用讯工捷使账号。如我们判断您账号的使用可能违反法律法规、危及您的账号安全及/或本软件信息安全的，我们可拒绝提供相应服务或终止本协议。我们有权视情况封禁或注销、回收该账号，由此带来的包括但不限于内容、虚拟财产清空等损失由您本人及该账号使用者自行承担。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">5.1.3 您注册获得的账号、设置的密码是您登录并以用户身份使用本软件的凭证，您的账户仅限您本人使用，您有责任妥善保管您的注册用户信息及用户密码的安全，您需要对您的注册用户以及密码下的行为承担法律责任。您同意在任何情况下不向他人泄露用户及密码信息。当在您怀疑他人在使用您的账号时，您应立即通知我们。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">5.1.4 用户注册笔墨讯使账号后如果长期（超过【6】个月）不登录该账号，我们有权回收该账号，以免造成资源浪费，由此带来的任何损失均由用户自行承担。</span></p>
|
||||
<p class="p1"><span class="s1">5.1.4 用户注册讯工捷使账号后如果长期（超过【6】个月）不登录该账号，我们有权回收该账号，以免造成资源浪费，由此带来的任何损失均由用户自行承担。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">5.1.5 我们特别提醒，您应妥善保管您的账号和密码。当您使用完本服务后，应安全退出。您须对账号进行的所有活动和行为负责，因您保管不善导致遭受账号或密码失窃、账号被非法使用，相关责任由您自行承担，您可通过司法、行政等救济途径向侵权行为人追偿。</span></p>
|
||||
<p class="p1"><span class="s1">5.1.6您不得恶意注册账号，包括但不限于频繁注册、批量注册、使用他人身份注册或其他不以正常使用本软件服务为目的的账号注册行为。除此外，您的账户不得以任何方式转让，否则我们有权追究您的违约责任，且由此产生的责任及后果均由您自行承担。</span></p>
|
||||
@@ -120,9 +120,9 @@
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">5.2.1 您理解并同意：为了向您提供有效的服务，本软件会利用您终端设备的处理器和带宽等资源。本软件使用过程中可能产生数据流量的费用，用户需自行向运营商了解相关资费信息，并自行承担相关费用。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">5.2.2 您理解并同意：本软件的某些功能可能会让第三方知晓用户的信息，例如：用户选择笔墨讯使与其他软件或硬件信息互通后，其他软件或硬件的提供方可以获取用户在笔墨讯使主动公开或传输的相关信息，用户在选择前应充分了解其他软件或硬件的产品功能及信息保护策略。</span></p>
|
||||
<p class="p1"><span class="s1">5.2.2 您理解并同意：本软件的某些功能可能会让第三方知晓用户的信息，例如：用户选择讯工捷使与其他软件或硬件信息互通后，其他软件或硬件的提供方可以获取用户在讯工捷使主动公开或传输的相关信息，用户在选择前应充分了解其他软件或硬件的产品功能及信息保护策略。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">5.2.3 您在使用本软件某一特定服务时，该服务可能会另有单独的协议、相关业务规则等（以下统称为“单独协议”），您在使用该项服务前请阅读并同意相关的单独协议；您使用前述特定服务，即视为您接受相关单独协议。例如：如果您需要使用笔墨讯使支付相关服务（包括但不限于笔墨讯使红包、转账、收款、充值、提现等支付服务）时，需阅读并遵守《笔墨讯使支付服务协议》；您使用笔墨讯使支付相关服务，即视为您接受《笔墨讯使支付服务协议》。</span></p>
|
||||
<p class="p1"><span class="s1">5.2.3 您在使用本软件某一特定服务时，该服务可能会另有单独的协议、相关业务规则等（以下统称为“单独协议”），您在使用该项服务前请阅读并同意相关的单独协议；您使用前述特定服务，即视为您接受相关单独协议。例如：如果您需要使用讯工捷使支付相关服务（包括但不限于讯工捷使红包、转账、收款、充值、提现等支付服务）时，需阅读并遵守《讯工捷使支付服务协议》；您使用讯工捷使支付相关服务，即视为您接受《讯工捷使支付服务协议》。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">5.2.4 您理解并同意我们将尽其商业上的合理努力保障您在本软件及服务中的数据存储安全，但是，我们并不能就此提供完全保证，包括但不限于以下情形：</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
@@ -142,20 +142,24 @@
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">5.2.5.4 用户发布的内容被他人转发、分享，因此类传播可能带来的风险和责任；</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">5.2.5.5 由于无线网络信号不稳定、无线网络带宽小等原因，所引起的笔墨讯使登录失败、资料同步不完整、页面打开速度慢等风险。</span></p>
|
||||
<p class="p1"><span class="s1">5.2.5.5 由于无线网络信号不稳定、无线网络带宽小等原因，所引起的讯工捷使登录失败、资料同步不完整、页面打开速度慢等风险。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">5.2.6 您理解并同意，笔墨讯使可能会不时发布并执行一些用户奖励活动，相关活动的具体规则的制定权和最终解释权归我们所有，我们有权根据自己的判断决定提前终止活动或延长活动时间。同一笔墨讯使用户名、同一手机号、同一身份证号码、同一手机设备均视为同一用户。如果我们发现有用户利用不正当手段参与活动，或者如果用户存在或涉嫌违法违规行为情况（包括不限于洗钱、外挂、作弊、套现、赌博、刷奖励类的活动、其他风险行为），我们有权取消您参与活动的资格，并有权追回您已经获得的奖励或代币或其他奖品。如果因非实名认证导致您无法参与活动的，视为您自动放弃活动资格。</span></p>
|
||||
<p class="p1"><span class="s1">5.2.6 您理解并同意，讯工捷使可能会不时发布并执行一些用户奖励活动，相关活动的具体规则的制定权和最终解释权归我们所有，我们有权根据自己的判断决定提前终止活动或延长活动时间。同一讯工捷使用户名、同一手机号、同一身份证号码、同一手机设备均视为同一用户。如果我们发现有用户利用不正当手段参与活动，或者如果用户存在或涉嫌违法违规行为情况（包括不限于洗钱、外挂、作弊、套现、赌博、刷奖励类的活动、其他风险行为），我们有权取消您参与活动的资格，并有权追回您已经获得的奖励或代币或其他奖品。如果因非实名认证导致您无法参与活动的，视为您自动放弃活动资格。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">5.3 第三方产品和服务</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">5.3.1 您在使用本软件第三方提供的产品或服务时，除遵守本协议约定外，还应遵守第三方的用户协议。我们和第三方对可能出现的纠纷在法律规定和约定的范围内各自承担责任。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">5.3.2 因用户使用本软件或要求我们提供特定服务时，本软件可能会调用第三方系统或者通过第三方支持用户的使用或访问，使用或访问的结果由该第三方提供（包括但不限于第三方通过笔墨讯使开放平台接入的内容等），我们不保证通过第三方提供服务及内容的安全性、准确性、有效性及其它不确定的风险，您应自行判断此类第三方网站或者服务的安全性和可用性，并自行承担相关风险和责任，由此若引发的任何争议及损害，与我们无关，我们不承担任何责任。</span></p>
|
||||
<p class="p1"><span class="s1">5.3.2 因用户使用本软件或要求我们提供特定服务时，本软件可能会调用第三方系统或者通过第三方支持用户的使用或访问，使用或访问的结果由该第三方提供（包括但不限于第三方通过讯工捷使开放平台接入的内容等），我们不保证通过第三方提供服务及内容的安全性、准确性、有效性及其它不确定的风险，您应自行判断此类第三方网站或者服务的安全性和可用性，并自行承担相关风险和责任，由此若引发的任何争议及损害，与我们无关，我们不承担任何责任。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">5.3.3 第三方服务提供者可以通过本软件向您提供产品或者服务，比如您可以购买并使用第三方开发的应用程序、接受第三方提供的服务及产品，或者通过笔墨讯使授权登录能力登录并使用其他互联网服务。在此过程中，第三方服务提供者可能会收集、使用和存储您的相关数据或信息，我们通过与第三方服务提供者签订的有关协议、笔墨讯使相关规范以及合理的产品流程设计，严格要求第三方服务提供者获取您的任何数据均应遵守相关法律法规的规定，必须事先获得您的明确同意，采取必要的数据保护措施，且仅为产品或服务之目的合理使用您的相关数据，向您提供修改或删除自己数据的方式，并在停止服务时必须删除全部数据等，尽最大可能保障您的数据和隐私不受侵犯。您在接受或使用第三方产品或服务前应充分了解第三方产品或服务的条款及政策。如果您发现有第三方服务提供者存在相关违法违规行为，可以向我们投诉，我们将查实后予以处理。</span></p>
|
||||
<p class="p1"><span class="s1">5.3.3 第三方服务提供者可以通过本软件向您提供产品或者服务，比如您可以购买并使用第三方开发的应用程序、接受第三方提供的服务及产品，或者通过讯工捷使授权登录能力登录并使用其他互联网服务。在此过程中，第三方服务提供者可能会收集、使用和存储您的相关数据或信息，我们通过与第三方服务提供者签订的有关协议、讯工捷使相关规范以及合理的产品流程设计，严格要求第三方服务提供者获取您的任何数据均应遵守相关法律法规的规定，必须事先获得您的明确同意，采取必要的数据保护措施，且仅为产品或服务之目的合理使用您的相关数据，向您提供修改或删除自己数据的方式，并在停止服务时必须删除全部数据等，尽最大可能保障您的数据和隐私不受侵犯。您在接受或使用第三方产品或服务前应充分了解第三方产品或服务的条款及政策。如果您发现有第三方服务提供者存在相关违法违规行为，可以向我们投诉，我们将查实后予以处理。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">5.3.4 您理解并同意，我们有权决定将本软件作商业用途，包括但不限于开发、使用本软件的部分服务为第三方作推广等，我们承诺在推广过程中严格按照本协议约定保护您的个人信息，同时您也可以根据系统设置选择屏蔽、拒绝接收相关推广信息。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">5.3.5 AI智能服务。我们可能在本软件中接入DeepSeek AI服务或我们部署、接入的其他人工智能模型，为您提供智能回复、内容生成、内容总结、智能客服、搜索推荐等AI辅助能力。您理解并同意，AI生成内容基于算法模型自动生成，可能存在不准确、不完整、不及时、不适合特定场景或与事实不一致的情况，相关内容仅供参考，不构成法律、医疗、金融、投资、职业等专业建议。您应结合实际情况独立判断并谨慎使用AI生成内容。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">5.3.6 您使用AI功能时，应遵守法律法规、本协议及相关平台规则，不得输入、生成、传播违法违规、侵害他人合法权益、泄露国家秘密、商业秘密、个人隐私或含有恶意代码、欺诈、歧视、暴力、色情低俗等内容的信息。您应确保输入内容来源合法并已取得必要授权，不得将AI功能用于自动化批量骚扰、虚假信息生成、绕过安全限制、侵犯知识产权或其他不当用途。因您输入、使用、发布或传播AI生成内容产生的责任，由您依法自行承担；我们有权依法依约对违规内容或违规账号采取限制、删除、暂停或终止服务等措施。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">5.4账号信息设置及维护</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">5.4.1您在注册或使用本软件时设置的名称、头像和简介等账号信息资料应遵守法律法规、社会道德风尚和信息真实性等原则，不得出现不良信息，不得冒用他人姓名、名称、字号、头像等或采取其他足以引起混淆的方式设置账号，不得侵害第三方的合法权益。</span></p>
|
||||
@@ -179,9 +183,9 @@
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">6.1 信息内容规范</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">6.1.1 本条所述信息内容是指用户使用本软件及服务过程中所制作、复制、发布、传播的任何内容，包括但不限于笔墨讯使账号、昵称、头像、姓名、用户说明等注册信息，或文字、语音、图片等发送、回复、图文和相关链接页面，以及其他使用笔墨讯使账号或本软件及服务所产生的内容。</span></p>
|
||||
<p class="p1"><span class="s1">6.1.1 本条所述信息内容是指用户使用本软件及服务过程中所制作、复制、发布、传播的任何内容，包括但不限于讯工捷使账号、昵称、头像、姓名、用户说明等注册信息，或文字、语音、图片等发送、回复、图文和相关链接页面，以及其他使用讯工捷使账号或本软件及服务所产生的内容。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">6.1.2 您理解并同意，笔墨讯使一直致力于为用户提供文明健康、规范有序的网络环境，您不得利用笔墨讯使账号或本软件及服务制作、复制、发布、传播如下干扰笔墨讯使正常运行，以及侵犯其他用户或第三方合法权益的内容，包括但不限于：</span></p>
|
||||
<p class="p1"><span class="s1">6.1.2 您理解并同意，讯工捷使一直致力于为用户提供文明健康、规范有序的网络环境，您不得利用讯工捷使账号或本软件及服务制作、复制、发布、传播如下干扰讯工捷使正常运行，以及侵犯其他用户或第三方合法权益的内容，包括但不限于：</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">6.1.2.1 发布、传送、传播、存储违反国家法律法规禁止的内容：</span></p>
|
||||
<p class="p1"><span class="s1">（1）违反宪法规定的基本原则，煽动抗拒或者破坏宪法等法律法规实施的；</span></p>
|
||||
@@ -205,18 +209,18 @@
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">6.1.2.4 发表、传送、传播骚扰、广告信息、过度营销信息及垃圾信息或含有任何性或性暗示的；</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">6.1.2.5 其他违反法律法规、政策及公序良俗、社会公德或干扰笔墨讯使正常运行和侵犯其他用户或第三方合法权益内容的信息。</span></p>
|
||||
<p class="p1"><span class="s1">6.1.2.5 其他违反法律法规、政策及公序良俗、社会公德或干扰讯工捷使正常运行和侵犯其他用户或第三方合法权益内容的信息。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">6.1.2.6 您通过本服务发送或传播的内容（包括但不限于网页、文字、图片、音频、视频、图表等）均由您自行承担责任。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">6.1.2.6.1 您发送或传播的内容应有合法来源，相关内容为您所有或您已获得权利人的授权。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">6.1.2.6.2 您同意笔墨讯使可为履行本协议或提供本服务的目的而使用您发送或传播的内容。</span></p>
|
||||
<p class="p1"><span class="s1">6.1.2.6.2 您同意讯工捷使可为履行本协议或提供本服务的目的而使用您发送或传播的内容。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">6.1.2.6.3 如果笔墨讯使收到权利人通知，主张您发送或传播的内容侵犯其相关权利的，您同意笔墨讯使有权进行独立判断并采取删除、屏蔽或断开链接等措施。</span></p>
|
||||
<p class="p1"><span class="s1">6.1.2.6.3 如果讯工捷使收到权利人通知，主张您发送或传播的内容侵犯其相关权利的，您同意讯工捷使有权进行独立判断并采取删除、屏蔽或断开链接等措施。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">6.1.2.7 您使用本服务时不得违反国家法律法规、侵犯他人合法权益。</span></p>
|
||||
<p class="p1"><span class="s1">您理解并同意，如果您被他人投诉侵权或您投诉他人侵权，笔墨讯使有权将争议中相关方的主体、联系方式、投诉相关内容等必要信息提供给其他争议方或相关部门，以便及时解决投诉纠纷，保护他人合法权益。</span></p>
|
||||
<p class="p1"><span class="s1">您理解并同意，如果您被他人投诉侵权或您投诉他人侵权，讯工捷使有权将争议中相关方的主体、联系方式、投诉相关内容等必要信息提供给其他争议方或相关部门，以便及时解决投诉纠纷，保护他人合法权益。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">6.2 软件使用规范</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
@@ -242,7 +246,7 @@
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">6.2.1.10 其他以任何不合法的方式、为任何不合法的目的、或与我们为此制定的其他规范和标准不一致的方式使用本软件及未经我们明确授权的行为。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">6.2.2 您理解并同意，基于用户体验、笔墨讯使或相关服务平台运营安全、平台规则要求及健康发展等综合因素，我们有权选择提供服务的对象，有权决定功能设置，有权决定功能开放、数据接口和相关数据披露的对象和范围。针对以下情形，有权视具体情况中止或终止提供本服务，包括但不限于：</span></p>
|
||||
<p class="p1"><span class="s1">6.2.2 您理解并同意，基于用户体验、讯工捷使或相关服务平台运营安全、平台规则要求及健康发展等综合因素，我们有权选择提供服务的对象，有权决定功能设置，有权决定功能开放、数据接口和相关数据披露的对象和范围。针对以下情形，有权视具体情况中止或终止提供本服务，包括但不限于：</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">6.2.2.1 违反法律法规或本协议规定的；</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
@@ -250,19 +254,19 @@
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">6.2.2.3 存在安全隐患的；</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">6.2.2.4 与笔墨讯使或其服务平台已有主要功能或功能组件相似、相同，或可实现上述功能或功能组件的主要效用的；</span></p>
|
||||
<p class="p1"><span class="s1">6.2.2.4 与讯工捷使或其服务平台已有主要功能或功能组件相似、相同，或可实现上述功能或功能组件的主要效用的；</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">6.2.2.5 界面、风格、功能、描述或使用者体验与笔墨讯使或其服务平台相似，可能造成笔墨讯使用户认为其所使用的功能或服务来源于我们或经我们授权的；</span></p>
|
||||
<p class="p1"><span class="s1">6.2.2.5 界面、风格、功能、描述或使用者体验与讯工捷使或其服务平台相似，可能造成讯工捷使用户认为其所使用的功能或服务来源于我们或经我们授权的；</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">6.2.2.6 违反笔墨讯使或其服务平台运营原则，或不符合我们其他管理要求的。</span></p>
|
||||
<p class="p1"><span class="s1">6.2.2.6 违反讯工捷使或其服务平台运营原则，或不符合我们其他管理要求的。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">6.3 服务运营规范</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">6.3.1 除非您与笔墨讯使另有约定，您同意本服务仅为您个人非商业性质的使用。</span></p>
|
||||
<p class="p1"><span class="s1">6.3.1 除非您与讯工捷使另有约定，您同意本服务仅为您个人非商业性质的使用。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">6.3.2 您应当通过笔墨讯使提供或认可的方式使用本服务。您依本协议条款所获得的权利不可转让。</span></p>
|
||||
<p class="p1"><span class="s1">6.3.2 您应当通过讯工捷使提供或认可的方式使用本服务。您依本协议条款所获得的权利不可转让。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">6.3.3 您不得使用未经笔墨讯使授权的插件、外挂或第三方工具对本协议项下的服务进行干扰、破坏、修改或施加其他影响。</span></p>
|
||||
<p class="p1"><span class="s1">6.3.3 您不得使用未经讯工捷使授权的插件、外挂或第三方工具对本协议项下的服务进行干扰、破坏、修改或施加其他影响。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">6.3.4 除非法律允许或我们书面许可，您使用本服务过程中不得从事下列行为：</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
@@ -274,15 +278,15 @@
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">4) 侵犯他人名誉权、肖像权、知识产权、商业秘密等合法权利的；</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">5) 未经我们书面许可利用笔墨讯使账号和任何功能，以及第三方运营平台进行推广或相互推广的；</span></p>
|
||||
<p class="p1"><span class="s1">5) 未经我们书面许可利用讯工捷使账号和任何功能，以及第三方运营平台进行推广或相互推广的；</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">6) 利用笔墨讯使账号或本软件及服务从事任何违法犯罪活动的；</span></p>
|
||||
<p class="p1"><span class="s1">6) 利用讯工捷使账号或本软件及服务从事任何违法犯罪活动的；</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">8) 其他违反法律法规规定、侵犯其他用户合法权益、干扰产品正常运营或我们未明确授权的行为。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">6.3.5 您在使用本服务过程中应当遵守当地相关的法律法规，并尊重当地的道德和风俗习惯。如果您的行为违反了当地法律法规或道德风俗，您应当为此单独承担责任。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">6.3.6 您应避免因使用本服务而使笔墨讯使卷入政治和公共事件，否则笔墨讯使有权暂停或终止对您的服务。</span></p>
|
||||
<p class="p1"><span class="s1">6.3.6 您应避免因使用本服务而使讯工捷使卷入政治和公共事件，否则讯工捷使有权暂停或终止对您的服务。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">6.4 对自己行为负责</span></p>
|
||||
<p class="p1"><span class="s1">您充分了解并同意，您必须为自己注册账号下的一切行为负责，包括您所发表的任何内容以及由此产生的任何后果。您应对本服务中的内容自行加以判断，并承担因使用内容而引起的所有风险，包括因对内容的正确性、完整性或实用性的依赖而产生的风险。我们无法且不会对因前述风险而导致的任何损失或损害承担责任。</span></p>
|
||||
@@ -307,19 +311,19 @@
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">七、知识产权声明</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">7.1 笔墨讯使在本服务中提供的内容（包括但不限于网页、文字、图片、音频、视频、图表、文字表述及其组合、界面设计、版面框架、图标、商标等）的知识产权归笔墨讯使所有，受中华人民共和国著作权法、商标法、专利法、反不正当竞争法及国际条约等法律法规的保护。未经我们书面允许，用户不得为任何目的擅自使用、复制、再造这些内容、或创造与内容有关的派生产品。用户在使用本服务中所产生的内容的知识产权归用户或相关权利人所有。</span></p>
|
||||
<p class="p1"><span class="s1">7.1 讯工捷使在本服务中提供的内容（包括但不限于网页、文字、图片、音频、视频、图表、文字表述及其组合、界面设计、版面框架、图标、商标等）的知识产权归讯工捷使所有，受中华人民共和国著作权法、商标法、专利法、反不正当竞争法及国际条约等法律法规的保护。未经我们书面允许，用户不得为任何目的擅自使用、复制、再造这些内容、或创造与内容有关的派生产品。用户在使用本服务中所产生的内容的知识产权归用户或相关权利人所有。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">7.2 除另有特别声明外，笔墨讯使提供本服务时所依据软件的著作权、专利权及其他知识产权均归笔墨讯使所有。</span></p>
|
||||
<p class="p1"><span class="s1">7.2 除另有特别声明外，讯工捷使提供本服务时所依据软件的著作权、专利权及其他知识产权均归讯工捷使所有。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">7.3 笔墨讯使在本服务中所使用的“笔墨讯使”等形象等商业标识，其著作权或商标权归笔墨讯使所有。</span></p>
|
||||
<p class="p1"><span class="s1">7.3 讯工捷使在本服务中所使用的“讯工捷使”等形象等商业标识，其著作权或商标权归讯工捷使所有。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">7.4 上述及其他任何本服务包含的内容的知识产权均受到法律保护，未经笔墨讯使、用户或相关权利人书面许可，任何人不得以任何形式进行使用或创作相关衍生作品。</span></p>
|
||||
<p class="p1"><span class="s1">7.4 上述及其他任何本服务包含的内容的知识产权均受到法律保护，未经讯工捷使、用户或相关权利人书面许可，任何人不得以任何形式进行使用或创作相关衍生作品。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">八、终端安全责任</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">8.1 您理解并同意，本软件同大多数互联网软件一样，可能会受多种因素影响，包括但不限于用户原因、网络服务质量、社会环境等；也可能会受各种安全问题的侵扰，包括但不限于他人非法利用用户资料，进行现实中的骚扰；用户下载安装的其他软件或访问的其他网站中可能含有病毒、木马程序或其他恶意程序，威胁您的终端设备信息和数据安全，继而影响本软件的正常使用等。因此，您应加强信息安全及个人信息的保护意识，注意密码保护，以免遭受损失。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">8.2 您不得制作、发布、使用、传播用于窃取笔墨讯使账号及他人个人信息、财产的恶意程序。</span></p>
|
||||
<p class="p1"><span class="s1">8.2 您不得制作、发布、使用、传播用于窃取讯工捷使账号及他人个人信息、财产的恶意程序。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">8.3 维护软件安全与正常使用是我们和您的共同责任，我们将按照行业标准合理审慎地采取必要技术措施保护您的终端设备信息和数据安全，但是您承认和同意我们并不能就此提供完全保证。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
@@ -407,21 +411,21 @@
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1"><span class="Apple-converted-space"> </span>十、不可抗力及其他免责事由</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">10.1 您理解并同意，在使用本服务的过程中，可能会遇到不可抗力等风险因素，使本服务发生中断。不可抗力是指不能预见、不能克服并不能避免且对一方或双方造成重大影响的客观事件，包括但不限于自然灾害如水灾、地震、瘟疫流行和风暴等以及社会事件如战争、动乱、政府行为等。出现上述情况时，笔墨讯使将努力在第一时间与相关单位配合，及时进行修复，但由此给您造成的损失笔墨讯使在法律允许的范围内免责。</span></p>
|
||||
<p class="p1"><span class="s1">10.1 您理解并同意，在使用本服务的过程中，可能会遇到不可抗力等风险因素，使本服务发生中断。不可抗力是指不能预见、不能克服并不能避免且对一方或双方造成重大影响的客观事件，包括但不限于自然灾害如水灾、地震、瘟疫流行和风暴等以及社会事件如战争、动乱、政府行为等。出现上述情况时，讯工捷使将努力在第一时间与相关单位配合，及时进行修复，但由此给您造成的损失讯工捷使在法律允许的范围内免责。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">10.2 在法律允许的范围内，笔墨讯使对以下情形导致的服务中断或受阻不承担责任：</span></p>
|
||||
<p class="p1"><span class="s1">10.2 在法律允许的范围内，讯工捷使对以下情形导致的服务中断或受阻不承担责任：</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">（1）受到计算机病毒、木马或其他恶意程序、黑客攻击的破坏；</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">（2）用户或笔墨讯使的电脑软件、系统、硬件和通信线路出现故障；</span></p>
|
||||
<p class="p1"><span class="s1">（2）用户或讯工捷使的电脑软件、系统、硬件和通信线路出现故障；</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">（3）用户操作不当；</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">（4）用户通过非笔墨讯使授权的方式使用本服务；</span></p>
|
||||
<p class="p1"><span class="s1">（4）用户通过非讯工捷使授权的方式使用本服务；</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">（5）其他笔墨讯使无法控制或合理预见的情形。</span></p>
|
||||
<p class="p1"><span class="s1">（5）其他讯工捷使无法控制或合理预见的情形。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">10.3 您理解并同意，在使用本服务的过程中，可能会遇到网络信息或其他用户行为带来的风险，笔墨讯使不对任何信息的真实性、适用性、合法性承担责任，也不对因侵权行为给您造成的损害负责。这些风险包括但不限于：</span></p>
|
||||
<p class="p1"><span class="s1">10.3 您理解并同意，在使用本服务的过程中，可能会遇到网络信息或其他用户行为带来的风险，讯工捷使不对任何信息的真实性、适用性、合法性承担责任，也不对因侵权行为给您造成的损害负责。这些风险包括但不限于：</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">（1）来自他人匿名或冒名的含有威胁、诽谤、令人反感或非法内容的信息；</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
@@ -429,19 +433,19 @@
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">（3）其他因网络信息或用户行为引起的风险。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">10.4 您理解并同意，本服务并非为某些特定目的而设计，包括但不限于核设施、军事用途、医疗设施、交通通信等重要领域。如果因为软件或服务的原因导致上述操作失败而带来的人员伤亡、财产损失和环境污染等，笔墨讯使不承担法律责任。</span></p>
|
||||
<p class="p1"><span class="s1">10.4 您理解并同意，本服务并非为某些特定目的而设计，包括但不限于核设施、军事用途、医疗设施、交通通信等重要领域。如果因为软件或服务的原因导致上述操作失败而带来的人员伤亡、财产损失和环境污染等，讯工捷使不承担法律责任。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">10.5 笔墨讯使根据本协议约定获得处理违法违规内容的权利，该权利不构成笔墨讯使的义务或承诺，笔墨讯使不能保证及时发现违法行为或进行相应处理。</span></p>
|
||||
<p class="p1"><span class="s1">10.5 讯工捷使根据本协议约定获得处理违法违规内容的权利，该权利不构成讯工捷使的义务或承诺，讯工捷使不能保证及时发现违法行为或进行相应处理。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">10.6 在任何情况下，您不应轻信借款、索要密码或其他涉及财产的网络信息。涉及财产操作的，请一定先核实对方身份，并请经常留意笔墨讯使有关防范诈骗犯罪的提示。</span></p>
|
||||
<p class="p1"><span class="s1">10.6 在任何情况下，您不应轻信借款、索要密码或其他涉及财产的网络信息。涉及财产操作的，请一定先核实对方身份，并请经常留意讯工捷使有关防范诈骗犯罪的提示。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1"><span class="Apple-converted-space"> </span>十一、服务的变更、中断、终止</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">11.1 笔墨讯使可能会对服务内容进行变更，也可能会中断、中止或终止服务。</span></p>
|
||||
<p class="p1"><span class="s1">11.1 讯工捷使可能会对服务内容进行变更，也可能会中断、中止或终止服务。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">11.2 您理解并同意，笔墨讯使有权自主决定经营策略。在笔墨讯使发生合并、分立、收购、资产转让时，笔墨讯使可向第三方转让本服务下相关资产；笔墨讯使也可在单方通知您后，将本协议下部分或全部服务转交由第三方经营或履行。具体受让主体以笔墨讯使通知的为准。</span></p>
|
||||
<p class="p1"><span class="s1">11.2 您理解并同意，讯工捷使有权自主决定经营策略。在讯工捷使发生合并、分立、收购、资产转让时，讯工捷使可向第三方转让本服务下相关资产；讯工捷使也可在单方通知您后，将本协议下部分或全部服务转交由第三方经营或履行。具体受让主体以讯工捷使通知的为准。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">11.3 如发生下列任何一种情形，笔墨讯使有权不经通知而中断或终止向您提供的服务：</span></p>
|
||||
<p class="p1"><span class="s1">11.3 如发生下列任何一种情形，讯工捷使有权不经通知而中断或终止向您提供的服务：</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">（1）根据法律规定您应提交真实信息，而您提供的个人资料不真实、或与注册时信息不一致又未能提供合理证明；</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
@@ -451,13 +455,13 @@
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">（4）出于安全的原因或其他必要的情形。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">11.4 笔墨讯使有权对部分服务进行收费。若您未按时足额付费，笔墨讯使有权中断、中止或终止提供服务。</span></p>
|
||||
<p class="p1"><span class="s1">11.4 讯工捷使有权对部分服务进行收费。若您未按时足额付费，讯工捷使有权中断、中止或终止提供服务。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">11.5 我们始终在不断变更和改进本软件服务，我们会尽最大努力向您提供服务，确保服务的连贯性和安全性。基于此，我们可能会对本软件进行更新，您应将软件更新到最新版本，否则我们不保证您可正常使用本软件。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">11.6 您理解并同意，我公司根据自身商业决策、政府行为、不可抗力等原因可能会选择中止、中断及终止本软件的部分或全部服务。如有此等情形发生，我们会采取合理的方式并以合理的期限提前通知您，除法律法规另有规定外，我们不承担由此对您造成的损失。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">11.7 您有责任自行备份存储在本服务中的数据。如果您的服务被终止，笔墨讯使可以从服务器上永久地删除您的数据，但法律法规另有规定的除外。服务终止后，笔墨讯使没有义务向您返还数据。</span></p>
|
||||
<p class="p1"><span class="s1">11.7 您有责任自行备份存储在本服务中的数据。如果您的服务被终止，讯工捷使可以从服务器上永久地删除您的数据，但法律法规另有规定的除外。服务终止后，讯工捷使没有义务向您返还数据。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">十二、未成年人使用条款</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
@@ -517,7 +521,7 @@
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">15.6 本协议条款无论因何种原因部分无效或不可执行，其余条款仍有效，对双方具有约束力。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">15.7 如您对本协议有任何意见或建议，您可通过本平台联系我们，联系方式为【fengqunwangluo2025@163.com】，我们会给予您必要的帮助。</span></p>
|
||||
<p class="p1"><span class="s1">15.7 如您对本协议有任何意见或建议，您可通过本平台联系我们，联系方式为【support@heshunming.top】，我们会给予您必要的帮助。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1"><span class="Apple-converted-space"> </span>自启动或关联启动说明</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
|
||||
@@ -62,8 +62,8 @@ type legalDocument struct {
|
||||
}
|
||||
|
||||
var legalDocuments = map[string]legalDocument{
|
||||
"/legal/user-terms": loadLegalDocument("legal/user-terms.html", "笔墨讯使软件服务协议"),
|
||||
"/legal/privacy-policy": loadLegalDocument("legal/privacy-policy.html", "笔墨讯使隐私政策"),
|
||||
"/legal/user-terms": loadLegalDocument("legal/user-terms.html", "讯工捷使软件服务协议"),
|
||||
"/legal/privacy-policy": loadLegalDocument("legal/privacy-policy.html", "讯工捷使隐私政策"),
|
||||
}
|
||||
|
||||
func loadLegalDocument(name, title string) legalDocument {
|
||||
|
||||
@@ -12,8 +12,8 @@ func TestServeLegalDocument(t *testing.T) {
|
||||
path string
|
||||
title string
|
||||
}{
|
||||
{path: "/legal/user-terms", title: "笔墨讯使软件服务协议"},
|
||||
{path: "/legal/privacy-policy", title: "笔墨讯使隐私政策"},
|
||||
{path: "/legal/user-terms", title: "讯工捷使软件服务协议"},
|
||||
{path: "/legal/privacy-policy", title: "讯工捷使隐私政策"},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.path, func(t *testing.T) {
|
||||
|
||||
@@ -322,6 +322,10 @@ type configType struct {
|
||||
// If true, do not attempt to negotiate websocket per message compression (RFC 7692.4).
|
||||
// It should be disabled (set to true) if you are using MSFT IIS as a reverse proxy.
|
||||
WSCompressionDisabled bool `json:"ws_compression_disabled"`
|
||||
// WebSocket heartbeat period and the maximum time to wait for a Pong, in seconds.
|
||||
// Zero values use the built-in defaults.
|
||||
WSPingPeriodSeconds int `json:"ws_ping_period_seconds"`
|
||||
WSPongWaitSeconds int `json:"ws_pong_wait_seconds"`
|
||||
// Address:port to listen for gRPC clients. If blank gRPC support will not be initialized.
|
||||
// Could be overridden from the command line with --grpc_listen.
|
||||
GrpcListen string `json:"grpc_listen"`
|
||||
@@ -716,6 +720,10 @@ func main() {
|
||||
|
||||
// Websocket compression.
|
||||
globals.wsCompression = !config.WSCompressionDisabled
|
||||
if err = configureWSHeartbeat(config.WSPingPeriodSeconds, config.WSPongWaitSeconds); err != nil {
|
||||
logs.Err.Fatal("Invalid WebSocket heartbeat configuration: ", err)
|
||||
}
|
||||
logs.Info.Printf("WebSocket heartbeat ping=%s pong_wait=%s", pingPeriod, pongWait)
|
||||
|
||||
if config.Media != nil {
|
||||
if config.Media.UseHandler == "" {
|
||||
@@ -919,6 +927,8 @@ func main() {
|
||||
mux.HandleFunc(config.ApiPath+"v1/im/me", serveIMMe)
|
||||
mux.HandleFunc(config.ApiPath+"v1/im/me/profile", serveIMMeProfile)
|
||||
mux.HandleFunc(config.ApiPath+"v1/im/me/privacy", serveIMMePrivacy)
|
||||
mux.HandleFunc(config.ApiPath+"v1/im/me/password", serveIMMePasswordChange)
|
||||
mux.HandleFunc(config.ApiPath+"v1/im/me/deletion", serveIMMeDeletion)
|
||||
mux.HandleFunc(config.ApiPath+"v1/im/settings/youth-mode/enable", serveIMYouthModeEnable)
|
||||
mux.HandleFunc(config.ApiPath+"v1/im/settings/youth-mode/disable", serveIMYouthModeDisable)
|
||||
mux.HandleFunc(config.ApiPath+"v1/im/settings/youth-mode", serveIMYouthMode)
|
||||
|
||||
@@ -89,7 +89,7 @@
|
||||
"max_idle_conns": 8,
|
||||
"registry_max_open_conns": 4,
|
||||
"registry_max_idle_conns": 2,
|
||||
"conn_max_lifetime": 60,
|
||||
"conn_max_lifetime": 1800,
|
||||
"sql_timeout": 10
|
||||
}
|
||||
},
|
||||
@@ -122,7 +122,7 @@
|
||||
"write_timeout_ms": 3000,
|
||||
"read_timeout_ms": 3000,
|
||||
"batch_timeout_ms": 10,
|
||||
"required_acks": 1,
|
||||
"required_acks": -1,
|
||||
"allow_auto_topic_creation": false,
|
||||
"tls_enabled": false,
|
||||
"sasl_enabled": true,
|
||||
@@ -229,6 +229,8 @@
|
||||
"enabled": false,
|
||||
"api_domain": "https://api.jpush.cn",
|
||||
"push_path": "/v3/push",
|
||||
"workers": 8,
|
||||
"device_cache_ttl_seconds": 30,
|
||||
"default_ttl_seconds": 86400,
|
||||
"call_ttl_seconds": 60,
|
||||
"apns_production": false,
|
||||
|
||||
+197
-3
@@ -29,6 +29,10 @@ var (
|
||||
Help: "Database operation duration by driver, operation, and status.",
|
||||
Buckets: []float64{0.001, 0.003, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5},
|
||||
}, []string{"driver", "operation", "status"})
|
||||
DBInvalidConnectionRetryTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "im_db_invalid_connection_retry_total",
|
||||
Help: "Retries after an invalid database connection by operation and outcome.",
|
||||
}, []string{"operation", "outcome"})
|
||||
|
||||
RealtimeFanoutEventsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "realtime_fanout_events_total",
|
||||
@@ -54,6 +58,58 @@ var (
|
||||
}, func() float64 {
|
||||
return float64(expvarIntValue("LiveSessions"))
|
||||
})
|
||||
IMWSWriteTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "im_ws_write_total",
|
||||
Help: "WebSocket frame write and pre-write rejection outcomes by pod, frame type, result, and bounded reason.",
|
||||
}, []string{"pod", "frame", "result", "reason"})
|
||||
IMWSWriteDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Name: "im_ws_write_duration_seconds",
|
||||
Help: "WebSocket frame write duration by pod, frame type, and result.",
|
||||
Buckets: []float64{0.0005, 0.001, 0.003, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10},
|
||||
}, []string{"pod", "frame", "result"})
|
||||
IMWSWriteBytesTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "im_ws_write_bytes_total",
|
||||
Help: "WebSocket payload bytes submitted to the socket by pod and frame type.",
|
||||
}, []string{"pod", "frame"})
|
||||
IMWSDisconnectTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "im_ws_disconnect_total",
|
||||
Help: "WebSocket read-loop disconnects by pod and bounded reason.",
|
||||
}, []string{"pod", "reason"})
|
||||
IMWSHeartbeatTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "im_ws_heartbeat_total",
|
||||
Help: "WebSocket heartbeat events by pod and event.",
|
||||
}, []string{"pod", "event"})
|
||||
IMWSHeartbeatSeconds = prometheus.NewGaugeVec(prometheus.GaugeOpts{
|
||||
Name: "im_ws_heartbeat_seconds",
|
||||
Help: "Effective WebSocket heartbeat timing by pod and setting.",
|
||||
}, []string{"pod", "setting"})
|
||||
IMWSSendQueueCurrent = prometheus.NewGaugeVec(prometheus.GaugeOpts{
|
||||
Name: "im_ws_send_queue_current",
|
||||
Help: "Current number of queued WebSocket outbound channel entries across sessions on a pod.",
|
||||
}, []string{"pod"})
|
||||
IMWSSendQueueCapacity = prometheus.NewGaugeVec(prometheus.GaugeOpts{
|
||||
Name: "im_ws_send_queue_capacity",
|
||||
Help: "Configured outbound channel capacity of one WebSocket session by pod.",
|
||||
}, []string{"pod"})
|
||||
IMWSSendQueueEnqueueTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "im_ws_send_queue_enqueue_total",
|
||||
Help: "WebSocket outbound channel enqueue attempts by pod and result.",
|
||||
}, []string{"pod", "result"})
|
||||
IMWSSendQueueDepth = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Name: "im_ws_send_queue_depth",
|
||||
Help: "Per-session WebSocket outbound channel depth observed after enqueue attempts.",
|
||||
Buckets: []float64{0, 1, 4, 8, 16, 32, 64, 96, 128, 144, 160},
|
||||
}, []string{"pod", "result"})
|
||||
IMWSSendQueueWaitDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Name: "im_ws_send_queue_wait_duration_seconds",
|
||||
Help: "Time a WebSocket outbound channel entry waited before dequeue or purge.",
|
||||
Buckets: []float64{0.0005, 0.001, 0.003, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120},
|
||||
}, []string{"pod", "result"})
|
||||
IMCtrlACKDeliveryDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Name: "im_ctrl_ack_delivery_duration_seconds",
|
||||
Help: "Accepted control-response age from request receipt to WebSocket enqueue or completed socket write.",
|
||||
Buckets: []float64{0.0005, 0.001, 0.003, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 20, 30, 60, 120},
|
||||
}, []string{"pod", "stage"})
|
||||
IMClusterLivePeers = prometheus.NewGaugeFunc(prometheus.GaugeOpts{
|
||||
Name: "im_cluster_live_peers",
|
||||
Help: "Current connected cluster peers.",
|
||||
@@ -115,6 +171,10 @@ var (
|
||||
Name: "im_internal_queue_capacity",
|
||||
Help: "Configured internal message queue capacity by pod and queue.",
|
||||
}, []string{"pod", "queue"})
|
||||
InternalQueueCurrent = prometheus.NewGaugeVec(prometheus.GaugeOpts{
|
||||
Name: "im_internal_queue_current",
|
||||
Help: "Current entries in an internal deferred queue by pod and queue.",
|
||||
}, []string{"pod", "queue"})
|
||||
MessagePersistDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Name: "im_message_persist_duration_seconds",
|
||||
Help: "Kafka message persistence duration by message kind and result.",
|
||||
@@ -125,6 +185,34 @@ var (
|
||||
Help: "Message publish hot-path duration by message kind, stage, and result.",
|
||||
Buckets: []float64{0.001, 0.003, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10},
|
||||
}, []string{"kind", "stage", "result"})
|
||||
TopicQueueWaitDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Name: "im_topic_queue_wait_duration_seconds",
|
||||
Help: "Time a client publish waited in a topic queue before processing by topic kind.",
|
||||
Buckets: []float64{0.0005, 0.001, 0.003, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60},
|
||||
}, []string{"pod", "kind"})
|
||||
PushDispatchQueueEnqueueTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "im_push_dispatch_queue_enqueue_total",
|
||||
Help: "Asynchronous push dispatch queue enqueue attempts by pod and result.",
|
||||
}, []string{"pod", "result"})
|
||||
PushDispatchQueueDepth = prometheus.NewGaugeVec(prometheus.GaugeOpts{
|
||||
Name: "im_push_dispatch_queue_depth",
|
||||
Help: "Current asynchronous push dispatch queue depth by pod.",
|
||||
}, []string{"pod"})
|
||||
PushDispatchQueueWaitDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Name: "im_push_dispatch_queue_wait_duration_seconds",
|
||||
Help: "Time a push dispatch job waited before a worker started processing it.",
|
||||
Buckets: []float64{0.0005, 0.001, 0.003, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60},
|
||||
}, []string{"pod"})
|
||||
PushDispatchDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Name: "im_push_dispatch_duration_seconds",
|
||||
Help: "Asynchronous push presence resolution and dispatch duration by pod and result.",
|
||||
Buckets: []float64{0.001, 0.003, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60},
|
||||
}, []string{"pod", "result"})
|
||||
PushRecipientCount = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Name: "im_push_recipient_count",
|
||||
Help: "Recipients in an asynchronously dispatched push receipt by pod.",
|
||||
Buckets: []float64{1, 2, 10, 50, 100, 300, 600, 900, 1200, 2000},
|
||||
}, []string{"pod"})
|
||||
KafkaMessagePersistTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "im_kafka_message_persist_total",
|
||||
Help: "Kafka message persistence outcomes by message kind and result.",
|
||||
@@ -144,12 +232,18 @@ var (
|
||||
)
|
||||
|
||||
func init() {
|
||||
Registry.MustRegister(WSConnectionsActive, IMWSConnectionsActive, IMClusterLivePeers, IMClusterExpectedPeers,
|
||||
HTTPRequestsTotal, HTTPRequestDuration, DBQueryDuration,
|
||||
Registry.MustRegister(WSConnectionsActive, IMWSConnectionsActive, IMWSWriteTotal, IMWSWriteDuration,
|
||||
IMWSWriteBytesTotal, IMWSDisconnectTotal, IMWSHeartbeatTotal, IMWSHeartbeatSeconds,
|
||||
IMWSSendQueueCurrent, IMWSSendQueueCapacity,
|
||||
IMWSSendQueueEnqueueTotal, IMWSSendQueueDepth, IMWSSendQueueWaitDuration, IMCtrlACKDeliveryDuration,
|
||||
IMClusterLivePeers, IMClusterExpectedPeers,
|
||||
HTTPRequestsTotal, HTTPRequestDuration, DBQueryDuration, DBInvalidConnectionRetryTotal,
|
||||
RealtimeFanoutEventsTotal, RealtimeFanoutDeliveryDuration, ClusterP2MQueueLength,
|
||||
ClusterP2MQueueCapacity, ClusterP2MEnqueueTotal, ClusterRPCInflight, ClusterRPCDuration,
|
||||
ClusterRouteTotal, ClusterProxyResponseTotal, InternalQueueEnqueueTotal, InternalQueueDepth,
|
||||
InternalQueueCapacity, MessagePersistDuration, MessagePublishStageDuration,
|
||||
InternalQueueCapacity, InternalQueueCurrent, MessagePersistDuration, MessagePublishStageDuration, TopicQueueWaitDuration,
|
||||
PushDispatchQueueEnqueueTotal, PushDispatchQueueDepth, PushDispatchQueueWaitDuration,
|
||||
PushDispatchDuration, PushRecipientCount,
|
||||
KafkaMessagePersistTotal)
|
||||
Registry.MustRegister(KafkaConsumerLag, KafkaBacklogOldestAge, KafkaDeadLettersTotal)
|
||||
}
|
||||
@@ -178,6 +272,39 @@ func ObserveMessagePublishStage(kind, stage string, started time.Time, err error
|
||||
Observe(time.Since(started).Seconds())
|
||||
}
|
||||
|
||||
func ObserveTopicQueueWait(kind string, queuedAt time.Time) {
|
||||
if queuedAt.IsZero() {
|
||||
return
|
||||
}
|
||||
TopicQueueWaitDuration.WithLabelValues(metricPodName(), normalizeLabel(kind)).Observe(time.Since(queuedAt).Seconds())
|
||||
}
|
||||
|
||||
func ObservePushDispatchEnqueue(depth int, accepted bool) {
|
||||
result := "ok"
|
||||
if !accepted {
|
||||
result = "full"
|
||||
}
|
||||
PushDispatchQueueEnqueueTotal.WithLabelValues(metricPodName(), result).Inc()
|
||||
PushDispatchQueueDepth.WithLabelValues(metricPodName()).Set(float64(depth))
|
||||
}
|
||||
|
||||
func SetPushDispatchQueueDepth(depth int) {
|
||||
PushDispatchQueueDepth.WithLabelValues(metricPodName()).Set(float64(depth))
|
||||
}
|
||||
|
||||
func ObservePushDispatch(queuedAt time.Time, recipients int, started time.Time, err error) {
|
||||
pod := metricPodName()
|
||||
if !queuedAt.IsZero() {
|
||||
PushDispatchQueueWaitDuration.WithLabelValues(pod).Observe(started.Sub(queuedAt).Seconds())
|
||||
}
|
||||
result := "ok"
|
||||
if err != nil {
|
||||
result = "error"
|
||||
}
|
||||
PushDispatchDuration.WithLabelValues(pod, result).Observe(time.Since(started).Seconds())
|
||||
PushRecipientCount.WithLabelValues(pod).Observe(float64(recipients))
|
||||
}
|
||||
|
||||
func IncRealtimeFanout(stage, kind, targetType, status string) {
|
||||
RealtimeFanoutEventsTotal.WithLabelValues(normalizeLabel(stage), normalizeLabel(kind),
|
||||
normalizeLabel(targetType), normalizeLabel(status)).Inc()
|
||||
@@ -203,6 +330,73 @@ func ObserveInternalQueueEnqueue(queue string, depth, capacity int, accepted boo
|
||||
InternalQueueCapacity.WithLabelValues(pod, queue).Set(float64(capacity))
|
||||
}
|
||||
|
||||
func SetInternalQueueCurrent(queue string, depth int) {
|
||||
InternalQueueCurrent.WithLabelValues(metricPodName(), normalizeLabel(queue)).Set(float64(depth))
|
||||
}
|
||||
|
||||
func ObserveWSWrite(frame, result, reason string, started time.Time, payloadBytes int) {
|
||||
pod := metricPodName()
|
||||
frame = normalizeLabel(frame)
|
||||
result = normalizeLabel(result)
|
||||
reason = normalizeLabel(reason)
|
||||
IMWSWriteTotal.WithLabelValues(pod, frame, result, reason).Inc()
|
||||
IMWSWriteDuration.WithLabelValues(pod, frame, result).Observe(time.Since(started).Seconds())
|
||||
if payloadBytes > 0 {
|
||||
IMWSWriteBytesTotal.WithLabelValues(pod, frame).Add(float64(payloadBytes))
|
||||
}
|
||||
}
|
||||
|
||||
func IncWSDisconnect(reason string) {
|
||||
IMWSDisconnectTotal.WithLabelValues(metricPodName(), normalizeLabel(reason)).Inc()
|
||||
}
|
||||
|
||||
func IncWSHeartbeat(event string) {
|
||||
IMWSHeartbeatTotal.WithLabelValues(metricPodName(), normalizeLabel(event)).Inc()
|
||||
}
|
||||
|
||||
func IncDBInvalidConnectionRetry(operation, outcome string) {
|
||||
DBInvalidConnectionRetryTotal.WithLabelValues(normalizeLabel(operation), normalizeLabel(outcome)).Inc()
|
||||
}
|
||||
|
||||
func SetWSHeartbeatConfig(pingPeriod, pongWait time.Duration) {
|
||||
pod := metricPodName()
|
||||
IMWSHeartbeatSeconds.WithLabelValues(pod, "ping_period").Set(pingPeriod.Seconds())
|
||||
IMWSHeartbeatSeconds.WithLabelValues(pod, "pong_wait").Set(pongWait.Seconds())
|
||||
}
|
||||
|
||||
func ObserveWSSendQueueEnqueue(depth, capacity int, accepted bool) {
|
||||
result := "ok"
|
||||
if !accepted {
|
||||
result = "full"
|
||||
}
|
||||
pod := metricPodName()
|
||||
IMWSSendQueueEnqueueTotal.WithLabelValues(pod, result).Inc()
|
||||
IMWSSendQueueDepth.WithLabelValues(pod, result).Observe(float64(depth))
|
||||
IMWSSendQueueCapacity.WithLabelValues(pod).Set(float64(capacity))
|
||||
if accepted {
|
||||
IMWSSendQueueCurrent.WithLabelValues(pod).Inc()
|
||||
}
|
||||
}
|
||||
|
||||
func ObserveWSSendQueueDequeue(queuedAt time.Time, purged bool) {
|
||||
result := "dequeued"
|
||||
if purged {
|
||||
result = "purged"
|
||||
}
|
||||
pod := metricPodName()
|
||||
IMWSSendQueueCurrent.WithLabelValues(pod).Dec()
|
||||
if !queuedAt.IsZero() {
|
||||
IMWSSendQueueWaitDuration.WithLabelValues(pod, result).Observe(time.Since(queuedAt).Seconds())
|
||||
}
|
||||
}
|
||||
|
||||
func ObserveCtrlACKDelivery(stage string, requestAt time.Time) {
|
||||
if requestAt.IsZero() {
|
||||
return
|
||||
}
|
||||
IMCtrlACKDeliveryDuration.WithLabelValues(metricPodName(), normalizeLabel(stage)).Observe(time.Since(requestAt).Seconds())
|
||||
}
|
||||
|
||||
func normalizeLabel(value string) string {
|
||||
if value == "" {
|
||||
return "unknown"
|
||||
|
||||
@@ -91,6 +91,15 @@ func TestServeMetricsIncludesWSConnections(t *testing.T) {
|
||||
immetrics.KafkaConsumerLag.WithLabelValues("test", "topic", "group").Set(0)
|
||||
immetrics.KafkaDeadLettersTotal.WithLabelValues("p2p", "test").Add(0)
|
||||
immetrics.ObserveInternalQueueEnqueue("test", 0, 1, true)
|
||||
immetrics.SetInternalQueueCurrent("test", 0)
|
||||
immetrics.ObserveWSWrite("text", "ok", "none", time.Now(), 10)
|
||||
immetrics.IncWSDisconnect("close_normal")
|
||||
immetrics.ObserveWSSendQueueEnqueue(1, 160, true)
|
||||
immetrics.ObserveWSSendQueueDequeue(time.Now(), false)
|
||||
immetrics.ObserveCtrlACKDelivery("write", time.Now())
|
||||
immetrics.ObserveTopicQueueWait("group", time.Now())
|
||||
immetrics.ObservePushDispatchEnqueue(1, true)
|
||||
immetrics.ObservePushDispatch(time.Now(), 1, time.Now(), nil)
|
||||
wrt := httptest.NewRecorder()
|
||||
serveMetrics(wrt, httptest.NewRequest(http.MethodGet, "/metrics", nil))
|
||||
|
||||
@@ -102,8 +111,15 @@ func TestServeMetricsIncludesWSConnections(t *testing.T) {
|
||||
t.Fatalf("unexpected body %s", body)
|
||||
}
|
||||
for _, metric := range []string{"im_ws_connections_active", "im_cluster_p2m_queue_length",
|
||||
"im_ws_write_total", "im_ws_write_duration_seconds", "im_ws_write_bytes_total",
|
||||
"im_ws_disconnect_total", "im_ws_send_queue_current", "im_ws_send_queue_capacity",
|
||||
"im_ws_send_queue_enqueue_total", "im_ws_send_queue_depth", "im_ws_send_queue_wait_duration_seconds",
|
||||
"im_ctrl_ack_delivery_duration_seconds",
|
||||
"im_cluster_p2m_enqueue_total", "im_cluster_rpc_duration_seconds", "im_cluster_route_total",
|
||||
"im_internal_queue_enqueue_total", "im_internal_queue_depth", "im_internal_queue_capacity",
|
||||
"im_internal_queue_enqueue_total", "im_internal_queue_depth", "im_internal_queue_capacity", "im_internal_queue_current",
|
||||
"im_topic_queue_wait_duration_seconds", "im_push_dispatch_queue_enqueue_total",
|
||||
"im_push_dispatch_queue_depth", "im_push_dispatch_queue_wait_duration_seconds",
|
||||
"im_push_dispatch_duration_seconds", "im_push_recipient_count",
|
||||
"im_kafka_message_persist_total", "im_kafka_consumer_lag", "im_kafka_dead_letters_total"} {
|
||||
if !strings.Contains(body, "# HELP "+metric+" ") {
|
||||
t.Fatalf("metric %s was not exported", metric)
|
||||
|
||||
+95
-9
@@ -8,14 +8,109 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/tinode/chat/server/logs"
|
||||
immetrics "github.com/tinode/chat/server/metrics"
|
||||
"github.com/tinode/chat/server/push"
|
||||
"github.com/tinode/chat/server/store"
|
||||
"github.com/tinode/chat/server/store/types"
|
||||
)
|
||||
|
||||
const (
|
||||
pushDispatchQueueCapacity = 4096
|
||||
pushDispatchWorkers = 4
|
||||
)
|
||||
|
||||
type pushDispatchJob struct {
|
||||
rcpt *push.Receipt
|
||||
resolvePresence bool
|
||||
queuedAt time.Time
|
||||
}
|
||||
|
||||
var asyncPushDispatcher struct {
|
||||
once sync.Once
|
||||
queues [pushDispatchWorkers]chan pushDispatchJob
|
||||
depth atomic.Int64
|
||||
}
|
||||
|
||||
func initAsyncPushDispatcher() {
|
||||
asyncPushDispatcher.once.Do(func() {
|
||||
for i := range pushDispatchWorkers {
|
||||
asyncPushDispatcher.queues[i] = make(chan pushDispatchJob, pushDispatchQueueCapacity/pushDispatchWorkers)
|
||||
go func(queue <-chan pushDispatchJob) {
|
||||
for job := range queue {
|
||||
depth := asyncPushDispatcher.depth.Add(-1)
|
||||
immetrics.SetPushDispatchQueueDepth(int(depth))
|
||||
processPushDispatch(job)
|
||||
}
|
||||
}(asyncPushDispatcher.queues[i])
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func pushDispatchShard(rcpt *push.Receipt) int {
|
||||
hash := uint32(2166136261)
|
||||
for _, value := range []byte(rcpt.Payload.Topic) {
|
||||
hash ^= uint32(value)
|
||||
hash *= 16777619
|
||||
}
|
||||
hash ^= uint32(rcpt.TenantID)
|
||||
hash *= 16777619
|
||||
return int(hash % pushDispatchWorkers)
|
||||
}
|
||||
|
||||
func dispatchPushAsync(rcpt *push.Receipt, resolvePresence bool) bool {
|
||||
if rcpt == nil || globals.usersUpdate == nil {
|
||||
return false
|
||||
}
|
||||
initAsyncPushDispatcher()
|
||||
queue := asyncPushDispatcher.queues[pushDispatchShard(rcpt)]
|
||||
job := pushDispatchJob{rcpt: rcpt, resolvePresence: resolvePresence, queuedAt: time.Now()}
|
||||
depth := asyncPushDispatcher.depth.Add(1)
|
||||
select {
|
||||
case queue <- job:
|
||||
immetrics.ObservePushDispatchEnqueue(int(asyncPushDispatcher.depth.Load()), true)
|
||||
return true
|
||||
default:
|
||||
depth = asyncPushDispatcher.depth.Add(-1)
|
||||
immetrics.ObservePushDispatchEnqueue(int(depth), false)
|
||||
logs.Err.Printf("push dispatch queue full: tenant=%d topic=%s recipients=%d shard_len=%d shard_cap=%d total_depth=%d",
|
||||
rcpt.TenantID, rcpt.Payload.Topic, len(rcpt.To), len(queue), cap(queue), depth)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func processPushDispatch(job pushDispatchJob) {
|
||||
started := time.Now()
|
||||
var dispatchErr error
|
||||
if job.resolvePresence {
|
||||
unresolved := make([]types.Uid, 0, len(job.rcpt.To))
|
||||
for uid, recipient := range job.rcpt.To {
|
||||
if recipient.Delivered == 0 {
|
||||
unresolved = append(unresolved, uid)
|
||||
}
|
||||
}
|
||||
presences, err := store.GetUsersPresence(job.rcpt.TenantID, unresolved)
|
||||
if err != nil {
|
||||
dispatchErr = err
|
||||
logs.Warn.Printf("push dispatch: failed to read presence topic=%s recipients=%d err=%v",
|
||||
job.rcpt.Payload.Topic, len(unresolved), err)
|
||||
}
|
||||
for uid, presence := range presences {
|
||||
if presence != nil && presence.Online && presence.SessionCount > 0 {
|
||||
recipient := job.rcpt.To[uid]
|
||||
recipient.Delivered = presence.SessionCount
|
||||
job.rcpt.To[uid] = recipient
|
||||
}
|
||||
}
|
||||
}
|
||||
sendPush(job.rcpt)
|
||||
immetrics.ObservePushDispatch(job.queuedAt, len(job.rcpt.To), started, dispatchErr)
|
||||
}
|
||||
|
||||
// Subscribe or unsubscribe user to/from FCM topic (channel).
|
||||
func (t *Topic) channelSubUnsub(uid types.Uid, sub bool) {
|
||||
push.ChannelSub(&push.ChannelReq{
|
||||
@@ -75,15 +170,6 @@ func (t *Topic) pushForData(fromUid types.Uid, data *MsgServerData, msgMarkedAsR
|
||||
// Make sure the sender's devices receive a silent push.
|
||||
online = 1
|
||||
}
|
||||
if online == 0 {
|
||||
presence, err := store.GetUserPresence(t.tenantID, uid)
|
||||
if err != nil {
|
||||
logs.Warn.Printf("topic[%s]: failed to read user presence for push: uid=%s err=%v", t.name, uid.UserId(), err)
|
||||
} else if presence != nil && presence.Online && presence.SessionCount > 0 {
|
||||
online = presence.SessionCount
|
||||
}
|
||||
}
|
||||
|
||||
// Send only to those who have notifications enabled.
|
||||
mode := pud.modeWant & pud.modeGiven
|
||||
if mode.IsPresencer() && mode.IsReader() && !pud.deleted && !pud.isChan {
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/tinode/chat/server/logs"
|
||||
@@ -22,15 +23,38 @@ import (
|
||||
|
||||
const (
|
||||
defaultBufferSize = 1024
|
||||
defaultWorkers = 8
|
||||
defaultAPIDomain = "https://api.jpush.cn"
|
||||
defaultPushPath = "/v3/push"
|
||||
defaultTTL = 86400
|
||||
defaultCallTTL = 30
|
||||
defaultDeviceCacheTTL = 30
|
||||
callCategory = "im_call"
|
||||
)
|
||||
|
||||
var handler Handler
|
||||
|
||||
type deviceCacheKey struct {
|
||||
tenantID t.TenantID
|
||||
provider string
|
||||
uid t.Uid
|
||||
}
|
||||
|
||||
type deviceCacheEntry struct {
|
||||
devices []t.DeviceDef
|
||||
expiresAt time.Time
|
||||
staleUntil time.Time
|
||||
}
|
||||
|
||||
type providerDeviceCache struct {
|
||||
mu sync.Mutex
|
||||
entries map[deviceCacheKey]deviceCacheEntry
|
||||
retryAfter time.Time
|
||||
nextCleanup time.Time
|
||||
}
|
||||
|
||||
var jpushDeviceCache = providerDeviceCache{entries: make(map[deviceCacheKey]deviceCacheEntry)}
|
||||
|
||||
type Handler struct {
|
||||
input chan *push.Receipt
|
||||
channel chan *push.ChannelReq
|
||||
@@ -47,8 +71,10 @@ type configType struct {
|
||||
APIDomain string `json:"api_domain"`
|
||||
PushPath string `json:"push_path"`
|
||||
Buffer int `json:"buffer"`
|
||||
Workers int `json:"workers"`
|
||||
DefaultTTLSeconds int `json:"default_ttl_seconds"`
|
||||
CallTTLSeconds int `json:"call_ttl_seconds"`
|
||||
DeviceCacheTTLSeconds int `json:"device_cache_ttl_seconds"`
|
||||
APNsProduction bool `json:"apns_production"`
|
||||
UseAlias *bool `json:"use_alias"`
|
||||
Classification int `json:"classification"`
|
||||
@@ -154,10 +180,19 @@ func (Handler) Init(jsonconf json.RawMessage) (bool, error) {
|
||||
handler.config = normalized
|
||||
|
||||
go func() {
|
||||
workers := make(chan struct{}, normalized.Workers)
|
||||
for {
|
||||
select {
|
||||
case rcpt := <-handler.input:
|
||||
go sendReceipt(rcpt, &handler.config, handler.client)
|
||||
select {
|
||||
case workers <- struct{}{}:
|
||||
go func(receipt *push.Receipt) {
|
||||
defer func() { <-workers }()
|
||||
sendReceipt(receipt, &handler.config, handler.client)
|
||||
}(rcpt)
|
||||
case <-handler.stop:
|
||||
return
|
||||
}
|
||||
case req := <-handler.channel:
|
||||
logs.Info.Println("jpush: channel subscription ignored", req.Uid.UserId(), req.Channel)
|
||||
case <-handler.stop:
|
||||
@@ -186,12 +221,18 @@ func normalizeConfig(config configType) (configType, error) {
|
||||
if config.Buffer <= 0 {
|
||||
config.Buffer = defaultBufferSize
|
||||
}
|
||||
if config.Workers <= 0 {
|
||||
config.Workers = defaultWorkers
|
||||
}
|
||||
if config.DefaultTTLSeconds <= 0 {
|
||||
config.DefaultTTLSeconds = defaultTTL
|
||||
}
|
||||
if config.CallTTLSeconds <= 0 {
|
||||
config.CallTTLSeconds = defaultCallTTL
|
||||
}
|
||||
if config.DeviceCacheTTLSeconds <= 0 {
|
||||
config.DeviceCacheTTLSeconds = defaultDeviceCacheTTL
|
||||
}
|
||||
if config.Title == "" {
|
||||
config.Title = "MeetingIM"
|
||||
}
|
||||
@@ -260,6 +301,8 @@ func loadThirdPartyConfig(config *configType) {
|
||||
"jpush_master_secret",
|
||||
"jpush_api_domain",
|
||||
"jpush_push_path",
|
||||
"jpush_workers",
|
||||
"jpush_device_cache_ttl_seconds",
|
||||
"jpush_default_ttl_seconds",
|
||||
"jpush_call_ttl_seconds",
|
||||
"jpush_apns_production",
|
||||
@@ -285,6 +328,12 @@ func loadThirdPartyConfig(config *configType) {
|
||||
if v := strings.TrimSpace(values["jpush_push_path"]); v != "" {
|
||||
config.PushPath = v
|
||||
}
|
||||
if v, ok := parseConfigInt(values["jpush_workers"]); ok {
|
||||
config.Workers = v
|
||||
}
|
||||
if v, ok := parseConfigInt(values["jpush_device_cache_ttl_seconds"]); ok {
|
||||
config.DeviceCacheTTLSeconds = v
|
||||
}
|
||||
if v, ok := parseConfigInt(values["jpush_default_ttl_seconds"]); ok {
|
||||
config.DefaultTTLSeconds = v
|
||||
}
|
||||
@@ -344,7 +393,8 @@ func preparePayloads(rcpt *push.Receipt, config *configType) ([]*jpushPayload, [
|
||||
}
|
||||
var devices map[t.Uid][]t.DeviceDef
|
||||
if !useAlias {
|
||||
devices, _, err = providerDevicesForUsers(rcpt.TenantID, "jpush", uids...)
|
||||
devices, _, err = cachedProviderDevicesForUsers(rcpt.TenantID, "jpush",
|
||||
time.Duration(config.DeviceCacheTTLSeconds)*time.Second, uids...)
|
||||
if err != nil {
|
||||
logs.Warn.Println("jpush: db error", err)
|
||||
return nil, nil
|
||||
@@ -699,6 +749,99 @@ func providerDevicesForUsers(tenantID t.TenantID, provider string, uids ...t.Uid
|
||||
return map[t.Uid][]t.DeviceDef{}, 0, nil
|
||||
}
|
||||
|
||||
func cachedProviderDevicesForUsers(tenantID t.TenantID, provider string, ttl time.Duration,
|
||||
uids ...t.Uid) (map[t.Uid][]t.DeviceDef, int, error) {
|
||||
if ttl <= 0 {
|
||||
ttl = defaultDeviceCacheTTL * time.Second
|
||||
}
|
||||
provider = strings.ToLower(strings.TrimSpace(provider))
|
||||
now := time.Now()
|
||||
cache := &jpushDeviceCache
|
||||
cache.mu.Lock()
|
||||
defer cache.mu.Unlock()
|
||||
|
||||
result := make(map[t.Uid][]t.DeviceDef, len(uids))
|
||||
misses := make([]t.Uid, 0, len(uids))
|
||||
for _, uid := range uids {
|
||||
key := deviceCacheKey{tenantID: tenantID, provider: provider, uid: uid}
|
||||
entry, ok := cache.entries[key]
|
||||
if ok && now.Before(entry.expiresAt) {
|
||||
result[uid] = append([]t.DeviceDef(nil), entry.devices...)
|
||||
continue
|
||||
}
|
||||
misses = append(misses, uid)
|
||||
}
|
||||
if len(misses) == 0 {
|
||||
return result, countDevices(result), nil
|
||||
}
|
||||
if cache.nextCleanup.IsZero() || !now.Before(cache.nextCleanup) {
|
||||
for key, entry := range cache.entries {
|
||||
if !now.Before(entry.staleUntil) {
|
||||
delete(cache.entries, key)
|
||||
}
|
||||
}
|
||||
cache.nextCleanup = now.Add(time.Minute)
|
||||
}
|
||||
|
||||
// A failed lookup used to trigger one identical database query per group
|
||||
// message. Back off briefly and use stale entries where possible instead.
|
||||
if now.Before(cache.retryAfter) {
|
||||
cache.addStale(result, tenantID, provider, now, misses)
|
||||
return result, countDevices(result), nil
|
||||
}
|
||||
|
||||
loaded, _, err := providerDevicesForUsers(tenantID, provider, misses...)
|
||||
if err != nil {
|
||||
cache.retryAfter = now.Add(time.Second)
|
||||
if cache.addStale(result, tenantID, provider, now, misses) == len(misses) {
|
||||
return result, countDevices(result), nil
|
||||
}
|
||||
return result, countDevices(result), err
|
||||
}
|
||||
cache.retryAfter = time.Time{}
|
||||
staleTTL := 10 * ttl
|
||||
if staleTTL < 5*time.Minute {
|
||||
staleTTL = 5 * time.Minute
|
||||
}
|
||||
for _, uid := range misses {
|
||||
devices := append([]t.DeviceDef(nil), loaded[uid]...)
|
||||
cache.entries[deviceCacheKey{tenantID: tenantID, provider: provider, uid: uid}] = deviceCacheEntry{
|
||||
devices: devices, expiresAt: now.Add(ttl), staleUntil: now.Add(staleTTL),
|
||||
}
|
||||
result[uid] = append([]t.DeviceDef(nil), devices...)
|
||||
}
|
||||
return result, countDevices(result), nil
|
||||
}
|
||||
|
||||
func (c *providerDeviceCache) addStale(result map[t.Uid][]t.DeviceDef, tenantID t.TenantID,
|
||||
provider string, now time.Time, uids []t.Uid) int {
|
||||
covered := 0
|
||||
for _, uid := range uids {
|
||||
entry, ok := c.entries[deviceCacheKey{tenantID: tenantID, provider: provider, uid: uid}]
|
||||
if ok && now.Before(entry.staleUntil) {
|
||||
result[uid] = append([]t.DeviceDef(nil), entry.devices...)
|
||||
covered++
|
||||
}
|
||||
}
|
||||
return covered
|
||||
}
|
||||
|
||||
func countDevices(devices map[t.Uid][]t.DeviceDef) int {
|
||||
count := 0
|
||||
for _, entries := range devices {
|
||||
count += len(entries)
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
func resetDeviceCache() {
|
||||
jpushDeviceCache.mu.Lock()
|
||||
jpushDeviceCache.entries = make(map[deviceCacheKey]deviceCacheEntry)
|
||||
jpushDeviceCache.retryAfter = time.Time{}
|
||||
jpushDeviceCache.nextCleanup = time.Time{}
|
||||
jpushDeviceCache.mu.Unlock()
|
||||
}
|
||||
|
||||
func (Handler) IsReady() bool {
|
||||
return handler.input != nil
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -27,18 +28,23 @@ func (s testThirdPartyConfigStore) Get(_ string, keys ...string) (map[string]str
|
||||
|
||||
type testDevicePushTokenStore struct {
|
||||
devices map[types.Uid][]types.DeviceDef
|
||||
calls int
|
||||
}
|
||||
|
||||
func (s *testDevicePushTokenStore) Upsert(types.TenantID, *types.DevicePushToken) (*types.DevicePushToken, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (s *testDevicePushTokenStore) GetAllByProvider(types.TenantID, string, ...types.Uid) (map[types.Uid][]types.DeviceDef, int, error) {
|
||||
func (s *testDevicePushTokenStore) GetAllByProvider(_ types.TenantID, _ string, uids ...types.Uid) (map[types.Uid][]types.DeviceDef, int, error) {
|
||||
s.calls++
|
||||
result := make(map[types.Uid][]types.DeviceDef, len(uids))
|
||||
count := 0
|
||||
for _, devices := range s.devices {
|
||||
for _, uid := range uids {
|
||||
devices := s.devices[uid]
|
||||
result[uid] = devices
|
||||
count += len(devices)
|
||||
}
|
||||
return s.devices, count, nil
|
||||
return result, count, nil
|
||||
}
|
||||
|
||||
func TestNormalizeConfigLoadsThirdPartyConfig(t *testing.T) {
|
||||
@@ -49,6 +55,8 @@ func TestNormalizeConfigLoadsThirdPartyConfig(t *testing.T) {
|
||||
"jpush_master_secret": "master",
|
||||
"jpush_api_domain": "https://push.example.com/",
|
||||
"jpush_push_path": "v3/push",
|
||||
"jpush_workers": "4",
|
||||
"jpush_device_cache_ttl_seconds": "45",
|
||||
"jpush_default_ttl_seconds": "123",
|
||||
"jpush_call_ttl_seconds": "45",
|
||||
"jpush_use_alias": "false",
|
||||
@@ -68,6 +76,9 @@ func TestNormalizeConfigLoadsThirdPartyConfig(t *testing.T) {
|
||||
if config.APIDomain != "https://push.example.com" || config.PushPath != "/v3/push" {
|
||||
t.Fatalf("unexpected endpoint: %#v", config)
|
||||
}
|
||||
if config.Workers != 4 || config.DeviceCacheTTLSeconds != 45 {
|
||||
t.Fatalf("unexpected worker/cache config: %#v", config)
|
||||
}
|
||||
if config.DefaultTTLSeconds != 123 || config.CallTTLSeconds != 45 || config.UseAlias == nil || *config.UseAlias {
|
||||
t.Fatalf("unexpected ttl/use_alias: %#v", config)
|
||||
}
|
||||
@@ -119,6 +130,8 @@ func TestAliasForUidUsesJPushSafeCharacters(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestPreparePayloadsCanUseRegistrationID(t *testing.T) {
|
||||
resetDeviceCache()
|
||||
t.Cleanup(resetDeviceCache)
|
||||
oldTokens := store.DevicePushTokens
|
||||
uid := types.Uid(42)
|
||||
store.DevicePushTokens = &testDevicePushTokenStore{devices: map[types.Uid][]types.DeviceDef{
|
||||
@@ -147,6 +160,8 @@ func TestPreparePayloadsCanUseRegistrationID(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestPreparePayloadsDoesNotDropRecipientWithDeliveredSession(t *testing.T) {
|
||||
resetDeviceCache()
|
||||
t.Cleanup(resetDeviceCache)
|
||||
oldTokens := store.DevicePushTokens
|
||||
uid := types.Uid(42)
|
||||
store.DevicePushTokens = &testDevicePushTokenStore{devices: map[types.Uid][]types.DeviceDef{
|
||||
@@ -170,6 +185,57 @@ func TestPreparePayloadsDoesNotDropRecipientWithDeliveredSession(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparePayloadsCachesEmptyDeviceLookups(t *testing.T) {
|
||||
resetDeviceCache()
|
||||
t.Cleanup(resetDeviceCache)
|
||||
oldTokens := store.DevicePushTokens
|
||||
tokens := &testDevicePushTokenStore{devices: map[types.Uid][]types.DeviceDef{}}
|
||||
store.DevicePushTokens = tokens
|
||||
t.Cleanup(func() { store.DevicePushTokens = oldTokens })
|
||||
useAlias := false
|
||||
config := &configType{Title: "MeetingIM", DefaultTTLSeconds: 86400, CallTTLSeconds: 60,
|
||||
DeviceCacheTTLSeconds: 30, UseAlias: &useAlias}
|
||||
receipt := &push.Receipt{
|
||||
TenantID: 7,
|
||||
To: map[types.Uid]push.Recipient{42: {}, 43: {}},
|
||||
Payload: push.Payload{What: push.ActMsg, Topic: "grpTopic", SeqId: 9, Timestamp: time.Now()},
|
||||
}
|
||||
preparePayloads(receipt, config)
|
||||
preparePayloads(receipt, config)
|
||||
if tokens.calls != 1 {
|
||||
t.Fatalf("device store calls=%d want=1", tokens.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparePayloadsCoalescesConcurrentDeviceLookups(t *testing.T) {
|
||||
resetDeviceCache()
|
||||
t.Cleanup(resetDeviceCache)
|
||||
oldTokens := store.DevicePushTokens
|
||||
tokens := &testDevicePushTokenStore{devices: map[types.Uid][]types.DeviceDef{}}
|
||||
store.DevicePushTokens = tokens
|
||||
t.Cleanup(func() { store.DevicePushTokens = oldTokens })
|
||||
useAlias := false
|
||||
config := &configType{Title: "MeetingIM", DefaultTTLSeconds: 86400, CallTTLSeconds: 60,
|
||||
DeviceCacheTTLSeconds: 30, UseAlias: &useAlias}
|
||||
receipt := &push.Receipt{
|
||||
TenantID: 7,
|
||||
To: map[types.Uid]push.Recipient{42: {}, 43: {}},
|
||||
Payload: push.Payload{What: push.ActMsg, Topic: "grpTopic", SeqId: 9, Timestamp: time.Now()},
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
for range 20 {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
preparePayloads(receipt, config)
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
if tokens.calls != 1 {
|
||||
t.Fatalf("device store calls=%d want=1", tokens.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildPayloadUsesShortTTLForCalls(t *testing.T) {
|
||||
expiresAt := time.Now().Add(30 * time.Second).UTC().Format(time.RFC3339Nano)
|
||||
config := &configType{Title: "MeetingIM", DefaultTTLSeconds: 86400, CallTTLSeconds: 60}
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/tinode/chat/server/push"
|
||||
"github.com/tinode/chat/server/store/types"
|
||||
)
|
||||
|
||||
func TestDispatchPushAsyncDeliversToUserUpdater(t *testing.T) {
|
||||
oldUsersUpdate := globals.usersUpdate
|
||||
globals.usersUpdate = make(chan *UserCacheReq, 1)
|
||||
t.Cleanup(func() { globals.usersUpdate = oldUsersUpdate })
|
||||
|
||||
uid := types.Uid(101)
|
||||
rcpt := &push.Receipt{
|
||||
TenantID: 1,
|
||||
To: map[types.Uid]push.Recipient{uid: {Delivered: 1}},
|
||||
Payload: push.Payload{What: push.ActMsg, Topic: "grpAsync"},
|
||||
}
|
||||
if !dispatchPushAsync(rcpt, false) {
|
||||
t.Fatal("push dispatch was not enqueued")
|
||||
}
|
||||
|
||||
select {
|
||||
case req := <-globals.usersUpdate:
|
||||
if req.PushRcpt != rcpt {
|
||||
t.Fatal("user updater received the wrong receipt")
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("timed out waiting for asynchronous push dispatch")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeferredPresenceCoalescesAndRetries(t *testing.T) {
|
||||
tenantID := types.TenantID(1)
|
||||
topicName := "grpDeferred"
|
||||
topic := &Topic{tenantID: tenantID, name: topicName, serverMsg: make(chan *ServerComMessage, 1)}
|
||||
topic.serverMsg <- &ServerComMessage{TenantID: tenantID, RcptTo: topicName}
|
||||
|
||||
hub := &Hub{
|
||||
topics: &sync.Map{},
|
||||
deferredPresence: make(chan *ServerComMessage, 4),
|
||||
deferredPresenceStop: make(chan struct{}),
|
||||
}
|
||||
hub.topicPut(types.TopicKey{TenantID: tenantID, Topic: topicName}, topic)
|
||||
go hub.runDeferredPresence()
|
||||
t.Cleanup(func() { close(hub.deferredPresenceStop) })
|
||||
|
||||
off := &ServerComMessage{TenantID: tenantID, RcptTo: topicName,
|
||||
Pres: &MsgServerPres{What: "off", Src: "usrA"}}
|
||||
on := &ServerComMessage{TenantID: tenantID, RcptTo: topicName,
|
||||
Pres: &MsgServerPres{What: "on", Src: "usrA"}}
|
||||
if !hub.deferPresence(off) || !hub.deferPresence(on) {
|
||||
t.Fatal("failed to enqueue deferred presence")
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
<-topic.serverMsg
|
||||
|
||||
select {
|
||||
case got := <-topic.serverMsg:
|
||||
if got.Pres == nil || got.Pres.What != "on" {
|
||||
t.Fatalf("expected latest coalesced presence, got %#v", got.Pres)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("timed out waiting for deferred presence retry")
|
||||
}
|
||||
}
|
||||
+112
-10
@@ -35,6 +35,11 @@ import (
|
||||
// Maximum number of queued messages before session is considered stale and dropped.
|
||||
const sendQueueLimit = 128
|
||||
|
||||
// Multiplex sessions may receive a short burst of presence updates while many
|
||||
// remote connections are closing. Give the cluster drain a bounded chance to
|
||||
// free one slot before treating the proxy as stuck.
|
||||
const multiplexEnqueueWait = 250 * time.Millisecond
|
||||
|
||||
// Time given to a background session to terminate to avoid tiggering presence notifications.
|
||||
// If session terminates (or unsubscribes from topic) in this time frame notifications are not sent at all.
|
||||
const deferredNotificationsTimeout = time.Second * 5
|
||||
@@ -141,6 +146,10 @@ type Session struct {
|
||||
// Outbound mesages, buffered.
|
||||
// The content must be serialized in format suitable for the session.
|
||||
send chan any
|
||||
// WebSocket queue timestamps are kept separately to preserve the send channel's payload contract.
|
||||
wsQueueMu sync.Mutex
|
||||
wsQueueTimes []time.Time
|
||||
wsQueueHead int
|
||||
|
||||
// Channel for shutting down the session, buffer 1.
|
||||
// Content in the same format as for 'send'
|
||||
@@ -164,6 +173,10 @@ type Session struct {
|
||||
|
||||
// Type of proxy to master request being handled.
|
||||
proxyReq ProxyReqType
|
||||
// Ensures that one multiplexing session is drained by at most one cluster
|
||||
// write task at a time, preserving per-topic response order.
|
||||
clusterWriteScheduled atomic.Bool
|
||||
clusterQueueMu sync.Mutex
|
||||
}
|
||||
|
||||
// Subscription is a mapper of sessions to topics.
|
||||
@@ -182,6 +195,91 @@ type Subscription struct {
|
||||
supd chan<- *sessionUpdate
|
||||
}
|
||||
|
||||
func (s *Session) enqueueOutbound(payload any) bool {
|
||||
if s.isMultiplex() {
|
||||
return s.enqueueMultiplexOutbound(payload, multiplexEnqueueWait)
|
||||
}
|
||||
if s.proto != WEBSOCK {
|
||||
select {
|
||||
case s.send <- payload:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
s.wsQueueMu.Lock()
|
||||
defer s.wsQueueMu.Unlock()
|
||||
select {
|
||||
case s.send <- payload:
|
||||
s.wsQueueTimes = append(s.wsQueueTimes, time.Now())
|
||||
immetrics.ObserveWSSendQueueEnqueue(len(s.send), cap(s.send), true)
|
||||
return true
|
||||
default:
|
||||
immetrics.ObserveWSSendQueueEnqueue(len(s.send), cap(s.send), false)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Session) enqueueMultiplexOutbound(payload any, wait time.Duration) bool {
|
||||
deadline := time.NewTimer(wait)
|
||||
defer deadline.Stop()
|
||||
retry := time.NewTicker(time.Millisecond)
|
||||
defer retry.Stop()
|
||||
|
||||
drainScheduled := false
|
||||
for {
|
||||
s.clusterQueueMu.Lock()
|
||||
if atomic.LoadInt32(&s.terminating) > 0 {
|
||||
s.clusterQueueMu.Unlock()
|
||||
return true
|
||||
}
|
||||
select {
|
||||
case s.send <- payload:
|
||||
s.clusterQueueMu.Unlock()
|
||||
return true
|
||||
default:
|
||||
s.clusterQueueMu.Unlock()
|
||||
}
|
||||
|
||||
if !drainScheduled {
|
||||
s.scheduleClusterWriteLoop()
|
||||
drainScheduled = true
|
||||
}
|
||||
select {
|
||||
case <-retry.C:
|
||||
case <-deadline.C:
|
||||
s.clusterQueueMu.Lock()
|
||||
terminating := atomic.LoadInt32(&s.terminating) > 0
|
||||
s.clusterQueueMu.Unlock()
|
||||
return terminating
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Session) observeOutboundDequeue(purged bool) {
|
||||
if s.proto != WEBSOCK {
|
||||
return
|
||||
}
|
||||
s.wsQueueMu.Lock()
|
||||
if s.wsQueueHead >= len(s.wsQueueTimes) {
|
||||
s.wsQueueMu.Unlock()
|
||||
return
|
||||
}
|
||||
queuedAt := s.wsQueueTimes[s.wsQueueHead]
|
||||
s.wsQueueHead++
|
||||
if s.wsQueueHead == len(s.wsQueueTimes) {
|
||||
s.wsQueueTimes = s.wsQueueTimes[:0]
|
||||
s.wsQueueHead = 0
|
||||
} else if s.wsQueueHead >= 128 {
|
||||
copy(s.wsQueueTimes, s.wsQueueTimes[s.wsQueueHead:])
|
||||
s.wsQueueTimes = s.wsQueueTimes[:len(s.wsQueueTimes)-s.wsQueueHead]
|
||||
s.wsQueueHead = 0
|
||||
}
|
||||
s.wsQueueMu.Unlock()
|
||||
immetrics.ObserveWSSendQueueDequeue(queuedAt, purged)
|
||||
}
|
||||
|
||||
func (s *Session) addSub(topic string, sub *Subscription) {
|
||||
if s.multi != nil {
|
||||
s.multi.addSub(topic, sub)
|
||||
@@ -282,7 +380,7 @@ func (s *Session) isCluster() bool {
|
||||
}
|
||||
|
||||
func (s *Session) scheduleClusterWriteLoop() {
|
||||
if globals.cluster != nil && globals.cluster.proxyEventQueue != nil {
|
||||
if globals.cluster != nil && globals.cluster.proxyEventQueue != nil && s.clusterWriteScheduled.CompareAndSwap(false, true) {
|
||||
globals.cluster.proxyEventQueue.Schedule(
|
||||
func() { s.clusterWriteLoop(s.proxiedTopic) })
|
||||
}
|
||||
@@ -333,9 +431,7 @@ func (s *Session) queueOutBatch(msgs []*ServerComMessage) bool {
|
||||
}
|
||||
|
||||
if s.supportsMessageBatching() {
|
||||
select {
|
||||
case s.send <- msgs:
|
||||
default:
|
||||
if !s.enqueueOutbound(msgs) {
|
||||
// Never block here since it may also block the topic's run() goroutine.
|
||||
logs.Err.Println("s.queueOut: session's send queue2 full", s.sid)
|
||||
return false
|
||||
@@ -394,19 +490,24 @@ func (s *Session) queueOut(msg *ServerComMessage) bool {
|
||||
}
|
||||
}
|
||||
|
||||
select {
|
||||
case s.send <- msg:
|
||||
default:
|
||||
if !s.enqueueOutbound(msg) {
|
||||
// Never block here since it may also block the topic's run() goroutine.
|
||||
logs.Err.Println("s.queueOut: session's send queue full", s.sid)
|
||||
return false
|
||||
}
|
||||
if isAcceptedCtrl(msg) && s.proto == WEBSOCK {
|
||||
immetrics.ObserveCtrlACKDelivery("enqueue", msg.Timestamp)
|
||||
}
|
||||
if s.isMultiplex() {
|
||||
s.scheduleClusterWriteLoop()
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func isAcceptedCtrl(msg *ServerComMessage) bool {
|
||||
return msg != nil && msg.Ctrl != nil && msg.Ctrl.Code == http.StatusAccepted && msg.Ctrl.Id != ""
|
||||
}
|
||||
|
||||
// queueOutBytes attempts to send a ServerComMessage already serialized to []byte.
|
||||
// If the send buffer is full, it fails.
|
||||
func (s *Session) queueOutBytes(data []byte) bool {
|
||||
@@ -414,9 +515,7 @@ func (s *Session) queueOutBytes(data []byte) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
select {
|
||||
case s.send <- data:
|
||||
default:
|
||||
if !s.enqueueOutbound(data) {
|
||||
logs.Err.Println("s.queueOutBytes: session's send queue full", s.sid)
|
||||
return false
|
||||
}
|
||||
@@ -451,6 +550,7 @@ func (s *Session) stopSession(data any) {
|
||||
func (s *Session) purgeChannels() {
|
||||
for len(s.send) > 0 {
|
||||
<-s.send
|
||||
s.observeOutboundDequeue(true)
|
||||
}
|
||||
for len(s.stop) > 0 {
|
||||
<-s.stop
|
||||
@@ -859,6 +959,7 @@ func (s *Session) publish(msg *ClientComMessage) {
|
||||
|
||||
if sub := s.getSub(msg.RcptTo); sub != nil {
|
||||
// This is a post to a subscribed topic. The message is sent to the topic only
|
||||
msg.queuedAt = time.Now()
|
||||
select {
|
||||
case sub.broadcast <- msg:
|
||||
immetrics.ObserveInternalQueueEnqueue("session_topic_publish", len(sub.broadcast), cap(sub.broadcast), true)
|
||||
@@ -1677,6 +1778,7 @@ func (s *Session) note(msg *ClientComMessage) {
|
||||
|
||||
if sub := s.getSub(msg.RcptTo); sub != nil {
|
||||
// Pings can be sent to subscribed topics only
|
||||
msg.queuedAt = time.Now()
|
||||
select {
|
||||
case sub.broadcast <- msg:
|
||||
default:
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/tinode/chat/server/store/types"
|
||||
)
|
||||
|
||||
func TestFileLinkUserIDAllowsOpaqueTopicReferences(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
topic string
|
||||
isUser bool
|
||||
wantUID types.Uid
|
||||
}{
|
||||
{name: "cultivate reference", topic: "cvpbzxw1aio4ni8x0"},
|
||||
{name: "short opaque reference", topic: "x"},
|
||||
{name: "empty reference", topic: ""},
|
||||
{name: "user reference", topic: types.Uid(42).UserId(), isUser: true, wantUID: types.Uid(42)},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
gotUID, gotIsUser := fileLinkUserID(test.topic)
|
||||
if gotIsUser != test.isUser || gotUID != test.wantUID {
|
||||
t.Fatalf("fileLinkUserID(%q) = (%v, %v), want (%v, %v)", test.topic, gotUID, gotIsUser, test.wantUID, test.isUser)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -111,6 +111,7 @@ type groupMessageRuntime interface {
|
||||
RefreshUserPresence(tenantID types.TenantID, uid types.Uid, sid, nodeID string) error
|
||||
MarkUserOffline(tenantID types.TenantID, uid types.Uid, sid, nodeID string) (int, error)
|
||||
GetUserPresence(tenantID types.TenantID, uid types.Uid) (*UserPresence, error)
|
||||
GetUsersPresence(tenantID types.TenantID, uids []types.Uid) (map[types.Uid]*UserPresence, error)
|
||||
MarkTopicMemberOnline(tenantID types.TenantID, topic string, uid types.Uid, nodeID string) error
|
||||
MarkTopicMemberOffline(tenantID types.TenantID, topic string, uid types.Uid, nodeID string) error
|
||||
GetTopicNodeOnlineMembers(tenantID types.TenantID, topic, nodeID string) ([]types.Uid, error)
|
||||
@@ -339,6 +340,14 @@ func GetUserPresence(tenantID types.TenantID, uid types.Uid) (*UserPresence, err
|
||||
return groupMsgRuntime.GetUserPresence(tenantID, uid)
|
||||
}
|
||||
|
||||
func GetUsersPresence(tenantID types.TenantID, uids []types.Uid) (map[types.Uid]*UserPresence, error) {
|
||||
result := make(map[types.Uid]*UserPresence, len(uids))
|
||||
if groupMsgRuntime == nil || !tenantID.IsValid() || len(uids) == 0 {
|
||||
return result, nil
|
||||
}
|
||||
return groupMsgRuntime.GetUsersPresence(tenantID, uids)
|
||||
}
|
||||
|
||||
func UserPresenceEnabled() bool {
|
||||
return groupMsgRuntime != nil
|
||||
}
|
||||
@@ -1027,6 +1036,40 @@ func (r *redisGroupMessageRuntime) GetUserPresence(tenantID types.TenantID, uid
|
||||
return presence, nil
|
||||
}
|
||||
|
||||
func (r *redisGroupMessageRuntime) GetUsersPresence(tenantID types.TenantID, uids []types.Uid) (map[types.Uid]*UserPresence, error) {
|
||||
result := make(map[types.Uid]*UserPresence, len(uids))
|
||||
if len(uids) == 0 {
|
||||
return result, nil
|
||||
}
|
||||
ctx := context.Background()
|
||||
pipe := r.client.Pipeline()
|
||||
commands := make(map[types.Uid]*redis.MapStringStringCmd, len(uids))
|
||||
for _, uid := range uids {
|
||||
if !uid.IsZero() {
|
||||
commands[uid] = pipe.HGetAll(ctx, r.presenceUserKey(tenantID, uid))
|
||||
}
|
||||
}
|
||||
if _, err := pipe.Exec(ctx); err != nil && err != redis.Nil {
|
||||
return nil, err
|
||||
}
|
||||
for uid, command := range commands {
|
||||
raw, err := command.Result()
|
||||
if err != nil && err != redis.Nil {
|
||||
return nil, err
|
||||
}
|
||||
presence := &UserPresence{}
|
||||
if len(raw) > 0 {
|
||||
presence.Online = raw["online"] == "1"
|
||||
presence.NodeID = raw["node_id"]
|
||||
presence.SessionCount, _ = strconv.Atoi(raw["session_count"])
|
||||
presence.LastActiveAt, _ = time.Parse(time.RFC3339Nano, raw["last_active_at"])
|
||||
presence.LastSeenAt, _ = time.Parse(time.RFC3339Nano, raw["last_seen_at"])
|
||||
}
|
||||
result[uid] = presence
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (r *redisGroupMessageRuntime) MarkTopicMemberOnline(tenantID types.TenantID, topic string, uid types.Uid, nodeID string) error {
|
||||
if nodeID == "" {
|
||||
nodeID = "local"
|
||||
@@ -1368,7 +1411,8 @@ func (r *redisGroupMessageRuntime) hotSeqIDs(tenantID types.TenantID, topic stri
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if len(raw) < limit {
|
||||
expected := hotSeqExpectedCount(opts, limit)
|
||||
if len(raw) < expected {
|
||||
return nil, false, nil
|
||||
}
|
||||
seqs := make([]int64, 0, len(raw))
|
||||
@@ -1382,6 +1426,23 @@ func (r *redisGroupMessageRuntime) hotSeqIDs(tenantID types.TenantID, topic stri
|
||||
return seqs, true, nil
|
||||
}
|
||||
|
||||
func hotSeqExpectedCount(opts *types.QueryOpt, limit int) int {
|
||||
if limit <= 0 {
|
||||
return 0
|
||||
}
|
||||
if opts == nil || opts.Since <= 0 || opts.Before <= opts.Since {
|
||||
return limit
|
||||
}
|
||||
expected := opts.Before - opts.Since
|
||||
if expected <= 0 {
|
||||
return limit
|
||||
}
|
||||
if expected > int64(limit) {
|
||||
return limit
|
||||
}
|
||||
return int(expected)
|
||||
}
|
||||
|
||||
func expandRanges(ranges []types.Range, limit int) []int64 {
|
||||
if limit <= 0 {
|
||||
return nil
|
||||
|
||||
@@ -3,6 +3,7 @@ package store
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -126,6 +127,10 @@ func (r *testGroupMessageRuntime) GetUserPresence(types.TenantID, types.Uid) (*U
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (r *testGroupMessageRuntime) GetUsersPresence(types.TenantID, []types.Uid) (map[types.Uid]*UserPresence, error) {
|
||||
return map[types.Uid]*UserPresence{}, nil
|
||||
}
|
||||
|
||||
func (r *testGroupMessageRuntime) MarkTopicMemberOnline(types.TenantID, string, types.Uid, string) error {
|
||||
return nil
|
||||
}
|
||||
@@ -443,6 +448,34 @@ func TestInvalidateGroupSettingsUsesTopicCacheKey(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetGroupMuteUsesPositiveAndNegativeCache(t *testing.T) {
|
||||
oldRuntime := groupMsgRuntime
|
||||
runtime := newTestGroupMessageRuntime()
|
||||
groupMsgRuntime = runtime
|
||||
t.Cleanup(func() { groupMsgRuntime = oldRuntime })
|
||||
|
||||
tenantID := types.TenantID(7)
|
||||
topic := "grpAlpha"
|
||||
uid := types.Uid(42)
|
||||
cacheID := groupMemberCacheID(topic, uid)
|
||||
expires := time.Now().UTC().Add(time.Minute)
|
||||
want := &types.GroupMute{TenantID: tenantID, Topic: topic, UserID: uid, Status: types.GroupMuteStatusActive, ExpiresAt: &expires}
|
||||
if err := CacheObject(tenantID, cacheNamespaceGroupMute, cacheID, groupMuteCacheValue{Found: true, Mute: want}); err != nil {
|
||||
t.Fatalf("cache positive mute: %v", err)
|
||||
}
|
||||
got, err := (groupBusinessMapper{}).GetMute(tenantID, topic, uid)
|
||||
if err != nil || got == nil || got.UserID != uid || got.Status != types.GroupMuteStatusActive {
|
||||
t.Fatalf("positive cache result: mute=%#v err=%v", got, err)
|
||||
}
|
||||
|
||||
if err = CacheObject(tenantID, cacheNamespaceGroupMute, cacheID, groupMuteCacheValue{}); err != nil {
|
||||
t.Fatalf("cache negative mute: %v", err)
|
||||
}
|
||||
if got, err = (groupBusinessMapper{}).GetMute(tenantID, topic, uid); !errors.Is(err, types.ErrNotFound) || got != nil {
|
||||
t.Fatalf("negative cache result: mute=%#v err=%v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTopicAndSubscriptionCacheTTLIsShort(t *testing.T) {
|
||||
runtime := &redisGroupMessageRuntime{cfg: redisConfig{CacheTTL: 300, ShortTTL: 600}}
|
||||
for _, namespace := range []string{cacheNamespaceTopic, cacheNamespaceSub} {
|
||||
@@ -455,6 +488,21 @@ func TestTopicAndSubscriptionCacheTTLIsShort(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHotSeqExpectedCountUsesRangeSizeWhenSmallerThanLimit(t *testing.T) {
|
||||
if got := hotSeqExpectedCount(&types.QueryOpt{Since: 41, Before: 54, Limit: 20}, 20); got != 13 {
|
||||
t.Fatalf("expected range-sized count, got %d", got)
|
||||
}
|
||||
if got := hotSeqExpectedCount(&types.QueryOpt{Since: 41, Before: 80, Limit: 20}, 20); got != 20 {
|
||||
t.Fatalf("expected limit-sized count, got %d", got)
|
||||
}
|
||||
if got := hotSeqExpectedCount(&types.QueryOpt{Since: 0, Before: 54, Limit: 20}, 20); got != 20 {
|
||||
t.Fatalf("expected fallback limit count, got %d", got)
|
||||
}
|
||||
if got := hotSeqExpectedCount(nil, 20); got != 20 {
|
||||
t.Fatalf("expected nil opts limit count, got %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInvalidateSubscriptionCacheWithZeroUserDeletesTopicPrefix(t *testing.T) {
|
||||
oldRuntime := groupMsgRuntime
|
||||
runtime := newTestGroupMessageRuntime()
|
||||
|
||||
+77
-6
@@ -142,6 +142,7 @@ const (
|
||||
cacheNamespaceSub = "sub"
|
||||
cacheNamespaceGroupSettings = "group_settings"
|
||||
cacheNamespaceGroupMember = "group_member:v2"
|
||||
cacheNamespaceGroupMute = "group_mute:v1"
|
||||
cacheNamespaceVerification = "verification"
|
||||
cacheNamespaceActionToken = "action_token"
|
||||
cacheNamespaceFriends = "friends"
|
||||
@@ -2422,11 +2423,32 @@ func (groupBusinessMapper) GetMute(tenantID types.TenantID, topic string, uid ty
|
||||
if strings.TrimSpace(topic) == "" || uid.IsZero() {
|
||||
return nil, types.ErrMalformed
|
||||
}
|
||||
topic = strings.TrimSpace(topic)
|
||||
cacheID := groupMemberCacheID(topic, uid)
|
||||
var cached groupMuteCacheValue
|
||||
if ok, err := GetCachedObject(tenantID, cacheNamespaceGroupMute, cacheID, &cached); err != nil {
|
||||
logs.Warn.Printf("cache group mute[%s]: read failed: %v", cacheID, err)
|
||||
_ = DeleteCachedObject(tenantID, cacheNamespaceGroupMute, cacheID)
|
||||
} else if ok {
|
||||
if !cached.Found || cached.Mute == nil {
|
||||
return nil, types.ErrNotFound
|
||||
}
|
||||
if cached.Mute.ExpiresAt == nil || cached.Mute.ExpiresAt.After(time.Now().UTC()) {
|
||||
return cached.Mute, nil
|
||||
}
|
||||
_ = DeleteCachedObject(tenantID, cacheNamespaceGroupMute, cacheID)
|
||||
}
|
||||
tenantAdp, err := tenantBusinessAdapter(tenantID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return tenantAdp.TenantGroupMuteGet(tenantID, strings.TrimSpace(topic), uid)
|
||||
mute, err := tenantAdp.TenantGroupMuteGet(tenantID, topic, uid)
|
||||
if err == nil && mute != nil {
|
||||
_ = CacheObject(tenantID, cacheNamespaceGroupMute, cacheID, groupMuteCacheValue{Found: true, Mute: mute})
|
||||
} else if errors.Is(err, types.ErrNotFound) {
|
||||
_ = CacheObject(tenantID, cacheNamespaceGroupMute, cacheID, groupMuteCacheValue{})
|
||||
}
|
||||
return mute, err
|
||||
}
|
||||
|
||||
func (groupBusinessMapper) UpsertMute(tenantID types.TenantID, mute *types.GroupMute) (*types.GroupMute, error) {
|
||||
@@ -2440,6 +2462,12 @@ func (groupBusinessMapper) UpsertMute(tenantID types.TenantID, mute *types.Group
|
||||
updated, err := tenantAdp.TenantGroupMuteUpsert(tenantID, mute)
|
||||
if err == nil {
|
||||
invalidateGroupMemberPages(tenantID, mute.Topic)
|
||||
cacheID := groupMemberCacheID(mute.Topic, mute.UserID)
|
||||
if updated != nil {
|
||||
_ = CacheObject(tenantID, cacheNamespaceGroupMute, cacheID, groupMuteCacheValue{Found: true, Mute: updated})
|
||||
} else {
|
||||
_ = DeleteCachedObject(tenantID, cacheNamespaceGroupMute, cacheID)
|
||||
}
|
||||
}
|
||||
return updated, err
|
||||
}
|
||||
@@ -2456,6 +2484,7 @@ func (groupBusinessMapper) DeleteMute(tenantID types.TenantID, topic string, uid
|
||||
return err
|
||||
}
|
||||
invalidateGroupMemberPages(tenantID, topic)
|
||||
_ = CacheObject(tenantID, cacheNamespaceGroupMute, groupMemberCacheID(topic, uid), groupMuteCacheValue{})
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -2581,11 +2610,32 @@ func (groupBusinessMapper) CanPost(tenantID types.TenantID, topic string, uid ty
|
||||
if strings.TrimSpace(topic) == "" || uid.IsZero() || now.IsZero() {
|
||||
return false, types.ErrMalformed
|
||||
}
|
||||
tenantAdp, err := tenantBusinessAdapter(tenantID)
|
||||
if admin {
|
||||
return true, nil
|
||||
}
|
||||
topic = strings.TrimSpace(topic)
|
||||
mapper := groupBusinessMapper{}
|
||||
settings, err := mapper.GetSettings(tenantID, topic)
|
||||
if err != nil && !errors.Is(err, types.ErrNotFound) {
|
||||
return false, err
|
||||
}
|
||||
if settings != nil && settings.GroupStatus == types.GroupStatusActive && settings.MuteAll {
|
||||
return false, nil
|
||||
}
|
||||
mute, err := mapper.GetMute(tenantID, topic, uid)
|
||||
if errors.Is(err, types.ErrNotFound) {
|
||||
return true, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return tenantAdp.TenantGroupCanPost(tenantID, strings.TrimSpace(topic), uid, admin, now)
|
||||
return mute == nil || mute.Status != types.GroupMuteStatusActive ||
|
||||
(mute.ExpiresAt != nil && !mute.ExpiresAt.After(now)), nil
|
||||
}
|
||||
|
||||
type groupMuteCacheValue struct {
|
||||
Found bool `json:"found"`
|
||||
Mute *types.GroupMute `json:"mute,omitempty"`
|
||||
}
|
||||
|
||||
func groupMemberCacheID(topic string, uid types.Uid) string {
|
||||
@@ -3170,13 +3220,26 @@ func (relationsMapper) CanSendP2P(tenantID types.TenantID, userID, peerID types.
|
||||
return true, types.RelationshipFriend, nil
|
||||
}
|
||||
}
|
||||
ctx, err := (relationsMapper{}).GetRelationshipContext(tenantID, userID, peerID, "")
|
||||
tenantAdp, err := tenantBusinessAdapter(tenantID)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
ctx, err := relationshipContext(tenantAdp, tenantID, userID, peerID, "")
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
if ctx.Relationship == types.RelationshipFriend {
|
||||
return true, ctx.Relationship, nil
|
||||
}
|
||||
if ctx.Relationship == types.RelationshipNone {
|
||||
shared, err := tenantAdp.TenantUsersShareOpenPrivateGroup(tenantID, userID, peerID)
|
||||
if err != nil {
|
||||
return false, ctx.Relationship, err
|
||||
}
|
||||
if shared {
|
||||
return true, ctx.Relationship, nil
|
||||
}
|
||||
}
|
||||
return false, ctx.Relationship, nil
|
||||
}
|
||||
|
||||
@@ -4634,8 +4697,8 @@ func (fileMapper) LinkAttachments(tenantID types.TenantID, topic string, msgId t
|
||||
return err
|
||||
}
|
||||
userId := types.ZeroUid
|
||||
if topic != "" && types.GetTopicCat(topic) == types.TopicCatMe {
|
||||
userId = types.ParseUserId(topic)
|
||||
if parsedUserID, ok := fileLinkUserID(topic); ok {
|
||||
userId = parsedUserID
|
||||
topic = ""
|
||||
}
|
||||
return tenantAdp.TenantFileLinkAttachments(tenantID, topic, userId, msgId, fids)
|
||||
@@ -4643,6 +4706,14 @@ func (fileMapper) LinkAttachments(tenantID types.TenantID, topic string, msgId t
|
||||
return nil
|
||||
}
|
||||
|
||||
// File link topics may also be opaque retention keys, such as cultivate's cvp prefix.
|
||||
func fileLinkUserID(topic string) (types.Uid, bool) {
|
||||
if !strings.HasPrefix(topic, "usr") {
|
||||
return types.ZeroUid, false
|
||||
}
|
||||
return types.ParseUserId(topic), true
|
||||
}
|
||||
|
||||
func (fileMapper) UserHasAccess(tenantID types.TenantID, fid string, uid types.Uid) (bool, error) {
|
||||
if fid == "" || uid.IsZero() {
|
||||
return false, types.ErrMalformed
|
||||
|
||||
@@ -5,6 +5,7 @@ import "time"
|
||||
const (
|
||||
PublicUploadPurposeEnterpriseLicense = "enterprise_license"
|
||||
PublicUploadPurposeTenantFeedback = "tenant_entry_feedback"
|
||||
PublicUploadPurposeFeedback = "feedback"
|
||||
PublicUploadPurposeUserAvatar = "user_avatar"
|
||||
PublicUploadPurposeGroupAvatar = "group_avatar"
|
||||
|
||||
|
||||
@@ -1502,9 +1502,8 @@ type FileDef struct {
|
||||
User string
|
||||
// Type of the file.
|
||||
MimeType string
|
||||
// OriginalName is the client-provided file name. It is not persisted; media handlers use it
|
||||
// only to keep the public URL extension consistent with the uploaded file.
|
||||
OriginalName string `db:"-" bson:"-" json:"-"`
|
||||
// OriginalName is the client-provided file name.
|
||||
OriginalName string `db:"original_name" bson:"original_name" json:"-"`
|
||||
// Size of the file in bytes.
|
||||
Size int64
|
||||
// Internal file location, i.e. path on disk or an S3 blob address.
|
||||
|
||||
+4
-2
@@ -88,7 +88,7 @@
|
||||
"max_idle_conns": 8,
|
||||
"registry_max_open_conns": 4,
|
||||
"registry_max_idle_conns": 2,
|
||||
"conn_max_lifetime": 60,
|
||||
"conn_max_lifetime": 1800,
|
||||
"sql_timeout": 10
|
||||
}
|
||||
},
|
||||
@@ -121,7 +121,7 @@
|
||||
"write_timeout_ms": 3000,
|
||||
"read_timeout_ms": 3000,
|
||||
"batch_timeout_ms": 10,
|
||||
"required_acks": 1,
|
||||
"required_acks": -1,
|
||||
"allow_auto_topic_creation": false,
|
||||
"tls_enabled": false,
|
||||
"sasl_enabled": true,
|
||||
@@ -228,6 +228,8 @@
|
||||
"enabled": false,
|
||||
"api_domain": "https://api.jpush.cn",
|
||||
"push_path": "/v3/push",
|
||||
"workers": 8,
|
||||
"device_cache_ttl_seconds": 30,
|
||||
"default_ttl_seconds": 86400,
|
||||
"call_ttl_seconds": 60,
|
||||
"apns_production": false,
|
||||
|
||||
+78
-3
@@ -650,6 +650,14 @@ func (t *Topic) runLocal(hub *Hub) {
|
||||
|
||||
// handleClientMsg is the top-level handler of messages received by the topic from sessions.
|
||||
func (t *Topic) handleClientMsg(msg *ClientComMessage) {
|
||||
kind := "other"
|
||||
if t.cat == types.TopicCatP2P {
|
||||
kind = "p2p"
|
||||
} else if t.cat == types.TopicCatGrp {
|
||||
kind = "group"
|
||||
}
|
||||
immetrics.ObserveTopicQueueWait(kind, msg.queuedAt)
|
||||
msg.queuedAt = time.Time{}
|
||||
if msg.TenantID.IsZero() {
|
||||
msg.TenantID = t.tenantID
|
||||
} else if msg.TenantID != t.tenantID {
|
||||
@@ -1348,7 +1356,9 @@ func (t *Topic) saveAndBroadcastMessage(msg *ClientComMessage, asUid types.Uid,
|
||||
if t.cat == types.TopicCatGrp && msg.sess != nil && msg.sess.authLvl != auth.LevelRoot &&
|
||||
store.GroupBusiness != nil && store.Store != nil && store.Store.IsOpen() {
|
||||
access := pud.modeWant & pud.modeGiven
|
||||
started := time.Now()
|
||||
canPost, err := store.GroupBusiness.CanPost(t.tenantID, t.name, asUid, access.IsAdmin(), msg.Timestamp)
|
||||
immetrics.ObserveMessagePublishStage("group", "permission_check", started, err)
|
||||
if err != nil {
|
||||
logs.Warn.Printf("topic[%s]: failed to check group mute policy: %v", t.name, err)
|
||||
msg.sess.queueOut(ErrUnknown(msg.Id, t.original(asUid), msg.Timestamp))
|
||||
@@ -1457,12 +1467,18 @@ func (t *Topic) saveAndBroadcastMessage(msg *ClientComMessage, asUid types.Uid,
|
||||
// Tell the plugins that a message was accepted for delivery
|
||||
pluginMessage(t.tenantID, data.Data, plgActCreate)
|
||||
|
||||
fanoutStarted := time.Now()
|
||||
t.broadcastToSessions(data)
|
||||
publishRealtimeTopicMessage(t.name, data)
|
||||
immetrics.ObserveMessagePublishStage(kind, "realtime_fanout", fanoutStarted, nil)
|
||||
|
||||
// sendPush will update unread message count and send push notification.
|
||||
if pushRcpt := t.pushForData(asUid, data.Data, markedReadBySender); pushRcpt != nil {
|
||||
sendPush(pushRcpt)
|
||||
// Push preparation copies immutable topic state. Presence resolution, cluster
|
||||
// recipient partitioning and unread updates run outside the topic loop.
|
||||
pushStarted := time.Now()
|
||||
pushRcpt := t.pushForData(asUid, data.Data, markedReadBySender)
|
||||
immetrics.ObserveMessagePublishStage(kind, "push_prepare", pushStarted, nil)
|
||||
if pushRcpt != nil {
|
||||
dispatchPushAsync(pushRcpt, true)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -3519,6 +3535,13 @@ func (t *Topic) replyGetData(sess *Session, asUid types.Uid, asChan bool, req *M
|
||||
} else if ok {
|
||||
messages = filterHotMessagesForUser(hotMessages, asUid)
|
||||
}
|
||||
} else if shouldTryHotMessageSupplement(t.cat, opts, len(messages)) {
|
||||
hotMessages, ok, cacheErr := store.GetHotMessages(t.tenantID, t.name, opts, hotMessageRangeLimit(opts))
|
||||
if cacheErr != nil {
|
||||
logs.Warn.Printf("get.data hot message supplement failed topic=%s user=%s: %v", t.name, asUid.UserId(), cacheErr)
|
||||
} else if ok {
|
||||
messages = mergeHotMessageSupplement(messages, filterHotMessagesForUser(hotMessages, asUid), opts)
|
||||
}
|
||||
}
|
||||
|
||||
// Push the list of messages to the client as {data}.
|
||||
@@ -3579,6 +3602,28 @@ func shouldTryHotMessageFallback(cat types.TopicCat, lastID int64, opts *types.Q
|
||||
return lastID <= 0 || opts.Since >= floor
|
||||
}
|
||||
|
||||
func shouldTryHotMessageSupplement(cat types.TopicCat, opts *types.QueryOpt, got int) bool {
|
||||
if cat != types.TopicCatGrp || opts == nil || opts.IfModifiedSince != nil ||
|
||||
!opts.User.IsZero() || opts.Topic != "" || len(opts.IdRanges) > 0 ||
|
||||
opts.Since <= 0 || opts.Before <= opts.Since {
|
||||
return false
|
||||
}
|
||||
expected := hotMessageRangeLimit(opts)
|
||||
return expected > 0 && got > 0 && got < expected
|
||||
}
|
||||
|
||||
func hotMessageRangeLimit(opts *types.QueryOpt) int {
|
||||
if opts == nil || opts.Since <= 0 || opts.Before <= opts.Since {
|
||||
return hotMessageFallbackLimit(opts)
|
||||
}
|
||||
expected := opts.Before - opts.Since
|
||||
limit := hotMessageFallbackLimit(opts)
|
||||
if limit <= 0 || int64(limit) > expected {
|
||||
return int(expected)
|
||||
}
|
||||
return limit
|
||||
}
|
||||
|
||||
func hotMessageFallbackLimit(opts *types.QueryOpt) int {
|
||||
if opts != nil && opts.Limit > 0 {
|
||||
return opts.Limit
|
||||
@@ -3586,6 +3631,36 @@ func hotMessageFallbackLimit(opts *types.QueryOpt) int {
|
||||
return 1
|
||||
}
|
||||
|
||||
func mergeHotMessageSupplement(messages, hotMessages []types.Message, opts *types.QueryOpt) []types.Message {
|
||||
if len(messages) == 0 || len(hotMessages) == 0 {
|
||||
return messages
|
||||
}
|
||||
bySeq := make(map[int64]types.Message, len(hotMessages)+len(messages))
|
||||
for _, msg := range hotMessages {
|
||||
bySeq[msg.SeqId] = msg
|
||||
}
|
||||
for _, msg := range messages {
|
||||
bySeq[msg.SeqId] = msg
|
||||
}
|
||||
merged := make([]types.Message, 0, len(bySeq))
|
||||
for _, msg := range bySeq {
|
||||
if opts != nil {
|
||||
if opts.Since > 0 && msg.SeqId < opts.Since {
|
||||
continue
|
||||
}
|
||||
if opts.Before > 0 && msg.SeqId >= opts.Before {
|
||||
continue
|
||||
}
|
||||
}
|
||||
merged = append(merged, msg)
|
||||
}
|
||||
sort.Slice(merged, func(i, j int) bool { return merged[i].SeqId > merged[j].SeqId })
|
||||
if opts != nil && opts.Limit > 0 && len(merged) > opts.Limit {
|
||||
merged = merged[:opts.Limit]
|
||||
}
|
||||
return merged
|
||||
}
|
||||
|
||||
func filterHotMessagesForUser(messages []types.Message, asUid types.Uid) []types.Message {
|
||||
if len(messages) == 0 {
|
||||
return messages
|
||||
|
||||
@@ -1272,6 +1272,55 @@ func TestShouldTryHotMessageFallback(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestShouldTryHotMessageSupplement(t *testing.T) {
|
||||
opts := &types.QueryOpt{Since: 41, Before: 54, Limit: 20}
|
||||
if !shouldTryHotMessageSupplement(types.TopicCatGrp, opts, 9) {
|
||||
t.Fatal("expected hot message supplement for partial recent group range")
|
||||
}
|
||||
if shouldTryHotMessageSupplement(types.TopicCatGrp, opts, 13) {
|
||||
t.Fatal("did not expect supplement when range is complete")
|
||||
}
|
||||
if shouldTryHotMessageSupplement(types.TopicCatP2P, opts, 9) {
|
||||
t.Fatal("did not expect supplement for p2p topic")
|
||||
}
|
||||
if shouldTryHotMessageSupplement(types.TopicCatGrp, &types.QueryOpt{Since: 41, Before: 54, Limit: 20, IdRanges: []types.Range{{Low: 41, Hi: 54}}}, 9) {
|
||||
t.Fatal("did not expect supplement for explicit id ranges")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeHotMessageSupplementFillsRecentRangeGaps(t *testing.T) {
|
||||
opts := &types.QueryOpt{Since: 41, Before: 54, Limit: 20}
|
||||
messages := []types.Message{
|
||||
{SeqId: 53, Content: "db-53"},
|
||||
{SeqId: 52, Content: "db-52"},
|
||||
{SeqId: 51, Content: "db-51"},
|
||||
{SeqId: 50, Content: "db-50"},
|
||||
{SeqId: 49, Content: "db-49"},
|
||||
{SeqId: 47, Content: "db-47"},
|
||||
{SeqId: 46, Content: "db-46"},
|
||||
{SeqId: 42, Content: "db-42"},
|
||||
{SeqId: 41, Content: "db-41"},
|
||||
}
|
||||
hotMessages := make([]types.Message, 0, 13)
|
||||
for seq := int64(53); seq >= 41; seq-- {
|
||||
hotMessages = append(hotMessages, types.Message{SeqId: seq, Content: "hot"})
|
||||
}
|
||||
|
||||
merged := mergeHotMessageSupplement(messages, hotMessages, opts)
|
||||
if len(merged) != 13 {
|
||||
t.Fatalf("merged len = %d, want 13: %#v", len(merged), merged)
|
||||
}
|
||||
for i, msg := range merged {
|
||||
wantSeq := int64(53 - i)
|
||||
if msg.SeqId != wantSeq {
|
||||
t.Fatalf("merged[%d].SeqId = %d, want %d", i, msg.SeqId, wantSeq)
|
||||
}
|
||||
if msg.SeqId == 53 && msg.Content != "db-53" {
|
||||
t.Fatalf("database message should win duplicate seqs, got %#v", msg.Content)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilterHotMessagesForUserSkipsDeleted(t *testing.T) {
|
||||
uid := types.Uid(1)
|
||||
now := time.Now()
|
||||
@@ -3794,6 +3843,30 @@ func TestMessageAttachmentURLsDeduplicatesExplicitAndContentFileID(t *testing.T)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMessageAttachmentURLsFindsDraftyEntityRefs(t *testing.T) {
|
||||
msg := &ClientComMessage{
|
||||
Pub: &MsgClientPub{
|
||||
Content: map[string]any{
|
||||
"txt": "video",
|
||||
"ent": []any{
|
||||
map[string]any{
|
||||
"tp": "EX",
|
||||
"data": map[string]any{
|
||||
"mime": "video/mp4",
|
||||
"ref": "/v0/file/s/KJ0Qlrk4Gzw.mp4",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
attachments := messageAttachmentURLs(msg)
|
||||
if len(attachments) != 1 || attachments[0] != "/v0/file/s/KJ0Qlrk4Gzw" {
|
||||
t.Fatalf("unexpected Drafty attachments: %#v", attachments)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleBroadcastInfoChannelWithMultipleReaders(t *testing.T) {
|
||||
topicName := "grpTest"
|
||||
chanName := "chnTest"
|
||||
|
||||
Reference in New Issue
Block a user