Compare commits
25
Commits
7b25365ec3
...
027dda786e
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
027dda786e | ||
|
|
87ec8f9833 | ||
|
|
6930b23cce | ||
|
|
e8804883fc | ||
|
|
43808c36d1 | ||
|
|
b126b13b77 | ||
|
|
f3b1a06da5 | ||
|
|
6be7796049 | ||
|
|
0a33d21eb8 | ||
|
|
5d792f9f96 | ||
|
|
c219ef6455 | ||
|
|
1e0b4c181d | ||
|
|
54c594f39b | ||
|
|
2e1ceec717 | ||
|
|
1d8f9e3be4 | ||
|
|
8228ceb89f | ||
|
|
f24133f5de | ||
|
|
e30b70a3a7 | ||
|
|
ac6cd3c6e8 | ||
|
|
58c15efad5 | ||
|
|
2732e05a83 | ||
|
|
d3054f62b5 | ||
|
|
965f94f154 | ||
|
|
b05e8f60dd | ||
|
|
c46209f4e7 |
@@ -45,7 +45,8 @@ jobs:
|
||||
MYSQL_MAX_OPEN_CONNS=24 \
|
||||
MYSQL_MAX_IDLE_CONNS=8 \
|
||||
MYSQL_REGISTRY_MAX_OPEN_CONNS=4 \
|
||||
MYSQL_REGISTRY_MAX_IDLE_CONNS=2
|
||||
MYSQL_REGISTRY_MAX_IDLE_CONNS=2 \
|
||||
MYSQL_CONN_MAX_LIFETIME_SECONDS=1800
|
||||
kubectl patch statefulset/im-server -n im-server-system --type=strategic -p \
|
||||
'{"spec":{"template":{"spec":{"containers":[{"name":"im-server","readinessProbe":{"httpGet":{"path":"/healthz","port":6060},"timeoutSeconds":3,"periodSeconds":10,"failureThreshold":3},"livenessProbe":{"httpGet":{"path":"/livez","port":6060},"timeoutSeconds":3,"periodSeconds":20,"failureThreshold":3}}]}}}}'
|
||||
if kubectl auth can-i get services -n im-server-system >/dev/null 2>&1 && \
|
||||
@@ -64,7 +65,25 @@ jobs:
|
||||
echo "::warning::ci-deployer cannot manage Ingress/im-server or it does not exist; skipped least_conn update"
|
||||
fi
|
||||
kubectl rollout restart statefulset/im-server -n im-server-system
|
||||
kubectl rollout status statefulset/im-server -n im-server-system --timeout=300s
|
||||
if ! kubectl rollout status statefulset/im-server -n im-server-system --timeout=300s; then
|
||||
echo "::error::im-server rollout did not become ready within 300 seconds"
|
||||
kubectl get statefulset/im-server -n im-server-system -o wide || true
|
||||
kubectl get pods -n im-server-system -l app=im-server -o wide || true
|
||||
kubectl describe statefulset/im-server -n im-server-system || true
|
||||
for pod in $(kubectl get pods -n im-server-system -l app=im-server -o name 2>/dev/null); do
|
||||
echo "===== ${pod} describe ====="
|
||||
kubectl describe -n im-server-system "${pod}" || true
|
||||
echo "===== ${pod} current log ====="
|
||||
kubectl logs -n im-server-system "${pod}" -c im-server --tail=200 || true
|
||||
echo "===== ${pod} previous log ====="
|
||||
kubectl logs -n im-server-system "${pod}" -c im-server --previous --tail=200 || true
|
||||
done
|
||||
if kubectl auth can-i list events -n im-server-system >/dev/null 2>&1; then
|
||||
echo "===== recent events ====="
|
||||
kubectl get events -n im-server-system --sort-by=.lastTimestamp | tail -100 || true
|
||||
fi
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Waiting for ArgoCD reconciliation before validating effective settings"
|
||||
sleep 60
|
||||
@@ -86,12 +105,14 @@ jobs:
|
||||
jsonpath='{.spec.template.spec.containers[?(@.name=="im-server")].env[?(@.name=="MYSQL_REGISTRY_MAX_OPEN_CONNS")].value}')"
|
||||
mysql_registry_max_idle="$(kubectl get statefulset/im-server -n im-server-system -o \
|
||||
jsonpath='{.spec.template.spec.containers[?(@.name=="im-server")].env[?(@.name=="MYSQL_REGISTRY_MAX_IDLE_CONNS")].value}')"
|
||||
echo "effective requests=${request_cpu}/${request_memory} limits=${limit_cpu}/${limit_memory} liveness=${liveness} mysql=${mysql_max_open}/${mysql_max_idle} registry=${mysql_registry_max_open}/${mysql_registry_max_idle}"
|
||||
mysql_conn_max_lifetime="$(kubectl get statefulset/im-server -n im-server-system -o \
|
||||
jsonpath='{.spec.template.spec.containers[?(@.name=="im-server")].env[?(@.name=="MYSQL_CONN_MAX_LIFETIME_SECONDS")].value}')"
|
||||
echo "effective requests=${request_cpu}/${request_memory} limits=${limit_cpu}/${limit_memory} liveness=${liveness} mysql=${mysql_max_open}/${mysql_max_idle} registry=${mysql_registry_max_open}/${mysql_registry_max_idle} lifetime=${mysql_conn_max_lifetime}"
|
||||
if [ "${request_cpu}" != "1" ] || [ "${request_memory}" != "512Mi" ] || \
|
||||
[ "${limit_cpu}" != "2" ] || [ "${limit_memory}" != "1Gi" ] || \
|
||||
[ "${liveness}" != "/livez" ] || [ "${mysql_max_open}" != "24" ] || \
|
||||
[ "${mysql_max_idle}" != "8" ] || [ "${mysql_registry_max_open}" != "4" ] || \
|
||||
[ "${mysql_registry_max_idle}" != "2" ]; then
|
||||
[ "${mysql_registry_max_idle}" != "2" ] || [ "${mysql_conn_max_lifetime}" != "1800" ]; then
|
||||
echo "::error::ArgoCD reverted StatefulSet resources/probes/MySQL pools. Apply deploy/argocd/im-server-test/statefulset-performance-patch.yaml to the im-server-test source."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -32,6 +32,12 @@ spec:
|
||||
value: "3"
|
||||
- name: CLUSTER_ROUTE_RETRY_BACKOFF_MS
|
||||
value: "25"
|
||||
- name: TOPIC_CLIENT_QUEUE_BUFFER
|
||||
value: "4096"
|
||||
- name: TOPIC_SERVER_QUEUE_BUFFER
|
||||
value: "512"
|
||||
- name: TOPIC_PUBLISH_ENQUEUE_WAIT_MS
|
||||
value: "200"
|
||||
- name: MYSQL_MAX_OPEN_CONNS
|
||||
value: "24"
|
||||
- name: MYSQL_MAX_IDLE_CONNS
|
||||
|
||||
@@ -193,6 +193,9 @@ You can specify the following environment variables when issuing `docker run` co
|
||||
| `CLUSTER_PROXY_TO_MASTER_ENQUEUE_TIMEOUT_MS` | number | `300` | Maximum wait for a full proxy-to-master shard before returning 503. |
|
||||
| `CLUSTER_ROUTE_RETRIES` | number | `3` | Number of retries after a remote server-message route is rejected as overloaded. |
|
||||
| `CLUSTER_ROUTE_RETRY_BACKOFF_MS` | number | `25` | Initial exponential retry delay for remote server-message routes. |
|
||||
| `TOPIC_CLIENT_QUEUE_BUFFER` | number | `4096` | Per-topic client publish queue capacity. Increase for large groups with bursty publishers. |
|
||||
| `TOPIC_SERVER_QUEUE_BUFFER` | number | `512` | Per-topic server-message queue capacity. |
|
||||
| `TOPIC_PUBLISH_ENQUEUE_WAIT_MS` | number | `200` | Maximum wait for an attached session publish to enter a full topic queue before returning 503. |
|
||||
| `DEBUG_EMAIL_VERIFICATION_CODE`[^2] | string | | Enable dummy email verification code, e.g. `123456`. Disabled by default (empty string). |
|
||||
| `DEFAULT_COUNTRY_CODE`[^2] | string | `US` | 2-letter country code to assign to sessions by default when the country isn't specified by the client explicitly and it's impossible to infer it. |
|
||||
| `EXT_CONFIG`[^1] | string | | Path to external config file to use instead of the built-in one. If this parameter is used, most other variables are ignored[^1]. |
|
||||
|
||||
+21
-11
@@ -636,8 +636,14 @@ X-MeetingIM-APIKey: <预置的普通 API Key>
|
||||
| WebSocket `login basic` | `{"login":{"id":"login-1","scheme":"basic","secret":"base64(username:password)"}}` | `{"ctrl":{"id":"login-1","code":200,"params":{"user":"usr...","token":"token...","expires":"2026-09-03T09:00:00Z","authlvl":"auth"}}}` |
|
||||
| WebSocket `acc new` | `{"acc":{"id":"acc-1","user":"new","scheme":"basic","username":"alice","phone":"15500000000","registration_type":1,"secret":"base64(alice:password)","login":true,"cred":[{"meth":"tel","val":"15500000000","resp":"verification_token","params":{"challenge_id":"chg_123"}}]}}` | `{"ctrl":{"id":"acc-1","code":200,"params":{"user":"usr...","username":"alice","token":"token...","expires":"2026-09-03T09:00:00Z"}}}` |
|
||||
| `POST /v0/channels/lp` | Long Polling 首包 `{"hi":{"id":"hi-1","ver":"0.25","tenant":"default"}}` | `{"ctrl":{"id":"hi-1","code":201,"params":{"sid":"lp_sid_123"}}}` |
|
||||
| `POST /v0/file/u/` | Multipart 文件;Header `X-MeetingIM-Auth: token <base64_token>`;业务表单可带 `purpose=chat_file&topic=grp...` | `{"ctrl":{"code":200,"params":{"url":"/v0/file/s/file123","name":"a.pdf","size":1024,"mime":"application/pdf"}}}` |
|
||||
| `GET /v0/file/s/{file_id}` | Header `X-MeetingIM-Auth: token <base64_token>`,可带 `Range: bytes=0-` | 对象存储模式下鉴权通过后返回 302 到短期签名 URL;客户端跟随 Location 获取文件 |
|
||||
| `POST /v0/file/u/` | Multipart 文件;Header `X-MeetingIM-Auth: token <base64_token>`;聊天附件必须带 `topic=<消息所在会话 topic>`(`p2p.../grp...`),可选 `purpose=chat_file`、`id=<幂等ID>` | `{"ctrl":{"code":200,"params":{"expires":"2026-10-13T06:21:20Z","file_id":"s6yhMyiOfaU","url":"/v0/file/s/s6yhMyiOfaU.txt"}}}` |
|
||||
| `GET /v0/file/s/{file_id}` | Header `X-MeetingIM-Auth: token <base64_token>`,可带 `Range: bytes=0-`;URL 可带文件扩展名(服务端自动剥离) | 对象存储模式下鉴权通过后返回 302 到短期签名 URL;客户端跟随 Location 获取文件 |
|
||||
|
||||
**文件上传/下载鉴权与回收规则(已实现,客户端必须遵守):**
|
||||
|
||||
1. 上传聊天附件(图片/视频/语音/文件)必须携带 `topic=<消息将要发送到的会话 topic>`。服务端校验上传者是该 `p2p`/`grp` 会话的订阅成员(`modewant`/`modegiven` 含读权限),不是成员返回 `403 permission denied`;校验通过则在服务端建立「文件-会话」引用,该会话全体成员凭有效 Token 即可下载。
|
||||
2. 不带 `topic` 的上传只有上传者本人可下载(可用于头像等场景)。任何文件若一直未被任何会话/消息引用,服务端在创建 1 小时后由后台任务物理回收;上传后必须及时发送消息或携带 `topic`,否则对端将拿到 404。
|
||||
3. 下载放行范围:上传者本人、文件所属会话成员、消息级引用可见者(未撤回未删除且未被执行删除)、群成员、好友关系、收藏引用。非成员返回 `403 permission denied`(即 `FILE_FORBIDDEN`),客户端不自动重试。
|
||||
|
||||
### 0.6 Topic 与产品消息
|
||||
|
||||
@@ -1073,8 +1079,9 @@ X-MeetingIM-APIKey: <预置的普通 API Key>
|
||||
**状态:需改造。**
|
||||
|
||||
- `GET /v1/im/system-notifications?cursor=` 返回 `id,event_type,title,summary,created_at,read_at,action`;实时刷新以 0.5.4.6 的 WS `{info what:"im|group"}` 业务事件为准,系统通知列表记录不等同于每一条实时事件。
|
||||
- `summary` 为服务端生成的可直接展示文案(如「对方已同意你的好友申请」「对方已婉拒你的好友申请」),前端直接渲染,无需再做状态码到文案的映射;`action` 读路径统一归一为 `{type,target_id,payload,...原始键}`。
|
||||
- 新设备登录事件内容为 `device_name,platform,ip_masked,login_at,security_action`;IP 只下发必要脱敏值。
|
||||
- 已读:`POST /v1/im/system-notifications/{id}/read`,带幂等键;跳转目标只接受服务端白名单 `action.type` 和内部参数。
|
||||
- 已读:`POST /v1/im/system-notifications/{id}/read`,带幂等键;重复已读返回首次持久化的 `read_at`,不回显本次提交值。跳转目标只接受服务端白名单 `action.type` 和内部参数。
|
||||
- 当前普通认证用户可向 `sys` Topic 发布的路径必须封禁。客户端不得渲染普通用户伪造的系统卡片为可信通知。
|
||||
|
||||
### 2.6 接口示例
|
||||
@@ -1093,7 +1100,7 @@ X-MeetingIM-APIKey: <预置的普通 API Key>
|
||||
| --- | --- | --- |
|
||||
| `GET /v1/im/bootstrap` | Query only | `{"features":{"create_group":true,"user_search":true,"scan_qr":true,"system_notifications":true,"message_recall":true,"message_edit":true,"audio_call":true,"video_call":true},"permissions":{"create_group":true,"user_search":true,"scan_qr":true,"audio_call":true,"video_call":true},"entries":{"new_friends":{"enabled":true,"route":"/contacts/new-friends","unread_count":0},"groups":{"enabled":true,"route":"/contacts/groups","unread_count":0},"file_assistant":{"enabled":true,"route":"/chat/file-assistant","topic":"slf","unread_count":0},"system_messages":{"enabled":true,"route":"/chat/system","topic":"sys","unread_count":0}},"entry_badges":{"new_friends_pending_count":0,"groups_unread_count":0,"file_assistant_unread_count":0,"system_unread_count":0,"workbench_total_unread":0},"system_topic":{"topic":"sys","type":"system"},"message_policy":{"recall_window_seconds":120,"edit_window_seconds":300,"supported_actions":["copy","quote","delete","recall","edit"]},"security":{"realname":{"status":"unverified","required":false,"allow_skip":true,"remind_after":"24h"}}}` |
|
||||
| `GET /v1/im/search/conversations?q=&cursor=` | Query only | `{"items":[{"topic":"grp...","public":{"name":"项目群"},"private":{"pinned":false,"mute_notifications":false},"seq":12,"read":10,"recv":12,"unread":2,"updated_at":"2026-08-20T09:00:00Z"}],"next_cursor":""}` |
|
||||
| `GET /v1/im/system-notifications?cursor=&limit=20` | Query only | `{"items":[{"id":"evt_1","event_type":"friend.request.created","title":"新的好友申请","summary":"张三请求添加你为好友","action":{"type":"friend_request","request_id":"frq_1"},"read_at":null,"created_at":"2026-08-20T09:00:00Z"}],"next_cursor":""}` |
|
||||
| `GET /v1/im/system-notifications?cursor=&limit=20` | Query only | `{"items":[{"id":"evt_1","event_type":"friend.request.created","title":"新的好友申请","summary":"张三请求添加你为好友","action":{"type":"friend_request","target_id":"usr...","payload":{"request_id":"frq_1"},"request_id":"frq_1"},"read_at":null,"created_at":"2026-08-20T09:00:00Z"}],"next_cursor":""}` |
|
||||
| `POST /v1/im/system-notifications/{id}/read` | `{"read_at":"2026-08-20T09:00:00Z"}` | `{"status":"ok","id":"evt_1","read_at":"2026-08-20T09:00:00Z"}` |
|
||||
|
||||
## 3. 单聊会话、会话建立与消息状态
|
||||
@@ -1152,11 +1159,12 @@ X-MeetingIM-APIKey: <预置的普通 API Key>
|
||||
|
||||
<img src="images/双方沟通状态.png" alt="双方沟通状态" width="360">
|
||||
|
||||
**状态:上传基础已实现;媒体消息 Schema、ACL 和恢复需改造。**
|
||||
**状态:上传带 `topic` 与下载按会话授权已在服务端实现;客户端按本节接入。**
|
||||
|
||||
- 选择媒体后本地生成 `client_mid` 和稳定占位;调用文件上传,得到 `file_id` 后再 `pub x-im-type=image|video|voice`。
|
||||
- 选择媒体后本地生成 `client_mid` 和稳定占位;调用文件上传(**必须带 `topic=<该会话 topic>`**,服务端校验成员资格并自动建立文件-会话引用),得到 `file_id` 后再 `pub x-im-type=image|video|voice`。
|
||||
- `content` 至少含 `file_id,name,size,mime,width,height,duration_ms,thumbnail_file_id` 中适用字段;上传进度是客户端任务状态,不写成聊天消息。
|
||||
- 服务端发布成功后关联文件与 Topic/消息;文件下载必须校验消息可见性。
|
||||
- 服务端在上传阶段即建立文件与会话的引用(带 `topic`);消息发布成功后按消息附件引用建立消息级 ACL;文件下载校验会话成员身份与消息可见性。
|
||||
- 媒体文件(缩略图、原图、视频、语音)一律走同一次上传流程并带同一个 `topic`;缩略图也必须挂到会话,否则对端只能看到占位图。
|
||||
- 上传失败(网络中断、超时、5xx)按**整文件重传**处理:重新调用 `POST /v0/file/u/` 上传完整文件,不做分片/断点续传;每次重传得到新 `file_id`,旧 `file_id` 直接放弃、由服务端 TTL 回收,客户端不得再引用。
|
||||
- 上传成功但 `pub` 失败时不重新上传,复用已上传 `file_id` 和原 `client_mid` 重发消息,服务端按 `client_mid` 幂等去重。
|
||||
|
||||
@@ -1832,9 +1840,10 @@ GET /v1/im/topics/{topic}/message-mutations?since_version={localMutationVersion}
|
||||
|
||||
<img src="images/黑名单提示.png" alt="黑名单提示" width="360">
|
||||
|
||||
**状态:图片文件名与画面语义不一致;按“清除聊天记录”实施,需改造。**
|
||||
**状态:图片文件名与画面语义不一致;按“清除聊天记录”实施。服务端已实现并实测(2026-10-09),剩余前端改造。**
|
||||
|
||||
- 确认调用 MeetingIM `{del what:"msg",topic,delseq:[{"low":1,"hi":last_seq+1}],hard:false}`,仅删除当前用户视图;或封装 `DELETE /v1/im/conversations/{topic}/messages`。
|
||||
- 服务端现状(2026-10-09 测试环境实测通过):REST 接口仅写 `im_dellog` 本人区间并递增 `deleted_id`,不改动消息行、对方视角不受影响,本人其他设备经 `deleted_id` 同步感知清空;WS `{del}` 语义相同,两条路径二选一即可。对非成员或不存在会话返回 403 FORBIDDEN。
|
||||
- 不解除好友、不影响对方历史、不删除收藏;附件仅在无任何有效引用且过保留期后回收。
|
||||
- 前端需在确认文案明确“仅清除自己的聊天记录”,成功后清本地 Topic 消息缓存并保留会话关系。
|
||||
|
||||
@@ -2329,10 +2338,11 @@ GET /v1/im/topics/{topic}/message-mutations?since_version={localMutationVersion}
|
||||
|
||||
<img src="images/文件-下载.png" alt="文件-下载" width="360">
|
||||
|
||||
**状态:底层下载已实现租户校验;资源 ACL 需改造。**
|
||||
**状态:下载授权已实现(按上传者/会话成员/消息引用/群成员/好友/收藏放行)。**
|
||||
|
||||
- 客户端使用 `GET /v0/file/s/{file_id}`,Header 传 API Key 与 `X-MeetingIM-Auth: token <base64_token>`;禁止 `?auth=&secret=`。
|
||||
- 服务端除 Token 租户外,还必须校验该用户通过消息、群文件或收藏拥有有效引用,并支持 `Range`、`ETag`、内容长度和安全文件名。
|
||||
- 客户端使用 `GET /v0/file/s/{file_id}`,Header 传 API Key 与 `X-MeetingIM-Auth: token <base64_token>`;禁止 `?auth=&secret=`。URL 可带扩展名,服务端自动剥离。
|
||||
- 服务端按文件 ACL 放行:上传者本人、文件所属 `p2p`/`grp` 会话成员、消息级引用可见者(未撤回未删除且未被执行删除)、群成员、好友关系、收藏引用;非成员返回 `403 permission denied`(`FILE_FORBIDDEN`)。对象存储模式下鉴权通过返回 302 短期签名 URL。
|
||||
- 前提:发送方上传附件时带了 `topic`(见 0.5.5.1)。若对端下载一直 404,说明该文件从未关联会话/消息且已被 1 小时回收任务删除,应让发送方重新上传并携带 `topic`。
|
||||
- 下载前客户端校验本地空间和网络策略;`FILE_FORBIDDEN` 不自动重试,`FILE_UNAVAILABLE` 显示失效。
|
||||
|
||||
### 7.10 文件-下载中.png
|
||||
|
||||
@@ -0,0 +1,226 @@
|
||||
# IM R11 Traefik 入口故障运维排查手册
|
||||
|
||||
## 1. 目标与已知事实
|
||||
|
||||
需要定位 2026-10-04 北京时间 `10:39:30–10:42:30`,即 UTC `02:39:30–02:42:30` 的入口整体中断。
|
||||
|
||||
已知现象:
|
||||
|
||||
- `api.118im.com/metrics` 先后出现超时、HTTP 521 和 HTTP 503。
|
||||
- 三个源 IP 的 443 在约 10:40:16 同时 `connection refused`。
|
||||
- 2000 个 WebSocket 几乎同时断开,A/B 两台压测机均受影响。
|
||||
- Tinode cluster 采样仍为 2/2 live peers。
|
||||
- 故障后三个源 IP 返回同一张 `TRAEFIK DEFAULT CERT`,证书 `notBefore` 为 `2026-10-04 10:41:32 +08:00`,与入口恢复窗口重合。
|
||||
- 运维已确认 `216.236.31.189:6443` 按安全策略不对公网开放,只允许集群网络访问;外部调试机 TCP 超时属于预期行为。
|
||||
|
||||
排查过程中不要先重启 Traefik、节点或 k3s,也不要清理容器和日志。先完成只读取证。
|
||||
|
||||
## 2. 运维进入方式
|
||||
|
||||
优先在控制面节点直接执行 `kubectl`。如果控制面节点没有配置 kubeconfig,k3s 通常可使用:
|
||||
|
||||
```bash
|
||||
sudo k3s kubectl get nodes -o wide
|
||||
```
|
||||
|
||||
普通 Kubernetes 可使用:
|
||||
|
||||
```bash
|
||||
kubectl get nodes -o wide
|
||||
kubectl cluster-info
|
||||
```
|
||||
|
||||
不建议把 6443 直接向公网完全开放。长期方案应采用 VPN/堡垒机或仅允许固定运维出口 IP。短期也可以让运维直接在控制面节点采集下面的结果。
|
||||
|
||||
## 3. 第一优先级:确认 Traefik 是否重启
|
||||
|
||||
先找到 Traefik 所在 namespace、工作负载和 Pod:
|
||||
|
||||
```bash
|
||||
kubectl get deploy,ds,sts,pod -A -o wide | grep -i traefik
|
||||
kubectl get pods -A -l app.kubernetes.io/name=traefik -o wide
|
||||
kubectl get pods -A -l app=traefik -o wide
|
||||
```
|
||||
|
||||
对每个 Traefik Pod 执行:
|
||||
|
||||
```bash
|
||||
kubectl -n kube-system get pod TRAEFIK_POD -o jsonpath='{.metadata.name}{"\nstarted="}{.status.startTime}{"\nrestarts="}{.status.containerStatuses[*].restartCount}{"\nlastReason="}{.status.containerStatuses[*].lastState.terminated.reason}{"\nlastExit="}{.status.containerStatuses[*].lastState.terminated.exitCode}{"\nlastFinished="}{.status.containerStatuses[*].lastState.terminated.finishedAt}{"\n"}'
|
||||
|
||||
kubectl -n kube-system describe pod TRAEFIK_POD
|
||||
kubectl -n kube-system logs TRAEFIK_POD --all-containers --timestamps --since-time=2026-10-04T02:39:30Z
|
||||
kubectl -n kube-system logs TRAEFIK_POD --all-containers --previous --timestamps --since-time=2026-10-04T02:39:30Z
|
||||
```
|
||||
|
||||
重点检查:
|
||||
|
||||
- `OOMKilled`、exit code 137、liveness/readiness failure。
|
||||
- `Killing`、`Evicted`、`NodeNotReady`、`Preempted`。
|
||||
- `Configuration loaded`、provider reconnect、TLS store/default certificate 重建。
|
||||
- 10:41:32 附近是否有容器启动日志。
|
||||
|
||||
如果 Traefik 不在 `kube-system`,将命令中的 namespace 替换为实际值。
|
||||
|
||||
## 4. 第二优先级:查滚动发布和配置变更
|
||||
|
||||
```bash
|
||||
kubectl -n kube-system get deploy,ds,rs -l app.kubernetes.io/name=traefik -o wide
|
||||
kubectl -n kube-system rollout history deploy/traefik
|
||||
kubectl -n kube-system get deploy traefik -o yaml
|
||||
kubectl -n kube-system get cm,secret | grep -i traefik
|
||||
```
|
||||
|
||||
若由 Helm 管理:
|
||||
|
||||
```bash
|
||||
helm list -A | grep -i traefik
|
||||
helm history traefik -n kube-system
|
||||
helm get values traefik -n kube-system -a
|
||||
```
|
||||
|
||||
若由 ArgoCD 管理:
|
||||
|
||||
```bash
|
||||
kubectl get applications.argoproj.io -A
|
||||
kubectl get applications.argoproj.io -A -o yaml | grep -n -i -C 5 traefik
|
||||
kubectl get pods -n argocd -o wide
|
||||
kubectl logs -n argocd -l app.kubernetes.io/name=argocd-application-controller --all-containers --timestamps --since-time=2026-10-04T02:39:30Z
|
||||
```
|
||||
|
||||
同时检查 CI/CD 平台在北京时间 10:39–10:42 是否执行过 Traefik、Ingress、Service、证书或节点部署。需要记录操作者、commit、同步 revision 和开始/结束时间。
|
||||
|
||||
## 5. 第三优先级:节点是否重启或资源耗尽
|
||||
|
||||
```bash
|
||||
kubectl get nodes -o wide
|
||||
kubectl describe nodes
|
||||
kubectl get events -A --sort-by=.metadata.creationTimestamp
|
||||
kubectl get events -A --field-selector type=Warning --sort-by=.metadata.creationTimestamp
|
||||
```
|
||||
|
||||
在承载 Traefik 的节点执行:
|
||||
|
||||
```bash
|
||||
uptime -s
|
||||
last -x | head -30
|
||||
sudo journalctl --list-boots
|
||||
sudo journalctl --since '2026-10-04 10:39:30' --until '2026-10-04 10:42:30' -u k3s -u k3s-agent -u kubelet -u containerd --no-pager
|
||||
sudo journalctl -k --since '2026-10-04 10:39:30' --until '2026-10-04 10:42:30' --no-pager
|
||||
sudo dmesg -T | grep -Ei 'oom|killed process|memory cgroup|conntrack|nf_conntrack|link.*down|reset|I/O error'
|
||||
```
|
||||
|
||||
重点判断:
|
||||
|
||||
- 节点是否在 10:41 左右重启。
|
||||
- 内核是否 OOM kill Traefik、containerd 或 k3s。
|
||||
- 磁盘是否满、只读或发生 I/O error。
|
||||
- 网卡是否 link down、驱动 reset。
|
||||
- kubelet 是否因 liveness probe 杀掉 Traefik。
|
||||
|
||||
## 6. 第四优先级:Service 与 Endpoint 是否短暂清空
|
||||
|
||||
```bash
|
||||
kubectl get svc,endpoints,endpointslices -A | grep -Ei 'traefik|im-server'
|
||||
kubectl -n kube-system describe svc traefik
|
||||
kubectl -n kube-system get endpoints traefik -o yaml
|
||||
kubectl -n kube-system get endpointslices -l kubernetes.io/service-name=traefik -o yaml
|
||||
|
||||
kubectl -n im-server-system get pod,svc,endpoints,endpointslices -o wide
|
||||
kubectl -n im-server-system describe svc im-server
|
||||
kubectl -n im-server-system get endpoints im-server -o yaml
|
||||
kubectl -n im-server-system get endpointslices -l kubernetes.io/service-name=im-server -o yaml
|
||||
```
|
||||
|
||||
检查 10:40–10:41 是否发生:
|
||||
|
||||
- Traefik Service 没有可用 endpoint。
|
||||
- `im-server` 三个 Pod 同时 NotReady,导致上游 503。
|
||||
- selector、port、targetPort 或 Ingress backend 被 ArgoCD/Helm 临时改写。
|
||||
- StatefulSet 滚动时 readiness 配置错误,所有 endpoint 被同时摘除。
|
||||
|
||||
## 7. 第五优先级:连接数和 conntrack
|
||||
|
||||
在入口节点执行:
|
||||
|
||||
```bash
|
||||
sudo ss -lntp | grep -E ':443|:6443'
|
||||
sudo ss -s
|
||||
sudo sysctl net.netfilter.nf_conntrack_count net.netfilter.nf_conntrack_max
|
||||
sudo conntrack -S 2>/dev/null || true
|
||||
sudo nstat -az | grep -Ei 'ListenDrops|ListenOverflows|RetransSegs|InErrors|OutDiscards'
|
||||
sudo ip -s link
|
||||
```
|
||||
|
||||
如果有节点监控,导出 10:35–10:45 的:
|
||||
|
||||
- CPU usage、load、CPU throttling。
|
||||
- 内存、OOM、working set。
|
||||
- conntrack 使用率。
|
||||
- TCP listen overflow、reset、retransmit。
|
||||
- 网卡吞吐、drop、error。
|
||||
- Traefik active connection、open connection、5xx、upstream duration。
|
||||
|
||||
conntrack 达到 `nf_conntrack_max` 通常更偏向 timeout/drop;本轮观察到短时间 `connection refused`,因此还应优先确认 443 监听进程是否消失或入口规则是否被重建。
|
||||
|
||||
## 8. im-server 同期状态
|
||||
|
||||
```bash
|
||||
kubectl -n im-server-system get pods -o wide
|
||||
kubectl -n im-server-system get sts im-server -o yaml
|
||||
kubectl -n im-server-system describe sts im-server
|
||||
|
||||
for pod in $(kubectl -n im-server-system get pods -l app=im-server -o name); do
|
||||
kubectl -n im-server-system logs "$pod" --all-containers --timestamps --since-time=2026-10-04T02:39:30Z > "${pod##*/}.log"
|
||||
kubectl -n im-server-system logs "$pod" --all-containers --previous --timestamps --since-time=2026-10-04T02:39:30Z > "${pod##*/}.previous.log" 2>&1 || true
|
||||
done
|
||||
```
|
||||
|
||||
如果 `app=im-server` 标签不匹配,先执行 `kubectl -n im-server-system get pods --show-labels`,再替换 selector。
|
||||
|
||||
需要确认三个 im-server Pod 的:
|
||||
|
||||
- `startTime` 和 `restartCount`。
|
||||
- readiness/liveness failure。
|
||||
- 10:40 左右是否仍持续处理 Kafka、cluster RPC 和 WS。
|
||||
- 是否有 `too many open files`、OOM、accept error、DNS error、MySQL/Kafka timeout。
|
||||
|
||||
## 9. Kubernetes API 访问方式
|
||||
|
||||
从当前调试机访问三个源 IP 的 443 均正常。运维已确认 `216.236.31.189:6443` 只允许集群网络访问,因此外部连接超时符合既定安全策略,也与 VPN/TUN 无关。
|
||||
|
||||
运维应检查控制面节点:
|
||||
|
||||
```bash
|
||||
sudo ss -lntp | grep 6443
|
||||
sudo nft list ruleset | grep -n -C 3 6443
|
||||
sudo iptables -S | grep 6443
|
||||
sudo iptables -t nat -S | grep 6443
|
||||
```
|
||||
|
||||
后续排查采用以下方式之一:
|
||||
|
||||
1. 让排查命令直接在控制面节点运行并回传结果包。
|
||||
2. 如确需远程调试,为固定运维出口 IP 临时增加 6443 allowlist,完成后撤销。
|
||||
3. 通过堡垒机/VPN 进入控制面网络。
|
||||
|
||||
不要为了调试把 6443 对 `0.0.0.0/0` 永久开放。
|
||||
|
||||
## 10. 运维需要回传的结果
|
||||
|
||||
至少回传以下内容:
|
||||
|
||||
1. Traefik Pod YAML、describe、当前日志和 previous 日志。
|
||||
2. Traefik Deployment/DaemonSet、ReplicaSet 和 rollout history。
|
||||
3. 10:39:30–10:42:30 的 k3s/kubelet/containerd/kernel journal。
|
||||
4. 全集群 Warning Event、节点 describe。
|
||||
5. Traefik 和 im-server 的 Service、EndpointSlice、Ingress YAML。
|
||||
6. ArgoCD/Helm/CI 在故障窗口的发布历史。
|
||||
7. 入口节点 CPU、内存、conntrack、TCP overflow/retransmit 和网卡 drop。
|
||||
|
||||
拿到这些材料后,按照以下优先级定责:
|
||||
|
||||
1. Traefik 容器或节点在 10:41:32 重启。
|
||||
2. ArgoCD/Helm/CI 触发 Traefik 滚动或配置重建。
|
||||
3. Traefik/im-server endpoint 短暂清空。
|
||||
4. OOM、liveness、conntrack、监听队列或节点网络故障。
|
||||
5. 其他上游负载均衡或云网络事件。
|
||||
@@ -22,7 +22,6 @@ _2026-09-14 全面复核:本次对现有33条逐条重新核实(能实测的
|
||||
|---|---|---|
|
||||
| [问题25](#问题25) | `POST /messages/forward`、`/favorites/forward` | 带附言转发时 `head` 声明类型与 `content` 实际结构对不上 |
|
||||
| [问题27](#问题27) | `GET /reports/daily` | `status` 参数不做取值校验,传任何值都返回200+空列表 |
|
||||
| [问题30](#问题30) | `GET /system-notifications` | `summary`/`action`结构不一致,且会议响应事件被写成两条重复通知 |
|
||||
| [问题33](#问题33) | `GET /reports/daily` | `permissions` 四个字段恒为同一组硬编码值,与用户/参数无关 |
|
||||
| [问题35](#问题35) | `GET /users/{id}/profile` | 设了备注之后拿不到对方的原昵称(`GET /friends`一侧已修复) |
|
||||
| [新增3](#新增3) | `PATCH /conversations/{topic}/settings` | 单聊置顶没有批量查询接口,会话列表永远看不出置顶效果 |
|
||||
@@ -32,7 +31,6 @@ _2026-09-14 全面复核:本次对现有33条逐条重新核实(能实测的
|
||||
| [新增25](#新增25) | 原生WS `{pub}` | link类型消息协议层可构造,但没有真实用户可从UI触达的发送路径 |
|
||||
| [新增30](#新增30) | `GET /settings/youth-mode` | 开启后不对任何功能做实际限制,`enforced` 恒为硬编码false(前端已移除此功能,仅保留记录完整性) |
|
||||
| [新增31](#新增31) | `POST /reports/abuse` | 不校验 `requires_description`,可对必填原因提交空描述 |
|
||||
| [新增32](#新增32) | `POST /system-notifications/{id}/read` | 对已读通知重复标记,响应回显请求值但真实存储值不变 |
|
||||
| [新增34](#新增34) | `POST /tenant-applications` | 统一社会信用代码无去重机制,可无限重复提交 |
|
||||
| [新增41](#新增41) | `POST /groups/{topic}/mutes` | 无角色层级保护,管理员可禁言群主、任何角色可自我禁言 |
|
||||
| [新增47](#新增47) | `POST /v0/file/u/` | 真实文件上传上限是32MB而非声明的100MB,且从未用于前端预校验 |
|
||||
@@ -68,6 +66,8 @@ _2026-09-14 全面复核:本次对现有33条逐条重新核实(能实测的
|
||||
- **新增42**(群公告按id查询恒定返回最新公告):**已修复**。代码确认现在正确解析路径里的`announcement_id`并调用`GetAnnouncement`按id查询,不再无视这个参数。
|
||||
- **新增43**(群文件登记接口无长度/存在性校验):**已修复**。代码确认现在对`file_name`(255)/`mime_type`(128)做长度校验,对`file_size`做负数校验,且会调用`store.Files.Get`验证`file_id`真实存在。
|
||||
- **新增45**(收藏/搜索/宫格分页off-by-one永久丢失边界记录):**已修复**。实测:24条真实收藏,用limit=10分3页拉取(10+10+4),合并后24条一条不少、无重复。
|
||||
- **问题30**(系统通知`summary`/`action`结构不一致 + 会议响应事件双写):**已修复/已失效**(2026-10-09复核)。①会议响应双写随上游`0e2f4e7`(2026-09-17把整个workbench/会议模块从im_server移除)消失,现仅剩Kafka单路径写入且按固定event_id幂等upsert,不再适用;②`action`由读路径`normalizeSystemNotificationAction`统一归一为`{type,target_id,payload,...}`;③空`summary`读路径回退`title`;④机器码`summary`(`accepted`/`rejected`/`continue`/`cancel`)已在4处写入点改为人类可读文案(`serveIMFriendRequestAction`/`serveIMRiskChallengeConfirm`/`TenantFriendRequestAccept`/`TenantFriendRequestResolve`)。实测:A对B发好友申请、B拒绝后,A的通知列表返回`summary:"对方已婉拒你的好友申请"`、`action`为归一化结构。
|
||||
- **新增32**(系统通知重复已读,响应回显请求值但存储不变):**已修复**(2026-10-09复核)。存储层`TenantSystemNotificationMarkRead`在`COALESCE`更新后回读真实行返回,handler用返回值构造响应(单测`TestServeIMSystemNotificationsListAndRead`覆盖)。实测:同一通知先以`read_at=T1`标记已读、再以`T2`重复标记,两次响应均返回持久化的`T1`,不再回显本次提交值。
|
||||
|
||||
## 本次复核确认仍是历史误判(从未是真正的后端bug,不应再作为后端问题提出)
|
||||
|
||||
@@ -163,21 +163,6 @@ _2026-09-14 全面复核:本次对现有33条逐条重新核实(能实测的
|
||||
|
||||
---
|
||||
|
||||
<a id="问题30"></a>
|
||||
## 问题 30(P2):系统通知的 `summary` 要么是空串、要么是机器码,`action` 的结构因来源而异,且会议响应事件会被写成两条重复通知
|
||||
|
||||
**证据强度**:✅ 实测确认(三种summary情况+两条重复通知具体案例)+代码确认(重复通知根因)
|
||||
|
||||
**现象**:`summary`有时空串、有时机器码(如`accepted`/`rejected`)、有时真实数据;`action`是自由结构,不同来源放的键完全不同;会议响应更新会被两条独立代码路径各写一条内容几乎相同、结构不同的重复通知。
|
||||
|
||||
**根因(重复通知)**:`serveIMMeetingRespond`同时调用受开关`imReliableHTTPEventsEnabled`控制的`publishIMMeetingEvent`(生成`{meeting_id}`形态)和无条件执行的`publishIMWorkbenchAndAudit`(经Kafka产生`{module,resource_id}`形态),该开关对标准请求头调用恒为true。
|
||||
|
||||
**接口**:`GET /v1/im/system-notifications`
|
||||
|
||||
**前端现状**:`summary`能在已知事件码表里查到就替换成中文,查不到就原样显示;跳转按钮按action里出现的键分派;重复通知没有做任何去重(不能安全启发式判断"内容相同就是重复")。
|
||||
|
||||
---
|
||||
|
||||
<a id="问题33"></a>
|
||||
## 问题 33(P2):`permissions` 的四个字段在不同参数下恒为同一组值
|
||||
|
||||
@@ -285,19 +270,6 @@ _2026-09-14 全面复核:本次对现有33条逐条重新核实(能实测的
|
||||
|
||||
---
|
||||
|
||||
<a id="新增32"></a>
|
||||
## 新增 32(P2):系统通知已读接口对已有 `read_at` 的通知,响应会原样回显客户端提交的值,但真实存储值不变
|
||||
|
||||
**证据强度**:✅ 实测确认+代码确认(2026-09-14重读源码确认未变)
|
||||
|
||||
**现象**:对已有read_at的通知重复调用已读接口,响应200并原样回显本次提交的read_at(哪怕故意传明显不同值),但真实存储的read_at纹丝不动。
|
||||
|
||||
**根因**:`TenantSystemNotificationMarkRead`执行`UPDATE...SET read_at=COALESCE(read_at,?)`,已非空时保留原值;`serveIMSystemNotificationRead`(`server/im_http.go:3576`)直接回显请求体局部变量`readAt`,从未回读真实值。
|
||||
|
||||
**接口**:`POST /v1/im/system-notifications/{id}/read`
|
||||
|
||||
---
|
||||
|
||||
<a id="新增34"></a>
|
||||
## 新增 34(P2):企业注册申请接口对同一个统一社会信用代码没有任何去重/防重复提交机制
|
||||
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
-- IM file-message link msgid Snowflake compatibility migration.
|
||||
-- Scope: the shared business database (for example meeting_im).
|
||||
-- Fixes: im_filemsglinks.msgid was INT (max 2147483647) while message ids are
|
||||
-- snowflake int64 values (~1.58e18), so every INSERT into the link table
|
||||
-- failed with MySQL error 1264 "Out of range value for column 'msgid'".
|
||||
-- Under Kafka persistence that failure retried forever and stalled the
|
||||
-- consumer partition, blocking persistence for every message queued
|
||||
-- behind it.
|
||||
-- Notes: im_filemsglinks is a global (non tenant-sharded) table, but in
|
||||
-- tenant-scoped-DB deployments every tenant database has its own copy —
|
||||
-- run this script once per database.
|
||||
-- Re-running is safe (MODIFY to the same shape is idempotent).
|
||||
-- NOT REVERSIBLE: converting back to INT would truncate snowflake ids.
|
||||
-- Review and back up before execution. This script does not replay dead letters.
|
||||
|
||||
-- 1. Preflight: confirm the live column shape before touching anything.
|
||||
SELECT TABLE_SCHEMA, TABLE_NAME, COLUMN_TYPE, IS_NULLABLE
|
||||
FROM information_schema.COLUMNS
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = 'im_filemsglinks'
|
||||
AND COLUMN_NAME = 'msgid';
|
||||
|
||||
-- 2. Upgrade the column (keeps the nullable semantics of the original schema).
|
||||
ALTER TABLE `im_filemsglinks`
|
||||
MODIFY COLUMN `msgid` BIGINT DEFAULT NULL;
|
||||
|
||||
-- 3. Post-check: this query must return zero rows.
|
||||
SELECT TABLE_SCHEMA, TABLE_NAME, COLUMN_TYPE
|
||||
FROM information_schema.COLUMNS
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = 'im_filemsglinks'
|
||||
AND COLUMN_NAME = 'msgid'
|
||||
AND DATA_TYPE <> 'bigint';
|
||||
|
||||
-- 4. Observe pending recovery work caused by the overflow (and, before the
|
||||
-- startup schema-contract check existed, by any other drift). Replay only
|
||||
-- after the post-check passes and the referenced files still exist.
|
||||
SELECT status, last_reason, COUNT(*) AS entry_count,
|
||||
MIN(created_at) AS first_created_at, MAX(created_at) AS last_created_at
|
||||
FROM im_kafka_dead_letters
|
||||
WHERE last_error LIKE '%Out of range value for column %'
|
||||
OR last_error LIKE '%file lifecycle enqueue failed: not found%'
|
||||
GROUP BY status, last_reason
|
||||
ORDER BY status, last_reason;
|
||||
@@ -0,0 +1,159 @@
|
||||
# GROUP-2000 R10 双机群聊压测分析与改进
|
||||
|
||||
## 1. 测试结论
|
||||
|
||||
`group2k-r10-20261004-01` 的 A/B 同步有效,可以作为 2000 人群聊的真实双机容量结论。本轮持久化链路基本正常,但实时扇出已明显超过当前服务容量,整体判定为 **失败**。
|
||||
|
||||
- A/B 各建立 1000 个 WS 连接,共 2000/2000 就绪;两机实际开始时间相差约 84ms。
|
||||
- 2000 条发送尝试中,1984 条收到 202 ACK,16 条因 socket 中断而结果未知,无显式 403/502/503 拒绝。
|
||||
- 数据库最终找到 1999 条,无重复、无字段不一致;仅 `g-a-0098-02` 未落库。
|
||||
- 按 1985 条已在服务端确认的消息计算,期望实时投递 3,970,000 次,实际收到 1,979,444 次,到达率 49.86%,实时缺失率 50.14%。
|
||||
- 1,990,556 次实时缺失全部可从历史消息查回,说明主要问题是实时投递容量,不是 Kafka/MySQL 丢数据。
|
||||
- 负载期间共 152 次 WS 断连:147 次 connection reset、4 次 timeout、1 次 closed connection。
|
||||
- ACK P95 为 71.18s,本地实时投递 P95 为 329.68s,已不是普通抖动,而是持续性排队。
|
||||
|
||||
JPush 空设备查询放大、MySQL 连接频繁回收和 A/B 起跑错位等 R9 问题在本轮已解决。当前的主要容量瓶颈是大群每条消息的单 Topic 有序处理、跨 Pod multiplex 转发和各 Pod WS 逐连接序列化/写出组合成的实时扇出链路。
|
||||
|
||||
## 2. 四层对账
|
||||
|
||||
| 层级 | 结果 | 结论 |
|
||||
| --- | ---: | --- |
|
||||
| 发送尝试 | 2000 | A/B 各 1000 |
|
||||
| 即时收到 202 | 1733 | 251 条在超时后迟到 |
|
||||
| 最终收到 202 | 1984 | 发送成功率 99.2% |
|
||||
| 发送结果未知 | 16 | 全部为 socket 中断,无业务拒绝 |
|
||||
| 历史回查 | 1985/1985 | 服务端可确认消息全部可查 |
|
||||
| 数据库实际落库 | 1999/2000 | 14 条“结果未知”也已落库 |
|
||||
| 真正未落库 | 1 | A 机 `g-a-0098-02` |
|
||||
| 实时投递 | 1,979,444/3,970,000 | 到达率 49.86% |
|
||||
| 重复 / 乱序 | 0 / 0 | 已收到数据的正确性正常 |
|
||||
|
||||
16 条结果未知中,14 条 B 机消息已在数据库找到;A 机 `g-a-0102-02` 被实时观测到且已落库;只有 `g-a-0098-02` 在连接断开前未被服务端处理。因此不能把“客户端未收到 ACK”直接等同于“消息未落库”。
|
||||
|
||||
## 3. A/B 表现
|
||||
|
||||
| 指标 | 机器 A | 机器 B |
|
||||
| --- | ---: | ---: |
|
||||
| 就绪连接 | 1000 | 1000 |
|
||||
| 发送尝试 | 1000 | 1000 |
|
||||
| 最终观测 ACK | 998 | 986 |
|
||||
| 结果未知 | 2 | 14 |
|
||||
| 收到群消息 | 996,563 | 982,881 |
|
||||
| 断连 | 7 | 145 |
|
||||
|
||||
B 机的断连明显更多,但 reset 分布在多个公网源 IP,没有单一坏 IP 的证据。机器差异会影响结果,但 A 机同样出现大量投递缺失和长尾,不能将本轮失败只归因于 B 机网络。
|
||||
|
||||
## 4. 指标与时间线
|
||||
|
||||
- 负载窗口约为 09:21:43–09:26:32。
|
||||
- A 机断连集中在 09:23:20–09:25:24;B 机采样到的 reset 集中在 09:24:20–09:25:23。
|
||||
- 服务端 `/metrics` 在约 09:23:01–09:25:37 多次超过 10s 采集超时;233 次 im-server 采样中 14 次失败,17 次超过 5s。
|
||||
- Kafka consumer lag 和 oldest age 在收敛阶段均为 0。
|
||||
- `im_kafka_message_persist_total{kind="group",result="batch_ok"}` 合计 1999,与数据库实际条数一致。
|
||||
- `im_push_dispatch_queue_enqueue_total{result="ok"}` 为 1999,未观测到 push queue full/error;推送平均处理约 22ms。
|
||||
- Topic group queue wait 未超过 1s,Cluster P2M/route 计数未观测到 full/error。
|
||||
- 三个 Pod 的 ACK enqueue/write 指标都在 1s 内。注意该计时从服务端 WS 读循环完成解析后开始,因此客户端几十秒至数分钟的 ACK 延迟,主要发生在服务端 dispatch 之前的读取/调度阶段,或写入 socket 后的网络/客户端接收阶段。
|
||||
|
||||
`/metrics` 与 WS 同时出现长时间停顿,说明不只是某个业务队列满,而是 Pod 的 CPU/调度/网络写路径在大规模扇出下被长时间占用。本轮无法通过集群 API 读取 Pod Event 和完整日志,因此不将 CPU throttling、Ingress 和节点网卡中的任何一项单独定为唯一根因。
|
||||
|
||||
## 5. 代码链路分析
|
||||
|
||||
### 5.1 Topic 主节点
|
||||
|
||||
`Topic.broadcastToSessions` 遍历主 Topic 上的本地 session 和远程 Pod multiplex session,为每个目标复制、整理消息并入队。一个 2000 人群的 2000 条消息会生成约 400 万次终端投递。
|
||||
|
||||
当 multiplex 队列暂时满时,Topic goroutine 会等待最多 250ms;一个主 Topic 又需要保持消息顺序,所以无法通过对单条消息无界并发来解决。
|
||||
|
||||
### 5.2 跨 Pod multiplex
|
||||
|
||||
每个远程 Pod 使用一个 multiplex session。它的 drain 为保证 Topic 顺序,对每条 `Cluster.TopicProxy` RPC 同步等待远程返回。这保证了顺序,但对单条 RPC 往返、gob 编解码和远程 proxy Topic 处理的抖动很敏感。
|
||||
|
||||
### 5.3 Pod 内 WS 扇出
|
||||
|
||||
proxy Topic 收到一条群消息后,仍然遍历本 Pod 上的所有订阅 session,为每个 WS 连接复制消息并入队。每个 WS 只有单个 write loop 串行做序列化和 socket write。当扇出速度超过写出速度,session 队列积压,随后出现连接 reset、客户端重连以及更大的订阅恢复压力。
|
||||
|
||||
## 6. 已排除和未排除项
|
||||
|
||||
### 已排除为本轮主因
|
||||
|
||||
- Kafka 积压或持久化丢失:最终 lag=0,1999 条 batch_ok 与 DB 一致。
|
||||
- MySQL 批量落库能力:无重复、无字段不一致,只有 1 条在服务端未接收。
|
||||
- JPush 设备查询放大:1999 次异步入队成功,无队列满,平均耗时低。
|
||||
- A/B 起跑错位:本轮最终屏障通过,两机实际开始差距小于 100ms。
|
||||
- 显式限流或内部队列拒绝:没有 429/502/503,未观测到 Cluster、Topic、push 队列 full/error。
|
||||
|
||||
### 仍需补齐证据
|
||||
|
||||
- 每个 Topic 单条消息的完整处理耗时和扇出目标数。
|
||||
- multiplex 队列等待、RPC 耗时、远程 proxy 接收到入队完成的分段耗时。
|
||||
- WS send queue 深度、队列等待 P95/P99、full/purge 和按 Pod 的发生时间。
|
||||
- Pod CPU usage/throttling、Go scheduler latency、GC pause、goroutine 数、网卡吞吐/丢包/retransmit。
|
||||
- Ingress/LB 上游 reset、upstream response time、连接数和每 Pod 连接分布。
|
||||
|
||||
## 7. 改进方案
|
||||
|
||||
### P0:先定位每段长尾
|
||||
|
||||
1. 增加 Topic 处理总耗时、单条扇出目标数和 `broadcastToSessions` 耗时直方图,区分 master/proxy。
|
||||
2. 增加 multiplex 入队等待、队列深度、`Cluster.TopicProxy` RPC 耗时和远程处理耗时。
|
||||
3. 补齐 WS send queue P95/P99/max、full、purged,并区分 data/ctrl。ACK 必须能从大量 data 帧中被单独观测。
|
||||
4. 增加 WS 读取到 dispatch 前的时间戳,避免现有 ACK 指标忽略服务端读循环前的停顿。
|
||||
5. 恢复压测期间的 Pod/Event/CPU 可观测性,指标采集超时不应导致整段监控空白。
|
||||
|
||||
### P1:降低实时扇出成本
|
||||
|
||||
1. 将 master 到每个 proxy Pod 的多条消息改为有界批量 RPC,批内保持 SeqId 顺序,减少 2000 次同步 RPC 往返、gob 编码和调度开销。
|
||||
2. 为每个 Topic-Pod 建立单写者有界队列,允许不同 Pod 并行处理,同一 Pod 内严格按 SeqId 投递;不对单 Topic 做无界 goroutine 并发。
|
||||
3. 研究一次序列化、多 session 复用字节的可行性,避免同一条群消息在每个连接重复做等价 JSON 序列化。用户相关 Topic/权限字段必须先完成安全性分类,不可盲目全局复用。
|
||||
4. ctrl/ACK 与普通 data 帧分级调度,保证发送者在大群扇出积压时仍能快速获得明确结果。需保持单连接协议语义,不得导致客户端对 data/ctrl 的依赖关系错乱。
|
||||
5. 对慢消费 WS 定义明确策略:队列达阈值后记录可追踪原因、主动断开并依靠历史补拉,避免一批慢连接长时间拖住整个 Topic。
|
||||
|
||||
## 8. 实施和复测顺序
|
||||
|
||||
1. 先完成 P0 指标,用相同 2000 人、2000 条参数复现一轮,定量确认 Topic、multiplex RPC、WS queue 和 socket write 各占多少。
|
||||
2. 实施 multiplex 批量化和有界单写者,先做单元测试、顺序测试、队列满/节点断开/重连测试。
|
||||
3. 用 500、1000、1500、2000 人阶梯复测,每档至少两轮,不直接只跑最高档。
|
||||
4. 最后重复 R10 同参数双机测试,必须同时通过实时、历史、DB 和基础设施门禁。
|
||||
|
||||
## 9. 下一轮通过门禁
|
||||
|
||||
- 2000/2000 WS 就绪,A/B 实际开始差小于 1s。
|
||||
- 2000 条发送全部获得明确 ACK,无 outcome unknown,无 5xx。
|
||||
- 数据库精确找到 2000 条,缺失、重复、字段不一致均为 0,Kafka lag 最终归零。
|
||||
- 实时投递到达率不低于 99.99%,重复和乱序为 0。
|
||||
- ACK P95 小于 2s,投递 P95 小于 5s;本轮已明确不以 500ms 作为阻断门禁。
|
||||
- 断连、Topic/multiplex/WS queue full、Pod restart、OOM、liveness failure 均为 0。
|
||||
- 负载窗口内 `/metrics` 可持续采集,无连续 10s 超时;CPU throttling、GC、网络 retransmit 和 Ingress reset 有完整记录。
|
||||
|
||||
## 10. 产物
|
||||
|
||||
- 结果目录:`loadtest/im-pressure/results/group2k-r10-20261004-01/`
|
||||
- 汇总报告:`report.json`
|
||||
- 数据库核验:`db-verification.json`
|
||||
- A/B 明细:`agent-a-summary.json`、`agent-b-summary.json`
|
||||
- coordinator 台账:`coordinator.json`
|
||||
- 服务指标:`metrics.jsonl`
|
||||
- 代码版本:`c46209f`
|
||||
|
||||
## 11. 本轮已实施改造
|
||||
|
||||
R10 分析后已完成以下代码改造,待发布后进行 R11 复测:
|
||||
|
||||
- 普通非频道群消息在每个 Pod 仅执行一次 JSON 序列化,同 Pod 的所有 WS 连接复用同一份不可变字节。权限过滤、`SkipSid`、频道匿名、P2P 别名和非 WS 协议仍保持原路径。
|
||||
- 增加 Topic 完整处理耗时、广播耗时和按 websocket/multiplex/other 分类的扇出目标数。
|
||||
- 增加 master-to-proxy multiplex 入队深度和排队时间,以及远程 proxy Topic 接收后的队列等待时间。
|
||||
- 增加 WS 完成读帧后的 JSON 解析和本地 dispatch 耗时。
|
||||
- 新快路仍在 WS write loop 记录原有 `OutgoingMessageSize` 统计,不会因预序列化丢失原有监控口径。
|
||||
|
||||
本次没有直接引入新的批量 `net/rpc` 方法。StatefulSet 滚动发布时会短暂出现新旧 Pod 混合,新 RPC 端点与旧 Pod 不兼容,会产生发布窗口风险。R11 先用新增指标确认 multiplex RPC 在优化后是否仍为主瓶颈;若需要批量化,应采用“先发布双协议接收端,再开启批量发送端”的两阶段发布。
|
||||
|
||||
新增指标:
|
||||
|
||||
- `im_topic_process_duration_seconds`
|
||||
- `im_topic_broadcast_duration_seconds`
|
||||
- `im_topic_broadcast_targets`
|
||||
- `im_topic_shared_serialization_total`
|
||||
- `im_cluster_m2p_queue_wait_duration_seconds`
|
||||
- `im_cluster_m2p_queue_depth`
|
||||
- `im_cluster_proxy_queue_wait_duration_seconds`
|
||||
- `im_ws_read_dispatch_duration_seconds`
|
||||
@@ -0,0 +1,161 @@
|
||||
# GROUP-2000 R11 双机群聊复测分析
|
||||
|
||||
## 1. 结论
|
||||
|
||||
`group2k-r11-20261004-01` 判定为 **失败且不能作为稳定容量结论**。
|
||||
|
||||
本轮确认了 R10 后的群消息共享序列化优化已经生效,正常运行阶段的实时到达率明显高于 R10;但压测末段发生了约 1 分钟的公网入口/上游整体不可用,2000 个连接几乎同时断开。与此同时,Kafka 群消息持久化链路仅落库 1165 条,662 条服务端已确认或已观测消息未落库,暴露出批消费可靠性缺陷。
|
||||
|
||||
必须先修复 Kafka 批消费和查清入口整体中断,再重复相同参数。当前不能把 93.83% 到达率当作 2000 人容量通过,也不能继续直接提高压力。
|
||||
|
||||
## 2. 核心结果
|
||||
|
||||
| 指标 | R11 结果 | 判定 |
|
||||
| --- | ---: | --- |
|
||||
| WS 就绪 | 2000/2000 | 通过 |
|
||||
| A/B 实际起跑差 | 约 290ms | 通过 |
|
||||
| 发送尝试 | 2000 | A/B 各 1000 |
|
||||
| 最终明确 ACK | 1718 | 85.90% |
|
||||
| 发送结果未知 | 282 | 失败 |
|
||||
| ACK P95 | 58.37s | 失败 |
|
||||
| 期望实时投递 | 3,654,000 | 按 1827 条服务端确认/回查消息计算 |
|
||||
| 实际实时投递 | 3,428,697 | 到达率 93.83% |
|
||||
| 实时缺失 | 225,303 | 6.17% |
|
||||
| 数据库核验 | 1165/1827 | 缺失 662 条 |
|
||||
| 重复 / 乱序 | 0 / 0 | 已收到部分正确 |
|
||||
| WS 断连 | 2036 | 失败,包含重连后的再次断开 |
|
||||
|
||||
R10 的实时到达率为 49.86%,R11 提升到 93.83%,说明共享序列化方向有效;但 R11 同时出现了全局入口故障和持久化缺失,两个结果的运行条件不同,不能据此计算严谨的容量提升比例。
|
||||
|
||||
## 3. A/B 分机结果
|
||||
|
||||
| 指标 | 机器 A | 机器 B |
|
||||
| --- | ---: | ---: |
|
||||
| 就绪连接 | 1000 | 1000 |
|
||||
| 发送尝试 | 1000 | 1000 |
|
||||
| 最终观测 ACK | 919 | 799 |
|
||||
| 结果未知 | 81 | 201 |
|
||||
| 收到群消息 | 1,816,953 | 1,611,744 |
|
||||
| 断连 | 1005 | 1031 |
|
||||
| 主要断连原因 | abnormal close 996 | abnormal close 653、reset 378 |
|
||||
|
||||
B 的网络表现仍弱于 A,但 A 也几乎全量断开,因此本轮不能归因于单独一台压测机。
|
||||
|
||||
## 4. 故障时间线
|
||||
|
||||
时间均为北京时间 2026-10-04:
|
||||
|
||||
- 10:35:36:A/B 同步开始,2000 个连接全部就绪。
|
||||
- 10:35:44 起:公网 `/metrics` 出现高延迟和多次 10s 超时,表明服务在正式断开前已经出现调度或网络拥塞。
|
||||
- 10:40:06、10:40:12:`/metrics` 返回 HTTP 521。
|
||||
- 10:40:16 起:A 机对三个直连源 IP 的 443 均收到 `connection refused`,随后登录出现 500、WS 握手出现 503。
|
||||
- 10:40:17:A 的就绪连接从 1000 降到 8,约 992 个连接集中断开;B 同期发生同类大规模断开。
|
||||
- 10:40:48–10:41:03:公网入口持续返回 503。
|
||||
- 10:41:09 后:入口恢复,客户端自动重连;A 约在 10:43:37 恢复到 1000 个连接。
|
||||
|
||||
直连源 IP 同时拒绝 443,且公网出现 521/503,这不是 Cloudflare DNS 或压测机代理造成的普通客户端故障。采样期间 Tinode cluster 指标仍为 2/2 live peers,没有证据表明原生 cluster 先失联。
|
||||
|
||||
2026-10-04 11:31–11:32 重新探测时,三个源 IP 的 443 均已恢复并可通过 `api.118im.com` Host/SNI 返回健康检查 200。三个 IP 返回的是同一张 `TRAEFIK DEFAULT CERT`,SHA-256 指纹均为 `7C:9E:00:6F:56:D6:C9:87:44:62:7D:26:93:D8:92:51:F8:DA:43:66:61:DB:8B:F9:1F:47:17:C8:93:13:C4:8C`,证书 `notBefore` 为 **2026-10-04 10:41:32**。该时间与 R11 的 521/503 和源站拒绝连接恢复窗口精确重合,是 Traefik/入口组件在故障期发生重启、重建或重新初始化的强证据。
|
||||
|
||||
目前可以把故障范围收敛到 Traefik/入口层,但仅凭默认证书时间仍不能区分 Traefik Pod 重启、Deployment 滚动、节点重启、配置重载或入口服务重建。
|
||||
|
||||
运维已确认 Kubernetes API `216.236.31.189:6443` 按安全策略不对公网开放,只允许集群网络访问;当前调试机超时属于预期行为,不是本轮故障的一部分。Pod Event、Ingress controller 日志和节点事件必须由运维从控制面节点或集群内采集。在这些证据回传前,仍不把某一个基础设施组件写成唯一根因。
|
||||
|
||||
## 5. 共享序列化与实时链路
|
||||
|
||||
三个 Pod 的 `im_topic_shared_serialization_total` 在采集窗口分别增加约 5392、6664、6831,证明新快路已实际执行。
|
||||
|
||||
按采集窗口增量估算:
|
||||
|
||||
| 阶段 | 结果 |
|
||||
| --- | --- |
|
||||
| master Topic broadcast P95 | 不超过 5ms |
|
||||
| proxy-0 Topic broadcast P95 | 不超过 5ms |
|
||||
| proxy-1 Topic broadcast P95 | 不超过 10ms |
|
||||
| master 到 peer-0 的 multiplex 排队 P95 | 不超过 100ms |
|
||||
| master 到 peer-1 的 multiplex 排队 P95 | 不超过 250ms |
|
||||
| WS read-dispatch P95 | 三个 Pod 均不超过 50ms |
|
||||
|
||||
Topic 广播本身已不再呈现分钟级长尾,说明“同一群消息对每个 WS 重复 JSON 序列化”得到明显缓解。当前长尾更多出现在 multiplex 排队、socket/入口可用性以及故障后的重连恢复阶段。WS read-dispatch 仍有少量超过 5s 的样本,三个 Pod 分别约 13、10、18 次,需要保留观察,但不是本轮全量断开的充分解释。
|
||||
|
||||
## 6. Kafka 与数据库异常
|
||||
|
||||
### 6.1 对账证据
|
||||
|
||||
- 1827 条服务端确认或历史回查消息中,数据库精确找到 1165 条,缺失 662 条。
|
||||
- `im_kafka_message_persist_total{kind="group",result="batch_ok"}` 观测上限停在 1165,此后不再增长。该 counter 没有 `pod` 标签,公网采样又会随机落到不同 Pod,因此不能直接把它当作集群总数;这里只作为持久化停止推进的旁证。
|
||||
- 缺失消息全部来自第二轮:A 缺 339 条,序号 0085–0423;B 缺 323 条,序号 0085–0414。A 是完整连续区间,B 仅有少量间隙,不符合随机单条 MySQL 写失败的形态。
|
||||
- 采样结束时 group persist 的 lag 仍显示 0,但 `im_kafka_backlog_oldest_age_seconds` 在一个 Pod 上达到约 1082s。二者互相矛盾,现有 lag 指标不能作为“已消费完成”的依据。
|
||||
- 压测结束约 30 分钟后再次直连 MySQL 核验,结果仍为 1165/1827,缺失 662、错配 0、重复 0、额外消息 0。缺失没有后台追平,不能解释为普通消费延迟。
|
||||
|
||||
### 6.2 已确认的代码风险一:部分批次被丢弃
|
||||
|
||||
`fetchKafkaMessageBatch` 先连续调用 `FetchMessage` 组装批次。如果已经抓到若干消息,后续抓取返回非 `DeadlineExceeded` 错误,函数直接返回 `nil, err`,已经抓到的部分消息不会交给 handler,也不会立即提交。
|
||||
|
||||
Kafka reader 的内部读取位置已经前移。恢复后若继续抓取并提交更高 offset,之前被丢弃的消息可能随高 offset 一起被越过。这可以直接造成“生产成功、最终 lag 为 0、数据库却缺少一段连续消息”。
|
||||
|
||||
### 6.3 已确认的代码风险二:批量幂等 claim 可能活锁
|
||||
|
||||
`handleKafkaMessagePersistBatch` 在真正批量落库前逐条执行 `BeginKafkaConsume`。如果批次中途发生 Redis/基础设施错误,前面已经 claim 的消息不会释放;外层重试从批首重新开始,又会遇到 `processing` 状态。不同 claim 的过期时刻略有差异时,重试还可能反复续上前面的 claim,长期无法走到后面的消息。
|
||||
|
||||
这能解释 oldest age 长时间增长且持久化计数停止。当前“失败不提交 offset”的修复方向正确,但还没有覆盖“抓取半批失败”和“准备半批失败”两个边界。
|
||||
|
||||
### 6.4 lag 指标口径不可靠
|
||||
|
||||
当前指标直接读取 `kafka-go Reader.Stats().Lag`,同时在每次批处理结束后把 oldest age 清零。consumer group 模式下,该 lag 不是可靠的已提交 offset 差值;消息被 reader 抓取但尚未持久化时,也可能显示 0。必须改为按 partition 查询 high watermark 与 committed offset,并分别暴露 fetched、processed、committed offset。
|
||||
|
||||
## 7. 优先改进项
|
||||
|
||||
### P0:Kafka 持久化可靠性
|
||||
|
||||
1. `fetchKafkaMessageBatch` 在抓到部分消息后发生可恢复错误时,应返回已抓取批次供 handler 处理;只有一条都没抓到时才返回 fetch error。
|
||||
2. 批量准备阶段不能留下半批 claim。可先完成无副作用的解码与校验,再逐条 claim;claim 冲突时按消息拆分处理,不能让整个批次从头活锁。
|
||||
3. offset 只允许在该 partition 对应消息全部持久化成功后提交;增加 fetched/processed/committed offset 和 commit failure 指标。
|
||||
4. 增加回归测试:半批 fetch 后网络错误、准备第 N 条失败、Redis 短暂不可用、处理成功但 commit 失败、Pod 在 handler 中重启。
|
||||
5. 修复后先执行 2000 条“只验证 Kafka/MySQL”的持久化专项,要求 DB 2000/2000、lag/offset 对账一致,再恢复大群实时压测。
|
||||
|
||||
### P0:入口整体中断
|
||||
|
||||
1. 从 Ingress controller、Service endpoint、入口节点和云负载均衡拉取 10:40:00–10:41:10 的日志与事件。
|
||||
2. 核对是否发生 Ingress reload、endpoint 短暂清空、节点 conntrack/端口耗尽、进程监听中断、liveness 误杀或网络策略更新。
|
||||
3. 增加入口黑盒探针:Cloudflare 域名、源站 IP、Service、Pod 四层同时采样 TCP connect、TLS handshake、HTTP status 和耗时。
|
||||
4. 把“源站三个 IP 同时 connection refused”设为基础设施门禁,发生时本轮自动标记无效并保留现场。
|
||||
|
||||
### P1:实时链路
|
||||
|
||||
1. 保留共享序列化实现;从本轮数据看无需回退。
|
||||
2. 继续降低 master-to-proxy multiplex 排队,优先考虑兼容发布的有界批量 RPC 或单写者批发送。
|
||||
3. 补采 Pod CPU throttling、Go scheduler、GC、网卡 retransmit、Ingress upstream reset,解决 `/metrics` 本身在高压阶段失去可观测性的问题。
|
||||
|
||||
## 8. 下一轮顺序
|
||||
|
||||
1. 修复 Kafka 两个批消费边界并完成单元/故障注入测试。
|
||||
2. 运行持久化专项,确认 2000/2000 精确落库。
|
||||
3. 恢复 Kubernetes 与 Ingress 现场采集能力,查清 R11 的入口中断。
|
||||
4. 使用 R11 完全相同参数执行 R12,要求连续两轮无入口故障、无 outcome unknown、DB 2000/2000。
|
||||
5. 通过后再判断 2000 人容量;不要在问题未清零前提高并发。
|
||||
|
||||
## 9. 产物
|
||||
|
||||
- 结果目录:`loadtest/im-pressure/results/group2k-r11-20261004-01/`
|
||||
- 汇总:`report.json`
|
||||
- 数据库核验:`db-verification.json`
|
||||
- A/B 明细:`agent-a-summary.json`、`agent-b-summary.json`
|
||||
- 协调器台账:`coordinator.json`
|
||||
- 指标:`metrics.jsonl`
|
||||
- 被测代码:`b05e8f6 perf(group): reduce websocket fanout serialization`
|
||||
|
||||
## 10. 分析后已实施修复
|
||||
|
||||
已完成以下 Kafka 持久化修复,待发版后先执行持久化专项,再进行 R12:
|
||||
|
||||
1. 批量抓取在取得部分消息后遇到 broker/网络错误时,不再丢弃已抓消息;消费者会先处理并提交这部分消息,同时保留错误日志。
|
||||
2. 批准备过程中后续消息 claim 失败时,先持久化并完成前面已经 claim 的消息,再将准备错误交给外层重试,避免整批 processing claim 活锁。
|
||||
3. 新增半批 fetch error、首次 fetch error、部分 claim 后准备失败、刷盘与准备同时失败等回归测试。
|
||||
|
||||
验证结果:
|
||||
|
||||
- Kafka 定向单元测试通过。
|
||||
- `go test ./server ./server/store/... ./loadtest/im-pressure` 通过。
|
||||
- Kafka 定向 `go test -race` 通过。
|
||||
- `go vet` 仍报告 `server/internal_cultivate_test.go` 中既有的 `sync.Map` 锁值复制问题,与本次改动无关。
|
||||
@@ -0,0 +1,168 @@
|
||||
# GROUP-2000 R12 双机群聊复测分析
|
||||
|
||||
## 1. 结论
|
||||
|
||||
`group2k-r12-20261004-01` 判定为 **失败,2000 人群聊容量仍未通过**。
|
||||
|
||||
本轮有一个明确的正向结果:R11 后的 Kafka 部分批次修复已经生效。服务端实际接收并可从投递或历史中确认的 1694 条消息,MySQL 精确找到 1694 条,缺失、错配、重复和额外记录均为 0。
|
||||
|
||||
实时链路仍然失败。A/B 虽然均以 1000/1000 条连接起跑,但负载开始约 33 秒后公网 `/metrics` 开始连续超时,随后出现两波大规模 WebSocket reset。最终实时到达率只有 74.739%,ACK P95 为 74.72 秒,包含迟到 ACK 的控制响应 P95 达 407.90 秒。
|
||||
|
||||
与 R11 不同,本轮 Pod 没有重启,日志也没有再次出现 CoreDNS 超时。证据表明本轮先发生群 Topic 串行处理积压和入口响应失速,连接被成批 reset 后又触发重连风暴,三个 Pod 的业务库连接池均出现约 9,200-11,350 秒累计等待,最终形成放大回路。
|
||||
|
||||
## 2. 核心结果
|
||||
|
||||
| 指标 | R12 结果 | 判定 |
|
||||
| --- | ---: | --- |
|
||||
| WS 起跑就绪 | 2000/2000 | 通过 |
|
||||
| A/B 起跑差 | 约 33ms | 通过 |
|
||||
| 发送尝试 | 2000 | A/B 各 1000 |
|
||||
| 最终明确 ACK | 1501 | 75.05%,失败 |
|
||||
| 发送结果未知 | 499 | 失败 |
|
||||
| 服务端接收/回查确认消息 | 1694 | 另有 306 条未确认进入服务端 |
|
||||
| ACK P95 | 74.72s | 失败 |
|
||||
| 含迟到 ACK 的控制响应 P95 | 407.90s | 失败 |
|
||||
| 期望实时投递 | 3,388,000 | 1694 x 2000 |
|
||||
| 实际实时投递 | 2,532,160 | 到达率 74.739% |
|
||||
| 实时缺失 | 855,840 | 丢失率 25.261% |
|
||||
| MySQL 精确核验 | 1694/1694 | 通过 |
|
||||
| 重复 / 乱序 | 0 / 0 | 已收到部分正确 |
|
||||
| WS 断连 | 1257 | 失败 |
|
||||
| Pod 重启 | 0 | 无应用容器重启 |
|
||||
|
||||
`publish_final_accepted=1501` 只统计客户端最终观察到的 ACK。数据库和消息台账共同证明实际有 1694 条进入服务端,因此其中 193 条属于客户端未拿到 ACK、但服务端已经处理的消息。
|
||||
|
||||
## 3. A/B 分机结果
|
||||
|
||||
| 指标 | 机器 A | 机器 B |
|
||||
| --- | ---: | ---: |
|
||||
| 起跑连接 | 1000 | 1000 |
|
||||
| 发送尝试 | 1000 | 1000 |
|
||||
| 及时 ACK | 547 | 601 |
|
||||
| 迟到 ACK | 327 | 26 |
|
||||
| 结果未知 | 126 | 373 |
|
||||
| 收到群消息 | 1,389,624 | 1,142,536 |
|
||||
| 实时缺失 | 304,376 | 551,464 |
|
||||
| 断连 | 433 | 824 |
|
||||
| 主要断连原因 | reset 413、timeout 20 | abnormal close 405、reset 419 |
|
||||
|
||||
B 在第一波连接重置中受影响更重;第二波则同时影响 A/B,不能归因于单台压测机。
|
||||
|
||||
## 4. 故障时间线
|
||||
|
||||
时间均为 UTC,北京时间加 8 小时:
|
||||
|
||||
- 15:57:31:A/B 同步起跑,2000 个连接全部就绪。
|
||||
- 15:57:34-15:57:41:公网 `/metrics` 已从亚秒上升到约 2.5 秒。
|
||||
- 15:58:04:公网 `/metrics` 首次 10 秒超时;此后连续 28 次超时。
|
||||
- 15:58:24-15:58:59:第一波约 394 条服务端连接集中关闭,主要对应 B;B 汇总为 405 条 abnormal close。
|
||||
- 16:00:15 起:`im-server-0` 开始出现用户缓存更新队列满,窗口内共 78 次。
|
||||
- 16:01:50-16:02:10:第二波约 810 条服务端连接集中关闭;A/B 汇总的 reset 合计为 832 条。
|
||||
- 16:02:06 起:大量重连登录出现 `context deadline exceeded`。
|
||||
- 16:02:08:`im-server-1` readiness 请求本身超时;期间还出现一次 liveness 超时,但未达到重启阈值。
|
||||
- 16:02:39-16:02:47:三个 Pod 均产生 readiness 503 事件。
|
||||
- 16:02:48、16:02:50:`im-server-2`、`im-server-1` 恢复 Ready;`im-server-0` 未形成持续 NotReady。
|
||||
- 16:05:04:公网 `/metrics` 恢复 200,首次恢复请求仍耗时约 4.6 秒。
|
||||
- 16:11:55:A/B 汇总完成。
|
||||
|
||||
三个 Pod 的 `restartCount` 均为 0。R11 出现过的 `10.43.0.10:53`、`no such host`、DNS connection refused 在本轮窗口内均未出现,因此不能继续把本轮失败归因于 CoreDNS。
|
||||
|
||||
## 5. 容量瓶颈
|
||||
|
||||
群 Topic master 位于 `im-server-2`。最终指标显示:
|
||||
|
||||
| 指标 | 数值 |
|
||||
| --- | ---: |
|
||||
| 群发布处理次数 | 1694 |
|
||||
| Topic publish 处理总耗时 | 116.59s |
|
||||
| 单条平均处理耗时 | 约 68.8ms |
|
||||
| Topic 队列等待总耗时 | 2279.92s |
|
||||
| 单条平均排队 | 约 1.35s |
|
||||
| 队列等待最大桶 | 30s 内 |
|
||||
| master 广播总耗时 / 次数 | 21.33s / 4803 |
|
||||
| master 单次广播平均耗时 | 约 4.4ms |
|
||||
|
||||
按 68.8ms 的串行 Topic 处理耗时估算,单群持续处理能力约为 14.5 条/秒,低于本轮约 20 条/秒的目标发送速率。共享序列化后的广播本身仍然只有毫秒级,不是主要瓶颈;耗时主要位于 Topic publish 的保存、Kafka produce、权限检查和推送准备等关键路径。
|
||||
|
||||
这也解释了为何在正式断连前 ACK 已经出现 20-90 秒长尾,并且 `/metrics` 先失去响应。
|
||||
|
||||
## 6. 断连与入口
|
||||
|
||||
Pod 日志中的两波连接关闭均来自入口上游连接;客户端看到的是三个源站 IP 返回 `connection reset by peer`,不是主动退出压测程序:
|
||||
|
||||
- 第一波关闭连接的服务端会话寿命平均约 122 秒,集中影响 B。
|
||||
- 第二波同时影响 A/B;旧连接寿命主要集中在约 330 秒和 660 秒,说明它是同一时刻的批量重置,而非每条连接独立超时。
|
||||
- 三个源 IP 最终汇聚到同一个入口上游地址的现象仍然存在,入口层缺少副本、连接上限、reload/restart、upstream reset 和节点 conntrack 证据。
|
||||
|
||||
应用负载先失速,入口随后批量 reset,因此当前不能只改客户端重连参数来宣告通过。运维仍需补齐 Traefik、入口节点和 Service endpoint 的同窗口证据,判断 reset 是入口进程、节点网络、连接上限还是上游超时策略触发。
|
||||
|
||||
## 7. 重连放大与数据库
|
||||
|
||||
压测前后业务库连接池增量如下:
|
||||
|
||||
| Pod | 新增等待次数 | 新增累计等待 | 平均每次等待 |
|
||||
| --- | ---: | ---: | ---: |
|
||||
| im-server-0 | 817 | 9956.29s | 约 12.2s |
|
||||
| im-server-1 | 770 | 9218.27s | 约 12.0s |
|
||||
| im-server-2 | 768 | 11348.20s | 约 14.8s |
|
||||
|
||||
每个 Pod 的 `meeting_im` 最大连接数为 24。连接集中断开后,压测端自动重连触发登录、用户状态、IP 白名单和认证版本查询,连接池等待迅速升高;登录进一步超时,readiness 中的数据库检查也随之失败,Pod 被入口摘除,形成反馈回路。
|
||||
|
||||
不能简单地只提高连接池上限。下一轮前应先定位连接被长时间占用的 SQL/事务,并为登录重连增加服务端限流、排队和抖动;否则只是把压力转移到 MySQL。
|
||||
|
||||
## 8. Kafka 与落库
|
||||
|
||||
数据库直连核验结果:
|
||||
|
||||
| 项目 | 结果 |
|
||||
| --- | ---: |
|
||||
| tenant | allrich / tenant_id=8 |
|
||||
| 期望消息 | 1694 |
|
||||
| 精确找到 | 1694 |
|
||||
| 缺失 | 0 |
|
||||
| 字段错配 | 0 |
|
||||
| 重复行 | 0 |
|
||||
| 额外记录 | 0 |
|
||||
| 分表 | `im_008_group_messages_202610`、`im_008_messages` |
|
||||
|
||||
这证明 R11 的 Kafka 部分批次丢弃问题已修复。最后一次公网指标采样中某个 reader 的本地 `im_kafka_consumer_lag` 仍显示 111,但稍后的数据库核验已经完整;再次说明 `kafka-go Reader.Stats().Lag` 不能作为 consumer group 已提交 offset 的最终依据。
|
||||
|
||||
## 9. 改进优先级
|
||||
|
||||
### P0:群 Topic 关键路径
|
||||
|
||||
1. 对 Topic publish 的 save、Kafka produce、权限查询、推送准备分别补充 P95/P99 和并发占用,确认 68.8ms 的主要组成。
|
||||
2. 将不要求阻塞 ACK 的推送准备、通知类工作移出 Topic 单写者关键路径。
|
||||
3. 对必须保持顺序的持久化和序列分配设计有界流水线;保持同群顺序和失败可重试,不能重新引入已移除的业务 outbox。
|
||||
4. 优化后先跑 10、12、15、18、20 msg/s 阶梯,连续两轮确认 Topic 队列不持续增长,再重新判定 20 msg/s 容量。
|
||||
|
||||
### P0:入口批量 reset
|
||||
|
||||
1. 由运维提取 15:57:30-16:05:10 的 Traefik Pod 日志、事件、restart、reload、upstream reset 和打开连接数。
|
||||
2. 核对三个公网源 IP 是否最终只有一个 Traefik 实例承载;入口至少配置两个副本、反亲和和 PDB。
|
||||
3. 同步采集节点 conntrack、TCP retransmit、listen overflow、文件描述符、网卡丢包和 CPU throttling。
|
||||
4. 下一轮同时探测公网域名、每个源 IP、Service 和三个 Pod,避免 `/metrics` 单点失明。
|
||||
|
||||
### P1:重连保护
|
||||
|
||||
1. 客户端维持自动重连,但使用指数退避和随机抖动,避免 2000 条连接同步登录。
|
||||
2. 服务端对登录重建增加有界并发和快速过载响应,不让请求无限占用数据库连接池。
|
||||
3. 分解登录 SQL 指标,定位连接池长等待对应的查询或事务。
|
||||
4. readiness 的数据库检查应使用独立的小连接预算或短路缓存,避免业务池耗尽导致全部 Endpoint 同时被摘除;liveness 继续只检查进程存活。
|
||||
|
||||
## 10. 下一轮顺序
|
||||
|
||||
1. 先做 2000 连接、0 发消息的 15 分钟稳定性测试,排除入口固定连接寿命和 B 网络路径问题。
|
||||
2. 再做单群阶梯:10、12、15、18、20 msg/s,每档 5 分钟,观察 Topic queue 是否线性增长。
|
||||
3. 完成入口和数据库现场采集后,重复 R12 原参数两轮。
|
||||
4. 通过标准:2000/2000 连接全程稳定、发送成功率不低于 99.99%、实时到达率不低于 99.99%、DB 精确落库、无批量 reset、无持续 Topic/DB 队列增长。
|
||||
|
||||
## 11. 产物
|
||||
|
||||
- 结果目录:`loadtest/im-pressure/results/group2k-r12-20261004-01/`
|
||||
- 汇总:`report.json`
|
||||
- 数据库核验:`db-verification.json`
|
||||
- A/B 明细:`agent-a-summary.json`、`agent-b-summary.json`
|
||||
- 协调器台账:`coordinator.json`
|
||||
- 指标:`metrics.jsonl`
|
||||
- 被测代码:`965f94f fix(kafka): preserve partially fetched message batches`
|
||||
@@ -1038,8 +1038,9 @@ X-MeetingIM-APIKey: <预置的普通 API Key>
|
||||
**状态:需改造。**
|
||||
|
||||
- `GET /v1/im/system-notifications?cursor=` 返回 `id,event_type,title,summary,created_at,read_at,action`;实时刷新以 0.5.4.6 的 WS `{info what:"im|group"}` 业务事件为准,系统通知列表记录不等同于每一条实时事件。
|
||||
- `summary` 为服务端生成的可直接展示文案(如「对方已同意你的好友申请」「对方已婉拒你的好友申请」),前端直接渲染,无需再做状态码到文案的映射;`action` 读路径统一归一为 `{type,target_id,payload,...原始键}`。
|
||||
- 新设备登录事件内容为 `device_name,platform,ip_masked,login_at,security_action`;IP 只下发必要脱敏值。
|
||||
- 已读:`POST /v1/im/system-notifications/{id}/read`,带幂等键;跳转目标只接受服务端白名单 `action.type` 和内部参数。
|
||||
- 已读:`POST /v1/im/system-notifications/{id}/read`,带幂等键;重复已读返回首次持久化的 `read_at`,不回显本次提交值。跳转目标只接受服务端白名单 `action.type` 和内部参数。
|
||||
- 当前普通认证用户可向 `sys` Topic 发布的路径必须封禁。客户端不得渲染普通用户伪造的系统卡片为可信通知。
|
||||
|
||||
### 2.6 接口示例
|
||||
@@ -1058,7 +1059,7 @@ X-MeetingIM-APIKey: <预置的普通 API Key>
|
||||
| --- | --- | --- |
|
||||
| `GET /v1/im/bootstrap` | Query only | `{"features":{"create_group":true,"user_search":true,"scan_qr":true,"system_notifications":true,"message_recall":true,"message_edit":true,"audio_call":true,"video_call":true},"permissions":{"create_group":true,"user_search":true,"scan_qr":true,"audio_call":true,"video_call":true},"entries":{"new_friends":{"enabled":true,"route":"/contacts/new-friends","unread_count":0},"groups":{"enabled":true,"route":"/contacts/groups","unread_count":0},"file_assistant":{"enabled":true,"route":"/chat/file-assistant","topic":"slf","unread_count":0},"system_messages":{"enabled":true,"route":"/chat/system","topic":"sys","unread_count":0}},"entry_badges":{"new_friends_pending_count":0,"groups_unread_count":0,"file_assistant_unread_count":0,"system_unread_count":0,"workbench_total_unread":0},"system_topic":{"topic":"sys","type":"system"},"message_policy":{"recall_window_seconds":120,"edit_window_seconds":300,"supported_actions":["copy","quote","delete","recall","edit"]},"security":{"realname":{"status":"unverified","required":false,"allow_skip":true,"remind_after":"24h"}}}` |
|
||||
| `GET /v1/im/search/conversations?q=&cursor=` | Query only | `{"items":[{"topic":"grp...","public":{"name":"项目群"},"private":{"pinned":false,"mute_notifications":false},"seq":12,"read":10,"recv":12,"unread":2,"updated_at":"2026-08-20T09:00:00Z"}],"next_cursor":""}` |
|
||||
| `GET /v1/im/system-notifications?cursor=&limit=20` | Query only | `{"items":[{"id":"evt_1","event_type":"friend.request.created","title":"新的好友申请","summary":"张三请求添加你为好友","action":{"type":"friend_request","request_id":"frq_1"},"read_at":null,"created_at":"2026-08-20T09:00:00Z"}],"next_cursor":""}` |
|
||||
| `GET /v1/im/system-notifications?cursor=&limit=20` | Query only | `{"items":[{"id":"evt_1","event_type":"friend.request.created","title":"新的好友申请","summary":"张三请求添加你为好友","action":{"type":"friend_request","target_id":"usr...","payload":{"request_id":"frq_1"},"request_id":"frq_1"},"read_at":null,"created_at":"2026-08-20T09:00:00Z"}],"next_cursor":""}` |
|
||||
| `POST /v1/im/system-notifications/{id}/read` | `{"read_at":"2026-08-20T09:00:00Z"}` | `{"status":"ok","id":"evt_1","read_at":"2026-08-20T09:00:00Z"}` |
|
||||
|
||||
## 3. 单聊会话、会话建立与消息状态
|
||||
@@ -1773,9 +1774,10 @@ GET /v1/im/topics/{topic}/message-mutations?since_version={localMutationVersion}
|
||||
|
||||
<img src="images/黑名单提示.png" alt="黑名单提示" width="360">
|
||||
|
||||
**状态:图片文件名与画面语义不一致;按“清除聊天记录”实施,需改造。**
|
||||
**状态:图片文件名与画面语义不一致;按“清除聊天记录”实施。服务端已实现并实测(2026-10-09),剩余前端改造。**
|
||||
|
||||
- 确认调用 MeetingIM `{del what:"msg",topic,delseq:[{"low":1,"hi":last_seq+1}],hard:false}`,仅删除当前用户视图;或封装 `DELETE /v1/im/conversations/{topic}/messages`。
|
||||
- 服务端现状(2026-10-09 测试环境实测通过):REST 接口仅写 `im_dellog` 本人区间并递增 `deleted_id`,不改动消息行、对方视角不受影响,本人其他设备经 `deleted_id` 同步感知清空;WS `{del}` 语义相同,两条路径二选一即可。对非成员或不存在会话返回 403 FORBIDDEN。
|
||||
- 不解除好友、不影响对方历史、不删除收藏;附件仅在无任何有效引用且过保留期后回收。
|
||||
- 前端需在确认文案明确“仅清除自己的聊天记录”,成功后清本地 Topic 消息缓存并保留会话关系。
|
||||
|
||||
|
||||
@@ -197,8 +197,13 @@ mkdir -p "loadtest/im-pressure/results/${RUN_ID}"
|
||||
两个 agent 都建立 1000 条连接后,协调器下发统一的 UTC `start_at`。场景可替换为:
|
||||
|
||||
- `p2p`:1000 对用户两侧同时发送,默认每人 10 条、间隔 1 秒。
|
||||
- `group`:`users_per_machine=1000` 时每个 agent 有 500 个发送者,每人 2 条;全局每 100ms 一条,共 2000 条。其他容量按每个 agent 用户数的 50% 选发送者。
|
||||
- `mixed`:`users_per_machine=1000` 时使用 300 对私聊用户和每个 agent 250 个群发送者;其他容量分别按 30% 和 25% 同比计算。
|
||||
- `group`:默认每个 agent 有 `group_senders_per_machine` 个发送者,每人 `group_rounds` 条;两机合并后按 `group_publish_interval` 控制全局发送间隔。双机默认 500+500 发送者、100ms 一条,共 2000 条;单机 60% 默认 300+300 发送者。
|
||||
- `mixed`:默认使用 `mixed_p2p_users_per_machine` 对私聊用户和 `mixed_group_senders_per_machine` 个群发送者。双机默认每机 300 个私聊用户、250 个群发送者;单机 60% 默认每机 180 个私聊用户、150 个群发送者。
|
||||
- `idle`:只建立并维持 WS 连接,不发送业务消息,持续 `idle_duration`。该场景用于先确认 2000 长连接在入口、Pod、DNS、心跳和自动重连下是否稳定。
|
||||
|
||||
群聊容量阶梯建议只改 `group_publish_interval`,保持发送者和轮数不变:`125ms` 约等于 8 msg/s,`100ms` 约等于 10 msg/s,`83ms` 约等于 12 msg/s,`67ms` 约等于 15 msg/s,`56ms` 约等于 18 msg/s。先跑 `idle`,再从低速 `group` 逐级上探;某一级出现连接波动、ACK 长尾或 DB pool wait 明显上升时,重复该级确认后再进入下一档。
|
||||
|
||||
自动重连由 `reconnect_workers`、`reconnect_min_wait`、`reconnect_max_wait` 控制。断线账号会按指数退避并附带确定性抖动后重连,避免两台压测机在入口抖动时同时打出登录洪峰。
|
||||
|
||||
## 机器 A:回收并合并报告
|
||||
|
||||
|
||||
@@ -49,7 +49,7 @@ type agentRun struct {
|
||||
}
|
||||
|
||||
func validScenario(scenario string) bool {
|
||||
return scenario == "p2p" || scenario == "group" || scenario == "mixed"
|
||||
return scenario == "p2p" || scenario == "group" || scenario == "mixed" || scenario == "idle"
|
||||
}
|
||||
|
||||
func preflightP2P(ctx context.Context, cfg *Config, manifest *Manifest, local []AccountRecord, apiKey string) error {
|
||||
@@ -222,8 +222,12 @@ func runAgent(ctx context.Context, cfg *Config, manifest *Manifest, machine, coo
|
||||
quietPeriod, _ := configDuration(cfg.DeliveryQuiet)
|
||||
quietTimeout, _ := configDuration(cfg.DeliveryTimeout)
|
||||
log.Printf("agent=%s waiting for %s of delivery quiescence", machine, quietPeriod)
|
||||
if err = waitForDeliveryQuiescence(ctx, run.recorder, run.callbackQueue, quietPeriod, quietTimeout); err != nil {
|
||||
return err
|
||||
var quiescenceErr error
|
||||
if quiescenceErr = waitForDeliveryQuiescence(ctx, run.recorder, run.callbackQueue, quietPeriod, quietTimeout); quiescenceErr != nil {
|
||||
if ctx.Err() != nil {
|
||||
return quiescenceErr
|
||||
}
|
||||
log.Printf("agent=%s delivery quiescence wait timed out; continuing with observed submission: %v", machine, quiescenceErr)
|
||||
}
|
||||
observedLocal := run.recorder.unackedObservedMessages(ledger, start.Scenario)
|
||||
if err = coord.submitObserved(ctx, machine, observedLocal); err != nil {
|
||||
@@ -247,6 +251,9 @@ func runAgent(ctx context.Context, cfg *Config, manifest *Manifest, machine, coo
|
||||
run.mu.Lock()
|
||||
base.Errors = append(base.Errors, run.errors...)
|
||||
run.mu.Unlock()
|
||||
if quiescenceErr != nil {
|
||||
base.Errors = append(base.Errors, quiescenceErr.Error())
|
||||
}
|
||||
// Include messages observed over WebSocket without a timely ctrl ACK. They
|
||||
// are confirmed server-side publishes and must participate in realtime
|
||||
// delivery accounting instead of being reported as unexpected traffic.
|
||||
@@ -680,10 +687,25 @@ func (r *agentRun) connect(ctx context.Context, accounts []AccountRecord, groupT
|
||||
}
|
||||
|
||||
func (r *agentRun) maintainConnections(ctx context.Context, accounts []AccountRecord, groupTopic, apiKey, password string) {
|
||||
type reconnectState struct {
|
||||
attempt int
|
||||
nextAt time.Time
|
||||
}
|
||||
byIndex := make(map[int]AccountRecord, len(accounts))
|
||||
for _, account := range accounts {
|
||||
byIndex[account.Index] = account
|
||||
}
|
||||
states := make(map[int]reconnectState, len(accounts))
|
||||
var stateMu sync.Mutex
|
||||
minWait := mustDuration(r.cfg.ReconnectMinWait)
|
||||
maxWait := mustDuration(r.cfg.ReconnectMaxWait)
|
||||
if maxWait < minWait {
|
||||
maxWait = minWait
|
||||
}
|
||||
reconnectWorkers := r.cfg.ReconnectWorkers
|
||||
if reconnectWorkers <= 0 {
|
||||
reconnectWorkers = 8
|
||||
}
|
||||
ticker := time.NewTicker(time.Second)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
@@ -692,7 +714,7 @@ func (r *agentRun) maintainConnections(ctx context.Context, accounts []AccountRe
|
||||
}
|
||||
jobs := make(chan AccountRecord)
|
||||
var wg sync.WaitGroup
|
||||
workers := 16
|
||||
workers := reconnectWorkers
|
||||
if workers > len(accounts) {
|
||||
workers = len(accounts)
|
||||
}
|
||||
@@ -714,8 +736,17 @@ func (r *agentRun) maintainConnections(ctx context.Context, accounts []AccountRe
|
||||
if ctx.Err() == nil {
|
||||
log.Printf("account=%s reconnect failed: %v", account.Username, err)
|
||||
}
|
||||
stateMu.Lock()
|
||||
state := states[index]
|
||||
state.attempt++
|
||||
state.nextAt = time.Now().Add(reconnectBackoff(minWait, maxWait, state.attempt, index))
|
||||
states[index] = state
|
||||
stateMu.Unlock()
|
||||
continue
|
||||
}
|
||||
stateMu.Lock()
|
||||
delete(states, index)
|
||||
stateMu.Unlock()
|
||||
r.setClient(index, client)
|
||||
log.Printf("account=%s connection restored", account.Username)
|
||||
}
|
||||
@@ -732,6 +763,12 @@ func (r *agentRun) maintainConnections(ctx context.Context, accounts []AccountRe
|
||||
if current != nil && current.alive() {
|
||||
continue
|
||||
}
|
||||
stateMu.Lock()
|
||||
nextAt := states[index].nextAt
|
||||
stateMu.Unlock()
|
||||
if !nextAt.IsZero() && time.Now().Before(nextAt) {
|
||||
continue
|
||||
}
|
||||
select {
|
||||
case jobs <- account:
|
||||
case <-ctx.Done():
|
||||
@@ -750,6 +787,25 @@ func (r *agentRun) maintainConnections(ctx context.Context, accounts []AccountRe
|
||||
}
|
||||
}
|
||||
|
||||
func reconnectBackoff(minWait, maxWait time.Duration, attempt, index int) time.Duration {
|
||||
if attempt < 1 {
|
||||
attempt = 1
|
||||
}
|
||||
delay := minWait
|
||||
for i := 1; i < attempt && delay < maxWait; i++ {
|
||||
delay *= 2
|
||||
}
|
||||
if delay > maxWait {
|
||||
delay = maxWait
|
||||
}
|
||||
jitterBase := delay / 4
|
||||
if jitterBase <= 0 {
|
||||
return delay
|
||||
}
|
||||
jitter := time.Duration((index*97+attempt*53)%1000) * jitterBase / 1000
|
||||
return delay + jitter
|
||||
}
|
||||
|
||||
func (r *agentRun) connectAccountWithRetry(ctx context.Context, account AccountRecord, groupTopic, apiKey, password string) (*WSClient, error) {
|
||||
var lastErr error
|
||||
for attempt := 1; attempt <= 6; attempt++ {
|
||||
@@ -787,12 +843,25 @@ func (r *agentRun) execute(ctx context.Context, start time.Time, groupTopic stri
|
||||
case "p2p":
|
||||
return r.scheduleP2P(ctx, start, r.cfg.UsersPerMachine, r.cfg.P2PMessages, mustDuration(r.cfg.P2PInterval))
|
||||
case "group":
|
||||
return r.scheduleGroup(ctx, start, groupTopic, 1, scaledUsers(r.cfg.UsersPerMachine, 1, 2), 2)
|
||||
senders := r.cfg.GroupSenders
|
||||
if senders <= 0 {
|
||||
senders = scaledUsers(r.cfg.UsersPerMachine, 1, 2)
|
||||
}
|
||||
if senders > r.cfg.UsersPerMachine {
|
||||
return fmt.Errorf("group_senders_per_machine=%d exceeds users_per_machine=%d", senders, r.cfg.UsersPerMachine)
|
||||
}
|
||||
return r.scheduleGroup(ctx, start, groupTopic, 1, senders, r.cfg.GroupRounds, mustDuration(r.cfg.GroupInterval))
|
||||
case "mixed":
|
||||
p2pUsers := scaledUsers(r.cfg.UsersPerMachine, 3, 10)
|
||||
groupSenders := scaledUsers(r.cfg.UsersPerMachine, 1, 4)
|
||||
p2pUsers := r.cfg.MixedP2PUsers
|
||||
if p2pUsers <= 0 {
|
||||
p2pUsers = scaledUsers(r.cfg.UsersPerMachine, 3, 10)
|
||||
}
|
||||
groupSenders := r.cfg.MixedGroupSenders
|
||||
if groupSenders <= 0 {
|
||||
groupSenders = scaledUsers(r.cfg.UsersPerMachine, 1, 4)
|
||||
}
|
||||
if p2pUsers+groupSenders > r.cfg.UsersPerMachine {
|
||||
groupSenders = r.cfg.UsersPerMachine - p2pUsers
|
||||
return fmt.Errorf("mixed_p2p_users_per_machine + mixed_group_senders_per_machine exceeds users_per_machine")
|
||||
}
|
||||
if groupSenders <= 0 {
|
||||
return fmt.Errorf("users_per_machine=%d is too small for mixed scenario", r.cfg.UsersPerMachine)
|
||||
@@ -803,7 +872,7 @@ func (r *agentRun) execute(ctx context.Context, start time.Time, groupTopic stri
|
||||
go func() { defer wg.Done(); errCh <- r.scheduleP2P(ctx, start, p2pUsers, 5, 20*time.Second) }()
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
errCh <- r.scheduleGroup(ctx, start, groupTopic, p2pUsers+1, p2pUsers+groupSenders, 2)
|
||||
errCh <- r.scheduleGroup(ctx, start, groupTopic, p2pUsers+1, p2pUsers+groupSenders, r.cfg.GroupRounds, mustDuration(r.cfg.GroupInterval))
|
||||
}()
|
||||
wg.Wait()
|
||||
close(errCh)
|
||||
@@ -813,6 +882,8 @@ func (r *agentRun) execute(ctx context.Context, start time.Time, groupTopic stri
|
||||
}
|
||||
}
|
||||
return nil
|
||||
case "idle":
|
||||
return waitContext(ctx, mustDuration(r.cfg.IdleDuration))
|
||||
default:
|
||||
return fmt.Errorf("unknown scenario %s", r.scenario)
|
||||
}
|
||||
@@ -867,15 +938,19 @@ func (r *agentRun) scheduleP2P(ctx context.Context, start time.Time, users, mess
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
func (r *agentRun) scheduleGroup(ctx context.Context, start time.Time, topic string, first, last, rounds int) error {
|
||||
func (r *agentRun) scheduleGroup(ctx context.Context, start time.Time, topic string, first, last, rounds int, globalInterval time.Duration) error {
|
||||
if first < 1 || last > r.clientCount() || first > last {
|
||||
return fmt.Errorf("invalid group publisher range %d-%d", first, last)
|
||||
}
|
||||
count := last - first + 1
|
||||
roundDuration := time.Duration(count) * 200 * time.Millisecond
|
||||
if globalInterval <= 0 {
|
||||
return fmt.Errorf("group publish interval must be positive")
|
||||
}
|
||||
perMachineInterval := 2 * globalInterval
|
||||
roundDuration := time.Duration(count) * perMachineInterval
|
||||
offset := time.Duration(0)
|
||||
if r.machine == "b" {
|
||||
offset = 100 * time.Millisecond
|
||||
offset = globalInterval
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
for index := first; index <= last; index++ {
|
||||
@@ -884,7 +959,7 @@ func (r *agentRun) scheduleGroup(ctx context.Context, start time.Time, topic str
|
||||
go func(index, position int) {
|
||||
defer wg.Done()
|
||||
for round := 1; round <= rounds; round++ {
|
||||
target := start.Add(time.Duration(round-1)*roundDuration + time.Duration(position)*200*time.Millisecond + offset)
|
||||
target := start.Add(time.Duration(round-1)*roundDuration + time.Duration(position)*perMachineInterval + offset)
|
||||
if waitUntil(ctx, target) != nil {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -76,6 +76,33 @@ func TestScenarioUsersScaleWithConfiguredCapacity(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconnectBackoffCapsAndJitters(t *testing.T) {
|
||||
minWait := 2 * time.Second
|
||||
maxWait := 30 * time.Second
|
||||
first := reconnectBackoff(minWait, maxWait, 1, 10)
|
||||
if first < minWait || first >= minWait+minWait/4 {
|
||||
t.Fatalf("first backoff=%s outside expected jitter range", first)
|
||||
}
|
||||
capped := reconnectBackoff(minWait, maxWait, 10, 10)
|
||||
if capped < maxWait || capped >= maxWait+maxWait/4 {
|
||||
t.Fatalf("capped backoff=%s outside expected jitter range", capped)
|
||||
}
|
||||
if reconnectBackoff(minWait, maxWait, 3, 10) == reconnectBackoff(minWait, maxWait, 3, 11) {
|
||||
t.Fatal("different account indexes should receive different jitter")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidScenarioIncludesIdle(t *testing.T) {
|
||||
for _, scenario := range []string{"p2p", "group", "mixed", "idle"} {
|
||||
if !validScenario(scenario) {
|
||||
t.Fatalf("scenario %s should be valid", scenario)
|
||||
}
|
||||
}
|
||||
if validScenario("unknown") {
|
||||
t.Fatal("unknown scenario should be rejected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewHistoryQueryIncludesInterleavedSequenceRange(t *testing.T) {
|
||||
messages := make([]AcceptedMessage, 10)
|
||||
for i := range messages {
|
||||
|
||||
@@ -13,6 +13,9 @@
|
||||
"insecure_tls": false,
|
||||
"prepare_workers": 20,
|
||||
"connect_ramp": "20s",
|
||||
"reconnect_workers": 8,
|
||||
"reconnect_min_wait": "2s",
|
||||
"reconnect_max_wait": "30s",
|
||||
"command_timeout": "20s",
|
||||
"ws_ping_interval": "20s",
|
||||
"preflight_workers": 32,
|
||||
@@ -26,6 +29,12 @@
|
||||
"close_ramp": "10s",
|
||||
"p2p_messages_per_user": 10,
|
||||
"p2p_interval": "1s",
|
||||
"group_senders_per_machine": 500,
|
||||
"group_rounds": 2,
|
||||
"group_publish_interval": "100ms",
|
||||
"mixed_p2p_users_per_machine": 300,
|
||||
"mixed_group_senders_per_machine": 250,
|
||||
"idle_duration": "15m",
|
||||
"drain_p2p": "30s",
|
||||
"drain_group": "60s",
|
||||
"delivery_quiet_period": "10s",
|
||||
|
||||
@@ -17,6 +17,9 @@
|
||||
"insecure_tls": false,
|
||||
"prepare_workers": 20,
|
||||
"connect_ramp": "30s",
|
||||
"reconnect_workers": 8,
|
||||
"reconnect_min_wait": "2s",
|
||||
"reconnect_max_wait": "30s",
|
||||
"command_timeout": "90s",
|
||||
"ws_ping_interval": "20s",
|
||||
"preflight_workers": 32,
|
||||
@@ -30,6 +33,12 @@
|
||||
"close_ramp": "60s",
|
||||
"p2p_messages_per_user": 10,
|
||||
"p2p_interval": "1s",
|
||||
"group_senders_per_machine": 300,
|
||||
"group_rounds": 2,
|
||||
"group_publish_interval": "100ms",
|
||||
"mixed_p2p_users_per_machine": 180,
|
||||
"mixed_group_senders_per_machine": 150,
|
||||
"idle_duration": "15m",
|
||||
"drain_p2p": "30s",
|
||||
"drain_group": "60s",
|
||||
"delivery_quiet_period": "10s",
|
||||
|
||||
@@ -101,7 +101,7 @@ func coordinatorCommand(ctx context.Context, args []string) error {
|
||||
fs := flag.NewFlagSet("coordinator", flag.ContinueOnError)
|
||||
listen := fs.String("listen", ":8090", "coordinator listen address")
|
||||
runID := fs.String("run-id", "", "unique run ID, max 24 characters")
|
||||
scenario := fs.String("scenario", "", "p2p, group, or mixed")
|
||||
scenario := fs.String("scenario", "", "p2p, group, mixed, or idle")
|
||||
output := fs.String("output", "loadtest/im-pressure/results/coordinator.json", "coordinator result path")
|
||||
expected := fs.Int("expected-agents", 2, "number of agents")
|
||||
delay := fs.Duration("start-delay", 30*time.Second, "delay after all agents are ready")
|
||||
|
||||
@@ -26,10 +26,19 @@ type Config struct {
|
||||
InsecureTLS bool `json:"insecure_tls"`
|
||||
PrepareWorkers int `json:"prepare_workers"`
|
||||
ConnectRamp string `json:"connect_ramp"`
|
||||
ReconnectWorkers int `json:"reconnect_workers"`
|
||||
ReconnectMinWait string `json:"reconnect_min_wait"`
|
||||
ReconnectMaxWait string `json:"reconnect_max_wait"`
|
||||
CommandTimeout string `json:"command_timeout"`
|
||||
WSPingInterval string `json:"ws_ping_interval"`
|
||||
P2PMessages int `json:"p2p_messages_per_user"`
|
||||
P2PInterval string `json:"p2p_interval"`
|
||||
GroupSenders int `json:"group_senders_per_machine"`
|
||||
GroupRounds int `json:"group_rounds"`
|
||||
GroupInterval string `json:"group_publish_interval"`
|
||||
MixedP2PUsers int `json:"mixed_p2p_users_per_machine"`
|
||||
MixedGroupSenders int `json:"mixed_group_senders_per_machine"`
|
||||
IdleDuration string `json:"idle_duration"`
|
||||
DrainP2P string `json:"drain_p2p"`
|
||||
DrainGroup string `json:"drain_group"`
|
||||
DeliveryQuiet string `json:"delivery_quiet_period"`
|
||||
@@ -68,6 +77,29 @@ func loadConfig(path string) (*Config, error) {
|
||||
if cfg.ConnectRamp == "" {
|
||||
cfg.ConnectRamp = "20s"
|
||||
}
|
||||
if cfg.ReconnectWorkers <= 0 {
|
||||
cfg.ReconnectWorkers = 8
|
||||
}
|
||||
if cfg.ReconnectWorkers > 128 {
|
||||
return nil, errors.New("reconnect_workers must not exceed 128")
|
||||
}
|
||||
if cfg.ReconnectMinWait == "" {
|
||||
cfg.ReconnectMinWait = "2s"
|
||||
}
|
||||
if _, err = configDuration(cfg.ReconnectMinWait); err != nil {
|
||||
return nil, fmt.Errorf("reconnect_min_wait: %w", err)
|
||||
}
|
||||
if cfg.ReconnectMaxWait == "" {
|
||||
cfg.ReconnectMaxWait = "30s"
|
||||
}
|
||||
minReconnect, _ := configDuration(cfg.ReconnectMinWait)
|
||||
maxReconnect, err := configDuration(cfg.ReconnectMaxWait)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reconnect_max_wait: %w", err)
|
||||
}
|
||||
if maxReconnect < minReconnect {
|
||||
return nil, errors.New("reconnect_max_wait must be greater than or equal to reconnect_min_wait")
|
||||
}
|
||||
if cfg.CommandTimeout == "" {
|
||||
cfg.CommandTimeout = "20s"
|
||||
}
|
||||
@@ -83,6 +115,30 @@ func loadConfig(path string) (*Config, error) {
|
||||
if cfg.P2PInterval == "" {
|
||||
cfg.P2PInterval = "1s"
|
||||
}
|
||||
if cfg.GroupRounds <= 0 {
|
||||
cfg.GroupRounds = 2
|
||||
}
|
||||
if cfg.GroupInterval == "" {
|
||||
cfg.GroupInterval = "100ms"
|
||||
}
|
||||
if _, err = configDuration(cfg.GroupInterval); err != nil {
|
||||
return nil, fmt.Errorf("group_publish_interval: %w", err)
|
||||
}
|
||||
if cfg.GroupSenders < 0 {
|
||||
return nil, errors.New("group_senders_per_machine must not be negative")
|
||||
}
|
||||
if cfg.MixedP2PUsers < 0 {
|
||||
return nil, errors.New("mixed_p2p_users_per_machine must not be negative")
|
||||
}
|
||||
if cfg.MixedGroupSenders < 0 {
|
||||
return nil, errors.New("mixed_group_senders_per_machine must not be negative")
|
||||
}
|
||||
if cfg.IdleDuration == "" {
|
||||
cfg.IdleDuration = "15m"
|
||||
}
|
||||
if _, err = configDuration(cfg.IdleDuration); err != nil {
|
||||
return nil, fmt.Errorf("idle_duration: %w", err)
|
||||
}
|
||||
if cfg.DrainP2P == "" {
|
||||
cfg.DrainP2P = "30s"
|
||||
}
|
||||
|
||||
@@ -467,7 +467,7 @@ CREATE TABLE IF NOT EXISTS `im_filemsglinks` (
|
||||
`id` int NOT NULL AUTO_INCREMENT,
|
||||
`createdat` datetime(3) NOT NULL,
|
||||
`fileid` bigint NOT NULL,
|
||||
`msgid` int DEFAULT NULL,
|
||||
`msgid` bigint DEFAULT NULL,
|
||||
`topic` char(25) DEFAULT NULL,
|
||||
`userid` bigint DEFAULT NULL,
|
||||
PRIMARY KEY (`id`),
|
||||
|
||||
@@ -276,8 +276,9 @@ func (c *adminInternalClient) TenantLoginIPWhitelist(id types.TenantID) (*tenant
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
func (c *adminInternalClient) LatestAppRelease(platform, channel string) (*types.AppRelease, error) {
|
||||
func (c *adminInternalClient) LatestAppRelease(appCode, platform, channel string) (*types.AppRelease, error) {
|
||||
query := url.Values{}
|
||||
query.Set("app_code", strings.ToLower(strings.TrimSpace(appCode)))
|
||||
query.Set("platform", strings.ToLower(strings.TrimSpace(platform)))
|
||||
query.Set("channel", strings.ToLower(strings.TrimSpace(channel)))
|
||||
var dto struct {
|
||||
|
||||
@@ -38,7 +38,7 @@ func TestAdminInternalClientLatestAppRelease(t *testing.T) {
|
||||
config: adminInternalAPIConfig{BaseURL: server.URL, ServiceName: "im-server", Secret: "test-secret"},
|
||||
http: server.Client(),
|
||||
}
|
||||
release, err := client.LatestAppRelease(" IOS ", "STABLE")
|
||||
release, err := client.LatestAppRelease(" IM ", " IOS ", "STABLE")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -284,6 +284,9 @@ type ClusterResp struct {
|
||||
|
||||
// Original request type.
|
||||
OrigReqType ProxyReqType
|
||||
|
||||
// Set after RPC decoding on the receiving node; not serialized by gob.
|
||||
proxyQueuedAt time.Time
|
||||
}
|
||||
|
||||
// ClusterPing is used to detect node restarts.
|
||||
@@ -743,6 +746,7 @@ func (Cluster) TopicProxy(msg *ClusterResp, unused *bool) error {
|
||||
// Find appropriate topic, send the message to it.
|
||||
if t := globals.hub.topicGet(types.TopicKey{TenantID: msg.TenantID, Topic: msg.RcptTo}); t != nil {
|
||||
msg.SrvMsg.uid = types.ParseUserId(msg.SrvMsg.AsUser)
|
||||
msg.proxyQueuedAt = time.Now()
|
||||
select {
|
||||
case t.proxy <- msg:
|
||||
immetrics.ObserveInternalQueueEnqueue("cluster_topic_proxy", len(t.proxy), cap(t.proxy), true)
|
||||
@@ -1420,6 +1424,8 @@ func (sess *Session) clusterWriteLoop(forTopic string) {
|
||||
return
|
||||
}
|
||||
srvMsg := msg.(*ServerComMessage)
|
||||
immetrics.ObserveClusterM2PDequeue(sess.clnode.name, srvMsg.clusterQueuedAt)
|
||||
srvMsg.clusterQueuedAt = time.Time{}
|
||||
response := &ClusterResp{TenantID: srvMsg.TenantID, SrvMsg: srvMsg}
|
||||
if srvMsg.sess == nil {
|
||||
response.OrigSid = "*"
|
||||
|
||||
@@ -881,6 +881,8 @@ type ServerComMessage struct {
|
||||
PreserveTopicName bool `json:"-"`
|
||||
// User id affected by this message.
|
||||
uid types.Uid
|
||||
// Timestamp when this message entered a master-to-proxy multiplex queue.
|
||||
clusterQueuedAt time.Time
|
||||
}
|
||||
|
||||
// Deep-shallow copy of ServerComMessage. Deep copy of service fields,
|
||||
@@ -899,6 +901,7 @@ func (src *ServerComMessage) copy() *ServerComMessage {
|
||||
SkipSid: src.SkipSid,
|
||||
PreserveTopicName: src.PreserveTopicName,
|
||||
uid: src.uid,
|
||||
clusterQueuedAt: src.clusterQueuedAt,
|
||||
}
|
||||
|
||||
dst.Ctrl = src.Ctrl.copy()
|
||||
|
||||
@@ -405,6 +405,7 @@ type TenantBusinessAdapter interface {
|
||||
TenantGroupMemberUpsert(t.TenantID, *t.GroupMember) (*t.GroupMember, error)
|
||||
TenantGroupMemberGet(t.TenantID, string, t.Uid) (*t.GroupMember, error)
|
||||
TenantGroupMemberList(t.TenantID, string, t.GroupMemberListOptions) ([]t.GroupMember, string, error)
|
||||
TenantGroupMembersCount(t.TenantID, []string) (map[string]int64, error)
|
||||
TenantGroupMemberUpdate(t.TenantID, string, t.Uid, t.GroupMemberPatch) (*t.GroupMember, error)
|
||||
TenantUsersShareOpenPrivateGroup(t.TenantID, t.Uid, t.Uid) (bool, error)
|
||||
TenantGroupInvitationCreate(t.TenantID, *t.GroupInvitation) (*t.GroupInvitation, error)
|
||||
|
||||
@@ -2259,57 +2259,120 @@ func (a *adapter) TenantMessageSave(id t.TenantID, msg *t.Message) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// schemaColumnContract describes the required shape of one column.
|
||||
type schemaColumnContract struct {
|
||||
dataType string
|
||||
notNull bool
|
||||
autoInc bool
|
||||
}
|
||||
|
||||
func (c schemaColumnContract) describe() string {
|
||||
out := strings.ToUpper(c.dataType)
|
||||
if c.notNull {
|
||||
out += " NOT NULL"
|
||||
}
|
||||
if c.autoInc {
|
||||
out += " AUTO_INCREMENT"
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// TenantMessageSchemaValidate verifies the storage contracts asynchronous
|
||||
// persistence depends on: message ids and topic sequence counters are written
|
||||
// as explicit snowflake int64 values, and file-message links reference them.
|
||||
// A column that is still INT overflows on the first snowflake write and, under
|
||||
// Kafka persistence, stalls the consumer partition instead of failing fast.
|
||||
func (a *adapter) TenantMessageSchemaValidate(tenantIDs []t.TenantID) error {
|
||||
if len(tenantIDs) == 0 {
|
||||
return errors.New("mysql: no active tenants available for message schema validation")
|
||||
}
|
||||
// Message ids are explicit snowflake values stored in the auto-increment
|
||||
// id column; sequence counters and file ids are snowflake-derived too.
|
||||
var (
|
||||
bigintPK = schemaColumnContract{dataType: "bigint", notNull: true, autoInc: true}
|
||||
bigintReq = schemaColumnContract{dataType: "bigint", notNull: true}
|
||||
bigintOpt = schemaColumnContract{dataType: "bigint"}
|
||||
)
|
||||
type scopedCheck struct {
|
||||
table string
|
||||
column string
|
||||
want schemaColumnContract
|
||||
}
|
||||
perTenant := func(tenantID t.TenantID, scopedDB bool) []scopedCheck {
|
||||
msgTable, topicsTable, subsTable := tenantMessagesTable(tenantID), tenantTopicsTable(tenantID), tenantSubscriptionsTable(tenantID)
|
||||
if scopedDB {
|
||||
msgTable, topicsTable, subsTable = "im_messages", "im_topics", "im_subscriptions"
|
||||
}
|
||||
return []scopedCheck{
|
||||
{msgTable, "id", bigintPK},
|
||||
{topicsTable, "seqid", bigintReq},
|
||||
{subsTable, "recvseqid", bigintOpt},
|
||||
{subsTable, "readseqid", bigintOpt},
|
||||
// Global file tables exist once per database and are shared by all tenants.
|
||||
{"im_filemsglinks", "fileid", bigintReq},
|
||||
{"im_filemsglinks", "msgid", bigintOpt},
|
||||
{"im_fileuploads", "id", bigintReq},
|
||||
}
|
||||
}
|
||||
validated := make(map[*adapter]map[string]struct{})
|
||||
run := func(tenantAdp *adapter, chk scopedCheck) error {
|
||||
key := chk.table + "." + chk.column
|
||||
if _, ok := validated[tenantAdp][key]; ok {
|
||||
return nil
|
||||
}
|
||||
if err := tenantAdp.validateColumnContract(chk.table, chk.column, chk.want); err != nil {
|
||||
return err
|
||||
}
|
||||
if validated[tenantAdp] == nil {
|
||||
validated[tenantAdp] = make(map[string]struct{})
|
||||
}
|
||||
validated[tenantAdp][key] = struct{}{}
|
||||
return nil
|
||||
}
|
||||
for _, tenantID := range tenantIDs {
|
||||
tenantAdp, err := a.routedTenantAdapter(tenantID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("mysql: resolve tenant %d for message schema validation: %w", tenantID, err)
|
||||
}
|
||||
table := tenantMessagesTable(tenantID)
|
||||
if tenantAdp.tenantScopedDB {
|
||||
table = "im_messages"
|
||||
}
|
||||
if validated[tenantAdp] == nil {
|
||||
validated[tenantAdp] = make(map[string]struct{})
|
||||
}
|
||||
if _, ok := validated[tenantAdp][table]; ok {
|
||||
continue
|
||||
}
|
||||
if err = tenantAdp.validateMessageIDColumn(table); err != nil {
|
||||
for _, chk := range perTenant(tenantID, tenantAdp.tenantScopedDB) {
|
||||
if err = run(tenantAdp, chk); err != nil {
|
||||
return err
|
||||
}
|
||||
validated[tenantAdp][table] = struct{}{}
|
||||
}
|
||||
}
|
||||
if !a.tenantScopedDB {
|
||||
if err := a.validateMessageIDColumn("im_messages"); err != nil {
|
||||
for _, chk := range perTenant(0, true) {
|
||||
if err := run(a, chk); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *adapter) validateMessageIDColumn(table string) error {
|
||||
if !regexp.MustCompile(`^im_(?:[0-9]+_)?messages$`).MatchString(table) {
|
||||
return fmt.Errorf("mysql: unsafe message table name %q", table)
|
||||
// validateColumnContract fails when the live column shape does not match the
|
||||
// contract. The caller turns the error into a fatal startup failure so a
|
||||
// drifted schema is caught before the first write instead of as a stuck
|
||||
// persistence partition.
|
||||
func (a *adapter) validateColumnContract(table, column string, want schemaColumnContract) error {
|
||||
if !regexp.MustCompile(`^im_(?:[0-9]{3}_)?[a-z_]+$`).MatchString(table) {
|
||||
return fmt.Errorf("mysql: unsafe table name %q for schema validation", table)
|
||||
}
|
||||
var dataType, nullable, extra string
|
||||
err := a.db.QueryRow(`SELECT DATA_TYPE,IS_NULLABLE,EXTRA FROM information_schema.COLUMNS
|
||||
WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? AND COLUMN_NAME='id'`, table).
|
||||
WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? AND COLUMN_NAME=?`, table, column).
|
||||
Scan(&dataType, &nullable, &extra)
|
||||
if err == sql.ErrNoRows {
|
||||
return fmt.Errorf("mysql: required message table %s is missing", table)
|
||||
return fmt.Errorf("mysql: required table %s is missing", table)
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("mysql: inspect %s.id: %w", table, err)
|
||||
return fmt.Errorf("mysql: inspect %s.%s: %w", table, column, err)
|
||||
}
|
||||
if !strings.EqualFold(dataType, "bigint") || !strings.EqualFold(nullable, "NO") ||
|
||||
!strings.Contains(strings.ToLower(extra), "auto_increment") {
|
||||
return fmt.Errorf("mysql: incompatible %s.id schema: type=%s nullable=%s extra=%s; want BIGINT NOT NULL AUTO_INCREMENT",
|
||||
table, dataType, nullable, extra)
|
||||
if !strings.EqualFold(dataType, want.dataType) || want.notNull != strings.EqualFold(nullable, "NO") ||
|
||||
(want.autoInc && !strings.Contains(strings.ToLower(extra), "auto_increment")) {
|
||||
return fmt.Errorf("mysql: incompatible %s.%s schema: type=%s nullable=%s extra=%s; want %s "+
|
||||
"(apply scripts/IM-租户业务库初始化.sql equivalents or the matching BIGINT migration before starting)",
|
||||
table, column, dataType, nullable, extra, want.describe())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -6887,7 +6950,7 @@ func (a *adapter) TenantFriendRequestAccept(id t.TenantID, requestID string, act
|
||||
eventType := "friend.request." + t.FriendRequestAccepted
|
||||
applicantEventID := t.StableEventID("relation", row.RequestID+":"+t.FriendRequestAccepted, eventType, applicant.UserId())
|
||||
if err = insertSystemNotificationTx(tx, id, applicant, t.StableEventID(applicantEventID, "notification"), eventType,
|
||||
"好友申请已处理", t.FriendRequestAccepted, map[string]any{"type": "friend_request", "request_id": row.RequestID, "user_id": actor.UserId(), "topic": topic}); err != nil {
|
||||
"好友申请已处理", "对方已同意你的好友申请", map[string]any{"type": "friend_request", "request_id": row.RequestID, "user_id": actor.UserId(), "topic": topic}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err = tx.Commit(); err != nil {
|
||||
@@ -6939,8 +7002,12 @@ func (a *adapter) TenantFriendRequestResolve(id t.TenantID, requestID string, ac
|
||||
}
|
||||
eventType := "friend.request." + status
|
||||
eventID := t.StableEventID("relation", req.RequestID+":"+status, eventType, req.ApplicantID.UserId())
|
||||
summary := "对方已婉拒你的好友申请"
|
||||
if status == t.FriendRequestAccepted {
|
||||
summary = "对方已同意你的好友申请"
|
||||
}
|
||||
if err = insertSystemNotificationTx(tx, id, req.ApplicantID, t.StableEventID(eventID, "notification"), eventType,
|
||||
"好友申请已处理", status, map[string]any{"type": "friend_request", "request_id": req.RequestID, "user_id": actor.UserId()}); err != nil {
|
||||
"好友申请已处理", summary, map[string]any{"type": "friend_request", "request_id": req.RequestID, "user_id": actor.UserId()}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err = tx.Commit(); err != nil {
|
||||
|
||||
@@ -394,6 +394,50 @@ func (a *adapter) TenantGroupMemberList(id t.TenantID, topic string, opts t.Grou
|
||||
return items, next, rows.Err()
|
||||
}
|
||||
|
||||
// TenantGroupMembersCount counts active members for the given topics using the
|
||||
// same visibility rules as TenantGroupMemberList: membership active, subscription
|
||||
// not deleted, user not deleted. Returns counts keyed by topic; topics with no
|
||||
// active members are absent from the map.
|
||||
func (a *adapter) TenantGroupMembersCount(id t.TenantID, topics []string) (map[string]int64, error) {
|
||||
if tenantAdp, err := a.routeGroupTenant(id); err != nil {
|
||||
return nil, err
|
||||
} else if tenantAdp != a {
|
||||
return tenantAdp.TenantGroupMembersCount(id, topics)
|
||||
}
|
||||
if len(topics) == 0 {
|
||||
return map[string]int64{}, nil
|
||||
}
|
||||
placeholders := strings.TrimSuffix(strings.Repeat("?,", len(topics)), ",")
|
||||
args := make([]any, 0, len(topics)+3)
|
||||
args = append(args, t.StateDeleted, id, t.GroupMemberStatusActive)
|
||||
for _, topic := range topics {
|
||||
args = append(args, topic)
|
||||
}
|
||||
ctx, cancel := a.getContext()
|
||||
if cancel != nil {
|
||||
defer cancel()
|
||||
}
|
||||
rows, err := a.db.QueryContext(ctx, tenantShardSQL(id, `SELECT gm.topic, COUNT(DISTINCT gm.user_id) FROM im_group_members AS gm
|
||||
JOIN im_subscriptions AS s ON s.tenant_id=gm.tenant_id AND s.topic=gm.topic AND s.userid=gm.user_id AND s.deletedat IS NULL
|
||||
JOIN im_users AS u ON u.tenant_id=gm.tenant_id AND u.id=gm.user_id AND u.state!=?
|
||||
WHERE gm.tenant_id=? AND gm.member_status=? AND gm.topic IN (`+placeholders+`)
|
||||
GROUP BY gm.topic`), args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
counts := make(map[string]int64, len(topics))
|
||||
for rows.Next() {
|
||||
var topic string
|
||||
var count int64
|
||||
if err = rows.Scan(&topic, &count); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
counts[topic] = count
|
||||
}
|
||||
return counts, rows.Err()
|
||||
}
|
||||
|
||||
func (a *adapter) TenantUsersShareOpenPrivateGroup(id t.TenantID, userID, peerID t.Uid) (bool, error) {
|
||||
if tenantAdp, err := a.routeGroupTenant(id); err != nil {
|
||||
return false, err
|
||||
|
||||
+38
-1
@@ -239,6 +239,9 @@ func largeFileReceiveHTTP(wrt http.ResponseWriter, req *http.Request) {
|
||||
}
|
||||
|
||||
msgID := req.FormValue("id")
|
||||
// When the uploader passes a conversation topic, the file is linked to it so other
|
||||
// members of that conversation can download it (see TenantFileUserHasAccess).
|
||||
uploadTopic := strings.TrimSpace(req.FormValue("topic"))
|
||||
// Check authorization: either auth information or SID must be present
|
||||
authMethod, secret := getHttpAuth(req)
|
||||
principal, challenge, err := authFileRequest(authMethod, secret, req.FormValue("sid"), getRemoteAddr(req))
|
||||
@@ -250,11 +253,17 @@ func largeFileReceiveHTTP(wrt http.ResponseWriter, req *http.Request) {
|
||||
writeHttpResponse(InfoChallenge(msgID, now, challenge), nil)
|
||||
return
|
||||
}
|
||||
if principal.UID.IsZero() && req.FormValue("topic") != "newacc" {
|
||||
if principal.UID.IsZero() && uploadTopic != "newacc" {
|
||||
// Not authenticated and not signup.
|
||||
writeHttpResponse(ErrAuthRequired(msgID, "", now, now), nil)
|
||||
return
|
||||
}
|
||||
if !principal.UID.IsZero() && uploadTopic != "" {
|
||||
if err = authorizeUploadTopicLink(principal.TenantID, principal.UID, uploadTopic); err != nil {
|
||||
writeHttpResponse(decodeStoreError(err, msgID, now, nil), err)
|
||||
return
|
||||
}
|
||||
}
|
||||
// Check if uploads are handled elsewhere.
|
||||
headers, statusCode, err := mh.Headers(principal.TenantID, req.Method, req.URL, req.Header, true)
|
||||
if err != nil {
|
||||
@@ -361,6 +370,13 @@ func largeFileReceiveHTTP(wrt http.ResponseWriter, req *http.Request) {
|
||||
logs.Warn.Printf("media upload: failed to enqueue file lifecycle event file=%s: %v", fdef.Id, err)
|
||||
}
|
||||
|
||||
if uploadTopic != "" {
|
||||
if err = store.Files.LinkAttachments(principal.TenantID, uploadTopic, types.ZeroUid, []string{fdef.Uid().String()}); err != nil {
|
||||
// The file itself is valid; linking is best-effort so do not fail the upload.
|
||||
logs.Warn.Printf("media upload: failed to link file %s to topic %s: %v", fdef.Id, uploadTopic, err)
|
||||
}
|
||||
}
|
||||
|
||||
params := map[string]any{"url": url, "file_id": fdef.Uid().String()}
|
||||
if globals.mediaGcPeriod > 0 {
|
||||
// How long this file is guaranteed to exist without being attached to a message or a topic.
|
||||
@@ -484,6 +500,27 @@ func authorizeLargeFileRequestURL(tenantID types.TenantID, uid types.Uid, rawURL
|
||||
return authorizeLargeFileDownload(tenantID, uid, rawURL, fd)
|
||||
}
|
||||
|
||||
// authorizeUploadTopicLink verifies the uploader is a reading member of the conversation
|
||||
// topic, so the uploaded file may be linked to it and downloaded by other members.
|
||||
func authorizeUploadTopicLink(tenantID types.TenantID, uid types.Uid, topic string) error {
|
||||
switch types.GetTopicCat(topic) {
|
||||
case types.TopicCatP2P, types.TopicCatGrp:
|
||||
default:
|
||||
return types.ErrPermissionDenied
|
||||
}
|
||||
if store.Store == nil || !store.Store.IsOpen() {
|
||||
return types.ErrUnsupported
|
||||
}
|
||||
sub, err := store.Subs.Get(tenantID, topic, uid, false)
|
||||
if err != nil || sub == nil || sub.DeletedAt != nil {
|
||||
return types.ErrPermissionDenied
|
||||
}
|
||||
if !sub.ModeWant.IsReader() || !sub.ModeGiven.IsReader() {
|
||||
return types.ErrPermissionDenied
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func authorizeLargeFileDownload(tenantID types.TenantID, uid types.Uid, rawURL string, fd *types.FileDef) error {
|
||||
if fd == nil {
|
||||
return types.ErrNotFound
|
||||
|
||||
@@ -105,7 +105,9 @@ func (sess *Session) readLoop() {
|
||||
atomic.StoreInt64(&sess.lastAction, types.TimeNow().UnixNano())
|
||||
_ = sess.ws.SetReadDeadline(time.Now().Add(pongWait))
|
||||
statsInc("IncomingMessagesWebsockTotal", 1)
|
||||
dispatchStarted := time.Now()
|
||||
sess.dispatchRaw(raw)
|
||||
immetrics.ObserveWSReadDispatch(dispatchStarted)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -160,6 +162,11 @@ func (sess *Session) writeLoop() {
|
||||
if isAcceptedCtrl(v) {
|
||||
immetrics.ObserveCtrlACKDelivery("write", v.Timestamp)
|
||||
}
|
||||
case sharedServerMessage:
|
||||
statsAddHistSample("OutgoingMessageSize", float64(len(v)))
|
||||
if !sess.sendMessage([]byte(v)) {
|
||||
return
|
||||
}
|
||||
default: // serialized message
|
||||
if !sess.sendMessage(v) {
|
||||
return
|
||||
@@ -207,11 +214,22 @@ func (sess *Session) writeWSFrame(mt int, msg any) error {
|
||||
}
|
||||
reason := classifyWSError(err)
|
||||
immetrics.ObserveWSWrite(frame, "error", reason, started, payloadBytes)
|
||||
if shouldLogWSWriteError(reason) {
|
||||
logs.Err.Printf("ws: writeLoop sid=%s uid=%s frame=%s queue=%d capacity=%d reason=%s err=%v",
|
||||
sess.sid, sess.uid.UserId(), frame, len(sess.send), cap(sess.send), reason, err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func shouldLogWSWriteError(reason string) bool {
|
||||
switch reason {
|
||||
case "close_normal", "close_going_away", "closed", "eof", "broken_pipe", "connection_reset":
|
||||
return false
|
||||
default:
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
func classifyWSError(err error) string {
|
||||
if err == nil {
|
||||
return "none"
|
||||
|
||||
@@ -37,6 +37,31 @@ func TestClassifyWSError(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestShouldLogWSWriteError(t *testing.T) {
|
||||
tests := []struct {
|
||||
reason string
|
||||
want bool
|
||||
}{
|
||||
{reason: "close_normal", want: false},
|
||||
{reason: "close_going_away", want: false},
|
||||
{reason: "closed", want: false},
|
||||
{reason: "eof", want: false},
|
||||
{reason: "broken_pipe", want: false},
|
||||
{reason: "connection_reset", want: false},
|
||||
{reason: "timeout", want: true},
|
||||
{reason: "close_abnormal", want: true},
|
||||
{reason: "other", want: true},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.reason, func(t *testing.T) {
|
||||
if got := shouldLogWSWriteError(tc.reason); got != tc.want {
|
||||
t.Fatalf("shouldLogWSWriteError(%q)=%v want %v", tc.reason, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestWebSocketOutboundQueueTiming(t *testing.T) {
|
||||
sess := &Session{proto: WEBSOCK, send: make(chan any, 2)}
|
||||
payload := []byte("message")
|
||||
|
||||
+42
-2
@@ -10,6 +10,8 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -30,6 +32,44 @@ var requestLatencyDistribution = []float64{1, 2, 3, 4, 5, 6, 8, 10, 13, 16, 20,
|
||||
var outgoingMessageSizeDistribution = []float64{1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 16384,
|
||||
65536, 262144, 1048576, 4194304, 16777216, 67108864, 268435456, 1073741824, 4294967296}
|
||||
|
||||
const (
|
||||
defaultTopicClientQueue = 4096
|
||||
defaultTopicServerQueue = 512
|
||||
defaultTopicPublishWait = 200 * time.Millisecond
|
||||
topicClientQueueEnv = "TOPIC_CLIENT_QUEUE_BUFFER"
|
||||
topicServerQueueEnv = "TOPIC_SERVER_QUEUE_BUFFER"
|
||||
topicPublishEnqueueWaitEnv = "TOPIC_PUBLISH_ENQUEUE_WAIT_MS"
|
||||
)
|
||||
|
||||
var topicQueueConfig = loadTopicQueueConfig()
|
||||
|
||||
type topicQueueRuntimeConfig struct {
|
||||
clientBuffer int
|
||||
serverBuffer int
|
||||
publishEnqueueWait time.Duration
|
||||
}
|
||||
|
||||
func loadTopicQueueConfig() topicQueueRuntimeConfig {
|
||||
return topicQueueRuntimeConfig{
|
||||
clientBuffer: intEnvOrDefault(topicClientQueueEnv, defaultTopicClientQueue),
|
||||
serverBuffer: intEnvOrDefault(topicServerQueueEnv, defaultTopicServerQueue),
|
||||
publishEnqueueWait: time.Duration(intEnvOrDefault(topicPublishEnqueueWaitEnv, int(defaultTopicPublishWait/time.Millisecond))) * time.Millisecond,
|
||||
}
|
||||
}
|
||||
|
||||
func intEnvOrDefault(name string, fallback int) int {
|
||||
raw := strings.TrimSpace(os.Getenv(name))
|
||||
if raw == "" {
|
||||
return fallback
|
||||
}
|
||||
parsed, err := strconv.Atoi(raw)
|
||||
if err != nil || parsed <= 0 {
|
||||
logs.Warn.Printf("invalid %s=%q, using %d", name, raw, fallback)
|
||||
return fallback
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
|
||||
// Request to hub to remove the topic
|
||||
type topicUnreg struct {
|
||||
// Tenant which owns the topic.
|
||||
@@ -261,8 +301,8 @@ func (h *Hub) run() {
|
||||
// Indicates a proxy topic.
|
||||
isProxy: globals.cluster.isRemoteTopic(key),
|
||||
sessions: make(map[*Session]perSessionData),
|
||||
clientMsg: make(chan *ClientComMessage, 192),
|
||||
serverMsg: make(chan *ServerComMessage, 64),
|
||||
clientMsg: make(chan *ClientComMessage, topicQueueConfig.clientBuffer),
|
||||
serverMsg: make(chan *ServerComMessage, topicQueueConfig.serverBuffer),
|
||||
reg: make(chan *ClientComMessage, 256),
|
||||
unreg: make(chan *ClientComMessage, 256),
|
||||
meta: make(chan *ClientComMessage, 64),
|
||||
|
||||
+72
-14
@@ -1494,8 +1494,12 @@ func serveIMRiskChallengeConfirm(wrt http.ResponseWriter, req *http.Request) {
|
||||
if imReliableHTTPEventsEnabled(req) {
|
||||
eventType := "security.risk." + decision
|
||||
eventID := stableIMEventID("risk", challengeID, decision, challenge.State)
|
||||
summary := "你已取消本次操作"
|
||||
if decision == "continue" {
|
||||
summary = "你已确认继续执行该操作"
|
||||
}
|
||||
if err = createIMSystemNotification(principal.TenantID, principal.Uid,
|
||||
stableIMEventID(eventID, "notification"), eventType, "安全操作确认", decision,
|
||||
stableIMEventID(eventID, "notification"), eventType, "安全操作确认", summary,
|
||||
types.KVMap{"risk_challenge_id": challengeID}); err != nil {
|
||||
writeIMError(wrt, req, http.StatusInternalServerError, 90000, imReasonInternal, "internal error", nil)
|
||||
return
|
||||
@@ -2283,10 +2287,18 @@ func serveIMAppReleaseLatest(wrt http.ResponseWriter, req *http.Request) {
|
||||
}
|
||||
platform := strings.ToLower(strings.TrimSpace(req.URL.Query().Get("platform")))
|
||||
channel := strings.ToLower(strings.TrimSpace(req.URL.Query().Get("channel")))
|
||||
appCode := strings.ToLower(strings.TrimSpace(req.URL.Query().Get("app_code")))
|
||||
currentVersion := strings.TrimSpace(req.URL.Query().Get("current_version"))
|
||||
if channel == "" {
|
||||
channel = "stable"
|
||||
}
|
||||
if appCode == "" {
|
||||
appCode = "im"
|
||||
}
|
||||
if !isValidAppCode(appCode) {
|
||||
writeIMError(wrt, req, http.StatusBadRequest, 30000, imReasonMalformedRequest, "malformed request", map[string]any{"field": "app_code"})
|
||||
return
|
||||
}
|
||||
if platform != "ios" && platform != "android" && platform != "web" {
|
||||
writeIMError(wrt, req, http.StatusBadRequest, 30000, imReasonMalformedRequest, "malformed request", map[string]any{"field": "platform"})
|
||||
return
|
||||
@@ -2296,7 +2308,7 @@ func serveIMAppReleaseLatest(wrt http.ResponseWriter, req *http.Request) {
|
||||
writeIMError(wrt, req, http.StatusBadRequest, 30000, imReasonMalformedRequest, "malformed request", map[string]any{"field": "current_version"})
|
||||
return
|
||||
}
|
||||
release, err := store.AppReleases.Latest(platform, channel)
|
||||
release, err := store.AppReleases.Latest(appCode, platform, channel)
|
||||
if err != nil {
|
||||
if platform == "web" {
|
||||
logs.Warn.Println("im app release latest: using web fallback", err)
|
||||
@@ -4205,6 +4217,25 @@ func serveIMGroupList(wrt http.ResponseWriter, req *http.Request) {
|
||||
items = append(items, item)
|
||||
seen++
|
||||
}
|
||||
if len(items) > 0 {
|
||||
listTopics := make([]string, 0, len(items))
|
||||
for _, item := range items {
|
||||
if topic, ok := item["topic"].(string); ok && topic != "" {
|
||||
listTopics = append(listTopics, topic)
|
||||
}
|
||||
}
|
||||
if counts, err := store.GroupBusiness.MembersCount(principal.TenantID, listTopics); err == nil {
|
||||
for _, item := range items {
|
||||
if topic, ok := item["topic"].(string); ok {
|
||||
if count, ok := counts[topic]; ok {
|
||||
item["member_count"] = count
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
logs.Warn.Printf("serveIMGroupList: failed to count members for %d topics: %v", len(listTopics), err)
|
||||
}
|
||||
}
|
||||
writeIMSuccess(wrt, req, map[string]any{"items": items, "next_cursor": nextCursor, "has_more": nextCursor != ""})
|
||||
}
|
||||
|
||||
@@ -4298,7 +4329,7 @@ func serveIMGroupCreate(wrt http.ResponseWriter, req *http.Request) {
|
||||
writeIMRelationError(wrt, req, err)
|
||||
return
|
||||
}
|
||||
writeIMSuccess(wrt, req, groupDetailPayload(created, selfSub))
|
||||
writeIMSuccess(wrt, req, groupDetailPayload(created, selfSub, groupMemberCountFor(principal.TenantID, topicName)))
|
||||
}
|
||||
|
||||
func serveIMGroupResource(wrt http.ResponseWriter, req *http.Request) {
|
||||
@@ -4595,7 +4626,7 @@ func serveIMGroupDetail(wrt http.ResponseWriter, req *http.Request, topic string
|
||||
writeIMError(wrt, req, http.StatusNotFound, 30004, imReasonMalformedRequest, "group not found", nil)
|
||||
return
|
||||
}
|
||||
payload := groupDetailPayload(group, selfSub)
|
||||
payload := groupDetailPayload(group, selfSub, groupMemberCountFor(principal.TenantID, topic))
|
||||
if settings, err := store.GroupBusiness.GetSettings(principal.TenantID, topic); err == nil {
|
||||
if settings != nil && settings.GroupStatus == types.GroupStatusDissolved {
|
||||
writeIMError(wrt, req, http.StatusGone, 30004, imReasonGroupDissolved, "group dissolved", map[string]any{"topic": topic})
|
||||
@@ -4691,7 +4722,7 @@ func serveIMGroupDetail(wrt http.ResponseWriter, req *http.Request, topic string
|
||||
writeIMError(wrt, req, http.StatusInternalServerError, 90000, imReasonInternal, "internal error", nil)
|
||||
return
|
||||
}
|
||||
payload := groupDetailPayload(updated, selfSub)
|
||||
payload := groupDetailPayload(updated, selfSub, groupMemberCountFor(principal.TenantID, topic))
|
||||
if settingsUpdated {
|
||||
if settings, err := store.GroupBusiness.GetSettings(principal.TenantID, topic); err == nil {
|
||||
payload = mergeGroupSettingsPayload(payload, settings)
|
||||
@@ -4757,7 +4788,7 @@ func serveIMGroupSettings(wrt http.ResponseWriter, req *http.Request, topic stri
|
||||
writeIMError(wrt, req, http.StatusInternalServerError, 90000, imReasonInternal, "internal error", nil)
|
||||
return
|
||||
}
|
||||
payload := groupSettingsPayload(group, selfSub)
|
||||
payload := groupSettingsPayload(group, selfSub, groupMemberCountFor(principal.TenantID, topic))
|
||||
if settings, err := store.GroupBusiness.GetSettings(principal.TenantID, topic); err == nil {
|
||||
payload = mergeGroupSettingsPayload(payload, settings)
|
||||
payload["governance"] = groupGovernancePayloadFromSettings(settings, selfSub)
|
||||
@@ -4934,7 +4965,7 @@ func serveIMGroupQRCodes(wrt http.ResponseWriter, req *http.Request, topic strin
|
||||
writeIMError(wrt, req, http.StatusInternalServerError, 90000, imReasonInternal, "internal error", nil)
|
||||
return
|
||||
}
|
||||
groupPayload := groupDetailPayload(group, selfSub)
|
||||
groupPayload := groupDetailPayload(group, selfSub, groupMemberCountFor(principal.TenantID, topic))
|
||||
writeIMSuccess(wrt, req, map[string]any{
|
||||
"qr_token": token,
|
||||
"qr_payload": "im://qr?token=" + url.QueryEscape(token) + "&purpose=join_group",
|
||||
@@ -6076,7 +6107,7 @@ func serveIMQRCodesResolve(wrt http.ResponseWriter, req *http.Request) {
|
||||
if selfSub != nil && (selfSub.ModeGiven & selfSub.ModeWant).IsJoiner() {
|
||||
status = "joined"
|
||||
}
|
||||
groupPayload := groupDetailPayload(group, selfSub)
|
||||
groupPayload := groupDetailPayload(group, selfSub, groupMemberCountFor(principal.TenantID, claims.Topic))
|
||||
writeIMSuccess(wrt, req, map[string]any{
|
||||
"purpose": claims.Purpose,
|
||||
"expires_at": time.Unix(claims.Expires, 0).UTC().Format(time.RFC3339Nano),
|
||||
@@ -6410,11 +6441,13 @@ func serveIMFriendRequestAction(wrt http.ResponseWriter, req *http.Request) {
|
||||
if imReliableHTTPEventsEnabled(req) {
|
||||
eventType := "friend.request." + status
|
||||
action := types.KVMap{}
|
||||
summary := "对方已婉拒你的好友申请"
|
||||
if status == types.FriendRequestAccepted {
|
||||
action["topic"] = updated.ApplicantID.P2PName(updated.TargetID)
|
||||
summary = "对方已同意你的好友申请"
|
||||
}
|
||||
if err = publishIMRelationEventWithAction(principal.TenantID, principal.Uid, updated.ApplicantID,
|
||||
updated.RequestID+":"+status, eventType, "好友申请已处理", status, action); err != nil {
|
||||
updated.RequestID+":"+status, eventType, "好友申请已处理", summary, action); err != nil {
|
||||
writeIMError(wrt, req, http.StatusInternalServerError, 90000, imReasonInternal, "internal error", nil)
|
||||
return
|
||||
}
|
||||
@@ -7491,15 +7524,27 @@ func mergeGroupListSettingsPayload(payload map[string]any, settings *types.Group
|
||||
return payload
|
||||
}
|
||||
|
||||
func groupDetailPayload(group *types.Topic, selfSub *types.Subscription) map[string]any {
|
||||
// groupMemberCountFor returns the live member count for a single group topic,
|
||||
// mirroring the visibility rules of the group members list endpoint.
|
||||
func groupMemberCountFor(tenantID types.TenantID, topic string) int64 {
|
||||
if topic == "" {
|
||||
return 0
|
||||
}
|
||||
counts, err := store.GroupBusiness.MembersCount(tenantID, []string{topic})
|
||||
if err != nil {
|
||||
logs.Warn.Printf("group[%s]: failed to count members: %v", topic, err)
|
||||
return 0
|
||||
}
|
||||
return counts[topic]
|
||||
}
|
||||
|
||||
func groupDetailPayload(group *types.Topic, selfSub *types.Subscription, memberCount int64) map[string]any {
|
||||
public := map[string]any{}
|
||||
topic := ""
|
||||
memberCount := 0
|
||||
ownerID := ""
|
||||
if group != nil {
|
||||
public = groupPublicMap(group.Public)
|
||||
topic = group.Id
|
||||
memberCount = group.SubCnt
|
||||
if owner := types.ParseUid(group.Owner); !owner.IsZero() {
|
||||
ownerID = owner.UserId()
|
||||
}
|
||||
@@ -7528,8 +7573,8 @@ func groupDetailPayload(group *types.Topic, selfSub *types.Subscription) map[str
|
||||
}
|
||||
}
|
||||
|
||||
func groupSettingsPayload(group *types.Topic, selfSub *types.Subscription) map[string]any {
|
||||
data := groupDetailPayload(group, selfSub)
|
||||
func groupSettingsPayload(group *types.Topic, selfSub *types.Subscription, memberCount int64) map[string]any {
|
||||
data := groupDetailPayload(group, selfSub, memberCount)
|
||||
private := map[string]any{}
|
||||
if selfSub != nil {
|
||||
private = groupPrivateMap(selfSub.Private)
|
||||
@@ -8776,6 +8821,19 @@ func appReleasePayload(release *types.AppRelease, currentVersion string, hasUpda
|
||||
}
|
||||
}
|
||||
|
||||
func isValidAppCode(raw string) bool {
|
||||
if raw == "" || len(raw) > 32 {
|
||||
return false
|
||||
}
|
||||
for _, r := range raw {
|
||||
if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == '-' || r == '_' {
|
||||
continue
|
||||
}
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func parseNumericAppVersion(raw string) ([]uint64, error) {
|
||||
raw = strings.TrimPrefix(strings.ToLower(strings.TrimSpace(raw)), "v")
|
||||
parts := strings.Split(raw, ".")
|
||||
|
||||
+13
-1
@@ -42,6 +42,7 @@ type testVerificationStore struct {
|
||||
|
||||
type testHTTPInfrastructureStore struct {
|
||||
deadLetters []types.KafkaDeadLetter
|
||||
markSucceededErr error
|
||||
}
|
||||
|
||||
type testTenantEntryStore struct {
|
||||
@@ -110,6 +111,9 @@ func (s *testHTTPInfrastructureStore) BeginKafkaConsume(_ types.TenantID, record
|
||||
}
|
||||
|
||||
func (s *testHTTPInfrastructureStore) MarkKafkaConsumeSucceeded(_ types.TenantID, _, _ string, _ time.Time) error {
|
||||
if s.markSucceededErr != nil {
|
||||
return s.markSucceededErr
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -269,6 +273,7 @@ type testGroupBusinessStore struct {
|
||||
listMembers []types.GroupMember
|
||||
listMembersOpts types.GroupMemberListOptions
|
||||
listMembersNext string
|
||||
membersCounts map[string]int64
|
||||
updatedSettings types.GroupSettingsPatch
|
||||
updatedMembers []types.GroupMemberPatch
|
||||
updatedRoleMode types.AccessMode
|
||||
@@ -350,6 +355,13 @@ func (s *testGroupBusinessStore) ListMembers(_ types.TenantID, _ string, opts ty
|
||||
return s.listMembers, s.listMembersNext, nil
|
||||
}
|
||||
|
||||
func (s *testGroupBusinessStore) MembersCount(_ types.TenantID, _ []string) (map[string]int64, error) {
|
||||
if s.membersCounts == nil {
|
||||
return map[string]int64{}, nil
|
||||
}
|
||||
return s.membersCounts, nil
|
||||
}
|
||||
|
||||
func (s *testGroupBusinessStore) UpdateMember(_ types.TenantID, topic string, uid types.Uid, patch types.GroupMemberPatch) (*types.GroupMember, error) {
|
||||
if s.updateMemberErr != nil {
|
||||
return nil, s.updateMemberErr
|
||||
@@ -726,7 +738,7 @@ func (s *testProfilesStore) UpdatePrivacy(_ types.TenantID, uid types.Uid, patch
|
||||
return privacy, nil
|
||||
}
|
||||
|
||||
func (s *testAppReleaseStore) Latest(platform, channel string) (*types.AppRelease, error) {
|
||||
func (s *testAppReleaseStore) Latest(_, platform, channel string) (*types.AppRelease, error) {
|
||||
if s.err != nil {
|
||||
return nil, s.err
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/tinode/chat/server/logs"
|
||||
@@ -20,6 +21,9 @@ const (
|
||||
|
||||
imYouthActionMessage = "message"
|
||||
imYouthActionCall = "call"
|
||||
|
||||
imSecurityPolicyCacheTTL = 5 * time.Second
|
||||
imSecurityPolicyErrorCacheTTL = 2 * time.Second
|
||||
)
|
||||
|
||||
type imPolicyDecision struct {
|
||||
@@ -29,6 +33,17 @@ type imPolicyDecision struct {
|
||||
Details map[string]any
|
||||
}
|
||||
|
||||
type imSecurityPolicyCacheEntry struct {
|
||||
policy *types.SecurityPolicy
|
||||
err error
|
||||
expires time.Time
|
||||
}
|
||||
|
||||
var imSecurityPolicyCache = struct {
|
||||
sync.Mutex
|
||||
entries map[string]imSecurityPolicyCacheEntry
|
||||
}{entries: make(map[string]imSecurityPolicyCacheEntry)}
|
||||
|
||||
func imAllowDecision() imPolicyDecision {
|
||||
return imPolicyDecision{Allowed: true}
|
||||
}
|
||||
@@ -119,7 +134,28 @@ func imSecurityPolicy(tenantID types.TenantID, tenantCode string) (*types.Securi
|
||||
if tenantCode == "" {
|
||||
return nil, types.ErrNotFound
|
||||
}
|
||||
key := fmt.Sprintf("%d:%s", tenantID, tenantCode)
|
||||
now := time.Now()
|
||||
imSecurityPolicyCache.Lock()
|
||||
if cached, ok := imSecurityPolicyCache.entries[key]; ok && now.Before(cached.expires) {
|
||||
imSecurityPolicyCache.Unlock()
|
||||
return cached.policy, cached.err
|
||||
}
|
||||
if store.ModerationConfig == nil {
|
||||
imSecurityPolicyCache.Unlock()
|
||||
return nil, types.ErrUnsupported
|
||||
}
|
||||
policy, err := store.ModerationConfig.SecurityPolicy(tenantCode)
|
||||
ttl := imSecurityPolicyCacheTTL
|
||||
if err != nil {
|
||||
ttl = imSecurityPolicyErrorCacheTTL
|
||||
}
|
||||
imSecurityPolicyCache.entries[key] = imSecurityPolicyCacheEntry{
|
||||
policy: policy,
|
||||
err: err,
|
||||
expires: now.Add(ttl),
|
||||
}
|
||||
imSecurityPolicyCache.Unlock()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -93,6 +93,12 @@ func TestEnqueueKafkaAuditEventPublishesCrossSystemEvent(t *testing.T) {
|
||||
func TestEnqueueKafkaFileAttachmentsBindRunsSynchronously(t *testing.T) {
|
||||
ctrl := gomock.NewController(t)
|
||||
defer ctrl.Finish()
|
||||
var uidGenerator types.UidGenerator
|
||||
if err := uidGenerator.Init(1, []byte("0123456789abcdef")); err != nil {
|
||||
t.Fatalf("initialize UID generator: %v", err)
|
||||
}
|
||||
store.SetTestUidGenerator(uidGenerator)
|
||||
defer store.SetTestUidGenerator(types.UidGenerator{})
|
||||
|
||||
oldKafkaBus := kafkaBus
|
||||
oldFiles := store.Files
|
||||
@@ -104,7 +110,8 @@ func TestEnqueueKafkaFileAttachmentsBindRunsSynchronously(t *testing.T) {
|
||||
kafkaBus = &kafkaRuntime{}
|
||||
files := mock_store.NewMockFilePersistenceInterface(ctrl)
|
||||
store.Files = files
|
||||
msgID := types.Uid(101)
|
||||
databaseMsgID := types.Uid(101)
|
||||
msgID := store.EncodeUid(int64(databaseMsgID))
|
||||
msg := &types.Message{
|
||||
ObjHeader: types.ObjHeader{Id: msgID.String(), CreatedAt: time.Date(2026, 9, 1, 10, 0, 0, 0, time.UTC)},
|
||||
TenantID: 7,
|
||||
@@ -114,7 +121,7 @@ func TestEnqueueKafkaFileAttachmentsBindRunsSynchronously(t *testing.T) {
|
||||
Head: types.KVMap{"x-im-type": "image"},
|
||||
}
|
||||
attachments := []string{"https://api.118im.com/v1/im/files/file_001"}
|
||||
files.EXPECT().LinkAttachments(types.TenantID(7), "", msgID, attachments).Return(nil)
|
||||
files.EXPECT().LinkAttachments(types.TenantID(7), "", databaseMsgID, attachments).Return(nil)
|
||||
|
||||
if err := enqueueKafkaFileAttachmentsBind(context.Background(), 7, msg, attachments); err != nil {
|
||||
t.Fatalf("enqueueKafkaFileAttachmentsBind returned error: %v", err)
|
||||
|
||||
@@ -128,7 +128,14 @@ func executeKafkaFileAttachmentsBind(tenantID types.TenantID, payload kafkaFileL
|
||||
return types.ErrMalformed
|
||||
}
|
||||
if types.GetTopicCat(payload.Topic) != types.TopicCatGrp {
|
||||
return store.Files.LinkAttachments(tenantID, "", msgID, payload.AttachmentURLs)
|
||||
// Kafka persistence writes the decoded UID as the SQL message primary key,
|
||||
// while the lifecycle payload carries the externally encoded UID. The
|
||||
// file-link adapter expects the raw SQL ID, so decode it before binding.
|
||||
databaseMsgID := types.Uid(store.DecodeUid(msgID))
|
||||
if databaseMsgID.IsZero() {
|
||||
return types.ErrMalformed
|
||||
}
|
||||
return store.Files.LinkAttachments(tenantID, "", databaseMsgID, payload.AttachmentURLs)
|
||||
}
|
||||
uploader := types.ParseUserId(payload.UserID)
|
||||
if uploader.IsZero() {
|
||||
|
||||
@@ -159,13 +159,29 @@ type kafkaMessagePersistItem struct {
|
||||
envelope kafkaEnvelope
|
||||
tenantID types.TenantID
|
||||
payload kafkaMessagePersistPayload
|
||||
consume *types.KafkaConsumeRecord
|
||||
}
|
||||
|
||||
type kafkaMessagePersistPreparer func(context.Context, kafka.Message, string) (*kafkaMessagePersistItem, bool, error)
|
||||
type kafkaMessagePersistGroupsWriter func(context.Context, map[string][]*kafkaMessagePersistItem) error
|
||||
|
||||
func handleKafkaMessagePersistBatch(ctx context.Context, messages []kafka.Message, sourceTopicName string) error {
|
||||
return processKafkaMessagePersistBatch(ctx, messages, sourceTopicName,
|
||||
prepareKafkaMessagePersistItem, persistKafkaMessagePersistGroups)
|
||||
}
|
||||
|
||||
func processKafkaMessagePersistBatch(ctx context.Context, messages []kafka.Message, sourceTopicName string,
|
||||
prepare kafkaMessagePersistPreparer, persist kafkaMessagePersistGroupsWriter) error {
|
||||
groups := make(map[string][]*kafkaMessagePersistItem)
|
||||
for _, message := range messages {
|
||||
item, terminal, err := prepareKafkaMessagePersistItem(ctx, message, sourceTopicName)
|
||||
item, terminal, err := prepare(ctx, message, sourceTopicName)
|
||||
if err != nil {
|
||||
// Preparing an item acquires its idempotency claim. If a later item
|
||||
// fails, finish all earlier claimed items before retrying the batch;
|
||||
// otherwise retries can keep colliding with those processing claims.
|
||||
if flushErr := persist(ctx, groups); flushErr != nil {
|
||||
return fmt.Errorf("persist claimed Kafka messages after prepare failure: %w (prepare error: %v)", flushErr, err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
if terminal {
|
||||
@@ -177,7 +193,14 @@ func handleKafkaMessagePersistBatch(ctx context.Context, messages []kafka.Messag
|
||||
}
|
||||
groups[key] = append(groups[key], item)
|
||||
}
|
||||
return persist(ctx, groups)
|
||||
}
|
||||
|
||||
func persistKafkaMessagePersistGroups(ctx context.Context, groups map[string][]*kafkaMessagePersistItem) error {
|
||||
for _, items := range groups {
|
||||
if len(items) == 0 {
|
||||
continue
|
||||
}
|
||||
kind := items[0].payload.MessageKind
|
||||
if kind == "" {
|
||||
kind = "p2p"
|
||||
@@ -232,9 +255,12 @@ func prepareKafkaMessagePersistItem(ctx context.Context, message kafka.Message,
|
||||
return nil, false, err
|
||||
}
|
||||
if !claimed {
|
||||
if consume != nil && consume.Status != types.KafkaConsumeSucceeded && consume.Status != types.KafkaConsumeDead {
|
||||
return nil, false, fmt.Errorf("%w: event=%s status=%s", errKafkaConsumeInProgress, envelope.EventID, consume.Status)
|
||||
}
|
||||
return nil, true, nil
|
||||
}
|
||||
item := &kafkaMessagePersistItem{envelope: envelope, tenantID: tenantID}
|
||||
item := &kafkaMessagePersistItem{envelope: envelope, tenantID: tenantID, consume: consume}
|
||||
if err = json.Unmarshal(envelope.Payload, &item.payload); err != nil {
|
||||
return nil, true, markKafkaDeadLetter(ctx, tenantID, envelope, consume, "MALFORMED_PAYLOAD", err.Error())
|
||||
}
|
||||
@@ -283,12 +309,23 @@ func persistKafkaMessageIndividually(ctx context.Context, item *kafkaMessagePers
|
||||
}
|
||||
err := runKafkaMessageSingleRetries(ctx, kafkaBus.config.MessageRetries, operation, markFailed, waitKafkaRetry)
|
||||
if err != nil {
|
||||
// Message persistence failures are recoverable infrastructure failures. Returning
|
||||
// the error keeps the Kafka offset uncommitted so the outer consumer retries the
|
||||
// message until it is durable. Only malformed payloads are dead-lettered while
|
||||
// preparing the item.
|
||||
immetrics.KafkaMessagePersistTotal.WithLabelValues(kind, "retry_exhausted").Inc()
|
||||
immetrics.MessagePersistDuration.WithLabelValues(kind, "retry_exhausted").Observe(time.Since(started).Seconds())
|
||||
logs.Err.Printf("kafka message persist retry exhausted: tenant=%d event=%s topic=%s seq=%d kind=%s err=%v",
|
||||
item.tenantID, item.envelope.EventID, item.payload.Message.Topic, item.payload.Message.SeqId, kind, err)
|
||||
// Dead-letter the message instead of leaving the offset uncommitted: an
|
||||
// unresolvable failure (e.g. an attachment file already reclaimed by GC)
|
||||
// would otherwise be retried forever and stall every message behind it
|
||||
// on the partition. The dead letter keeps the envelope for replay once
|
||||
// the underlying issue is fixed.
|
||||
if deadErr := markKafkaDeadLetter(ctx, item.tenantID, item.envelope, item.consume, "PERSIST_RETRY_EXHAUSTED", err.Error()); deadErr != nil {
|
||||
logs.Err.Printf("kafka message persist dead letter failed, keeping offset uncommitted: tenant=%d event=%s topic=%s seq=%d err=%v",
|
||||
item.tenantID, item.envelope.EventID, item.payload.Message.Topic, item.payload.Message.SeqId, deadErr)
|
||||
return err
|
||||
}
|
||||
logs.Warn.Printf("kafka message persist dead-lettered after retries: tenant=%d event=%s topic=%s seq=%d kind=%s err=%v",
|
||||
item.tenantID, item.envelope.EventID, item.payload.Message.Topic, item.payload.Message.SeqId, kind, err)
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -319,8 +356,16 @@ func finishKafkaMessagePersist(ctx context.Context, item *kafkaMessagePersistIte
|
||||
return fmt.Errorf("file lifecycle enqueue failed: %w", err)
|
||||
}
|
||||
}
|
||||
return store.HTTPInfrastructure.MarkKafkaConsumeSucceeded(item.tenantID, item.envelope.EventID,
|
||||
kafkaBus.config.ConsumerGroup, time.Now().UTC().Round(time.Millisecond))
|
||||
if err := store.HTTPInfrastructure.MarkKafkaConsumeSucceeded(item.tenantID, item.envelope.EventID,
|
||||
kafkaBus.config.ConsumerGroup, time.Now().UTC().Round(time.Millisecond)); err != nil {
|
||||
if errors.Is(err, types.ErrNotFound) {
|
||||
logs.Warn.Printf("kafka message persist consume state missing after durable write: tenant=%d event=%s topic=%s seq=%d",
|
||||
item.tenantID, item.envelope.EventID, item.payload.Message.Topic, item.payload.Message.SeqId)
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func kafkaMessagePersistFailureReason(err error) string {
|
||||
|
||||
+20
-8
@@ -301,6 +301,7 @@ type kafkaMessageCommitter func(context.Context, kafka.Message) error
|
||||
type kafkaRetryWaiter func(context.Context, time.Duration) error
|
||||
type kafkaBatchHandler func(context.Context, []kafka.Message) error
|
||||
type kafkaBatchCommitter func(context.Context, ...kafka.Message) error
|
||||
type kafkaMessageFetcher func(context.Context) (kafka.Message, error)
|
||||
|
||||
func startKafkaConsumer(topicName string, handler kafkaMessageHandler) {
|
||||
startKafkaConsumerWithGroup(topicName, "", handler)
|
||||
@@ -389,19 +390,23 @@ func startKafkaBatchConsumer(topicName string, handler kafkaBatchHandler) {
|
||||
defer kafkaBus.wg.Done()
|
||||
defer reader.Close()
|
||||
for {
|
||||
messages, err := fetchKafkaMessageBatch(kafkaBus.ctx, reader, kafkaBus.config.MessageBatchSize,
|
||||
messages, fetchErr := fetchKafkaMessageBatch(kafkaBus.ctx, reader, kafkaBus.config.MessageBatchSize,
|
||||
time.Duration(kafkaBus.config.MessageBatchWait)*time.Millisecond)
|
||||
if err != nil {
|
||||
if errors.Is(err, context.Canceled) || errors.Is(kafkaBus.ctx.Err(), context.Canceled) {
|
||||
if fetchErr != nil && len(messages) == 0 {
|
||||
if errors.Is(fetchErr, context.Canceled) || errors.Is(kafkaBus.ctx.Err(), context.Canceled) {
|
||||
return
|
||||
}
|
||||
logs.Err.Println("Kafka batch fetch failed:", err)
|
||||
logs.Err.Println("Kafka batch fetch failed:", fetchErr)
|
||||
continue
|
||||
}
|
||||
if fetchErr != nil {
|
||||
logs.Warn.Printf("Kafka batch fetch interrupted; processing retained messages: topic=%s size=%d err=%v",
|
||||
topic, len(messages), fetchErr)
|
||||
}
|
||||
if len(messages) > 0 {
|
||||
readerMetrics.observe(messages[0].Time)
|
||||
}
|
||||
if err = consumeKafkaMessageBatch(kafkaBus.ctx, messages, handler, reader.CommitMessages, waitKafkaRetry); err != nil {
|
||||
if err := consumeKafkaMessageBatch(kafkaBus.ctx, messages, handler, reader.CommitMessages, waitKafkaRetry); err != nil {
|
||||
if errors.Is(err, context.Canceled) || errors.Is(kafkaBus.ctx.Err(), context.Canceled) {
|
||||
return
|
||||
}
|
||||
@@ -468,7 +473,11 @@ func (m *kafkaReaderMetrics) update() {
|
||||
}
|
||||
|
||||
func fetchKafkaMessageBatch(ctx context.Context, reader *kafka.Reader, maxSize int, maxWait time.Duration) ([]kafka.Message, error) {
|
||||
first, err := reader.FetchMessage(ctx)
|
||||
return fetchKafkaMessageBatchWith(ctx, maxSize, maxWait, reader.FetchMessage)
|
||||
}
|
||||
|
||||
func fetchKafkaMessageBatchWith(ctx context.Context, maxSize int, maxWait time.Duration, fetch kafkaMessageFetcher) ([]kafka.Message, error) {
|
||||
first, err := fetch(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -480,13 +489,16 @@ func fetchKafkaMessageBatch(ctx context.Context, reader *kafka.Reader, maxSize i
|
||||
break
|
||||
}
|
||||
fetchCtx, cancel := context.WithTimeout(ctx, remaining)
|
||||
message, fetchErr := reader.FetchMessage(fetchCtx)
|
||||
message, fetchErr := fetch(fetchCtx)
|
||||
cancel()
|
||||
if errors.Is(fetchErr, context.DeadlineExceeded) && ctx.Err() == nil {
|
||||
break
|
||||
}
|
||||
if fetchErr != nil {
|
||||
return nil, fetchErr
|
||||
// FetchMessage advances the reader's internal position before offsets are
|
||||
// committed. Preserve a partial batch so a later higher-offset commit cannot
|
||||
// skip messages already fetched into this batch.
|
||||
return messages, fetchErr
|
||||
}
|
||||
messages = append(messages, message)
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/segmentio/kafka-go"
|
||||
"github.com/tinode/chat/server/store"
|
||||
"github.com/tinode/chat/server/store/types"
|
||||
)
|
||||
|
||||
@@ -75,6 +76,103 @@ func TestConsumeKafkaMessageBatchDoesNotCommitWhileHandlerFails(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchKafkaMessageBatchWithRetainsPartialBatchOnFetchError(t *testing.T) {
|
||||
fetchCalls := 0
|
||||
fetchErr := errors.New("broker connection reset")
|
||||
messages, err := fetchKafkaMessageBatchWith(context.Background(), 10, time.Second,
|
||||
func(context.Context) (kafka.Message, error) {
|
||||
fetchCalls++
|
||||
if fetchCalls == 1 {
|
||||
return kafka.Message{Partition: 2, Offset: 41}, nil
|
||||
}
|
||||
return kafka.Message{}, fetchErr
|
||||
})
|
||||
|
||||
if !errors.Is(err, fetchErr) {
|
||||
t.Fatalf("fetchKafkaMessageBatchWith() error = %v, want %v", err, fetchErr)
|
||||
}
|
||||
if len(messages) != 1 || messages[0].Partition != 2 || messages[0].Offset != 41 {
|
||||
t.Fatalf("fetchKafkaMessageBatchWith() messages = %#v, want retained offset 41", messages)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchKafkaMessageBatchWithReturnsInitialFetchErrorWithoutMessages(t *testing.T) {
|
||||
fetchErr := errors.New("broker unavailable")
|
||||
messages, err := fetchKafkaMessageBatchWith(context.Background(), 10, time.Second,
|
||||
func(context.Context) (kafka.Message, error) {
|
||||
return kafka.Message{}, fetchErr
|
||||
})
|
||||
|
||||
if !errors.Is(err, fetchErr) {
|
||||
t.Fatalf("fetchKafkaMessageBatchWith() error = %v, want %v", err, fetchErr)
|
||||
}
|
||||
if len(messages) != 0 {
|
||||
t.Fatalf("fetchKafkaMessageBatchWith() messages = %#v, want none", messages)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessKafkaMessagePersistBatchFlushesClaimsBeforePrepareRetry(t *testing.T) {
|
||||
prepareErr := errors.New("redis temporarily unavailable")
|
||||
prepareCalls := 0
|
||||
persistCalls := 0
|
||||
|
||||
err := processKafkaMessagePersistBatch(context.Background(), []kafka.Message{{Offset: 10}, {Offset: 11}}, "group_message_persist",
|
||||
func(_ context.Context, message kafka.Message, topic string) (*kafkaMessagePersistItem, bool, error) {
|
||||
prepareCalls++
|
||||
if topic != "group_message_persist" {
|
||||
t.Fatalf("prepare topic = %q", topic)
|
||||
}
|
||||
if message.Offset == 11 {
|
||||
return nil, false, prepareErr
|
||||
}
|
||||
return &kafkaMessagePersistItem{
|
||||
tenantID: types.TenantID(7),
|
||||
payload: kafkaMessagePersistPayload{
|
||||
MessageKind: "group",
|
||||
ShardMonth: "202610",
|
||||
Message: types.Message{ShardMonth: "202610"},
|
||||
},
|
||||
}, false, nil
|
||||
},
|
||||
func(_ context.Context, groups map[string][]*kafkaMessagePersistItem) error {
|
||||
persistCalls++
|
||||
items := groups["7:202610"]
|
||||
if len(items) != 1 {
|
||||
t.Fatalf("persisted groups = %#v, want one claimed item", groups)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
if !errors.Is(err, prepareErr) {
|
||||
t.Fatalf("processKafkaMessagePersistBatch() error = %v, want %v", err, prepareErr)
|
||||
}
|
||||
if prepareCalls != 2 || persistCalls != 1 {
|
||||
t.Fatalf("calls prepare=%d persist=%d, want 2/1", prepareCalls, persistCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessKafkaMessagePersistBatchReportsFlushAndPrepareErrors(t *testing.T) {
|
||||
prepareErr := errors.New("claim failed")
|
||||
persistErr := errors.New("database unavailable")
|
||||
prepareCalls := 0
|
||||
|
||||
err := processKafkaMessagePersistBatch(context.Background(), []kafka.Message{{Offset: 10}, {Offset: 11}}, "message_persist",
|
||||
func(_ context.Context, _ kafka.Message, _ string) (*kafkaMessagePersistItem, bool, error) {
|
||||
prepareCalls++
|
||||
if prepareCalls == 2 {
|
||||
return nil, false, prepareErr
|
||||
}
|
||||
return &kafkaMessagePersistItem{tenantID: types.TenantID(7)}, false, nil
|
||||
},
|
||||
func(context.Context, map[string][]*kafkaMessagePersistItem) error {
|
||||
return persistErr
|
||||
})
|
||||
|
||||
if !errors.Is(err, persistErr) || !strings.Contains(err.Error(), prepareErr.Error()) {
|
||||
t.Fatalf("processKafkaMessagePersistBatch() error = %v, want both flush and prepare errors", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConsumeKafkaMalformedMessageRetriesCommit(t *testing.T) {
|
||||
commitCalls := 0
|
||||
waitCalls := 0
|
||||
@@ -156,3 +254,30 @@ func TestRunKafkaMessageSingleRetriesStopsOnSuccess(t *testing.T) {
|
||||
t.Fatalf("calls operation=%d failed=%d, want 3/2", operationCalls, markFailedCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFinishKafkaMessagePersistTreatsMissingConsumeStateAsSucceeded(t *testing.T) {
|
||||
oldInfrastructure := store.HTTPInfrastructure
|
||||
oldKafkaBus := kafkaBus
|
||||
defer func() {
|
||||
store.HTTPInfrastructure = oldInfrastructure
|
||||
kafkaBus = oldKafkaBus
|
||||
}()
|
||||
|
||||
store.HTTPInfrastructure = &testHTTPInfrastructureStore{markSucceededErr: types.ErrNotFound}
|
||||
kafkaBus = &kafkaRuntime{config: kafkaConfig{ConsumerGroup: "test-consumer"}}
|
||||
|
||||
err := finishKafkaMessagePersist(context.Background(), &kafkaMessagePersistItem{
|
||||
envelope: kafkaEnvelope{EventID: "msg.persist.8.p2pA.1"},
|
||||
tenantID: types.TenantID(8),
|
||||
payload: kafkaMessagePersistPayload{
|
||||
MessageKind: "p2p",
|
||||
Message: types.Message{
|
||||
Topic: "p2pA",
|
||||
SeqId: 1,
|
||||
},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("finishKafkaMessagePersist() error = %v, want nil", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,156 @@
|
||||
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
|
||||
<meta http-equiv="Content-Style-Type" content="text/css">
|
||||
<title></title>
|
||||
<meta name="Generator" content="Cocoa HTML Writer">
|
||||
<meta name="CocoaVersion" content="2575.7">
|
||||
<style type="text/css">
|
||||
p.p1 {margin: 0.0px 0.0px 0.0px 0.0px; font: 12.0px Times; -webkit-text-stroke: #000000}
|
||||
p.p2 {margin: 0.0px 0.0px 0.0px 0.0px; font: 12.0px Times; -webkit-text-stroke: #000000; min-height: 14.0px}
|
||||
span.s1 {font-kerning: none}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<p class="p1"><span class="s1">禾圃集隐私政策</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">更新日期:2026年10月10日</span></p>
|
||||
<p class="p1"><span class="s1">生效日期:2026年10月10日</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">引言</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">禾圃集是由【南京赫顺名网络科技有限公司】(以下简称“我们”)提供的花园养成互动应用。我们深知个人信息对您的重要性,并会尽全力保护您的个人信息安全可靠。我们致力于维持您对我们的信任,恪守以下原则保护您的个人信息:权责一致原则、目的明确原则、选择同意原则、最少够用原则、确保安全原则、主体参与原则、公开透明原则等。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">同时,我们承诺:我们不会主动共享或提供您的个人信息给第三方;我们不会将您的个人信息用于本政策未明确说明的目的;我们只会收集实现功能所必需的信息。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">本政策将帮助您了解以下内容:</span></p>
|
||||
<p class="p1"><span class="s1">1、个人信息的定义与范围;</span></p>
|
||||
<p class="p1"><span class="s1">2、我们收集的信息;</span></p>
|
||||
<p class="p1"><span class="s1">3、我们如何使用收集的信息;</span></p>
|
||||
<p class="p1"><span class="s1">4、我们如何共享、转让、公开披露您的信息;</span></p>
|
||||
<p class="p1"><span class="s1">5、我们如何保护您的个人信息;</span></p>
|
||||
<p class="p1"><span class="s1">6、我们如何保存您的个人信息;</span></p>
|
||||
<p class="p1"><span class="s1">7、我们如何使用Cookie及同类技术;</span></p>
|
||||
<p class="p1"><span class="s1">8、您如何管理自己的信息;</span></p>
|
||||
<p class="p1"><span class="s1">9、您共享的信息;</span></p>
|
||||
<p class="p1"><span class="s1">10、未成年人保护。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">请您在使用本软件前仔细阅读本政策并确定了解我们对您个人信息的处理规则。对本政策中我们认为与您的权益存在重大关系的条款,我们采用加粗方式提示您注意。您点击“同意”或开始使用本软件,即视为您已阅读并同意本政策的全部内容。如有疑问,您可通过本政策公布的方式联系我们(联系邮箱:support@heshunming.top)。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1"><span class="Apple-converted-space"> </span>一、个人信息的定义与范围</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">1.1 个人信息是指以电子或者其他方式记录的能够单独或者与其他信息结合识别特定自然人身份或者反映特定自然人活动情况的各种信息。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">1.2 本政策中所涉及的个人信息包括:手机号码、密码、昵称、头像、花园养成数据(种植记录、养护动作记录、花朵成长阶段与成长值记录、收割记录、花园统计数据)、您主动发布的动态与分享内容、设备信息、日志信息等。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">1.3 个人敏感信息是指一旦泄露、非法提供或滥用可能危害人身和财产安全,极易导致个人名誉、身心健康受到损害或歧视性待遇等的个人信息。本政策中涉及的个人信息中符合个人敏感信息类型的,我们将以加粗形式提示您注意。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1"><span class="Apple-converted-space"> </span>二、我们收集的信息</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">2.1 禾圃集软件功能分为核心功能和附加功能,我们将详细向您说明各业务功能所收集的个人信息以及收集方法、目的、拒绝提供的影响等。业务功能具体包括:账号注册与登录、花园种植与养护、花朵成长与收割、花园统计与图鉴、动态分享与互动等。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">2.2 您使用禾圃集软件核心功能前,需要创建账号并完善个人资料。通过用户主动提供的方式,我们会收集您的以下信息:</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">2.2.1 账号注册与登录:您需要向我们提供您的手机号码,并设置您的用户密码。手机号码属于个人敏感信息,收集它是为了创建账号、验证身份并保护您的账号安全。如果您拒绝提供手机号码,禾圃集将无法为您创建账号并提供服务。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">2.2.2 个人资料:您可以选择向我们提供昵称、头像等信息。您可以在个人主页资料中选择是否填写上述信息。如您不提供这些个人信息,您依然可以使用禾圃集的核心功能,但可能影响您在动态互动等场景下的展示效果。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">2.3 花园种植与养护功能:当您使用种植、浇水、施肥、收割等功能时,我们会记录您的花园养成数据,包括但不限于您种植的花种、种植与养护时间、养护动作次数、花朵成长阶段、成长值变化、收割记录等。上述数据用于向您展示花朵成长状态、推送成长提醒、生成成长记录与花园统计。这是本服务的基础功能,若您拒绝使用相关功能,我们将不会产生相应的养成数据。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">2.4 动态分享功能:当您生成花园快照、花朵盛开或收割的分享素材、发布动态或对他人动态进行互动时,我们会收集您主动提交的分享文案、图片、互动行为记录及时间等信息。如果您拒绝提供上述信息,您将无法使用分享与互动功能,但不影响您使用花园养成核心功能。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">2.5 相机与相册权限:当您使用头像设置、动态图片上传、分享素材保存等功能时,我们会调用您的设备系统相册权限或系统摄像头权限。上述权限的调用需您在系统中主动授权,您可以随时在系统设置中关闭。您拒绝授权仅影响相关图片功能的使用,不影响其他功能。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">2.6 通知权限:为了向您推送花朵成长、盛开等养成提醒及系统通知,我们会向您申请设备通知权限。该权限需您主动授权,您可以随时在系统设置中关闭,关闭后您仍可正常使用软件的全部核心功能。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">2.7 剪贴板信息:为了读取您复制的花园分享链接或邀请码、便于您提交反馈与举报,本软件会在相应的功能场景下读取剪贴板信息。我们不会将剪贴板信息用于其他目的。您也可以随时在系统设置中关闭本软件的剪贴板读取权限。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">2.8 设备信息与日志信息:为了保障服务安全稳定运行、进行数据统计分析以及捕获崩溃问题,我们的产品在应用运行期间会收集您的设备信息(设备型号、操作系统及版本)以及服务运行时产生的日志信息(IP地址、访问时间、访问记录、崩溃记录等)。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">2.9 我们不会主动申请您的通讯录、精确位置、麦克风等与花园养成服务无关的权限,也不会收集您的银行账户、支付信息。如未来因新功能需要收集前述信息,我们将按照法律法规要求另行取得您的授权同意。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1"><span class="Apple-converted-space"> </span>三、我们如何使用收集的信息</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">3.1 我们会将收集的信息用于:创建账号与身份验证;提供花园种植、养护、成长、收割等养成服务;生成成长记录与花园统计;提供动态分享与互动功能;推送养成提醒与系统通知;改进产品功能与服务体验(例如优化花朵成长节奏);保障服务安全稳定运行(包括作弊、外挂与异常养成行为的监测)。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">3.2 我们会遵循隐私保护政策收集、使用您的信息,但不会仅因您同意本政策而采用强制捆绑的方式收集全部个人信息。我们会逐一说明所收集的您的个人信息类型及其对应的用途,以便您了解我们针对某一特定功能收集的具体个人信息类别、使用理由及收集方式。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">3.3 超出上述范围使用您的信息,或要将您的信息用于本政策未载明的其他目的,我们会再次征得您的同意。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">3.4 我们不会将您的花园养成数据用于任何形式的广告投放,也不会据此向您推送第三方商业广告。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1"><span class="Apple-converted-space"> </span>四、我们如何共享、转让、公开披露您的信息</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">4.1 共享:我们不会与任何第三方共享您的个人信息,除非获得您的明确同意,或属于以下情形:</span></p>
|
||||
<p class="p1"><span class="s1">(1)相关法律法规或法院、政府机关的要求;</span></p>
|
||||
<p class="p1"><span class="s1">(2)为完成合并、分立、收购或资产转让而转移;</span></p>
|
||||
<p class="p1"><span class="s1">(3)为提供您要求的服务所必需,例如使用第三方推送、崩溃统计服务时必要的设备信息。我们会与第三方签署严格的数据保护约定,要求其仅为本服务之目的处理您的信息。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">4.2 本软件接入的第三方服务详见《禾圃集软件服务协议》第九条关于第三方SDK的说明,包括:Bugly SDK(应用崩溃捕获,收集设备型号、操作系统版本、崩溃日志)、苹果系统推送APNs(收集设备推送令牌)。上述第三方服务的个人信息处理规则,请您阅读其官方隐私政策。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">4.3 转让:我们不会将您的个人信息转让给任何公司、组织和个人,除非获得您的明确同意,或发生合并、分立、收购、资产转让等情形且我们将确保信息在转让后的安全。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">4.4 公开披露:我们仅会在以下情形公开披露您的个人信息:获得您明确同意;基于法律法规、司法或行政机关的强制性要求。您在动态中主动分享的花园快照、分享文案等内容,将按您选择的范围向其他用户展示。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1"><span class="Apple-converted-space"> </span>五、我们如何保护您的个人信息</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">5.1 我们已使用符合行业标准的安全技术手段保护您提供的个人信息,包括数据加密存储与传输、访问权限控制、安全审计等,并建立完善的管理制度来防止您的个人信息遭到未经授权的访问、使用、篡改或泄露。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">5.2 我们会采取合理可行的措施,尽力避免收集无关的个人信息。我们只会在实现本政策所述目的所必需的期限内保留您的个人信息,除非法律另有规定。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">5.3 我们会举办安全和隐私保护培训课程,加强员工对于保护个人信息重要性的认识。我们建立了个人信息保护的专项制度、流程和组织,确保个人信息保护工作的持续有效。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">5.4 如不幸发生个人信息安全事件,我们将按照法律法规的要求,及时向您告知安全事件的基本情况和可能的影响、我们已采取或将要采取的处置措施、您可自主防范和降低风险的建议等,并以邮件、推送通知等方式告知您。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1"><span class="Apple-converted-space"> </span>六、我们如何保存您的个人信息</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">6.1 保存地点:我们在中华人民共和国境内运营中收集和产生的个人信息,将存储在中华人民共和国境内。如需跨境传输,我们会按照法律法规要求另行取得您的授权同意。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">6.2 保存期限:在您使用本服务期间,我们会持续保存您的账号信息与花园养成数据。当您注销账号后,我们将在合理的期限内删除或匿名化处理您的个人信息及花园养成数据,法律法规另有规定的除外。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1"><span class="Apple-converted-space"> </span>七、我们如何使用Cookie及同类技术</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">7.1 本软件为原生客户端应用,我们在本软件内不使用浏览器Cookie。为实现登录状态的维持、崩溃统计与系统推送,我们会使用设备标识符、本地存储、推送令牌等同类技术。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">7.2 上述技术仅用于实现本政策所述的功能目的。您可以通过卸载软件、在系统设置中关闭通知权限、重置设备广告标识符等方式删除或停用相关信息,但可能影响相关功能的使用。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1"><span class="Apple-converted-space"> </span>八、您如何管理自己的信息</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">8.1 查看与更正:您可以在软件内随时查看、更正您的昵称、头像等个人资料信息;您的花园养成数据(种植记录、成长记录、统计信息)可在花园页面及成长记录中查看。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">8.2 删除:您可以删除您主动发布的动态与分享内容。删除后,该内容将不再对外展示。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">8.3 账号注销:您可以依照软件内提供的流程申请注销您的账号。注销成功后,我们将停止为您提供服务,并在合理期限内删除或匿名化处理您的个人信息与花园养成数据,法律法规另有规定的除外。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">8.4 权限管理:您可以在移动设备系统设置中,随时对本软件的相机、相册、通知、剪贴板等权限进行开启或关闭,相关操作不影响本软件核心养成功能的正常使用。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">8.5 如果您无法通过上述方式访问、更正、删除您的个人信息,或需要撤回授权同意,您可以通过本政策公布的联系方式与我们联系,我们将在15个工作日内回复您的请求。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1"><span class="Apple-converted-space"> </span>九、您共享的信息</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">9.1 您可以通过本软件的分享功能,将花园快照、花朵盛开与收割的分享素材、您的动态内容分享给其他用户或第三方平台(如社交媒体)。您共享的信息中可能包含您的昵称、头像、花园养成成果等内容,请注意避免在分享内容中包含您的个人敏感信息。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">9.2 您可以随时删除您已发布的动态与分享内容。对于他人对您内容的互动记录,您可以在软件内查看或向我们申请处理。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">9.3 请您审慎考虑通过本服务共享信息的范围与对象。因您主动共享信息而导致的个人信息泄露等风险与后果,由您自行承担。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1"><span class="Apple-converted-space"> </span>十、未成年人保护</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">10.1 我们非常重视对未成年人个人信息的保护。若您是未满18周岁的未成年人,在使用本软件前,应事先取得您家长或法定监护人的同意,并在监护人的指导下使用本服务和阅读本政策。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">10.2 对于经监护人同意使用本服务的未成年人,我们将仅收集必要的个人信息,并在法律法规和监护人同意的范围内使用。我们不会向未成年人推送商业性信息。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">10.3 如果监护人发现未成年人未经其同意使用了本服务或提供了个人信息,您可以联系我们,我们将尽快删除相关数据。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1"><span class="Apple-converted-space"> </span>政策更新</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">我们可能适时对本政策进行修订。未经您明确同意,我们不会削减您按照本政策所应享有的权利。对于重大变更,我们会通过软件公告、系统消息或推送通知等方式向您说明,并在必要时再次征得您的同意。您可以在本软件内随时查阅最新版本的本政策。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1"><span class="Apple-converted-space"> </span>如何联系我们</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">如您对本政策有任何疑问、意见或建议,或需要行使您的个人信息权利,您可通过本平台联系我们,联系方式为【support@heshunming.top】。一般情况下,我们将在15个工作日内回复。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">如果您对我们的回复不满意,特别是我们的行为损害了您的合法权益,您还可以向网信、电信、公安及工商等主管机关投诉、举报,或向有管辖权的人民法院提起诉讼。</span></p>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,371 @@
|
||||
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
|
||||
<meta http-equiv="Content-Style-Type" content="text/css">
|
||||
<title></title>
|
||||
<meta name="Generator" content="Cocoa HTML Writer">
|
||||
<meta name="CocoaVersion" content="2575.7">
|
||||
<style type="text/css">
|
||||
p.p1 {margin: 0.0px 0.0px 0.0px 0.0px; font: 12.0px Times; -webkit-text-stroke: #000000}
|
||||
p.p2 {margin: 0.0px 0.0px 0.0px 0.0px; font: 12.0px Times; -webkit-text-stroke: #000000; min-height: 14.0px}
|
||||
span.s1 {font-kerning: none}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<p class="p1"><span class="s1"><span class="Apple-converted-space"> </span>禾圃集软件服务协议</span></p>
|
||||
<p class="p1"><span class="s1"><span class="Apple-converted-space"> </span>前言</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">欢迎您使用【南京赫顺名网络科技有限公司】公司(下称“我公司”或“我们”)为您提供的禾圃集的服务!</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">特别提示:</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">为使用禾圃集软件(以下简称“本软件”)及服务,您应当阅读并遵守《禾圃集软件服务协议》(以下简称“本协议”)。请您务必仔细阅读、充分理解本协议各条款内容,特别是豁免或者限制我公司责任的条款、对用户权利进行限制的条款、约定管辖与法律适用条款,以及开通或使用某项服务的单独协议。限制、免责条款或者其他涉及您重大权益的条款可能以加粗等形式提示您重点注意。如您对本协议有任何疑问,可通过文末联系方式联系我们。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">除非您已阅读并接受本协议所有条款,否则您无权注册或登录使用本软件及相关服务。您点击“同意”,或者您注册或登录本软件的行为,或者其他明示或者默示方式表示接受本协议的行为即视为您已阅读并同意接受本协议的约束。本协议即在您与我公司之间产生法律效力,成为对双方均具有约束力的法律文件。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">如果您未满18周岁,或存在其他不具备与用户行为相适应的民事行为能力的情形,请在法定监护人的陪同、指导下阅读本协议,并特别注意未成年人使用条款,在确保监护人同意本协议内容后使用本软件。您及您的监护人应依照法律法规规定,承担同意本协议及使用本软件而导致的相应后果。</span></p>
|
||||
<p class="p1"><span class="s1"><span class="Apple-converted-space"> </span>一、协议的范围</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">1.1 协议适用主体范围</span></p>
|
||||
<p class="p1"><span class="s1">本协议是您与我们之间关于您下载、安装、注册、登录、使用本软件以及使用我们提供的相关服务所订立的协议。包括本协议正文、《禾圃集隐私政策》及我公司已经发布的或将来可能发布/更新的并采取合理途径通知的各类规则、规范、通知等。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">1.2 协议关系</span></p>
|
||||
<p class="p1"><span class="s1">本协议内容同时包括我们可能不断发布的关于本服务的相关协议、业务规则、通知等内容。上述内容一经正式发布,即为本协议不可分割的组成部分,您同样应当遵守。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">1.3 相关定义</span></p>
|
||||
<p class="p1"><span class="s1">“我公司或我们”,是指禾圃集及相关服务可能存在的运营关联单位,即【南京赫顺名网络科技有限公司】公司及其关联公司。</span></p>
|
||||
<p class="p1"><span class="s1">“用户”,是指使用本软件及相关服务的使用者,在本协议中更多地称为“您”。</span></p>
|
||||
<p class="p1"><span class="s1">“服务”,是指我们向您提供的禾圃集软件相关服务,具体以我们实际提供的为准。同时,我们有权对提供的软件服务进行增加、关停或其他调整。</span></p>
|
||||
<p class="p1"><span class="s1">“账号”,指用户在使用服务时可能需要注册的账号。用户可在本软件注册并获得的账号,作为登录并使用本软件的凭证。</span></p>
|
||||
<p class="p1"><span class="s1">“花园养成内容”,指您在使用本服务过程中种植、养护的花朵及相关成长数据,包括但不限于花种、成长阶段、成长值、养护动作记录(浇水、施肥)、收割与分享记录、花园统计信息等。</span></p>
|
||||
<p class="p1"><span class="s1"><span class="Apple-converted-space"> </span>二、关于本服务</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">2.1 本服务的内容</span></p>
|
||||
<p class="p1"><span class="s1">本服务内容是指我们向用户提供的花园养成互动应用(以下简称“禾圃集”),核心功能包括但不限于:在花房中选择花种并种植;通过浇水、施肥等养护动作促进花朵成长;花朵按阶段自然成长(种子、发芽、花苞、盛开等);花朵盛开后的收割与成长记录;花园统计与花朵图鉴;生成花园快照、花朵盛开与收割的分享素材;以及查看好友的花园动态并进行互动(以下简称“本服务”)。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">2.2 本服务的形式</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">2.2.1 您使用本服务需要下载禾圃集客户端软件,对于这些软件,我们给予您一项个人的、不可转让及非排他性的许可。您仅可为访问或使用本服务的目的而使用这些软件及服务。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">2.2.2 本服务中禾圃集客户端软件可能会提供包括但不限于iOS、Android等多个应用版本,用户必须选择与所安装终端设备相匹配的软件版本。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">2.3 本服务许可的范围</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">2.3.1 我们给予您一项个人的、不可转让及非排他性的许可,以使用本软件。您可以为非商业目的在单一台终端设备上安装、使用、显示、运行本软件。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">2.3.2 本条及本协议其他条款未明确授权的其他一切权利仍由我们保留,您在行使这些权利时须另外取得我们的书面许可。我们如果未行使前述任何权利,并不构成对该权利的放弃。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">2.4 按现状提供服务</span></p>
|
||||
<p class="p1"><span class="s1">您理解并同意,禾圃集的服务是按照现有技术和条件所能达到的现状提供的。禾圃集会尽最大努力向您提供服务,确保服务的连续性和安全性;但禾圃集不能随时预见和防范法律、技术以及其他风险,包括但不限于不可抗力、病毒、木马、黑客攻击、系统不稳定、第三方服务缺陷、政府行为等原因可能导致的服务中断、数据丢失以及其他的损失和风险。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">2.5 虚拟养成内容的性质</span></p>
|
||||
<p class="p1"><span class="s1">2.5.1 您理解并同意,花朵、花种、成长值、养护次数等均为本服务中的虚拟养成内容,仅限在本服务内按照产品规则使用,不具有任何现实货币价值。我们不对任何用户之间就虚拟养成内容进行的现实交易提供保障,由此产生的纠纷与损失由您自行承担。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">2.5.2 养成内容的成长进度、阶段变化等均以我们的服务器记录的数据为准。您通过卸载、重装软件、更换设备等方式,仍可登录同一账号查看您的花园养成数据;但若您注销账号,相关养成数据将按本协议约定被删除且无法恢复。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1"><span class="Apple-converted-space"> </span>三、软件的获取与使用</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">3.1 您可以直接从我们的网站上获取本软件,也可以通过苹果App Store等得到我们授权的第三方应用商店获取。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">3.2 如果您从未经我们授权的第三方获取本软件或与本软件名称相同的安装程序,我们无法保证该软件能够正常使用,并对因此给您造成的损失不予负责。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">3.3 我们可能为不同的终端设备开发了不同的软件版本,您应当根据实际情况选择下载合适的版本进行安装。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">3.4 下载安装程序后,您需要按照该程序提示的步骤正确安装。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">3.5 为了改善用户体验、保证服务的安全性及产品功能的一致性,我们可能会不时对禾圃集软件进行更新。我们有权不经过向您特别通知而对软件进行更新,或者对软件的部分功能效果进行改变或限制。您应当将相关软件更新到最新版本,否则我们并不保证其能正常使用。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">3.6 如果您不再需要使用本软件或者需要安装新版软件,可以自行卸载。如果您愿意帮助我们改进产品服务,请告知卸载的原因。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1"><span class="Apple-converted-space"> </span>四、用户个人信息保护</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">4.1 保护用户个人信息是我们的一项基本原则,我们将按照本协议及《禾圃集隐私政策》的规定收集、使用、存储您的个人信息,并会采取合理的措施保护您的个人信息。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">4.2 您在注册账号或使用本服务的过程中,需要提供一些必要的信息,例如:为向您提供账号注册服务或进行用户身份识别,需要您提供手机号码并设置密码;本软件部分功能可能需要收集您的个人信息包括:昵称、头像、花园养成数据(种植记录、浇水施肥记录、花朵成长阶段与成长记录)、分享内容等。若国家法律法规或政策有特殊规定的,您需要提供真实的身份信息。若您提供的信息不完整,则无法使用本服务或在使用过程中受到限制。您了解并同意,您有义务保持您提供信息的真实性、有效性及完整性。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">4.3 一般情况下,您可随时查看、修改自己提交的信息,但出于安全性和身份识别的考虑,您可能无法修改注册时提供的初始注册信息及其他验证信息。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">4.4 我们将运用各种安全技术和程序建立完善的管理制度来保护您的个人信息,以免遭受未经授权的访问、使用或泄露。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">4.5 未经您的同意,我们不会将您的个人信息转移或披露给任何非关联的第三方,除非:</span></p>
|
||||
<p class="p1"><span class="s1">(1)相关法律法规或法院、政府机关要求;</span></p>
|
||||
<p class="p1"><span class="s1">(2)为完成合并、分立、收购或资产转让而转移;</span></p>
|
||||
<p class="p1"><span class="s1">(3)为提供您要求的服务所必需;</span></p>
|
||||
<p class="p1"><span class="s1">(4)依据其他相关协议规则可以转移或披露给任何第三方的情形。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">4.6 我们非常重视对未成年人个人信息的保护。若您是18周岁以下的未成年人,在使用我们的服务前,应事先取得您家长或法定监护人的书面同意。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">4.7 我们可能会在向您提供服务的过程之中所收集的信息用作下列用途:</span></p>
|
||||
<p class="p1"><span class="s1">(1) 向您提供花园养成及相关互动服务;</span></p>
|
||||
<p class="p1"><span class="s1">(2) 在我们提供服务时,用于身份验证、客户服务、安全防范、作弊与外挂监测、存档和备份用途,确保我们向您提供的产品和服务的安全性;</span></p>
|
||||
<p class="p1"><span class="s1">(3) 帮助我们设计新服务,改进我们现有服务,例如优化花朵成长节奏与养护体验;</span></p>
|
||||
<p class="p1"><span class="s1">(4) 使我们更加了解您如何接入和使用我们的服务,从而针对性地回应您的个性化需求,例如语言设置、个性化的帮助服务和指导;</span></p>
|
||||
<p class="p1"><span class="s1">(5) 软件认证或管理软件升级;</span></p>
|
||||
<p class="p1"><span class="s1">(6) 让您参与有关我们产品和服务的调查。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1"><span class="Apple-converted-space"> </span>五、主权利义务条款</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">5.1 账号使用规范</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">5.1.1 您在使用本服务前需要注册一个禾圃集账号。本软件为您提供注册及登录通道,禾圃集账号通过手机号码进行注册,您可按照页面提示填写信息、阅读并同意本协议,在完成全部注册登录程序后,成为本软件用户。另外我们有权根据用户需求或产品需要对账号注册的方式进行变更,关于您使用账号的具体规则,请遵守相关账号使用协议以及我们为此发布的专项规则。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">5.1.2 禾圃集账号的所有权归我们公司所有,用户完成申请注册手续后,仅获得禾圃集账号的使用权,且该使用权仅属于初始申请注册人。同时,初始申请注册人不得赠与、借用、出租、转让或售卖禾圃集账号或者以其他方式许可非初始申请注册人使用禾圃集账号。非初始申请注册人不得通过受赠、继承、承租、受让或者其他任何方式使用禾圃集账号。如我们判断您账号的使用可能违反法律法规、危及您的账号安全及/或本软件信息安全的,我们可拒绝提供相应服务或终止本协议。我们有权视情况封禁或注销、回收该账号,由此带来的包括但不限于花园养成内容清空等损失由您本人及该账号使用者自行承担。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">5.1.3 您注册获得的账号、设置的密码是您登录并以用户身份使用本软件的凭证,您的账户仅限您本人使用,您有责任妥善保管您的注册用户信息及用户密码的安全,您需要对您的注册用户以及密码下的行为承担法律责任。您同意在任何情况下不向他人泄露用户及密码信息。当在您怀疑他人在使用您的账号时,您应立即通知我们。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">5.1.4 用户注册禾圃集账号后如果长期(超过【6】个月)不登录该账号,我们有权回收该账号,以免造成资源浪费,由此带来的任何损失均由用户自行承担。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">5.1.5 我们特别提醒,您应妥善保管您的账号和密码。当您使用完本服务后,应安全退出。您须对账号进行的所有活动和行为负责,因您保管不善导致遭受账号或密码失窃、账号被非法使用,相关责任由您自行承担,您可通过司法、行政等救济途径向侵权行为人追偿。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">5.1.6 您不得恶意注册账号,包括但不限于频繁注册、批量注册、使用他人身份注册或其他不以正常使用本软件服务为目的的账号注册行为。除此外,您的账户不得以任何方式转让,否则我们有权追究您的违约责任,且由此产生的责任及后果均由您自行承担。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">5.2 用户注意事项</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">5.2.1 您理解并同意:为了向您提供有效的服务,本软件会利用您终端设备的处理器和带宽等资源。本软件使用过程中可能产生数据流量的费用,用户需自行向运营商了解相关资费信息,并自行承担相关费用。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">5.2.2 您理解并同意我们将尽其商业上的合理努力保障您在本软件及服务中的数据存储安全,但是,我们并不能就此提供完全保证,包括但不限于以下情形:</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">5.2.2.1 我们不对您在本软件及服务中相关数据的删除或存储失败负责;</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">5.2.2.2 我们有权根据实际情况自行决定单个用户在本软件及服务中数据的最长存储期限,并在服务器上为其分配数据最大存储空间等。您可根据自己的需要自行备份本软件及服务中的相关数据;</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">5.2.2.3 如果您停止使用本软件及服务或服务被终止或取消,我们可以从服务器上永久地删除您的数据。服务停止、终止或取消后,我们没有义务向您返还任何数据。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">5.2.3 您理解并同意,禾圃集可能会不时发布并执行一些用户奖励活动,相关活动的具体规则的制定权和最终解释权归我们所有,我们有权根据自己的判断决定提前终止活动或延长活动时间。同一禾圃集账号、同一手机号、同一手机设备均视为同一用户。如果我们发现有用户利用不正当手段参与活动,或者如果用户存在或涉嫌违法违规行为情况(包括不限于外挂、作弊、脚本刷取、虚假养成的活动、其他风险行为),我们有权取消您参与活动的资格,并有权追回您已经获得的奖励。如果因非实名认证导致您无法参与活动的,视为您自动放弃活动资格。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">5.3 第三方产品和服务</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">5.3.1 您在使用本软件第三方提供的产品或服务时,除遵守本协议约定外,还应遵守第三方的用户协议。我们和第三方对可能出现的纠纷在法律规定和约定的范围内各自承担责任。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">5.3.2 因用户使用本软件或要求我们提供特定服务时,本软件可能会调用第三方系统或者通过第三方支持用户的使用或访问,我们不保证通过第三方提供服务及内容的安全性、准确性、有效性及其它不确定的风险,您应自行判断此类第三方网站或者服务的安全性和可用性,并自行承担相关风险和责任,由此若引发的任何争议及损害,与我们无关,我们不承担任何责任。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">5.4 账号信息设置及维护</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">5.4.1 您在注册或使用本软件时设置的名称、头像和简介等账号信息资料应遵守法律法规、社会道德风尚和信息真实性等原则,不得出现不良信息,不得冒用他人姓名、名称、字号、头像等或采取其他足以引起混淆的方式设置账号,不得侵害第三方的合法权益。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">5.4.2 在使用本软件服务时,您应当按本软件页面的提示准确完整地提供您的信息,以便本软件或其他用户与您联系。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">5.4.3 如本软件按您最后一次提供的信息与您联系未果、您未按本软件的要求及时提供信息、您提供的信息存在明显不实或行政司法机关核实您提供的信息无效的,您将承担因此对您自身、他人及本软件造成的全部损失与不利后果。本软件可向您发出询问或要求整改的通知,并要求您进行重新认证,直至中止、终止对您提供部分或全部软件产品或服务,本软件对此不承担责任。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">5.5 账号注销及回收</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">5.5.1 您可以依照本软件的流程申请注销您的账号,但您仍应对您在注销账号前或使用本软件服务期间的行为承担相应责任。注销成功后,除非法律法规另有规定,我们不会为您提供账号记录、花园养成数据等恢复或提供服务,请您谨慎操作。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">5.5.2 您理解并同意,为了充分使用账号资源,如您在注册后未及时进行初次登录使用或连续超过【6】个月未登录账号并使用,且不存在未到期或未履行完毕的持续性本软件服务的,我们有权收回您的账号。如您的账号被收回,您无法通过您此前持有的账号登录并使用本软件,您账号下保存的花园养成数据和使用记录也将无法恢复。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">5.6 账号违规处置</span></p>
|
||||
<p class="p1"><span class="s1">若您在注册或使用本软件账号时,存在任何违反法律法规或不符合本协议约定的行为,我们有权不予注册;已经注册的,我们有权视情况要求用户限期改正、或单方采取短期封禁、永久封禁、注销账号等措施。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1"><span class="Apple-converted-space"> </span>六、用户行为规范</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">6.1 信息内容规范</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">6.1.1 本条所述信息内容是指用户使用本软件及服务过程中所制作、复制、发布、传播的任何内容,包括但不限于禾圃集账号、昵称、头像、用户说明等注册信息,或花园动态、分享文案、分享图片及相关链接页面,以及其他使用禾圃集账号或本软件及服务所产生的内容。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">6.1.2 您理解并同意,禾圃集一直致力于为用户提供文明健康、规范有序的网络环境,您不得利用禾圃集账号或本软件及服务制作、复制、发布、传播如下干扰禾圃集正常运行,以及侵犯其他用户或第三方合法权益的内容,包括但不限于:</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">6.1.2.1 发布、传送、传播、存储违反国家法律法规禁止的内容,包括但不限于危害国家安全、泄露国家秘密、破坏国家统一与主权和领土完整、损害国家荣誉和利益、煽动民族仇恨与歧视、宣扬邪教和封建迷信、散布谣言扰乱社会秩序、散布淫秽色情赌博暴力恐怖信息、诽谤或侮辱他人、侵害未成年人合法权益以及含有法律、行政法规禁止的其他内容的;</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">6.1.2.2 发布、传送、传播、存储侵犯他人名誉权、肖像权、知识产权、商业秘密等合法权利的内容;</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">6.1.2.3 涉及他人隐私、个人信息或资料的;</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">6.1.2.4 发表、传送、传播骚扰、广告信息、过度营销信息及垃圾信息或含有任何性或性暗示的;</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">6.1.2.5 其他违反法律法规、政策及公序良俗、社会公德或干扰禾圃集正常运行和侵犯其他用户或第三方合法权益内容的信息。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">6.1.2.6 您通过本服务发布或分享的内容(包括但不限于花园动态、文字、图片等)均由您自行承担责任。您发布或分享的内容应有合法来源,相关内容为您所有或您已获得权利人的授权。您同意禾圃集可为履行本协议或提供本服务的目的而使用您发布或分享的内容。如果我们收到权利人通知,主张您发布或分享的内容侵犯其相关权利的,您同意我们有权进行独立判断并采取删除、屏蔽等措施。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">6.2 软件使用规范</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">6.2.1 除非法律允许或我们书面许可,您使用本软件过程中不得从事下列行为:</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">6.2.1.1 删除本软件及其副本上关于著作权的信息;</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">6.2.1.2 对本软件进行反向工程、反向汇编、反向编译,或者以其他方式尝试发现本软件的源代码;</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">6.2.1.3 对我们享有知识产权的内容进行使用、出租、出借、复制、修改、链接、转载、汇编、发表、出版、建立镜像站点等;</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">6.2.1.4 通过修改或伪造软件运行中的指令、数据,或使用自动化脚本、插件、外挂等方式批量执行浇水、施肥等养护动作,或以其他方式异常刷取成长值、奖励或养成数据;</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">6.2.1.5 通过非我们开发、授权、许可的第三方软件、插件、外挂、系统,登录或使用我们的软件及服务,或制作、发布、传播上述工具,或对本软件的正常运行进行干扰、破坏、修改或施加其他影响;</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">6.2.1.6 以任何方式(包括但不限于盗链、冗余盗取、非法抓取、模拟下载、深度链接、假冒注册等)直接或间接盗取本软件的花种数据、分享内容、用户信息等信息内容;</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">6.2.1.7 利用或针对本软件进行任何危害计算机网络安全的行为,包括但不限于:非法侵入网络、干扰网络正常功能、窃取网络数据等危害网络安全的活动,或为上述活动提供程序、工具等支持与帮助;</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">6.2.1.8 其他以任何不合法的方式、为任何不合法的目的、或与我们为此制定的其他规范和标准不一致的方式使用本软件及未经我们明确授权的行为。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">6.3 服务运营规范</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">6.3.1 除非您与禾圃集另有约定,您同意本服务仅为您个人非商业性质的使用。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">6.3.2 您应当通过禾圃集提供或认可的方式使用本服务。您依本协议条款所获得的权利不可转让。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">6.3.3 您不得使用未经禾圃集授权的插件、外挂或第三方工具对本协议项下的服务进行干扰、破坏、修改或施加其他影响。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">6.4 对自己行为负责</span></p>
|
||||
<p class="p1"><span class="s1">您充分了解并同意,您必须为自己注册账号下的一切行为负责,包括您所发表的任何内容以及由此产生的任何后果。您应对本服务中的内容自行加以判断,并承担因使用内容而引起的所有风险,包括因对内容的正确性、完整性或实用性的依赖而产生的风险。我们无法且不会对因前述风险而导致的任何损失或损害承担责任。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">6.5 违约处理</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">6.5.1 如果我们发现或收到他人举报或投诉用户违反本协议约定的,我们有权不经通知随时对相关内容进行删除、屏蔽,并视行为情节对违规账号处以包括但不限于警告、限制或禁止使用部分或全部功能、清除异常养成数据、账号封禁直至注销的处罚,并公告处理结果。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">6.5.2 您理解并同意,因您违反本协议或相关服务条款的规定,导致或产生第三方主张的任何索赔、要求或损失,您应当独立承担责任;我们因此遭受损失的,您也应当一并赔偿,包括行政处罚或损害赔偿款等直接损失以及商誉损失、和解款、律师费、诉讼费等间接损失。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">6.5.3 我们可将对您上述违约行为的处置信息,以及其他经国家行政或司法机关生效法律文书确认的违法信息,在本软件上予以公示。对涉嫌违反法律法规的行为,我们有义务保存有关记录,向国家有关机关报告并配合调查。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1"><span class="Apple-converted-space"> </span>七、知识产权声明</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">7.1 禾圃集在本服务中提供的内容(包括但不限于花种形象与图鉴、花朵素材与动画、网页、文字、图片、图表、界面设计、版面框架、图标、商标等)的知识产权归禾圃集所有,受中华人民共和国著作权法、商标法、专利法、反不正当竞争法及国际条约等法律法规的保护。未经我们书面允许,用户不得为任何目的擅自使用、复制、再造这些内容、或创造与内容有关的派生产品。用户在使用本服务中所产生的自有内容的知识产权归用户或相关权利人所有。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">7.2 除另有特别声明外,禾圃集提供本服务时所依据软件的著作权、专利权及其他知识产权均归禾圃集所有。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">7.3 禾圃集在本服务中所使用的“禾圃集”等商业标识,其著作权或商标权归禾圃集所有。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">7.4 上述及其他任何本服务包含的内容的知识产权均受到法律保护,未经禾圃集、用户或相关权利人书面许可,任何人不得以任何形式进行使用或创作相关衍生作品。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1"><span class="Apple-converted-space"> </span>八、终端安全责任</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">8.1 您理解并同意,本软件同大多数互联网软件一样,可能会受多种因素影响,包括但不限于用户原因、网络服务质量、社会环境等;也可能会受各种安全问题的侵扰,包括但不限于他人非法利用用户资料,进行现实中的骚扰;您下载安装的其他软件或访问的其他网站中可能含有病毒、木马程序或其他恶意程序,威胁您的终端设备信息和数据安全,继而影响本软件的正常使用等。因此,您应加强信息安全及个人信息的保护意识,注意密码保护,以免遭受损失。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">8.2 您不得制作、发布、使用、传播用于窃取禾圃集账号及他人个人信息、财产的恶意程序。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">8.3 维护软件安全与正常使用是我们和您的共同责任,我们将按照行业标准合理审慎地采取必要技术措施保护您的终端设备信息和数据安全,但是您承认和同意我们并不能就此提供完全保证。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">8.4 在任何情况下,您不应轻信借款、索要密码或其他涉及财产的网络信息。涉及财产操作的,请一定先核实对方身份,并请经常留意我们有关防范诈骗犯罪的提示。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1"><span class="Apple-converted-space"> </span>九、第三方软件或技术</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">9.1 本软件可能会使用第三方软件或技术(包括本软件可能使用的开放源代码和公共领域代码等,下同),这种使用已经获得合法授权。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">9.2 本软件如果使用了第三方的软件或技术,我们将按照相关法规或约定,对相关的协议或其他文件,可能通过本协议附件、在本软件安装包特定文件夹中打包、或通过开源软件页面等形式进行展示,它们可能会以“软件使用许可协议”、“授权协议”、“开放源代码许可证”或其他形式来表达。前述通过各种形式展示的相关协议、其他文件及网页,均是本协议不可分割的组成部分,与本协议具有同等的法律效力,您应当遵守这些要求。如果您没有遵守这些要求,该第三方或者国家机关可能会对您提起诉讼、罚款或采取其他制裁措施,并要求我们给予配合,您应当自行承担法律责任。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">9.3 如因本软件使用的第三方软件或技术引发的任何纠纷,应由该第三方负责解决,我们不承担任何责任。我们不对第三方软件或技术提供客服支持,若您需要获取支持,请与第三方联系。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">APP涉及内嵌第三方代码、插件传输个人信息的情形逐项列举如下:</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">**Bugly SDK(iOS版)**</span></p>
|
||||
<p class="p1"><span class="s1">主体:深圳市腾讯计算机系统有限公司</span></p>
|
||||
<p class="p1"><span class="s1">用途:应用崩溃问题捕获</span></p>
|
||||
<p class="p1"><span class="s1">收集的个人信息类型:设备型号、操作系统及版本、设备传感器信息、应用崩溃日志、启动与卡顿日志;</span></p>
|
||||
<p class="p1"><span class="s1">官网链接:</span></p>
|
||||
<p class="p1"><span class="s1">https://privacy.qq.com/document/preview/fc748b3d96224fdb825ea79e132c1a56</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">**苹果系统推送(APNs)**</span></p>
|
||||
<p class="p1"><span class="s1">主体:Apple Inc.</span></p>
|
||||
<p class="p1"><span class="s1">用途:向您推送花园养成提醒(如花朵成长、盛开提醒)及系统通知</span></p>
|
||||
<p class="p1"><span class="s1">涉及个人信息:设备推送令牌(Device Token);推送能力由苹果系统提供,您可以随时在系统设置中关闭本软件的通知权限。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">如后续版本接入其他第三方服务,我们将按照法律法规要求在本协议及《禾圃集隐私政策》中更新说明并向您提示。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1"><span class="Apple-converted-space"> </span>十、不可抗力及其他免责事由</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">10.1 您理解并同意,在使用本服务的过程中,可能会遇到不可抗力等风险因素,使本服务发生中断。不可抗力是指不能预见、不能克服并不能避免且对一方或双方造成重大影响的客观事件,包括但不限于自然灾害如水灾、地震、瘟疫流行和风暴等以及社会事件如战争、动乱、政府行为等。出现上述情况时,禾圃集将努力在第一时间与相关单位配合,及时进行修复,但由此给您造成的损失禾圃集在法律允许的范围内免责。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">10.2 在法律允许的范围内,禾圃集对以下情形导致的服务中断或受阻不承担责任:</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">(1)受到计算机病毒、木马或其他恶意程序、黑客攻击的破坏;</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">(2)用户或禾圃集的电脑软件、系统、硬件和通信线路出现故障;</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">(3)用户操作不当;</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">(4)用户通过非禾圃集授权的方式使用本服务;</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">(5)其他禾圃集无法控制或合理预见的情形。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">10.3 您理解并同意,在使用本服务的过程中,可能会遇到网络信息或其他用户行为带来的风险,禾圃集不对任何信息的真实性、适用性、合法性承担责任,也不对因侵权行为给您造成的损害负责。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">10.4 禾圃集根据本协议约定获得处理违法违规内容的权利,该权利不构成禾圃集的义务或承诺,禾圃集不能保证及时发现违法行为或进行相应处理。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">10.5 在任何情况下,您不应轻信借款、索要密码或其他涉及财产的网络信息。涉及财产操作的,请一定先核实对方身份,并请经常留意禾圃集有关防范诈骗犯罪的提示。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1"><span class="Apple-converted-space"> </span>十一、服务的变更、中断、终止</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">11.1 禾圃集可能会对服务内容进行变更,也可能会中断、中止或终止服务。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">11.2 您理解并同意,禾圃集有权自主决定经营策略。在禾圃集发生合并、分立、收购、资产转让时,禾圃集可向第三方转让本服务下相关资产;禾圃集也可在单方通知您后,将本协议下部分或全部服务转交由第三方经营或履行。具体受让主体以禾圃集通知的为准。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">11.3 如发生下列任何一种情形,禾圃集有权不经通知而中断或终止向您提供的服务:</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">(1)根据法律规定您应提交真实信息,而您提供的个人资料不真实、或与注册时信息不一致又未能提供合理证明;</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">(2)您违反相关法律法规或本协议的约定;</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">(3)按照法律规定或主管部门的要求;</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">(4)出于安全的原因或其他必要的情形。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">11.4 我们始终在不断变更和改进本软件服务,我们会尽最大努力向您提供服务,确保服务的连贯性和安全性。基于此,我们可能会对本软件进行更新,您应将软件更新到最新版本,否则我们不保证您可正常使用本软件。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">11.5 您理解并同意,我公司根据自身商业决策、政府行为、不可抗力等原因可能会选择中止、中断及终止本软件的部分或全部服务。如有此等情形发生,我们会采取合理的方式并以合理的期限提前通知您,除法律法规另有规定外,我们不承担由此对您造成的损失。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">11.6 您有责任自行备份存储在本服务中的数据。如果您的服务被终止,禾圃集可以从服务器上永久地删除您的数据,但法律法规另有规定的除外。服务终止后,禾圃集没有义务向您返还数据。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1"><span class="Apple-converted-space"> </span>十二、未成年人使用条款</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">12.1 若用户未满18周岁,则为未成年人,应在监护人监护、指导下阅读本协议和使用本服务。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">12.2 未成年人用户涉世未深,容易被网络虚拟迷惑,且好奇心强,遇事缺乏随机应变的处理能力,很容易被别有用心的人利用而又缺乏自我保护能力。因此,未成年人用户在使用本服务时应注意以下事项,提高安全意识,加强自我保护:</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">(1)认清网络世界与现实世界的区别,合理安排使用时间,避免沉迷于网络,影响日常的学习生活;</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">(2)填写个人资料时,加强个人保护意识,以免不良分子对个人生活造成骚扰;</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">(3)在监护人或老师的指导下,学习正确使用网络;</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">(4)避免与陌生网友随意见面或参加联谊活动,以免不法分子有机可乘,危及自身安全。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">12.3 监护人、学校均应对未成年人使用本服务时多做引导。特别是家长应关心子女的成长,注意与子女的沟通,指导子女上网应该注意的安全问题,防患于未然。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1"><span class="Apple-converted-space"> </span>十三、协议变更与终止</span></p>
|
||||
<p class="p1"><span class="s1">13.1 协议变更通知</span></p>
|
||||
<p class="p1"><span class="s1">我们有权在必要时修改本协议内容,并将通过合理的方式(包括但不限于本软件公告、系统消息、推送通知、手机短信、电子邮件等)及合理的期限提前通知您,该等通知于发送之日视为已送达您。您可以通过软件相关页面查阅最新版本的协议内容。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">13.2 变更协议生效</span></p>
|
||||
<p class="p1"><span class="s1">本协议变更生效后,如您继续使用本软件,即视为您已接受修改后的协议。如您不接受修改后的协议,应当停止使用本软件,我们将根据公平原则与您终止协议。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">13.3 协议终止</span></p>
|
||||
<p class="p1"><span class="s1">您注销您的账号或我们通知您终止为您提供本软件服务(包括但不限于封禁账号、注销账号等)时,本协议将同时终止,我们将不再另行通知您终止本协议。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1"><span class="Apple-converted-space"> </span>十四、通知</span></p>
|
||||
<p class="p1"><span class="s1">14.1 联系方式</span></p>
|
||||
<p class="p1"><span class="s1">14.1.1 您在注册成为本软件用户,并接受本软件服务时,您应该向本软件提供真实有效的联系方式(包括您的联系电话、电子邮件地址等),对于联系方式发生变更的,您有义务及时更新有关信息,并保持可被联系的状态。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">14.1.2 您在注册本软件用户时生成的用于接收系统消息的账号,也作为您的有效联系方式。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">14.1.3 本软件将向您的上述联系方式向您送达各类通知,而此类通知的内容可能对您的权利义务产生重大的有利或不利影响,请您务必及时关注。您如果不愿意接收商业性信息,您有权通过本软件提供的相应的退订功能进行退订。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">14.2 通知送达</span></p>
|
||||
<p class="p1"><span class="s1">14.2.1 本软件通过上述联系方式向您发出通知,其中以电子的方式发出的书面通知,包括但不限于在本软件公告,向您提供的联系电话发送手机短信,向您提供的电子邮件地址发送电子邮件,向您的账号发送信息、系统消息以及推送通知,在发送成功后即视为送达;以纸质载体发出的书面通知,按照提供联系地址交邮后的第【7】个自然日即视为送达。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">14.2.2 对于在本软件上因使用本服务引起的任何纠纷,您同意司法机关(包括但不限于人民法院)可以通过手机短信、电子邮件等现代通讯方式或邮寄方式向您送达法律文书(包括但不限于诉讼文书)。您指定接收法律文书的手机号码、电子邮箱或软件账号等联系方式为您在本软件注册、更新时提供的联系方式以及在注册本软件用户时生成的账号,司法机关向上述联系方式发出法律文书即视为送达。您指定的邮寄地址为您的法定联系地址或您提供的有效联系地址。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">14.2.3 您同意司法机关可采取以上一种或多种送达方式向您送达法律文书,司法机关采取多种方式向您送达法律文书,送达时间以上述送达方式中最先送达的为准。您同意上述送达方式适用于各个司法程序阶段。如进入诉讼程序的,包括但不限于一审、二审、再审、执行以及督促程序等。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">14.2.4 您应当保证所提供的联系方式是准确、有效的,并进行实时更新。如果因提供的联系方式不确切,或不及时告知变更后的联系方式,使法律文书无法送达或未及时送达,由您自行承担由此可能产生的法律后果。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1"><span class="Apple-converted-space"> </span>十五、其他</span></p>
|
||||
<p class="p1"><span class="s1">15.1 您使用本软件即视为您已阅读并同意受本协议的约束。我们有权在必要时修改本协议条款。您可以在本软件的最新版本中查阅相关协议条款。本协议条款变更后,如果您继续使用本软件,即视为您已接受修改后的协议。如果您不接受修改后的协议,应当停止使用本软件。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">15.2 本协议的成立、生效、履行、解释及争议解决,适用中华人民共和国大陆地区法律法规;如法律无相关规定的,参照商业惯例及/或行业惯例。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">15.3 若您和我们之间发生任何纠纷或争议,首先应友好协商解决;协商不成的,您同意将纠纷或争议提交我公司所在地人民法院管辖。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">15.4 本协议所有条款的标题仅为阅读方便,本身并无实际含义,不能作为本协议含义解释的依据。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">15.5 本协议可能存在包括中文、英文在内的多种语言的版本,若存在中文版本与其他语言的版本相冲突的地方,以中文版本为准。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">15.6 本协议条款无论因何种原因部分无效或不可执行,其余条款仍有效,对双方具有约束力。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">15.7 如您对本协议有任何意见或建议,您可通过本平台联系我们,联系方式为【support@heshunming.top】,我们会给予您必要的帮助。</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1"><span class="Apple-converted-space"> </span>读取剪贴板信息说明</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">1、用于读取您复制的花园分享链接或邀请码,方便您快速进入对应花园或完成好友邀请;</span></p>
|
||||
<p class="p2"><span class="s1"></span><br></p>
|
||||
<p class="p1"><span class="s1">2、用于复制平台联系方式,方便您提交意见反馈、举报等。</span></p>
|
||||
</body>
|
||||
</html>
|
||||
@@ -64,6 +64,8 @@ type legalDocument struct {
|
||||
var legalDocuments = map[string]legalDocument{
|
||||
"/legal/user-terms": loadLegalDocument("legal/user-terms.html", "讯工捷使软件服务协议"),
|
||||
"/legal/privacy-policy": loadLegalDocument("legal/privacy-policy.html", "讯工捷使隐私政策"),
|
||||
"/legal/garden/user-terms": loadLegalDocument("legal/garden-user-terms.html", "禾圃集软件服务协议"),
|
||||
"/legal/garden/privacy-policy": loadLegalDocument("legal/garden-privacy-policy.html", "禾圃集隐私政策"),
|
||||
}
|
||||
|
||||
func loadLegalDocument(name, title string) legalDocument {
|
||||
|
||||
@@ -897,6 +897,9 @@ func main() {
|
||||
// Public legal pages are opened directly by clients before login.
|
||||
mux.HandleFunc("/legal/user-terms", serveLegalDocument)
|
||||
mux.HandleFunc("/legal/privacy-policy", serveLegalDocument)
|
||||
// Garden app (禾圃集) legal pages.
|
||||
mux.HandleFunc("/legal/garden/user-terms", serveLegalDocument)
|
||||
mux.HandleFunc("/legal/garden/privacy-policy", serveLegalDocument)
|
||||
|
||||
// Handle websocket clients.
|
||||
mux.HandleFunc(config.ApiPath+"v0/channels", serveWebSocket)
|
||||
|
||||
+112
-46
@@ -58,12 +58,80 @@ type readerCounter struct {
|
||||
reader io.Reader
|
||||
}
|
||||
|
||||
type ossReadSeekCloser struct {
|
||||
open func(offset int64) (io.ReadCloser, error)
|
||||
size int64
|
||||
pos int64
|
||||
body io.ReadCloser
|
||||
closed bool
|
||||
}
|
||||
|
||||
func (rc *readerCounter) Read(buf []byte) (int, error) {
|
||||
n, err := rc.reader.Read(buf)
|
||||
atomic.AddInt64(&rc.count, int64(n))
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (r *ossReadSeekCloser) Read(buf []byte) (int, error) {
|
||||
if r.closed {
|
||||
return 0, errors.New("read from closed OSS object")
|
||||
}
|
||||
if r.pos >= r.size {
|
||||
return 0, io.EOF
|
||||
}
|
||||
if r.body == nil {
|
||||
var err error
|
||||
r.body, err = r.open(r.pos)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
n, err := r.body.Read(buf)
|
||||
r.pos += int64(n)
|
||||
if err != nil {
|
||||
_ = r.body.Close()
|
||||
r.body = nil
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (r *ossReadSeekCloser) Seek(offset int64, whence int) (int64, error) {
|
||||
var next int64
|
||||
switch whence {
|
||||
case io.SeekStart:
|
||||
next = offset
|
||||
case io.SeekCurrent:
|
||||
next = r.pos + offset
|
||||
case io.SeekEnd:
|
||||
next = r.size + offset
|
||||
default:
|
||||
return 0, errors.New("invalid seek origin")
|
||||
}
|
||||
if next < 0 {
|
||||
return 0, errors.New("negative seek position")
|
||||
}
|
||||
if next == r.pos {
|
||||
return next, nil
|
||||
}
|
||||
if r.body != nil {
|
||||
_ = r.body.Close()
|
||||
r.body = nil
|
||||
}
|
||||
r.pos = next
|
||||
return next, nil
|
||||
}
|
||||
|
||||
func (r *ossReadSeekCloser) Close() error {
|
||||
if r.closed {
|
||||
return nil
|
||||
}
|
||||
r.closed = true
|
||||
if r.body != nil {
|
||||
return r.body.Close()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Init initializes the media handler.
|
||||
func (oh *ossHandler) Init(jsconf string) error {
|
||||
var err error
|
||||
@@ -207,7 +275,9 @@ func truthy(value string) bool {
|
||||
}
|
||||
}
|
||||
|
||||
// Headers adds CORS headers and redirects GET and HEAD requests to OSS.
|
||||
// Headers adds CORS and cache headers. Downloads stay on the IM origin: forwarding the authenticated
|
||||
// browser request through a signed OSS redirect also forwards custom headers, which forces an OSS
|
||||
// preflight and fails unless the bucket exposes those private headers through CORS.
|
||||
func (oh *ossHandler) Headers(tenantID types.TenantID, method string, url *url.URL, headers http.Header, serve bool) (http.Header, int, error) {
|
||||
headers, status := media.CORSHandler(method, headers, oh.corsOrigins, serve)
|
||||
if status != 0 || method == http.MethodPost || method == http.MethodPut {
|
||||
@@ -225,51 +295,15 @@ func (oh *ossHandler) Headers(tenantID types.TenantID, method string, url *url.U
|
||||
}
|
||||
|
||||
if fdef.ETag != "" && strings.Trim(headers.Get("If-None-Match"), "\"") == fdef.ETag {
|
||||
return http.Header{
|
||||
"ETag": {`"` + fdef.ETag + `"`},
|
||||
"Cache-Control": {oh.conf.CacheControl},
|
||||
}, http.StatusNotModified, nil
|
||||
headers.Set("ETag", `"`+fdef.ETag+`"`)
|
||||
headers.Set("Cache-Control", oh.conf.CacheControl)
|
||||
return headers, http.StatusNotModified, nil
|
||||
}
|
||||
objectKey, err := oh.resolveObjectKey(fdef.Location)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
if fdef.ETag != "" {
|
||||
headers.Set("ETag", `"`+fdef.ETag+`"`)
|
||||
}
|
||||
|
||||
var redirURL string
|
||||
switch method {
|
||||
case http.MethodGet:
|
||||
options := []aliyunoss.Option{
|
||||
aliyunoss.ResponseCacheControl(oh.conf.CacheControl),
|
||||
}
|
||||
if isAttachment, _ := strconv.ParseBool(url.Query().Get("asatt")); isAttachment {
|
||||
options = append(options, aliyunoss.ResponseContentDisposition("attachment"))
|
||||
}
|
||||
redirURL, err = oh.bucket.SignURL(objectKey, aliyunoss.HTTPGet, int64(oh.conf.PresignTTL), options...)
|
||||
case http.MethodHead:
|
||||
redirURL, err = oh.bucket.SignURL(objectKey, aliyunoss.HTTPHead, int64(oh.conf.PresignTTL))
|
||||
}
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
if redirURL != "" {
|
||||
return http.Header{
|
||||
"Location": {normalizeSignedRedirectURL(redirURL)},
|
||||
"ETag": {`"` + fdef.ETag + `"`},
|
||||
"Content-Type": {"application/json; charset=utf-8"},
|
||||
"Cache-Control": {oh.conf.CacheControl},
|
||||
}, http.StatusFound, nil
|
||||
}
|
||||
return nil, 0, nil
|
||||
}
|
||||
|
||||
func normalizeSignedRedirectURL(rawURL string) string {
|
||||
parsed, err := url.Parse(strings.TrimSpace(rawURL))
|
||||
if err != nil || parsed.Scheme != "http" || parsed.Host == "" {
|
||||
return rawURL
|
||||
}
|
||||
parsed.Scheme = "https"
|
||||
return parsed.String()
|
||||
headers.Set("Cache-Control", oh.conf.CacheControl)
|
||||
return headers, 0, nil
|
||||
}
|
||||
|
||||
// Upload processes request for a file upload.
|
||||
@@ -332,9 +366,41 @@ func (oh *ossHandler) UploadPublic(fdef *media.PublicFileDef, file io.Reader) (s
|
||||
return key, rc.count, nil
|
||||
}
|
||||
|
||||
// Download is unsupported because OSS objects are served through signed redirects.
|
||||
// Download streams a private OSS object through the authenticated IM endpoint. The seekable reader
|
||||
// reopens the object with a byte range after every seek, allowing http.ServeContent to handle HEAD
|
||||
// and Range requests without buffering an entire image, video, or document in server memory.
|
||||
func (oh *ossHandler) Download(tenantID types.TenantID, url string) (*types.FileDef, media.ReadSeekCloser, error) {
|
||||
return nil, nil, types.ErrUnsupported
|
||||
fid := oh.GetIdFromUrl(url)
|
||||
if fid.IsZero() {
|
||||
return nil, nil, types.ErrNotFound
|
||||
}
|
||||
fd, err := oh.getFileRecord(tenantID, fid)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
objectKey, err := oh.resolveObjectKey(fd.Location)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if fd.Size <= 0 {
|
||||
meta, err := oh.bucket.GetObjectDetailedMeta(objectKey)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
fd.Size, err = strconv.ParseInt(meta.Get("Content-Length"), 10, 64)
|
||||
if err != nil || fd.Size < 0 {
|
||||
return nil, nil, types.ErrMalformed
|
||||
}
|
||||
}
|
||||
|
||||
reader := &ossReadSeekCloser{size: fd.Size}
|
||||
reader.open = func(offset int64) (io.ReadCloser, error) {
|
||||
if offset <= 0 {
|
||||
return oh.bucket.GetObject(objectKey)
|
||||
}
|
||||
return oh.bucket.GetObject(objectKey, aliyunoss.Range(offset, fd.Size-1))
|
||||
}
|
||||
return fd, reader, nil
|
||||
}
|
||||
|
||||
// Delete deletes files from OSS by provided object keys.
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
package oss
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"net/http"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -58,14 +60,31 @@ func TestIsOSSObjectNotFound(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeSignedRedirectURL(t *testing.T) {
|
||||
rawURL := "http://im-oss-t1.oss-cn-shenzhen.aliyuncs.com/tenants/1/avatar.jpg?Expires=1&OSSAccessKeyId=ak&Signature=sig"
|
||||
want := "https://im-oss-t1.oss-cn-shenzhen.aliyuncs.com/tenants/1/avatar.jpg?Expires=1&OSSAccessKeyId=ak&Signature=sig"
|
||||
if got := normalizeSignedRedirectURL(rawURL); got != want {
|
||||
t.Fatalf("normalizeSignedRedirectURL()=%q, want %q", got, want)
|
||||
func TestOSSReadSeekCloser(t *testing.T) {
|
||||
data := []byte("0123456789")
|
||||
reader := &ossReadSeekCloser{
|
||||
size: int64(len(data)),
|
||||
open: func(offset int64) (io.ReadCloser, error) {
|
||||
return io.NopCloser(bytes.NewReader(data[offset:])), nil
|
||||
},
|
||||
}
|
||||
if got := normalizeSignedRedirectURL("/v0/file/s/avatar.jpg"); got != "/v0/file/s/avatar.jpg" {
|
||||
t.Fatalf("relative URL must be unchanged, got %q", got)
|
||||
defer reader.Close()
|
||||
|
||||
buf := make([]byte, 3)
|
||||
if n, err := reader.Read(buf); err != nil || n != 3 || string(buf) != "012" {
|
||||
t.Fatalf("initial read = %q, %d, %v", buf, n, err)
|
||||
}
|
||||
if pos, err := reader.Seek(-2, io.SeekEnd); err != nil || pos != 8 {
|
||||
t.Fatalf("seek end = %d, %v", pos, err)
|
||||
}
|
||||
if n, err := reader.Read(buf); err != nil && err != io.EOF || n != 2 || string(buf[:n]) != "89" {
|
||||
t.Fatalf("ranged read = %q, %d, %v", buf[:n], n, err)
|
||||
}
|
||||
if pos, err := reader.Seek(4, io.SeekStart); err != nil || pos != 4 {
|
||||
t.Fatalf("seek start = %d, %v", pos, err)
|
||||
}
|
||||
if n, err := reader.Read(buf[:2]); err != nil || n != 2 || string(buf[:2]) != "45" {
|
||||
t.Fatalf("second ranged read = %q, %d, %v", buf[:2], n, err)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -190,6 +190,49 @@ var (
|
||||
Help: "Time a client publish waited in a topic queue before processing by topic kind.",
|
||||
Buckets: []float64{0.0005, 0.001, 0.003, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60},
|
||||
}, []string{"pod", "kind"})
|
||||
TopicProcessDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Name: "im_topic_process_duration_seconds",
|
||||
Help: "End-to-end topic event processing duration by pod, topic kind, role, and operation.",
|
||||
Buckets: []float64{0.0005, 0.001, 0.003, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60},
|
||||
}, []string{"pod", "kind", "role", "operation"})
|
||||
TopicBroadcastDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Name: "im_topic_broadcast_duration_seconds",
|
||||
Help: "Time spent preparing and enqueueing one topic broadcast by pod, topic kind, and role.",
|
||||
Buckets: []float64{0.0005, 0.001, 0.003, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10},
|
||||
}, []string{"pod", "kind", "role"})
|
||||
TopicBroadcastTargets = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Name: "im_topic_broadcast_targets",
|
||||
Help: "Number of local WebSocket, multiplex, or other targets considered by one topic broadcast.",
|
||||
Buckets: []float64{0, 1, 10, 50, 100, 250, 500, 750, 1000, 1500, 2000, 3000},
|
||||
}, []string{"pod", "kind", "role", "target_type"})
|
||||
TopicSharedSerializationTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "im_topic_shared_serialization_total",
|
||||
Help: "Shared WebSocket serialization events by pod, topic kind, and result.",
|
||||
}, []string{"pod", "kind", "result"})
|
||||
TopicBroadcastDropTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "im_topic_broadcast_drop_total",
|
||||
Help: "Realtime topic broadcasts skipped by pod, topic kind, role, and reason.",
|
||||
}, []string{"pod", "kind", "role", "reason"})
|
||||
ClusterM2PQueueWaitDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Name: "im_cluster_m2p_queue_wait_duration_seconds",
|
||||
Help: "Time a master-to-proxy response waited in the multiplex outbound queue by pod and peer.",
|
||||
Buckets: []float64{0.0005, 0.001, 0.003, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10},
|
||||
}, []string{"pod", "peer"})
|
||||
ClusterM2PQueueDepth = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Name: "im_cluster_m2p_queue_depth",
|
||||
Help: "Master-to-proxy multiplex outbound queue depth observed after enqueue by pod and peer.",
|
||||
Buckets: []float64{0, 1, 4, 8, 16, 32, 64, 96, 128, 160},
|
||||
}, []string{"pod", "peer"})
|
||||
ClusterProxyQueueWaitDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Name: "im_cluster_proxy_queue_wait_duration_seconds",
|
||||
Help: "Time a cluster response waited in the proxy topic queue by pod and message type.",
|
||||
Buckets: []float64{0.0005, 0.001, 0.003, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60},
|
||||
}, []string{"pod", "message_type"})
|
||||
WSReadDispatchDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Name: "im_ws_read_dispatch_duration_seconds",
|
||||
Help: "Time from a completed WebSocket frame read through JSON decoding and local dispatch.",
|
||||
Buckets: []float64{0.0001, 0.0005, 0.001, 0.003, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5},
|
||||
}, []string{"pod"})
|
||||
PushDispatchQueueEnqueueTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "im_push_dispatch_queue_enqueue_total",
|
||||
Help: "Asynchronous push dispatch queue enqueue attempts by pod and result.",
|
||||
@@ -242,6 +285,8 @@ func init() {
|
||||
ClusterP2MQueueCapacity, ClusterP2MEnqueueTotal, ClusterRPCInflight, ClusterRPCDuration,
|
||||
ClusterRouteTotal, ClusterProxyResponseTotal, InternalQueueEnqueueTotal, InternalQueueDepth,
|
||||
InternalQueueCapacity, InternalQueueCurrent, MessagePersistDuration, MessagePublishStageDuration, TopicQueueWaitDuration,
|
||||
TopicProcessDuration, TopicBroadcastDuration, TopicBroadcastTargets, TopicSharedSerializationTotal,
|
||||
TopicBroadcastDropTotal, ClusterM2PQueueWaitDuration, ClusterM2PQueueDepth, ClusterProxyQueueWaitDuration, WSReadDispatchDuration,
|
||||
PushDispatchQueueEnqueueTotal, PushDispatchQueueDepth, PushDispatchQueueWaitDuration,
|
||||
PushDispatchDuration, PushRecipientCount,
|
||||
KafkaMessagePersistTotal)
|
||||
@@ -279,6 +324,51 @@ func ObserveTopicQueueWait(kind string, queuedAt time.Time) {
|
||||
TopicQueueWaitDuration.WithLabelValues(metricPodName(), normalizeLabel(kind)).Observe(time.Since(queuedAt).Seconds())
|
||||
}
|
||||
|
||||
func ObserveTopicProcess(kind, role, operation string, started time.Time) {
|
||||
TopicProcessDuration.WithLabelValues(metricPodName(), normalizeLabel(kind), normalizeLabel(role),
|
||||
normalizeLabel(operation)).Observe(time.Since(started).Seconds())
|
||||
}
|
||||
|
||||
func ObserveTopicBroadcast(kind, role string, started time.Time, websocketTargets, multiplexTargets, otherTargets int) {
|
||||
pod := metricPodName()
|
||||
kind = normalizeLabel(kind)
|
||||
role = normalizeLabel(role)
|
||||
TopicBroadcastDuration.WithLabelValues(pod, kind, role).Observe(time.Since(started).Seconds())
|
||||
TopicBroadcastTargets.WithLabelValues(pod, kind, role, "websocket").Observe(float64(websocketTargets))
|
||||
TopicBroadcastTargets.WithLabelValues(pod, kind, role, "multiplex").Observe(float64(multiplexTargets))
|
||||
TopicBroadcastTargets.WithLabelValues(pod, kind, role, "other").Observe(float64(otherTargets))
|
||||
}
|
||||
|
||||
func IncTopicSharedSerialization(kind, result string) {
|
||||
TopicSharedSerializationTotal.WithLabelValues(metricPodName(), normalizeLabel(kind), normalizeLabel(result)).Inc()
|
||||
}
|
||||
|
||||
func IncTopicBroadcastDrop(kind, role, reason string) {
|
||||
TopicBroadcastDropTotal.WithLabelValues(metricPodName(), normalizeLabel(kind), normalizeLabel(role), normalizeLabel(reason)).Inc()
|
||||
}
|
||||
|
||||
func ObserveClusterM2PEnqueue(peer string, depth int) {
|
||||
ClusterM2PQueueDepth.WithLabelValues(metricPodName(), normalizeLabel(peer)).Observe(float64(depth))
|
||||
}
|
||||
|
||||
func ObserveClusterM2PDequeue(peer string, queuedAt time.Time) {
|
||||
if queuedAt.IsZero() {
|
||||
return
|
||||
}
|
||||
ClusterM2PQueueWaitDuration.WithLabelValues(metricPodName(), normalizeLabel(peer)).Observe(time.Since(queuedAt).Seconds())
|
||||
}
|
||||
|
||||
func ObserveClusterProxyQueueWait(messageType string, queuedAt time.Time) {
|
||||
if queuedAt.IsZero() {
|
||||
return
|
||||
}
|
||||
ClusterProxyQueueWaitDuration.WithLabelValues(metricPodName(), normalizeLabel(messageType)).Observe(time.Since(queuedAt).Seconds())
|
||||
}
|
||||
|
||||
func ObserveWSReadDispatch(started time.Time) {
|
||||
WSReadDispatchDuration.WithLabelValues(metricPodName()).Observe(time.Since(started).Seconds())
|
||||
}
|
||||
|
||||
func ObservePushDispatchEnqueue(depth int, accepted bool) {
|
||||
result := "ok"
|
||||
if !accepted {
|
||||
|
||||
@@ -98,6 +98,14 @@ func TestServeMetricsIncludesWSConnections(t *testing.T) {
|
||||
immetrics.ObserveWSSendQueueDequeue(time.Now(), false)
|
||||
immetrics.ObserveCtrlACKDelivery("write", time.Now())
|
||||
immetrics.ObserveTopicQueueWait("group", time.Now())
|
||||
immetrics.ObserveTopicProcess("group", "master", "publish", time.Now())
|
||||
immetrics.ObserveTopicBroadcast("group", "master", time.Now(), 1, 0, 0)
|
||||
immetrics.IncTopicSharedSerialization("group", "serialized")
|
||||
immetrics.IncTopicBroadcastDrop("group", "master", "ws_backpressure")
|
||||
immetrics.ObserveClusterM2PEnqueue("peer", 1)
|
||||
immetrics.ObserveClusterM2PDequeue("peer", time.Now())
|
||||
immetrics.ObserveClusterProxyQueueWait("data", time.Now())
|
||||
immetrics.ObserveWSReadDispatch(time.Now())
|
||||
immetrics.ObservePushDispatchEnqueue(1, true)
|
||||
immetrics.ObservePushDispatch(time.Now(), 1, time.Now(), nil)
|
||||
wrt := httptest.NewRecorder()
|
||||
@@ -117,7 +125,12 @@ func TestServeMetricsIncludesWSConnections(t *testing.T) {
|
||||
"im_ctrl_ack_delivery_duration_seconds",
|
||||
"im_cluster_p2m_enqueue_total", "im_cluster_rpc_duration_seconds", "im_cluster_route_total",
|
||||
"im_internal_queue_enqueue_total", "im_internal_queue_depth", "im_internal_queue_capacity", "im_internal_queue_current",
|
||||
"im_topic_queue_wait_duration_seconds", "im_push_dispatch_queue_enqueue_total",
|
||||
"im_topic_queue_wait_duration_seconds", "im_topic_process_duration_seconds",
|
||||
"im_topic_broadcast_duration_seconds", "im_topic_broadcast_targets", "im_topic_shared_serialization_total",
|
||||
"im_topic_broadcast_drop_total",
|
||||
"im_cluster_m2p_queue_wait_duration_seconds", "im_cluster_m2p_queue_depth",
|
||||
"im_cluster_proxy_queue_wait_duration_seconds", "im_ws_read_dispatch_duration_seconds",
|
||||
"im_push_dispatch_queue_enqueue_total",
|
||||
"im_push_dispatch_queue_depth", "im_push_dispatch_queue_wait_duration_seconds",
|
||||
"im_push_dispatch_duration_seconds", "im_push_recipient_count",
|
||||
"im_kafka_message_persist_total", "im_kafka_consumer_lag", "im_kafka_dead_letters_total"} {
|
||||
|
||||
+1
-4
@@ -389,9 +389,6 @@ func sendPush(rcpt *push.Receipt) {
|
||||
}
|
||||
|
||||
if len(local.PushRcpt.To) > 0 || local.PushRcpt.Channel != "" {
|
||||
select {
|
||||
case globals.usersUpdate <- local:
|
||||
default:
|
||||
}
|
||||
enqueueUserUpdate(local, "push", false)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/tinode/chat/server/auth"
|
||||
@@ -27,6 +28,29 @@ const (
|
||||
realtimeTargetUsers = "users"
|
||||
)
|
||||
|
||||
const platformEvictFanoutDebounce = 750 * time.Millisecond
|
||||
|
||||
type platformEvictFanoutKey struct {
|
||||
tenantID types.TenantID
|
||||
uid types.Uid
|
||||
platf string
|
||||
reason string
|
||||
}
|
||||
|
||||
type platformEvictFanoutPending struct {
|
||||
skipSid string
|
||||
timer *time.Timer
|
||||
}
|
||||
|
||||
var platformEvictFanout = struct {
|
||||
sync.Mutex
|
||||
pending map[platformEvictFanoutKey]*platformEvictFanoutPending
|
||||
}{
|
||||
pending: make(map[platformEvictFanoutKey]*platformEvictFanoutPending),
|
||||
}
|
||||
|
||||
var publishRealtimeFanout = store.PublishRealtimeFanout
|
||||
|
||||
type realtimeGroupUnreadRecipient struct {
|
||||
User string `json:"user"`
|
||||
Source string `json:"source"`
|
||||
@@ -69,7 +93,7 @@ func publishRealtimeTopicMessage(topic string, msg *ServerComMessage) {
|
||||
return
|
||||
}
|
||||
dedupKey := realtimeMessageDedupKey(topic, msg)
|
||||
if err := store.PublishRealtimeFanout(currentNodeID(), msg.TenantID, realtimeKindMessage, realtimeTargetTopic, topic, msg.SkipSid, dedupKey, msg); err != nil {
|
||||
if err := publishRealtimeFanout(currentNodeID(), msg.TenantID, realtimeKindMessage, realtimeTargetTopic, topic, msg.SkipSid, dedupKey, msg); err != nil {
|
||||
immetrics.IncRealtimeFanout("publish", realtimeKindMessage, realtimeTargetTopic, "error")
|
||||
logs.Warn.Printf("redis realtime fanout: publish topic=%s failed: %v", topic, err)
|
||||
return
|
||||
@@ -89,7 +113,7 @@ func publishRealtimeUserMessage(uid types.Uid, msg *ServerComMessage) {
|
||||
return
|
||||
}
|
||||
dedupKey := realtimeMessageDedupKey(uid.UserId(), msg)
|
||||
if err := store.PublishRealtimeFanout(currentNodeID(), msg.TenantID, realtimeKindMessage, realtimeTargetUser, uid.UserId(), msg.SkipSid, dedupKey, msg); err != nil {
|
||||
if err := publishRealtimeFanout(currentNodeID(), msg.TenantID, realtimeKindMessage, realtimeTargetUser, uid.UserId(), msg.SkipSid, dedupKey, msg); err != nil {
|
||||
immetrics.IncRealtimeFanout("publish", realtimeKindMessage, realtimeTargetUser, "error")
|
||||
logs.Warn.Printf("redis realtime fanout: publish user=%s failed: %v", uid.UserId(), err)
|
||||
return
|
||||
@@ -109,7 +133,7 @@ func publishRealtimeGroupUnread(tenantID types.TenantID, topic, actor string, se
|
||||
Recipients: recipients,
|
||||
}
|
||||
dedupKey := "group-unread:" + topic + ":" + strconv.FormatInt(seqID, 10)
|
||||
if err := store.PublishRealtimeFanout(currentNodeID(), tenantID, realtimeKindGroupUnread,
|
||||
if err := publishRealtimeFanout(currentNodeID(), tenantID, realtimeKindGroupUnread,
|
||||
realtimeTargetUsers, topic, "", dedupKey, payload); err != nil {
|
||||
immetrics.IncRealtimeFanout("publish", realtimeKindGroupUnread, realtimeTargetUsers, "error")
|
||||
logs.Warn.Printf("redis realtime fanout: publish group unread topic=%s seq=%d failed: %v", topic, seqID, err)
|
||||
@@ -140,7 +164,7 @@ func publishRealtimeCallEvent(msg *ClientComMessage) {
|
||||
Timestamp: msg.Timestamp,
|
||||
}
|
||||
dedupKey := "call:" + msg.RcptTo + ":" + msg.AsUser + ":" + event.Event + ":" + strconv.FormatInt(msg.Note.SeqId, 10)
|
||||
if err := store.PublishRealtimeFanout(currentNodeID(), msg.TenantID, realtimeKindCallEvent, realtimeTargetTopic, msg.RcptTo, sid, dedupKey, event); err != nil {
|
||||
if err := publishRealtimeFanout(currentNodeID(), msg.TenantID, realtimeKindCallEvent, realtimeTargetTopic, msg.RcptTo, sid, dedupKey, event); err != nil {
|
||||
immetrics.IncRealtimeFanout("publish", realtimeKindCallEvent, realtimeTargetTopic, "error")
|
||||
logs.Warn.Printf("redis realtime fanout: publish call event topic=%s from=%s event=%s seq=%d failed: %v",
|
||||
msg.RcptTo, msg.AsUser, event.Event, msg.Note.SeqId, err)
|
||||
@@ -162,7 +186,7 @@ func publishRealtimeEvictUser(tenantID types.TenantID, uid types.Uid, reason str
|
||||
msg.Ctrl.Params = map[string]any{"reason": reason}
|
||||
}
|
||||
dedupKey := "evict:user:" + uid.UserId() + ":" + reason
|
||||
if err := store.PublishRealtimeFanout(currentNodeID(), tenantID, realtimeKindSessionControl, realtimeTargetUser, uid.UserId(), skipSid, dedupKey, msg); err != nil {
|
||||
if err := publishRealtimeFanout(currentNodeID(), tenantID, realtimeKindSessionControl, realtimeTargetUser, uid.UserId(), skipSid, dedupKey, msg); err != nil {
|
||||
immetrics.IncRealtimeFanout("publish", realtimeKindSessionControl, realtimeTargetUser, "error")
|
||||
logs.Warn.Printf("redis realtime fanout: publish evict user=%s failed: %v", uid.UserId(), err)
|
||||
return
|
||||
@@ -183,7 +207,7 @@ func publishRealtimeEvictDevice(tenantID types.TenantID, uid types.Uid, deviceID
|
||||
}
|
||||
target := uid.UserId() + ":" + strings.TrimSpace(deviceID)
|
||||
dedupKey := "evict:device:" + target + ":" + reason
|
||||
if err := store.PublishRealtimeFanout(currentNodeID(), tenantID, realtimeKindSessionControl, realtimeTargetDevice, target, "", dedupKey, msg); err != nil {
|
||||
if err := publishRealtimeFanout(currentNodeID(), tenantID, realtimeKindSessionControl, realtimeTargetDevice, target, "", dedupKey, msg); err != nil {
|
||||
immetrics.IncRealtimeFanout("publish", realtimeKindSessionControl, realtimeTargetDevice, "error")
|
||||
logs.Warn.Printf("redis realtime fanout: publish evict device=%s failed: %v", target, err)
|
||||
return
|
||||
@@ -191,24 +215,52 @@ func publishRealtimeEvictDevice(tenantID types.TenantID, uid types.Uid, deviceID
|
||||
immetrics.IncRealtimeFanout("publish", realtimeKindSessionControl, realtimeTargetDevice, "ok")
|
||||
}
|
||||
|
||||
// publishRealtimeEvictUserPlatform evicts all sessions of the user on the given
|
||||
// platform across cluster nodes. skipSid (the new login session) is part of the
|
||||
// dedup key because repeated logins on the same platform within the dedup window
|
||||
// must each reach the subscriber nodes.
|
||||
func publishRealtimeEvictUserPlatform(tenantID types.TenantID, uid types.Uid, platf, reason, skipSid string) {
|
||||
if !tenantID.IsValid() || uid.IsZero() || strings.TrimSpace(platf) == "" {
|
||||
immetrics.IncRealtimeFanout("publish", realtimeKindSessionControl, realtimeTargetPlatf, "skip_invalid")
|
||||
return
|
||||
}
|
||||
key := platformEvictFanoutKey{
|
||||
tenantID: tenantID,
|
||||
uid: uid,
|
||||
platf: strings.TrimSpace(platf),
|
||||
reason: reason,
|
||||
}
|
||||
platformEvictFanout.Lock()
|
||||
if pending := platformEvictFanout.pending[key]; pending != nil {
|
||||
pending.skipSid = skipSid
|
||||
pending.timer.Reset(platformEvictFanoutDebounce)
|
||||
platformEvictFanout.Unlock()
|
||||
immetrics.IncRealtimeFanout("publish", realtimeKindSessionControl, realtimeTargetPlatf, "coalesced")
|
||||
return
|
||||
}
|
||||
pending := &platformEvictFanoutPending{skipSid: skipSid}
|
||||
pending.timer = time.AfterFunc(platformEvictFanoutDebounce, func() {
|
||||
platformEvictFanout.Lock()
|
||||
current := platformEvictFanout.pending[key]
|
||||
if current != pending {
|
||||
platformEvictFanout.Unlock()
|
||||
return
|
||||
}
|
||||
delete(platformEvictFanout.pending, key)
|
||||
finalSkipSid := pending.skipSid
|
||||
platformEvictFanout.Unlock()
|
||||
publishRealtimeEvictUserPlatformNow(key.tenantID, key.uid, key.platf, key.reason, finalSkipSid)
|
||||
})
|
||||
platformEvictFanout.pending[key] = pending
|
||||
platformEvictFanout.Unlock()
|
||||
}
|
||||
|
||||
func publishRealtimeEvictUserPlatformNow(tenantID types.TenantID, uid types.Uid, platf, reason, skipSid string) {
|
||||
msg := NoErrEvicted("", "", types.TimeNow())
|
||||
msg.TenantID = tenantID
|
||||
msg.AsUser = uid.UserId()
|
||||
if reason != "" {
|
||||
msg.Ctrl.Params = map[string]any{"reason": reason}
|
||||
}
|
||||
target := uid.UserId() + ":" + strings.TrimSpace(platf)
|
||||
target := uid.UserId() + ":" + platf
|
||||
dedupKey := "evict:platform:" + target + ":" + skipSid
|
||||
if err := store.PublishRealtimeFanout(currentNodeID(), tenantID, realtimeKindSessionControl, realtimeTargetPlatf, target, skipSid, dedupKey, msg); err != nil {
|
||||
if err := publishRealtimeFanout(currentNodeID(), tenantID, realtimeKindSessionControl, realtimeTargetPlatf, target, skipSid, dedupKey, msg); err != nil {
|
||||
immetrics.IncRealtimeFanout("publish", realtimeKindSessionControl, realtimeTargetPlatf, "error")
|
||||
logs.Warn.Printf("redis realtime fanout: publish evict platform=%s failed: %v", target, err)
|
||||
return
|
||||
@@ -472,28 +524,33 @@ func deliverRealtimeSessionControl(envelope *store.RealtimeFanoutEnvelope, msg *
|
||||
immetrics.IncRealtimeFanout("deliver", envelope.Kind, envelope.TargetType, "skip_invalid_target")
|
||||
return
|
||||
}
|
||||
globals.sessionStore.EvictUserDevice(types.ParseUserId(uidText), deviceID, fanoutEvictReason(msg))
|
||||
immetrics.IncRealtimeFanout("deliver", envelope.Kind, envelope.TargetType, "evicted")
|
||||
recordRealtimeSessionControlEvict(envelope, globals.sessionStore.EvictUserDevice(types.ParseUserId(uidText), deviceID, fanoutEvictReason(msg)))
|
||||
case realtimeTargetPlatf:
|
||||
uidText, platf, ok := strings.Cut(envelope.Target, ":")
|
||||
if !ok {
|
||||
immetrics.IncRealtimeFanout("deliver", envelope.Kind, envelope.TargetType, "skip_invalid_target")
|
||||
return
|
||||
}
|
||||
globals.sessionStore.EvictUserPlatform(types.ParseUserId(uidText), platf, envelope.SkipSid, fanoutEvictReason(msg))
|
||||
immetrics.IncRealtimeFanout("deliver", envelope.Kind, envelope.TargetType, "evicted")
|
||||
recordRealtimeSessionControlEvict(envelope, globals.sessionStore.EvictUserPlatform(types.ParseUserId(uidText), platf, envelope.SkipSid, fanoutEvictReason(msg)))
|
||||
case realtimeTargetUser:
|
||||
uid := types.ParseUserId(envelope.Target)
|
||||
if uid.IsZero() {
|
||||
immetrics.IncRealtimeFanout("deliver", envelope.Kind, envelope.TargetType, "skip_invalid_target")
|
||||
return
|
||||
}
|
||||
globals.sessionStore.EvictUser(uid, envelope.SkipSid, fanoutEvictReason(msg))
|
||||
immetrics.IncRealtimeFanout("deliver", envelope.Kind, envelope.TargetType, "evicted")
|
||||
recordRealtimeSessionControlEvict(envelope, globals.sessionStore.EvictUser(uid, envelope.SkipSid, fanoutEvictReason(msg)))
|
||||
}
|
||||
_ = msg
|
||||
}
|
||||
|
||||
func recordRealtimeSessionControlEvict(envelope *store.RealtimeFanoutEnvelope, evicted int) {
|
||||
if evicted > 0 {
|
||||
immetrics.IncRealtimeFanout("deliver", envelope.Kind, envelope.TargetType, "evicted")
|
||||
return
|
||||
}
|
||||
immetrics.IncRealtimeFanout("deliver", envelope.Kind, envelope.TargetType, "no_local_session")
|
||||
}
|
||||
|
||||
func recordRealtimeFanoutDeliverResult(kind, targetType string, delivered int) {
|
||||
if delivered > 0 {
|
||||
immetrics.IncRealtimeFanout("deliver", kind, targetType, "local_delivered")
|
||||
|
||||
@@ -260,6 +260,61 @@ func TestRealtimeSessionControlStaleEvictDropped(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRealtimePlatformEvictFanoutCoalescesToLatestSession(t *testing.T) {
|
||||
oldPublish := publishRealtimeFanout
|
||||
t.Cleanup(func() {
|
||||
publishRealtimeFanout = oldPublish
|
||||
resetPlatformEvictFanoutForTest()
|
||||
})
|
||||
resetPlatformEvictFanoutForTest()
|
||||
|
||||
type call struct {
|
||||
target string
|
||||
skipSid string
|
||||
dedupKey string
|
||||
}
|
||||
calls := make(chan call, 3)
|
||||
publishRealtimeFanout = func(_ string, _ types.TenantID, kind, targetType, target, skipSid, dedupKey string, _ any) error {
|
||||
if kind == realtimeKindSessionControl && targetType == realtimeTargetPlatf {
|
||||
calls <- call{target: target, skipSid: skipSid, dedupKey: dedupKey}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
tenantID := types.TenantID(1)
|
||||
uid := types.Uid(1001)
|
||||
publishRealtimeEvictUserPlatform(tenantID, uid, "web", "replaced_by_new_login", "sid-old")
|
||||
publishRealtimeEvictUserPlatform(tenantID, uid, "web", "replaced_by_new_login", "sid-new")
|
||||
publishRealtimeEvictUserPlatform(tenantID, uid, "web", "replaced_by_new_login", "sid-final")
|
||||
|
||||
select {
|
||||
case got := <-calls:
|
||||
t.Fatalf("evict fanout published before debounce: %#v", got)
|
||||
case <-time.After(platformEvictFanoutDebounce / 2):
|
||||
}
|
||||
|
||||
select {
|
||||
case got := <-calls:
|
||||
if got.target != uid.UserId()+":web" {
|
||||
t.Fatalf("target = %q", got.target)
|
||||
}
|
||||
if got.skipSid != "sid-final" {
|
||||
t.Fatalf("skipSid = %q, want latest", got.skipSid)
|
||||
}
|
||||
if got.dedupKey != "evict:platform:"+uid.UserId()+":web:sid-final" {
|
||||
t.Fatalf("dedupKey = %q", got.dedupKey)
|
||||
}
|
||||
case <-time.After(2 * platformEvictFanoutDebounce):
|
||||
t.Fatal("timed out waiting for coalesced evict fanout")
|
||||
}
|
||||
|
||||
select {
|
||||
case got := <-calls:
|
||||
t.Fatalf("expected one coalesced fanout, got extra %#v", got)
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
func TestRealtimeP2PTopicFallbackRewritesVisibleInfoTopic(t *testing.T) {
|
||||
oldStore := globals.sessionStore
|
||||
oldHub := globals.hub
|
||||
@@ -314,6 +369,17 @@ func TestRealtimeP2PTopicFallbackRewritesVisibleInfoTopic(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func resetPlatformEvictFanoutForTest() {
|
||||
platformEvictFanout.Lock()
|
||||
defer platformEvictFanout.Unlock()
|
||||
for key, pending := range platformEvictFanout.pending {
|
||||
if pending.timer != nil {
|
||||
pending.timer.Stop()
|
||||
}
|
||||
delete(platformEvictFanout.pending, key)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRealtimeCallEventRoutesToLocalTopic(t *testing.T) {
|
||||
oldHub := globals.hub
|
||||
defer func() { globals.hub = oldHub }()
|
||||
|
||||
@@ -35,6 +35,11 @@ import (
|
||||
// Maximum number of queued messages before session is considered stale and dropped.
|
||||
const sendQueueLimit = 128
|
||||
|
||||
const (
|
||||
lossyBroadcastQueueLimit = sendQueueLimit / 2
|
||||
lossyBroadcastMaxWait = 2 * time.Second
|
||||
)
|
||||
|
||||
// Multiplex sessions may receive a short burst of presence updates while many
|
||||
// remote connections are closing. Give the cluster drain a bounded chance to
|
||||
// free one slot before treating the proxy as stuck.
|
||||
@@ -195,6 +200,9 @@ type Subscription struct {
|
||||
supd chan<- *sessionUpdate
|
||||
}
|
||||
|
||||
// sharedServerMessage is an immutable JSON payload reused by multiple WebSocket sessions.
|
||||
type sharedServerMessage []byte
|
||||
|
||||
func (s *Session) enqueueOutbound(payload any) bool {
|
||||
if s.isMultiplex() {
|
||||
return s.enqueueMultiplexOutbound(payload, multiplexEnqueueWait)
|
||||
@@ -234,8 +242,16 @@ func (s *Session) enqueueMultiplexOutbound(payload any, wait time.Duration) bool
|
||||
s.clusterQueueMu.Unlock()
|
||||
return true
|
||||
}
|
||||
if msg, ok := payload.(*ServerComMessage); ok {
|
||||
msg.clusterQueuedAt = time.Now()
|
||||
}
|
||||
select {
|
||||
case s.send <- payload:
|
||||
peer := "unknown"
|
||||
if s.clnode != nil {
|
||||
peer = s.clnode.name
|
||||
}
|
||||
immetrics.ObserveClusterM2PEnqueue(peer, len(s.send))
|
||||
s.clusterQueueMu.Unlock()
|
||||
return true
|
||||
default:
|
||||
@@ -525,6 +541,49 @@ func (s *Session) queueOutBytes(data []byte) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func (s *Session) queueOutSharedBytes(data []byte) bool {
|
||||
if s == nil || atomic.LoadInt32(&s.terminating) > 0 {
|
||||
return true
|
||||
}
|
||||
if !s.enqueueOutbound(sharedServerMessage(data)) {
|
||||
logs.Err.Println("s.queueOutSharedBytes: session's send queue full", s.sid)
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (s *Session) queueOutLossySharedBytes(data []byte) (queued bool, skipped bool) {
|
||||
if s == nil || atomic.LoadInt32(&s.terminating) > 0 {
|
||||
return true, false
|
||||
}
|
||||
if s.proto != WEBSOCK {
|
||||
return s.queueOutSharedBytes(data), false
|
||||
}
|
||||
|
||||
s.wsQueueMu.Lock()
|
||||
depth := len(s.send)
|
||||
stale := false
|
||||
if s.wsQueueHead < len(s.wsQueueTimes) {
|
||||
stale = time.Since(s.wsQueueTimes[s.wsQueueHead]) > lossyBroadcastMaxWait
|
||||
}
|
||||
if depth >= lossyBroadcastQueueLimit || stale {
|
||||
immetrics.ObserveWSSendQueueEnqueue(depth, cap(s.send), false)
|
||||
s.wsQueueMu.Unlock()
|
||||
return true, true
|
||||
}
|
||||
select {
|
||||
case s.send <- sharedServerMessage(data):
|
||||
s.wsQueueTimes = append(s.wsQueueTimes, time.Now())
|
||||
immetrics.ObserveWSSendQueueEnqueue(len(s.send), cap(s.send), true)
|
||||
s.wsQueueMu.Unlock()
|
||||
return true, false
|
||||
default:
|
||||
immetrics.ObserveWSSendQueueEnqueue(len(s.send), cap(s.send), false)
|
||||
s.wsQueueMu.Unlock()
|
||||
return true, true
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Session) maybeScheduleClusterWriteLoop() {
|
||||
if s.multi != nil {
|
||||
s.multi.scheduleClusterWriteLoop()
|
||||
@@ -960,6 +1019,24 @@ func (s *Session) publish(msg *ClientComMessage) {
|
||||
if sub := s.getSub(msg.RcptTo); sub != nil {
|
||||
// This is a post to a subscribed topic. The message is sent to the topic only
|
||||
msg.queuedAt = time.Now()
|
||||
if topicQueueConfig.publishEnqueueWait > 0 {
|
||||
timer := time.NewTimer(topicQueueConfig.publishEnqueueWait)
|
||||
select {
|
||||
case sub.broadcast <- msg:
|
||||
immetrics.ObserveInternalQueueEnqueue("session_topic_publish", len(sub.broadcast), cap(sub.broadcast), true)
|
||||
if !timer.Stop() {
|
||||
select {
|
||||
case <-timer.C:
|
||||
default:
|
||||
}
|
||||
}
|
||||
case <-timer.C:
|
||||
immetrics.ObserveInternalQueueEnqueue("session_topic_publish", len(sub.broadcast), cap(sub.broadcast), false)
|
||||
// Reply with a 503 to the user.
|
||||
s.queueOut(ErrServiceUnavailableReply(msg, msg.Timestamp))
|
||||
logs.Err.Println("s.publish: sub.broadcast channel full, topic ", msg.RcptTo, s.sid)
|
||||
}
|
||||
} else {
|
||||
select {
|
||||
case sub.broadcast <- msg:
|
||||
immetrics.ObserveInternalQueueEnqueue("session_topic_publish", len(sub.broadcast), cap(sub.broadcast), true)
|
||||
@@ -969,6 +1046,7 @@ func (s *Session) publish(msg *ClientComMessage) {
|
||||
s.queueOut(ErrServiceUnavailableReply(msg, msg.Timestamp))
|
||||
logs.Err.Println("s.publish: sub.broadcast channel full, topic ", msg.RcptTo, s.sid)
|
||||
}
|
||||
}
|
||||
} else if msg.RcptTo == "sys" {
|
||||
if s.authLvl != auth.LevelRoot {
|
||||
s.queueOut(ErrPermissionDeniedReply(msg, msg.Timestamp))
|
||||
|
||||
@@ -338,7 +338,7 @@ func (ss *SessionStore) Shutdown() {
|
||||
}
|
||||
|
||||
// EvictUser terminates all sessions of a given user.
|
||||
func (ss *SessionStore) EvictUser(uid types.Uid, skipSid string, reason ...string) {
|
||||
func (ss *SessionStore) EvictUser(uid types.Uid, skipSid string, reason ...string) int {
|
||||
ss.lock.Lock()
|
||||
defer ss.lock.Unlock()
|
||||
|
||||
@@ -348,6 +348,7 @@ func (ss *SessionStore) EvictUser(uid types.Uid, skipSid string, reason ...strin
|
||||
if len(reason) > 0 && reason[0] != "" {
|
||||
evicted.Ctrl.Params = map[string]any{"reason": reason[0]}
|
||||
}
|
||||
count := 0
|
||||
for _, s := range ss.sessCache {
|
||||
if s.uid == uid && !s.isMultiplex() && s.sid != skipSid {
|
||||
_, data := s.serialize(evicted)
|
||||
@@ -356,10 +357,12 @@ func (ss *SessionStore) EvictUser(uid types.Uid, skipSid string, reason ...strin
|
||||
if s.proto == LPOLL {
|
||||
ss.lru.Remove(s.lpTracker)
|
||||
}
|
||||
count++
|
||||
}
|
||||
}
|
||||
|
||||
statsSet("LiveSessions", int64(len(ss.sessCache)))
|
||||
return count
|
||||
}
|
||||
|
||||
// EvictUserDevice terminates all sessions of a given user connected with a device.
|
||||
|
||||
@@ -37,7 +37,7 @@ func (p localCryptoProvider) RegistrationSMSEnabled() (bool, error) {
|
||||
func (p localCryptoProvider) ThirdPartyConfig(string, ...string) (map[string]string, error) {
|
||||
return nil, types.ErrUnsupported
|
||||
}
|
||||
func (p localCryptoProvider) LatestAppRelease(string, string) (*types.AppRelease, error) {
|
||||
func (p localCryptoProvider) LatestAppRelease(string, string, string) (*types.AppRelease, error) {
|
||||
return nil, types.ErrUnsupported
|
||||
}
|
||||
func (p localCryptoProvider) ModerationProvider() (*types.ModerationProviderConfig, error) {
|
||||
|
||||
@@ -196,7 +196,9 @@ func encryptMessageContent(msg *types.Message) (*types.Message, error) {
|
||||
}
|
||||
|
||||
func decryptMessageContent(msg *types.Message) error {
|
||||
if msg == nil || msg.KeyVersion <= 0 {
|
||||
// 撤回消息的内容已被清空(recall 会把 head/content 置 NULL,但 content_nonce/key_version 残留),
|
||||
// 此类行没有密文可解,直接放行让上层按已撤回形态下发,否则整个 get data 窗口会被单条撤回消息毒化成 500。
|
||||
if msg == nil || msg.KeyVersion <= 0 || msg.Content == nil {
|
||||
return nil
|
||||
}
|
||||
entry, err := getMessageKey(msg.TenantID, msg.KeyVersion)
|
||||
|
||||
@@ -82,6 +82,27 @@ func TestMessageCryptoRejectsTampering(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// 撤回消息 content 已被清 NULL,但 recall UPDATE 保留了 content_nonce/key_version;
|
||||
// 读路径解密必须跳过此类行,否则单条撤回消息会把整个 get data 窗口毒化成 500(2026-10-08 测试环境实案)。
|
||||
func TestMessageCryptoSkipsRecalledContent(t *testing.T) {
|
||||
tenantID := types.TenantID(9004)
|
||||
key := bytes.Repeat([]byte{5}, 32)
|
||||
expires := time.Now().Add(time.Minute)
|
||||
messageKeyCache.Lock()
|
||||
messageKeyCache.items[messageKeyCacheID(tenantID, 0)] = cachedMessageKey{enabled: true, version: 2, key: key, expires: expires}
|
||||
messageKeyCache.items[messageKeyCacheID(tenantID, 2)] = cachedMessageKey{enabled: true, version: 2, key: key, expires: expires}
|
||||
messageKeyCache.Unlock()
|
||||
defer InvalidateMessageEncryptionKeys(tenantID)
|
||||
|
||||
recalled := &types.Message{TenantID: tenantID, Topic: "p2p-recalled", SeqId: 64, KeyVersion: 2, ContentNonce: bytes.Repeat([]byte{1}, 12), Content: nil}
|
||||
if err := decryptMessageContent(recalled); err != nil {
|
||||
t.Fatalf("expected recalled message with nil content to be skipped, got: %v", err)
|
||||
}
|
||||
if recalled.Content != nil {
|
||||
t.Fatalf("nil content should stay nil: %#v", recalled.Content)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMessageCryptoSearchMatchesDecryptedContent(t *testing.T) {
|
||||
content := map[string]any{"txt": "项目 验收通过", "nested": map[string]any{"title": "日报"}}
|
||||
if !messageContentMatchesSearch(content, "验收") {
|
||||
|
||||
+29
-6
@@ -171,7 +171,7 @@ type AdminInternalConfigProvider interface {
|
||||
ListActiveTenants(limit int) ([]types.Tenant, error)
|
||||
RegistrationSMSEnabled() (bool, error)
|
||||
ThirdPartyConfig(group string, keys ...string) (map[string]string, error)
|
||||
LatestAppRelease(platform, channel string) (*types.AppRelease, error)
|
||||
LatestAppRelease(appCode, platform, channel string) (*types.AppRelease, error)
|
||||
ModerationProvider() (*types.ModerationProviderConfig, error)
|
||||
ModerationPolicy(tenantCode string) (*types.ModerationPolicy, error)
|
||||
SensitiveWords(tenantCode string) (*types.SensitiveWordSet, error)
|
||||
@@ -1892,6 +1892,7 @@ type GroupBusinessPersistenceInterface interface {
|
||||
UpsertMember(types.TenantID, *types.GroupMember) (*types.GroupMember, error)
|
||||
GetMember(types.TenantID, string, types.Uid) (*types.GroupMember, error)
|
||||
ListMembers(types.TenantID, string, types.GroupMemberListOptions) ([]types.GroupMember, string, error)
|
||||
MembersCount(types.TenantID, []string) (map[string]int64, error)
|
||||
UpdateMember(types.TenantID, string, types.Uid, types.GroupMemberPatch) (*types.GroupMember, error)
|
||||
CreateInvitation(types.TenantID, *types.GroupInvitation) (*types.GroupInvitation, error)
|
||||
AcceptInvitation(types.TenantID, string, types.Uid, any, time.Time) (*types.GroupInvitation, error)
|
||||
@@ -2064,6 +2065,27 @@ func (groupBusinessMapper) GetMember(tenantID types.TenantID, topic string, uid
|
||||
return member, err
|
||||
}
|
||||
|
||||
func (groupBusinessMapper) MembersCount(tenantID types.TenantID, topics []string) (map[string]int64, error) {
|
||||
cleaned := make([]string, 0, len(topics))
|
||||
seen := make(map[string]bool, len(topics))
|
||||
for _, topic := range topics {
|
||||
topic = strings.TrimSpace(topic)
|
||||
if topic == "" || seen[topic] {
|
||||
continue
|
||||
}
|
||||
seen[topic] = true
|
||||
cleaned = append(cleaned, topic)
|
||||
}
|
||||
if len(cleaned) == 0 {
|
||||
return map[string]int64{}, nil
|
||||
}
|
||||
tenantAdp, err := tenantBusinessAdapter(tenantID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return tenantAdp.TenantGroupMembersCount(tenantID, cleaned)
|
||||
}
|
||||
|
||||
func (groupBusinessMapper) ListMembers(tenantID types.TenantID, topic string, opts types.GroupMemberListOptions) ([]types.GroupMember, string, error) {
|
||||
if strings.TrimSpace(topic) == "" {
|
||||
return nil, "", types.ErrMalformed
|
||||
@@ -2654,20 +2676,21 @@ type profileMapper struct{}
|
||||
var Profiles ProfilePersistenceInterface
|
||||
|
||||
type AppReleasePersistenceInterface interface {
|
||||
Latest(platform, channel string) (*types.AppRelease, error)
|
||||
Latest(appCode, platform, channel string) (*types.AppRelease, error)
|
||||
}
|
||||
|
||||
type appReleaseMapper struct{}
|
||||
|
||||
var AppReleases AppReleasePersistenceInterface
|
||||
|
||||
func (appReleaseMapper) Latest(platform, channel string) (*types.AppRelease, error) {
|
||||
func (appReleaseMapper) Latest(appCode, platform, channel string) (*types.AppRelease, error) {
|
||||
platform = strings.ToLower(strings.TrimSpace(platform))
|
||||
channel = strings.ToLower(strings.TrimSpace(channel))
|
||||
if platform == "" || channel == "" {
|
||||
appCode = strings.ToLower(strings.TrimSpace(appCode))
|
||||
if platform == "" || channel == "" || appCode == "" {
|
||||
return nil, types.ErrMalformed
|
||||
}
|
||||
cacheKey := platform + ":" + channel
|
||||
cacheKey := appCode + ":" + platform + ":" + channel
|
||||
if cached, ok := appReleaseConfigCache.get(cacheKey); ok {
|
||||
copy := *cached
|
||||
return ©, nil
|
||||
@@ -2675,7 +2698,7 @@ func (appReleaseMapper) Latest(platform, channel string) (*types.AppRelease, err
|
||||
if adminInternalConfigProvider == nil {
|
||||
return nil, types.ErrUnsupported
|
||||
}
|
||||
release, err := adminInternalConfigProvider.LatestAppRelease(platform, channel)
|
||||
release, err := adminInternalConfigProvider.LatestAppRelease(appCode, platform, channel)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -56,6 +56,7 @@ type UserPrivacyPatch struct {
|
||||
|
||||
type AppRelease struct {
|
||||
ID int64
|
||||
AppCode string
|
||||
Platform string
|
||||
Channel string
|
||||
LatestVersion string
|
||||
|
||||
+125
-2
@@ -202,6 +202,8 @@ const (
|
||||
StopRehashing
|
||||
)
|
||||
|
||||
const implicitGroupSubSelfOnlyThreshold = 1000
|
||||
|
||||
// Topic shutdown
|
||||
type shutDown struct {
|
||||
// Channel to report back completion of topic shutdown. Could be nil
|
||||
@@ -650,12 +652,20 @@ func (t *Topic) runLocal(hub *Hub) {
|
||||
|
||||
// handleClientMsg is the top-level handler of messages received by the topic from sessions.
|
||||
func (t *Topic) handleClientMsg(msg *ClientComMessage) {
|
||||
started := time.Now()
|
||||
kind := "other"
|
||||
if t.cat == types.TopicCatP2P {
|
||||
kind = "p2p"
|
||||
} else if t.cat == types.TopicCatGrp {
|
||||
kind = "group"
|
||||
}
|
||||
operation := "other"
|
||||
if msg.Pub != nil {
|
||||
operation = "publish"
|
||||
} else if msg.Note != nil {
|
||||
operation = "note"
|
||||
}
|
||||
defer immetrics.ObserveTopicProcess(kind, t.metricRole(), operation, started)
|
||||
immetrics.ObserveTopicQueueWait(kind, msg.queuedAt)
|
||||
msg.queuedAt = time.Time{}
|
||||
if msg.TenantID.IsZero() {
|
||||
@@ -1958,9 +1968,18 @@ func (t *Topic) handlePresence(msg *ServerComMessage) {
|
||||
|
||||
// broadcastToSessions writes message to attached sessions.
|
||||
func (t *Topic) broadcastToSessions(msg *ServerComMessage) {
|
||||
started := time.Now()
|
||||
kind := t.metricKind()
|
||||
role := t.metricRole()
|
||||
websocketTargets, multiplexTargets, otherTargets := 0, 0, 0
|
||||
defer func() {
|
||||
immetrics.ObserveTopicBroadcast(kind, role, started, websocketTargets, multiplexTargets, otherTargets)
|
||||
}()
|
||||
msg.TenantID = t.tenantID
|
||||
// List of sessions to be dropped.
|
||||
var dropSessions []*Session
|
||||
var sharedPayload []byte
|
||||
sharedPayloadReady := false
|
||||
// Broadcast the message. Only {data}, {pres}, {info} are broadcastable.
|
||||
// {meta} and {ctrl} are sent to the session only
|
||||
for sess, pssd := range t.sessions {
|
||||
@@ -2027,7 +2046,49 @@ func (t *Topic) broadcastToSessions(msg *ServerComMessage) {
|
||||
}
|
||||
}
|
||||
|
||||
// Make a copy of msg since messages sent to sessions differ.
|
||||
if sess.proto == WEBSOCK {
|
||||
websocketTargets++
|
||||
} else if sess.isMultiplex() {
|
||||
multiplexTargets++
|
||||
} else {
|
||||
otherTargets++
|
||||
}
|
||||
|
||||
// Ordinary group messages are identical for every local WebSocket subscriber.
|
||||
// Serialize once and reuse immutable bytes to avoid O(recipients) JSON work.
|
||||
if t.canShareSerializedBroadcast(msg, sess, pssd) {
|
||||
if !sharedPayloadReady {
|
||||
var err error
|
||||
sharedPayload, err = json.Marshal(msg)
|
||||
if err != nil {
|
||||
immetrics.IncTopicSharedSerialization(kind, "error")
|
||||
logs.Warn.Printf("topic[%s]: shared broadcast serialization failed: %v", t.name, err)
|
||||
sharedPayload = nil
|
||||
} else {
|
||||
immetrics.IncTopicSharedSerialization(kind, "serialized")
|
||||
}
|
||||
sharedPayloadReady = true
|
||||
}
|
||||
if sharedPayload != nil {
|
||||
if msg.Data != nil {
|
||||
queued, skipped := sess.queueOutLossySharedBytes(sharedPayload)
|
||||
if skipped {
|
||||
immetrics.IncTopicBroadcastDrop(kind, role, "ws_backpressure")
|
||||
}
|
||||
if queued {
|
||||
continue
|
||||
}
|
||||
} else if sess.queueOutSharedBytes(sharedPayload) {
|
||||
continue
|
||||
}
|
||||
immetrics.IncTopicSharedSerialization(kind, "full")
|
||||
logs.Warn.Printf("topic[%s]: connection stuck, detaching - %s", t.name, sess.sid)
|
||||
dropSessions = append(dropSessions, sess)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// Make a copy of msg since messages sent to sessions may differ.
|
||||
msgCopy := msg.copy()
|
||||
// Topic name may be different depending on the user to which the `sess` belongs.
|
||||
t.prepareBroadcastableMessage(msgCopy, pssd.uid, pssd.isChanSub)
|
||||
@@ -2046,6 +2107,48 @@ func (t *Topic) broadcastToSessions(msg *ServerComMessage) {
|
||||
}
|
||||
}
|
||||
|
||||
func (t *Topic) metricKind() string {
|
||||
switch t.cat {
|
||||
case types.TopicCatGrp:
|
||||
return "group"
|
||||
case types.TopicCatP2P:
|
||||
return "p2p"
|
||||
default:
|
||||
return "other"
|
||||
}
|
||||
}
|
||||
|
||||
func (t *Topic) metricRole() string {
|
||||
if t.isProxy {
|
||||
return "proxy"
|
||||
}
|
||||
return "master"
|
||||
}
|
||||
|
||||
func serverMessageType(msg *ServerComMessage) string {
|
||||
switch {
|
||||
case msg == nil:
|
||||
return "unknown"
|
||||
case msg.Data != nil:
|
||||
return "data"
|
||||
case msg.Ctrl != nil:
|
||||
return "ctrl"
|
||||
case msg.Pres != nil:
|
||||
return "pres"
|
||||
case msg.Info != nil:
|
||||
return "info"
|
||||
case msg.Meta != nil:
|
||||
return "meta"
|
||||
default:
|
||||
return "unknown"
|
||||
}
|
||||
}
|
||||
|
||||
func (t *Topic) canShareSerializedBroadcast(msg *ServerComMessage, sess *Session, pssd perSessionData) bool {
|
||||
return t.cat == types.TopicCatGrp && !t.isChan && sess.proto == WEBSOCK && !pssd.isChanSub &&
|
||||
(msg.Data != nil || msg.Pres != nil || msg.Info != nil)
|
||||
}
|
||||
|
||||
// subscriptionReply generates a response to a subscription request
|
||||
func (t *Topic) subscriptionReply(asChan bool, msg *ClientComMessage) error {
|
||||
// The topic is already initialized by the Hub
|
||||
@@ -2793,7 +2896,6 @@ func (t *Topic) replyGetDesc(sess *Session, asUid types.Uid, _ bool, opts *MsgGe
|
||||
if t.cat == types.TopicCatGrp {
|
||||
desc.IsChan = t.isChan
|
||||
desc.SubCnt = t.subCnt
|
||||
logs.Info.Println("replyGetDesc: grp topic", t.name, "subs", t.subCnt)
|
||||
}
|
||||
|
||||
if ifUpdated {
|
||||
@@ -3252,6 +3354,12 @@ func (t *Topic) replyGetSub(sess *Session, asUid types.Uid, authLevel auth.Level
|
||||
// Channel subscribers are using chnXXX topic name rather than grpXXX.
|
||||
topicName = msg.Original
|
||||
}
|
||||
if req = t.limitImplicitLargeGroupSubQuery(req, asUid, msg); req != nil {
|
||||
ifModified = time.Time{}
|
||||
if req.IfModifiedSince != nil {
|
||||
ifModified = *req.IfModifiedSince
|
||||
}
|
||||
}
|
||||
// Include sub.Public.
|
||||
if ifModified.IsZero() {
|
||||
// No cache management. Skip deleted subscriptions.
|
||||
@@ -3456,6 +3564,21 @@ func (t *Topic) replyGetSub(sess *Session, asUid types.Uid, authLevel auth.Level
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *Topic) limitImplicitLargeGroupSubQuery(req *MsgGetOpts, asUid types.Uid, msg *ClientComMessage) *MsgGetOpts {
|
||||
if t.cat != types.TopicCatGrp || msg == nil || msg.Sub == nil || t.subCnt < implicitGroupSubSelfOnlyThreshold {
|
||||
return req
|
||||
}
|
||||
if req != nil && (req.User != "" || req.Topic != "" || req.Limit != 0 || req.IfModifiedSince != nil) {
|
||||
return req
|
||||
}
|
||||
limited := &MsgGetOpts{}
|
||||
if req != nil {
|
||||
*limited = *req
|
||||
}
|
||||
limited.User = asUid.UserId()
|
||||
return limited
|
||||
}
|
||||
|
||||
// replySetSub is a response to new subscription request or an update to a subscription {set.sub}:
|
||||
// update topic metadata cache, save/update subs, reply to the caller as {ctrl} message,
|
||||
// generate a presence notification, if appropriate.
|
||||
|
||||
@@ -213,6 +213,7 @@ func (t *Topic) handleProxyLeaveRequest(msg *ClientComMessage, killTimer *time.T
|
||||
|
||||
// proxyMasterResponse at proxy topic processes a master topic response to an earlier request.
|
||||
func (t *Topic) proxyMasterResponse(msg *ClusterResp, killTimer *time.Timer) {
|
||||
immetrics.ObserveClusterProxyQueueWait(serverMessageType(msg.SrvMsg), msg.proxyQueuedAt)
|
||||
// Kills topic after a period of inactivity.
|
||||
keepAlive := idleProxyTopicTimeout
|
||||
|
||||
|
||||
@@ -2254,6 +2254,121 @@ func TestReplyGetDescInvalidOpts(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestReplyGetSubLimitsImplicitLargeGroupSubscriptionQuery(t *testing.T) {
|
||||
helper := TopicTestHelper{}
|
||||
helper.setUp(t, 2, types.TopicCatGrp, "grpLarge", true)
|
||||
defer helper.tearDown()
|
||||
|
||||
uid := helper.uids[0]
|
||||
helper.topic.subCnt = implicitGroupSubSelfOnlyThreshold
|
||||
helper.topic.perUser[uid] = perUserData{
|
||||
modeWant: types.ModeCReadOnly,
|
||||
modeGiven: types.ModeCReadOnly,
|
||||
}
|
||||
|
||||
helper.tt.EXPECT().
|
||||
GetUsers(types.TenantID(1), "grpLarge", gomock.AssignableToTypeOf(&types.QueryOpt{})).
|
||||
DoAndReturn(func(_ types.TenantID, _ string, opts *types.QueryOpt) ([]types.Subscription, error) {
|
||||
if opts == nil {
|
||||
t.Fatal("expected limited query options")
|
||||
}
|
||||
if opts.User != uid {
|
||||
t.Fatalf("expected query to be limited to %s, got %s", uid.UserId(), opts.User.UserId())
|
||||
}
|
||||
return []types.Subscription{{
|
||||
User: uid.String(),
|
||||
Topic: "grpLarge",
|
||||
ModeWant: types.ModeCReadOnly,
|
||||
ModeGiven: types.ModeCReadOnly,
|
||||
}}, nil
|
||||
})
|
||||
|
||||
msg := &ClientComMessage{
|
||||
Id: "sub1",
|
||||
Original: "grpLarge",
|
||||
AsUser: uid.UserId(),
|
||||
TenantID: 1,
|
||||
Sub: &MsgClientSub{Get: &MsgGetQuery{
|
||||
What: "sub",
|
||||
Sub: nil,
|
||||
}},
|
||||
}
|
||||
|
||||
if err := helper.topic.replyGetSub(helper.sessions[0], uid, auth.LevelAuth, false, msg); err != nil {
|
||||
t.Fatalf("replyGetSub failed: %v", err)
|
||||
}
|
||||
helper.finish()
|
||||
|
||||
if len(helper.results[0].messages) != 1 {
|
||||
t.Fatalf("expected one meta response, got %d", len(helper.results[0].messages))
|
||||
}
|
||||
resp := helper.results[0].messages[0].(*ServerComMessage)
|
||||
if resp.Meta == nil {
|
||||
t.Fatalf("expected meta response")
|
||||
}
|
||||
if len(resp.Meta.Sub) != 1 {
|
||||
t.Fatalf("expected one subscription, got %d", len(resp.Meta.Sub))
|
||||
}
|
||||
if got := resp.Meta.Sub[0].User; got != uid.UserId() {
|
||||
t.Fatalf("expected self subscription %s, got %s", uid.UserId(), got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReplyGetSubKeepsExplicitLargeGroupQuery(t *testing.T) {
|
||||
helper := TopicTestHelper{}
|
||||
helper.setUp(t, 2, types.TopicCatGrp, "grpLarge", true)
|
||||
defer helper.tearDown()
|
||||
|
||||
uid := helper.uids[0]
|
||||
peer := helper.uids[1]
|
||||
helper.topic.subCnt = implicitGroupSubSelfOnlyThreshold
|
||||
helper.topic.perUser[uid] = perUserData{
|
||||
modeWant: types.ModeCReadOnly,
|
||||
modeGiven: types.ModeCReadOnly,
|
||||
}
|
||||
|
||||
helper.tt.EXPECT().
|
||||
GetUsers(types.TenantID(1), "grpLarge", gomock.Nil()).
|
||||
Return([]types.Subscription{
|
||||
{
|
||||
User: uid.String(),
|
||||
Topic: "grpLarge",
|
||||
ModeWant: types.ModeCReadOnly,
|
||||
ModeGiven: types.ModeCReadOnly,
|
||||
},
|
||||
{
|
||||
User: peer.String(),
|
||||
Topic: "grpLarge",
|
||||
ModeWant: types.ModeCReadOnly,
|
||||
ModeGiven: types.ModeCReadOnly,
|
||||
},
|
||||
}, nil)
|
||||
|
||||
msg := &ClientComMessage{
|
||||
Id: "get1",
|
||||
Original: "grpLarge",
|
||||
AsUser: uid.UserId(),
|
||||
TenantID: 1,
|
||||
Get: &MsgClientGet{MsgGetQuery: MsgGetQuery{What: "sub"}},
|
||||
}
|
||||
|
||||
if err := helper.topic.replyGetSub(helper.sessions[0], uid, auth.LevelAuth, false, msg); err != nil {
|
||||
t.Fatalf("replyGetSub failed: %v", err)
|
||||
}
|
||||
helper.finish()
|
||||
|
||||
if len(helper.results[0].messages) != 1 {
|
||||
t.Fatalf("expected one meta response, got %d", len(helper.results[0].messages))
|
||||
}
|
||||
resp := helper.results[0].messages[0].(*ServerComMessage)
|
||||
if resp.Meta == nil {
|
||||
t.Fatalf("expected meta response")
|
||||
}
|
||||
if len(resp.Meta.Sub) != 2 {
|
||||
t.Fatalf("expected explicit query to return all subscriptions, got %d", len(resp.Meta.Sub))
|
||||
}
|
||||
}
|
||||
|
||||
// Verifies ctrl codes in session outputs.
|
||||
func registerSessionVerifyOutputs(t *testing.T, sessionOutput *responses, expectedCtrlCodes []int) {
|
||||
t.Helper()
|
||||
@@ -4334,3 +4449,75 @@ func TestMain(m *testing.M) {
|
||||
globals.maxSubscriberCount = 1_000_000_000
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
|
||||
func TestGroupBroadcastReusesSerializedWebSocketPayload(t *testing.T) {
|
||||
uid1, uid2, uid3 := types.Uid(1), types.Uid(2), types.Uid(3)
|
||||
sess1 := &Session{sid: "ws-1", proto: WEBSOCK, send: make(chan any, 2)}
|
||||
sess2 := &Session{sid: "ws-2", proto: WEBSOCK, send: make(chan any, 2)}
|
||||
sessNoRead := &Session{sid: "ws-no-read", proto: WEBSOCK, send: make(chan any, 2)}
|
||||
topic := &Topic{
|
||||
tenantID: 1,
|
||||
name: "grpShared",
|
||||
xoriginal: "grpShared",
|
||||
cat: types.TopicCatGrp,
|
||||
sessions: map[*Session]perSessionData{
|
||||
sess1: {uid: uid1},
|
||||
sess2: {uid: uid2},
|
||||
sessNoRead: {uid: uid3},
|
||||
},
|
||||
perUser: map[types.Uid]perUserData{
|
||||
uid1: {modeWant: types.ModeCFull, modeGiven: types.ModeCFull},
|
||||
uid2: {modeWant: types.ModeCFull, modeGiven: types.ModeCFull},
|
||||
uid3: {modeWant: types.ModeNone, modeGiven: types.ModeNone},
|
||||
},
|
||||
}
|
||||
msg := &ServerComMessage{Data: &MsgServerData{
|
||||
Topic: "grpShared", From: uid1.UserId(), SeqId: 42, Timestamp: time.Now(), Content: "hello",
|
||||
}}
|
||||
|
||||
topic.broadcastToSessions(msg)
|
||||
|
||||
payload1, ok := (<-sess1.send).(sharedServerMessage)
|
||||
if !ok {
|
||||
t.Fatal("first WebSocket did not receive a serialized payload")
|
||||
}
|
||||
payload2, ok := (<-sess2.send).(sharedServerMessage)
|
||||
if !ok {
|
||||
t.Fatal("second WebSocket did not receive a serialized payload")
|
||||
}
|
||||
if len(payload1) == 0 || len(payload2) == 0 || &payload1[0] != &payload2[0] {
|
||||
t.Fatal("group WebSocket recipients did not reuse the same serialized payload")
|
||||
}
|
||||
var decoded ServerComMessage
|
||||
if err := json.Unmarshal(payload1, &decoded); err != nil {
|
||||
t.Fatalf("invalid shared payload: %v", err)
|
||||
}
|
||||
if decoded.Data == nil || decoded.Data.Topic != "grpShared" || decoded.Data.SeqId != 42 || decoded.Data.Content != "hello" {
|
||||
t.Fatalf("unexpected shared payload: %#v", decoded.Data)
|
||||
}
|
||||
if len(sessNoRead.send) != 0 {
|
||||
t.Fatal("non-reader received a group broadcast")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSharedBroadcastEligibility(t *testing.T) {
|
||||
sess := &Session{proto: WEBSOCK}
|
||||
msg := &ServerComMessage{Data: &MsgServerData{Topic: "grpShared"}}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
topic Topic
|
||||
pssd perSessionData
|
||||
want bool
|
||||
}{
|
||||
{name: "ordinary group", topic: Topic{cat: types.TopicCatGrp}, want: true},
|
||||
{name: "channel", topic: Topic{cat: types.TopicCatGrp, isChan: true}},
|
||||
{name: "channel subscription", topic: Topic{cat: types.TopicCatGrp}, pssd: perSessionData{isChanSub: true}},
|
||||
{name: "p2p", topic: Topic{cat: types.TopicCatP2P}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := tc.topic.canShareSerializedBroadcast(msg, sess, tc.pssd); got != tc.want {
|
||||
t.Fatalf("eligibility=%v want=%v", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
+56
-21
@@ -6,12 +6,14 @@ import (
|
||||
"errors"
|
||||
"math/rand"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"slices"
|
||||
|
||||
"github.com/tinode/chat/server/auth"
|
||||
"github.com/tinode/chat/server/logs"
|
||||
immetrics "github.com/tinode/chat/server/metrics"
|
||||
"github.com/tinode/chat/server/push"
|
||||
"github.com/tinode/chat/server/store"
|
||||
"github.com/tinode/chat/server/store/types"
|
||||
@@ -23,8 +25,15 @@ const (
|
||||
unreadUpdateIOPending = -1
|
||||
// Counter initialization error.
|
||||
unreadUpdateError = -2
|
||||
|
||||
usersUpdateQueueCapacity = 8192
|
||||
usersUpdateEnqueueTimeout = 100 * time.Millisecond
|
||||
usersUpdateQueueFullLogEvery = 5 * time.Second
|
||||
usersUpdateInternalMetricQueue = "user_cache_update"
|
||||
)
|
||||
|
||||
var usersUpdateQueueFullLastLog atomic.Int64
|
||||
|
||||
// Process request for a new account.
|
||||
func replyCreateUser(s *Session, msg *ClientComMessage, rec *auth.Rec) {
|
||||
// The session cannot authenticate with the new account because it's already authenticated.
|
||||
@@ -1325,7 +1334,9 @@ func (pq *pendingReceiptsQueue) fix(index int) {
|
||||
|
||||
// Initialize users cache.
|
||||
func usersInit() {
|
||||
globals.usersUpdate = make(chan *UserCacheReq, 1024)
|
||||
globals.usersUpdate = make(chan *UserCacheReq, usersUpdateQueueCapacity)
|
||||
immetrics.SetInternalQueueCurrent(usersUpdateInternalMetricQueue, 0)
|
||||
immetrics.ObserveInternalQueueEnqueue(usersUpdateInternalMetricQueue, 0, usersUpdateQueueCapacity, true)
|
||||
|
||||
go userUpdater()
|
||||
}
|
||||
@@ -1347,10 +1358,7 @@ func usersUpdateUnread(tenantID types.TenantID, uid types.Uid, val int64, inc bo
|
||||
// Send request to remote node which owns the user.
|
||||
globals.cluster.routeUserReq(upd)
|
||||
} else {
|
||||
select {
|
||||
case globals.usersUpdate <- upd:
|
||||
default:
|
||||
}
|
||||
enqueueUserUpdate(upd, "unread", false)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1368,10 +1376,7 @@ func usersRegisterUser(tenantID types.TenantID, uid types.Uid, add bool) {
|
||||
// Send request to remote node which owns the user.
|
||||
globals.cluster.routeUserReq(upd)
|
||||
} else {
|
||||
select {
|
||||
case globals.usersUpdate <- upd:
|
||||
default:
|
||||
}
|
||||
enqueueUserUpdate(upd, "user", true)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1383,10 +1388,7 @@ func usersRemoveUser(tenantID types.TenantID, uid types.Uid) {
|
||||
|
||||
upd := &UserCacheReq{TenantID: tenantID, UserId: uid, Gone: true}
|
||||
if !globals.cluster.isRemoteTopic(types.TopicKey{TenantID: tenantID, Topic: uid.UserId()}) {
|
||||
select {
|
||||
case globals.usersUpdate <- upd:
|
||||
default:
|
||||
}
|
||||
enqueueUserUpdate(upd, "remove_user", true)
|
||||
}
|
||||
|
||||
if globals.cluster != nil {
|
||||
@@ -1431,24 +1433,56 @@ func usersRegisterTopic(t *Topic, add bool) {
|
||||
}
|
||||
|
||||
if len(local.UserIdList) > 0 {
|
||||
select {
|
||||
case globals.usersUpdate <- local:
|
||||
default:
|
||||
logs.Err.Println("User cache: globals.usersUpdate queue full: ", len(globals.usersUpdate))
|
||||
}
|
||||
enqueueUserUpdate(local, "topic", true)
|
||||
}
|
||||
}
|
||||
|
||||
// usersRequestFromCluster handles requests which came from other cluser nodes.
|
||||
func usersRequestFromCluster(req *UserCacheReq) {
|
||||
if globals.usersUpdate == nil {
|
||||
return
|
||||
enqueueUserUpdate(req, "cluster", true)
|
||||
}
|
||||
|
||||
func enqueueUserUpdate(req *UserCacheReq, kind string, waitOnFull bool) bool {
|
||||
q := globals.usersUpdate
|
||||
if req == nil || q == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
select {
|
||||
case globals.usersUpdate <- req:
|
||||
case q <- req:
|
||||
immetrics.ObserveInternalQueueEnqueue(usersUpdateInternalMetricQueue, len(q), cap(q), true)
|
||||
immetrics.SetInternalQueueCurrent(usersUpdateInternalMetricQueue, len(q))
|
||||
return true
|
||||
default:
|
||||
}
|
||||
|
||||
if waitOnFull {
|
||||
timer := time.NewTimer(usersUpdateEnqueueTimeout)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case q <- req:
|
||||
immetrics.ObserveInternalQueueEnqueue(usersUpdateInternalMetricQueue, len(q), cap(q), true)
|
||||
immetrics.SetInternalQueueCurrent(usersUpdateInternalMetricQueue, len(q))
|
||||
return true
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
|
||||
immetrics.ObserveInternalQueueEnqueue(usersUpdateInternalMetricQueue, len(q), cap(q), false)
|
||||
immetrics.SetInternalQueueCurrent(usersUpdateInternalMetricQueue, len(q))
|
||||
logUsersUpdateQueueFull(kind, q)
|
||||
return false
|
||||
}
|
||||
|
||||
func logUsersUpdateQueueFull(kind string, q chan *UserCacheReq) {
|
||||
now := time.Now().UnixNano()
|
||||
last := usersUpdateQueueFullLastLog.Load()
|
||||
if time.Duration(now-last) < usersUpdateQueueFullLogEvery ||
|
||||
!usersUpdateQueueFullLastLog.CompareAndSwap(last, now) {
|
||||
return
|
||||
}
|
||||
logs.Err.Printf("User cache: usersUpdate queue full kind=%s len=%d cap=%d wait=%s",
|
||||
kind, len(q), cap(q), usersUpdateEnqueueTimeout)
|
||||
}
|
||||
|
||||
var usersCache map[types.UserKey]userCacheEntry
|
||||
@@ -1590,6 +1624,7 @@ func userUpdater() {
|
||||
push.Push(rcpt)
|
||||
}
|
||||
case upd := <-globals.usersUpdate:
|
||||
immetrics.SetInternalQueueCurrent(usersUpdateInternalMetricQueue, len(globals.usersUpdate))
|
||||
if globals.shuttingDown {
|
||||
// If shutdown is in progress we don't care to process anything.
|
||||
// ignore all calls.
|
||||
|
||||
Reference in New Issue
Block a user