pro/im_server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(deploy): diagnose stalled statefulset rollouts

Browse Source
This commit is contained in:
richjohnrambo
2026-10-04 01:06:26 +08:00
parent 7b25365ec3
commit c46209f4e7
1 changed files with 25 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitea/workflows/deploy.yml
+25 -4
View File
@@ -45,7 +45,8 @@ jobs:
MYSQL_MAX_OPEN_CONNS=24 \ MYSQL_MAX_OPEN_CONNS=24 \
MYSQL_MAX_IDLE_CONNS=8 \ MYSQL_MAX_IDLE_CONNS=8 \
MYSQL_REGISTRY_MAX_OPEN_CONNS=4 \ MYSQL_REGISTRY_MAX_OPEN_CONNS=4 \
MYSQL_REGISTRY_MAX_IDLE_CONNS=2 MYSQL_REGISTRY_MAX_IDLE_CONNS=2 \
MYSQL_CONN_MAX_LIFETIME_SECONDS=1800
kubectl patch statefulset/im-server -n im-server-system --type=strategic -p \ kubectl patch statefulset/im-server -n im-server-system --type=strategic -p \
'{"spec":{"template":{"spec":{"containers":[{"name":"im-server","readinessProbe":{"httpGet":{"path":"/healthz","port":6060},"timeoutSeconds":3,"periodSeconds":10,"failureThreshold":3},"livenessProbe":{"httpGet":{"path":"/livez","port":6060},"timeoutSeconds":3,"periodSeconds":20,"failureThreshold":3}}]}}}}' '{"spec":{"template":{"spec":{"containers":[{"name":"im-server","readinessProbe":{"httpGet":{"path":"/healthz","port":6060},"timeoutSeconds":3,"periodSeconds":10,"failureThreshold":3},"livenessProbe":{"httpGet":{"path":"/livez","port":6060},"timeoutSeconds":3,"periodSeconds":20,"failureThreshold":3}}]}}}}'
if kubectl auth can-i get services -n im-server-system >/dev/null 2>&1 && \ if kubectl auth can-i get services -n im-server-system >/dev/null 2>&1 && \
@@ -64,7 +65,25 @@ jobs:
echo "::warning::ci-deployer cannot manage Ingress/im-server or it does not exist; skipped least_conn update" echo "::warning::ci-deployer cannot manage Ingress/im-server or it does not exist; skipped least_conn update"
fi fi
kubectl rollout restart statefulset/im-server -n im-server-system kubectl rollout restart statefulset/im-server -n im-server-system
kubectl rollout status statefulset/im-server -n im-server-system --timeout=300s if ! kubectl rollout status statefulset/im-server -n im-server-system --timeout=300s; then
echo "::error::im-server rollout did not become ready within 300 seconds"
kubectl get statefulset/im-server -n im-server-system -o wide || true
kubectl get pods -n im-server-system -l app=im-server -o wide || true
kubectl describe statefulset/im-server -n im-server-system || true
for pod in $(kubectl get pods -n im-server-system -l app=im-server -o name 2>/dev/null); do
echo "===== ${pod} describe ====="
kubectl describe -n im-server-system "${pod}" || true
echo "===== ${pod} current log ====="
kubectl logs -n im-server-system "${pod}" -c im-server --tail=200 || true
echo "===== ${pod} previous log ====="
kubectl logs -n im-server-system "${pod}" -c im-server --previous --tail=200 || true
done
if kubectl auth can-i list events -n im-server-system >/dev/null 2>&1; then
echo "===== recent events ====="
kubectl get events -n im-server-system --sort-by=.lastTimestamp | tail -100 || true
fi
exit 1
fi
echo "Waiting for ArgoCD reconciliation before validating effective settings" echo "Waiting for ArgoCD reconciliation before validating effective settings"
sleep 60 sleep 60
@@ -86,12 +105,14 @@ jobs:
jsonpath='{.spec.template.spec.containers[?(@.name=="im-server")].env[?(@.name=="MYSQL_REGISTRY_MAX_OPEN_CONNS")].value}')" jsonpath='{.spec.template.spec.containers[?(@.name=="im-server")].env[?(@.name=="MYSQL_REGISTRY_MAX_OPEN_CONNS")].value}')"
mysql_registry_max_idle="$(kubectl get statefulset/im-server -n im-server-system -o \ mysql_registry_max_idle="$(kubectl get statefulset/im-server -n im-server-system -o \
jsonpath='{.spec.template.spec.containers[?(@.name=="im-server")].env[?(@.name=="MYSQL_REGISTRY_MAX_IDLE_CONNS")].value}')" jsonpath='{.spec.template.spec.containers[?(@.name=="im-server")].env[?(@.name=="MYSQL_REGISTRY_MAX_IDLE_CONNS")].value}')"
echo "effective requests=${request_cpu}/${request_memory} limits=${limit_cpu}/${limit_memory} liveness=${liveness} mysql=${mysql_max_open}/${mysql_max_idle} registry=${mysql_registry_max_open}/${mysql_registry_max_idle}" mysql_conn_max_lifetime="$(kubectl get statefulset/im-server -n im-server-system -o \
jsonpath='{.spec.template.spec.containers[?(@.name=="im-server")].env[?(@.name=="MYSQL_CONN_MAX_LIFETIME_SECONDS")].value}')"
echo "effective requests=${request_cpu}/${request_memory} limits=${limit_cpu}/${limit_memory} liveness=${liveness} mysql=${mysql_max_open}/${mysql_max_idle} registry=${mysql_registry_max_open}/${mysql_registry_max_idle} lifetime=${mysql_conn_max_lifetime}"
if [ "${request_cpu}" != "1" ] || [ "${request_memory}" != "512Mi" ] || \ if [ "${request_cpu}" != "1" ] || [ "${request_memory}" != "512Mi" ] || \
[ "${limit_cpu}" != "2" ] || [ "${limit_memory}" != "1Gi" ] || \ [ "${limit_cpu}" != "2" ] || [ "${limit_memory}" != "1Gi" ] || \
[ "${liveness}" != "/livez" ] || [ "${mysql_max_open}" != "24" ] || \ [ "${liveness}" != "/livez" ] || [ "${mysql_max_open}" != "24" ] || \
[ "${mysql_max_idle}" != "8" ] || [ "${mysql_registry_max_open}" != "4" ] || \ [ "${mysql_max_idle}" != "8" ] || [ "${mysql_registry_max_open}" != "4" ] || \
[ "${mysql_registry_max_idle}" != "2" ]; then [ "${mysql_registry_max_idle}" != "2" ] || [ "${mysql_conn_max_lifetime}" != "1800" ]; then
echo "::error::ArgoCD reverted StatefulSet resources/probes/MySQL pools. Apply deploy/argocd/im-server-test/statefulset-performance-patch.yaml to the im-server-test source." echo "::error::ArgoCD reverted StatefulSet resources/probes/MySQL pools. Apply deploy/argocd/im-server-test/statefulset-performance-patch.yaml to the im-server-test source."
exit 1 exit 1
fi fi