diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index 5270cb7..88b49ba 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -45,7 +45,8 @@ jobs: MYSQL_MAX_OPEN_CONNS=24 \ MYSQL_MAX_IDLE_CONNS=8 \ MYSQL_REGISTRY_MAX_OPEN_CONNS=4 \ - MYSQL_REGISTRY_MAX_IDLE_CONNS=2 + MYSQL_REGISTRY_MAX_IDLE_CONNS=2 \ + MYSQL_CONN_MAX_LIFETIME_SECONDS=1800 kubectl patch statefulset/im-server -n im-server-system --type=strategic -p \ '{"spec":{"template":{"spec":{"containers":[{"name":"im-server","readinessProbe":{"httpGet":{"path":"/healthz","port":6060},"timeoutSeconds":3,"periodSeconds":10,"failureThreshold":3},"livenessProbe":{"httpGet":{"path":"/livez","port":6060},"timeoutSeconds":3,"periodSeconds":20,"failureThreshold":3}}]}}}}' if kubectl auth can-i get services -n im-server-system >/dev/null 2>&1 && \ @@ -64,7 +65,25 @@ jobs: echo "::warning::ci-deployer cannot manage Ingress/im-server or it does not exist; skipped least_conn update" fi kubectl rollout restart statefulset/im-server -n im-server-system - kubectl rollout status statefulset/im-server -n im-server-system --timeout=300s + if ! kubectl rollout status statefulset/im-server -n im-server-system --timeout=300s; then + echo "::error::im-server rollout did not become ready within 300 seconds" + kubectl get statefulset/im-server -n im-server-system -o wide || true + kubectl get pods -n im-server-system -l app=im-server -o wide || true + kubectl describe statefulset/im-server -n im-server-system || true + for pod in $(kubectl get pods -n im-server-system -l app=im-server -o name 2>/dev/null); do + echo "===== ${pod} describe =====" + kubectl describe -n im-server-system "${pod}" || true + echo "===== ${pod} current log =====" + kubectl logs -n im-server-system "${pod}" -c im-server --tail=200 || true + echo "===== ${pod} previous log =====" + kubectl logs -n im-server-system "${pod}" -c im-server --previous --tail=200 || true + done + if kubectl auth can-i list events -n im-server-system >/dev/null 2>&1; then + echo "===== recent events =====" + kubectl get events -n im-server-system --sort-by=.lastTimestamp | tail -100 || true + fi + exit 1 + fi echo "Waiting for ArgoCD reconciliation before validating effective settings" sleep 60 @@ -86,12 +105,14 @@ jobs: jsonpath='{.spec.template.spec.containers[?(@.name=="im-server")].env[?(@.name=="MYSQL_REGISTRY_MAX_OPEN_CONNS")].value}')" mysql_registry_max_idle="$(kubectl get statefulset/im-server -n im-server-system -o \ jsonpath='{.spec.template.spec.containers[?(@.name=="im-server")].env[?(@.name=="MYSQL_REGISTRY_MAX_IDLE_CONNS")].value}')" - echo "effective requests=${request_cpu}/${request_memory} limits=${limit_cpu}/${limit_memory} liveness=${liveness} mysql=${mysql_max_open}/${mysql_max_idle} registry=${mysql_registry_max_open}/${mysql_registry_max_idle}" + mysql_conn_max_lifetime="$(kubectl get statefulset/im-server -n im-server-system -o \ + jsonpath='{.spec.template.spec.containers[?(@.name=="im-server")].env[?(@.name=="MYSQL_CONN_MAX_LIFETIME_SECONDS")].value}')" + echo "effective requests=${request_cpu}/${request_memory} limits=${limit_cpu}/${limit_memory} liveness=${liveness} mysql=${mysql_max_open}/${mysql_max_idle} registry=${mysql_registry_max_open}/${mysql_registry_max_idle} lifetime=${mysql_conn_max_lifetime}" if [ "${request_cpu}" != "1" ] || [ "${request_memory}" != "512Mi" ] || \ [ "${limit_cpu}" != "2" ] || [ "${limit_memory}" != "1Gi" ] || \ [ "${liveness}" != "/livez" ] || [ "${mysql_max_open}" != "24" ] || \ [ "${mysql_max_idle}" != "8" ] || [ "${mysql_registry_max_open}" != "4" ] || \ - [ "${mysql_registry_max_idle}" != "2" ]; then + [ "${mysql_registry_max_idle}" != "2" ] || [ "${mysql_conn_max_lifetime}" != "1800" ]; then echo "::error::ArgoCD reverted StatefulSet resources/probes/MySQL pools. Apply deploy/argocd/im-server-test/statefulset-performance-patch.yaml to the im-server-test source." exit 1 fi