fix: harden multi-pod delivery and load verification

docker/README.md

@@ -188,6 +188,11 @@ You can specify the following environment variables when issuing `docker run` co
 | `AWS_S3_ENDPOINT`[^2] | string |  | An endpoint URL (hostname only or fully qualified URI) to override the default endpoint; can be of any S3-compatible service, such as `minio-api.x.io` |
 | `AWS_SECRET_ACCESS_KEY`[^2] | string |  | AWS [Secret Access Key](https://aws.amazon.com/blogs/security/wheres-my-secret-access-key/) when using `s3` media handler. |
 | `CLUSTER_SELF` | string |  | Node name if the server is running in a Tinode cluster. |
+| `CLUSTER_PROXY_TO_MASTER_BUFFER` | number | `4096` | Total bounded proxy-to-master queue capacity for each peer. |
+| `CLUSTER_PROXY_TO_MASTER_WORKERS` | number | `16` | Ordered queue shards and synchronous RPC workers for each peer. |
+| `CLUSTER_PROXY_TO_MASTER_ENQUEUE_TIMEOUT_MS` | number | `300` | Maximum wait for a full proxy-to-master shard before returning 503. |
+| `CLUSTER_ROUTE_RETRIES` | number | `3` | Number of retries after a remote server-message route is rejected as overloaded. |
+| `CLUSTER_ROUTE_RETRY_BACKOFF_MS` | number | `25` | Initial exponential retry delay for remote server-message routes. |
 | `DEBUG_EMAIL_VERIFICATION_CODE`[^2] | string |  | Enable dummy email verification code, e.g. `123456`. Disabled by default (empty string). |
 | `DEFAULT_COUNTRY_CODE`[^2] | string | `US` | 2-letter country code to assign to sessions by default when the country isn't specified by the client explicitly and it's impossible to infer it. |
 | `EXT_CONFIG`[^1] | string |  | Path to external config file to use instead of the built-in one. If this parameter is used, most other variables are ignored[^1]. |

docker/tinode/config.template

@@ -281,6 +281,11 @@
 	"cluster_config": {
 		"self": "",
 		"nodes": $CLUSTER_NODES_JSON,
+		"proxy_to_master_buffer": $CLUSTER_PROXY_TO_MASTER_BUFFER,
+		"proxy_to_master_workers": $CLUSTER_PROXY_TO_MASTER_WORKERS,
+		"proxy_to_master_enqueue_timeout_ms": $CLUSTER_PROXY_TO_MASTER_ENQUEUE_TIMEOUT_MS,
+		"route_retries": $CLUSTER_ROUTE_RETRIES,
+		"route_retry_backoff_ms": $CLUSTER_ROUTE_RETRY_BACKOFF_MS,
 		"failover": {
 			"enabled": true,
 			"heartbeat": 100,

docker/tinode/entrypoint.sh

@@ -13,6 +13,11 @@ else
 	CONFIG=working.config
 
 	: "${REDIS_KEY_PREFIX:=im}"
+	: "${CLUSTER_PROXY_TO_MASTER_BUFFER:=4096}"
+	: "${CLUSTER_PROXY_TO_MASTER_WORKERS:=16}"
+	: "${CLUSTER_PROXY_TO_MASTER_ENQUEUE_TIMEOUT_MS:=300}"
+	: "${CLUSTER_ROUTE_RETRIES:=3}"
+	: "${CLUSTER_ROUTE_RETRY_BACKOFF_MS:=25}"
 	if [ -z "$CLUSTER_NODES_JSON" ] ; then
 		CLUSTER_NODES_JSON='[{"name":"tinode-0","addr":"tinode-0:12000"},{"name":"tinode-1","addr":"tinode-1:12001"},{"name":"tinode-2","addr":"tinode-2:12002"}]'
 	fi