feat: add IM service health and metrics endpoints

docker/tinode/Dockerfile

@@ -223,9 +223,9 @@ RUN mkdir /botdata
 RUN chmod +x entrypoint.sh
 RUN chmod +x credentials.sh
 
-# Healthcheck: check if port 6060 is open
+# Healthcheck: verify the process and database dependency are responsive.
 HEALTHCHECK --interval=1m --timeout=3s --start-period=30s \
-  CMD nc -z localhost 6060 || exit 1
+  CMD wget -q -O /dev/null http://localhost:6060/healthz || exit 1
 
 # Generate config from template and run the server.
 ENTRYPOINT ["./entrypoint.sh"]

docker/tinode/Dockerfile.build

@@ -123,7 +123,7 @@ RUN mkdir -p /botdata static \
     && chmod +x entrypoint.sh credentials.sh tinode init-db apply-hotgo-tenant-db
 
 HEALTHCHECK --interval=1m --timeout=3s --start-period=30s \
-  CMD nc -z localhost 6060 || exit 1
+  CMD wget -q -O /dev/null http://localhost:6060/healthz || exit 1
 
 ENTRYPOINT ["./entrypoint.sh"]
 

server/db/mysql/adapter.go

@@ -149,7 +149,7 @@ func openMySQLDB(config configType, dsn, dbName string) (*sqlx.DB, string, error
 		dsn = cfg.FormatDSN()
 		dbName = cfg.DBName
 	}
-	db, err := sqlx.Open("mysql", dsn)
+	db, err := sqlx.Open(instrumentedMySQLDriverName, dsn)
 	if err != nil {
 		return nil, "", err
 	}
@@ -687,7 +687,7 @@ func (a *adapter) CreateDb(reset bool) error {
 	// Clear database name
 	cfg.DBName = ""
 
-	a.db, err = sqlx.Open("mysql", cfg.FormatDSN())
+	a.db, err = sqlx.Open(instrumentedMySQLDriverName, cfg.FormatDSN())
 	if err != nil {
 		return err
 	}

server/db/mysql/metrics_driver.go

@@ -0,0 +1,261 @@
+//go:build mysql
+// +build mysql
+
+package mysql
+
+import (
+	"context"
+	"database/sql"
+	"database/sql/driver"
+	"time"
+
+	ms "github.com/go-sql-driver/mysql"
+	immetrics "github.com/tinode/chat/server/metrics"
+)
+
+const instrumentedMySQLDriverName = "meetingim_mysql_metrics"
+
+func init() {
+	sql.Register(instrumentedMySQLDriverName, instrumentedMySQLDriver{})
+}
+
+type instrumentedMySQLDriver struct{}
+
+func (instrumentedMySQLDriver) Open(dsn string) (driver.Conn, error) {
+	connector, err := instrumentedMySQLDriver{}.OpenConnector(dsn)
+	if err != nil {
+		return nil, err
+	}
+	return connector.Connect(context.Background())
+}
+
+func (instrumentedMySQLDriver) OpenConnector(dsn string) (driver.Connector, error) {
+	cfg, err := ms.ParseDSN(dsn)
+	if err != nil {
+		return nil, err
+	}
+	connector, err := ms.NewConnector(cfg)
+	if err != nil {
+		return nil, err
+	}
+	return instrumentedConnector{connector: connector}, nil
+}
+
+type instrumentedConnector struct {
+	connector driver.Connector
+}
+
+func (c instrumentedConnector) Connect(ctx context.Context) (driver.Conn, error) {
+	conn, err := c.connector.Connect(ctx)
+	if err != nil {
+		return nil, err
+	}
+	return instrumentedConn{conn: conn}, nil
+}
+
+func (c instrumentedConnector) Driver() driver.Driver {
+	return instrumentedMySQLDriver{}
+}
+
+type instrumentedConn struct {
+	conn driver.Conn
+}
+
+func (c instrumentedConn) Prepare(query string) (driver.Stmt, error) {
+	stmt, err := c.conn.Prepare(query)
+	if err != nil {
+		return nil, err
+	}
+	return instrumentedStmt{stmt: stmt}, nil
+}
+
+func (c instrumentedConn) Close() error {
+	return c.conn.Close()
+}
+
+func (c instrumentedConn) Begin() (driver.Tx, error) {
+	started := time.Now()
+	tx, err := c.conn.Begin()
+	immetrics.ObserveDB("mysql", "begin", started, err)
+	if err != nil {
+		return nil, err
+	}
+	return instrumentedTx{tx: tx}, nil
+}
+
+func (c instrumentedConn) Ping(ctx context.Context) error {
+	pinger, ok := c.conn.(driver.Pinger)
+	if !ok {
+		return driver.ErrSkip
+	}
+	started := time.Now()
+	err := pinger.Ping(ctx)
+	immetrics.ObserveDB("mysql", "ping", started, err)
+	return err
+}
+
+func (c instrumentedConn) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) {
+	execer, ok := c.conn.(driver.ExecerContext)
+	if !ok {
+		return nil, driver.ErrSkip
+	}
+	started := time.Now()
+	result, err := execer.ExecContext(ctx, query, args)
+	immetrics.ObserveDB("mysql", "exec", started, err)
+	return result, err
+}
+
+func (c instrumentedConn) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) {
+	queryer, ok := c.conn.(driver.QueryerContext)
+	if !ok {
+		return nil, driver.ErrSkip
+	}
+	started := time.Now()
+	rows, err := queryer.QueryContext(ctx, query, args)
+	immetrics.ObserveDB("mysql", "query", started, err)
+	return rows, err
+}
+
+func (c instrumentedConn) BeginTx(ctx context.Context, opts driver.TxOptions) (driver.Tx, error) {
+	beginner, ok := c.conn.(driver.ConnBeginTx)
+	if !ok {
+		return nil, driver.ErrSkip
+	}
+	started := time.Now()
+	tx, err := beginner.BeginTx(ctx, opts)
+	immetrics.ObserveDB("mysql", "begin", started, err)
+	if err != nil {
+		return nil, err
+	}
+	return instrumentedTx{tx: tx}, nil
+}
+
+func (c instrumentedConn) PrepareContext(ctx context.Context, query string) (driver.Stmt, error) {
+	preparer, ok := c.conn.(driver.ConnPrepareContext)
+	if !ok {
+		return nil, driver.ErrSkip
+	}
+	stmt, err := preparer.PrepareContext(ctx, query)
+	if err != nil {
+		return nil, err
+	}
+	return instrumentedStmt{stmt: stmt}, nil
+}
+
+func (c instrumentedConn) CheckNamedValue(value *driver.NamedValue) error {
+	checker, ok := c.conn.(driver.NamedValueChecker)
+	if !ok {
+		return driver.ErrSkip
+	}
+	return checker.CheckNamedValue(value)
+}
+
+func (c instrumentedConn) ResetSession(ctx context.Context) error {
+	resetter, ok := c.conn.(driver.SessionResetter)
+	if !ok {
+		return nil
+	}
+	return resetter.ResetSession(ctx)
+}
+
+func (c instrumentedConn) IsValid() bool {
+	validator, ok := c.conn.(driver.Validator)
+	if !ok {
+		return true
+	}
+	return validator.IsValid()
+}
+
+type instrumentedStmt struct {
+	stmt driver.Stmt
+}
+
+func (s instrumentedStmt) Close() error {
+	return s.stmt.Close()
+}
+
+func (s instrumentedStmt) NumInput() int {
+	return s.stmt.NumInput()
+}
+
+func (s instrumentedStmt) Exec(args []driver.Value) (driver.Result, error) {
+	started := time.Now()
+	result, err := s.stmt.Exec(args)
+	immetrics.ObserveDB("mysql", "exec", started, err)
+	return result, err
+}
+
+func (s instrumentedStmt) Query(args []driver.Value) (driver.Rows, error) {
+	started := time.Now()
+	rows, err := s.stmt.Query(args)
+	immetrics.ObserveDB("mysql", "query", started, err)
+	return rows, err
+}
+
+func (s instrumentedStmt) ExecContext(ctx context.Context, args []driver.NamedValue) (driver.Result, error) {
+	execer, ok := s.stmt.(driver.StmtExecContext)
+	if !ok {
+		return nil, driver.ErrSkip
+	}
+	started := time.Now()
+	result, err := execer.ExecContext(ctx, args)
+	immetrics.ObserveDB("mysql", "exec", started, err)
+	return result, err
+}
+
+func (s instrumentedStmt) QueryContext(ctx context.Context, args []driver.NamedValue) (driver.Rows, error) {
+	queryer, ok := s.stmt.(driver.StmtQueryContext)
+	if !ok {
+		return nil, driver.ErrSkip
+	}
+	started := time.Now()
+	rows, err := queryer.QueryContext(ctx, args)
+	immetrics.ObserveDB("mysql", "query", started, err)
+	return rows, err
+}
+
+func (s instrumentedStmt) CheckNamedValue(value *driver.NamedValue) error {
+	checker, ok := s.stmt.(driver.NamedValueChecker)
+	if !ok {
+		return driver.ErrSkip
+	}
+	return checker.CheckNamedValue(value)
+}
+
+type instrumentedTx struct {
+	tx driver.Tx
+}
+
+func (tx instrumentedTx) Commit() error {
+	started := time.Now()
+	err := tx.tx.Commit()
+	immetrics.ObserveDB("mysql", "commit", started, err)
+	return err
+}
+
+func (tx instrumentedTx) Rollback() error {
+	started := time.Now()
+	err := tx.tx.Rollback()
+	immetrics.ObserveDB("mysql", "rollback", started, err)
+	return err
+}
+
+var (
+	_ driver.Driver             = instrumentedMySQLDriver{}
+	_ driver.DriverContext      = instrumentedMySQLDriver{}
+	_ driver.Connector          = instrumentedConnector{}
+	_ driver.Conn               = instrumentedConn{}
+	_ driver.Pinger             = instrumentedConn{}
+	_ driver.ExecerContext      = instrumentedConn{}
+	_ driver.QueryerContext     = instrumentedConn{}
+	_ driver.ConnBeginTx        = instrumentedConn{}
+	_ driver.ConnPrepareContext = instrumentedConn{}
+	_ driver.NamedValueChecker  = instrumentedConn{}
+	_ driver.SessionResetter    = instrumentedConn{}
+	_ driver.Validator          = instrumentedConn{}
+	_ driver.Stmt               = instrumentedStmt{}
+	_ driver.StmtExecContext    = instrumentedStmt{}
+	_ driver.StmtQueryContext   = instrumentedStmt{}
+	_ driver.NamedValueChecker  = instrumentedStmt{}
+	_ driver.Tx                 = instrumentedTx{}
+)

server/main.go

@@ -847,6 +847,8 @@ func main() {
 		logs.Info.Printf("Server status is available at '%s'", sspath)
 		mux.HandleFunc(sspath, serveStatus)
 	}
+	mux.HandleFunc("/healthz", serveHealthz)
+	mux.HandleFunc("/metrics", serveMetrics)
 
 	// Handle websocket clients.
 	mux.HandleFunc(config.ApiPath+"v0/channels", serveWebSocket)
@@ -946,7 +948,7 @@ func main() {
 		mux.HandleFunc("/", serve404)
 	}
 
-	if err = listenAndServe(config.Listen, withIMHTTPProtocol(config.ApiPath+"v1/im/", mux), tlsConfig, signalHandler()); err != nil {
+	if err = listenAndServe(config.Listen, monitorHTTP(withIMHTTPProtocol(config.ApiPath+"v1/im/", mux)), tlsConfig, signalHandler()); err != nil {
 		logs.Err.Fatal(err)
 	}
 }

server/metrics/metrics.go

@@ -0,0 +1,61 @@
+package metrics
+
+import (
+	"expvar"
+	"strconv"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+var (
+	Registry = prometheus.NewRegistry()
+
+	HTTPRequestsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name: "http_requests_total",
+		Help: "Total number of HTTP requests by method, normalized path, and status.",
+	}, []string{"method", "path", "status"})
+
+	HTTPRequestDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Name:    "http_request_duration_seconds",
+		Help:    "HTTP request duration by method, normalized path, and status.",
+		Buckets: []float64{0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10},
+	}, []string{"method", "path", "status"})
+
+	DBQueryDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Name:    "db_query_duration_seconds",
+		Help:    "Database operation duration by driver, operation, and status.",
+		Buckets: []float64{0.001, 0.003, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5},
+	}, []string{"driver", "operation", "status"})
+
+	WSConnectionsActive = prometheus.NewGaugeFunc(prometheus.GaugeOpts{
+		Name: "ws_connections_active",
+		Help: "Current active WebSocket connections.",
+	}, func() float64 {
+		return float64(expvarIntValue("LiveSessions"))
+	})
+)
+
+func init() {
+	Registry.MustRegister(WSConnectionsActive, HTTPRequestsTotal, HTTPRequestDuration, DBQueryDuration)
+}
+
+func ObserveDB(driverName, operation string, started time.Time, err error) {
+	status := "ok"
+	if err != nil {
+		status = "error"
+	}
+	DBQueryDuration.WithLabelValues(driverName, operation, status).Observe(time.Since(started).Seconds())
+}
+
+func expvarIntValue(name string) int64 {
+	value := expvar.Get(name)
+	if value == nil {
+		return 0
+	}
+	n, err := strconv.ParseInt(value.String(), 10, 64)
+	if err != nil {
+		return 0
+	}
+	return n
+}

server/monitoring_http.go

@@ -0,0 +1,151 @@
+package main
+
+import (
+	"bufio"
+	"context"
+	"encoding/json"
+	"io"
+	"net"
+	"net/http"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	immetrics "github.com/tinode/chat/server/metrics"
+	"github.com/tinode/chat/server/store"
+)
+
+var (
+	metricsHandler        = promhttp.HandlerFor(immetrics.Registry, promhttp.HandlerOpts{})
+	uuidLikeMetricSegment = regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F-]{13,}$`)
+)
+
+type healthzResponse struct {
+	Status string            `json:"status"`
+	Checks map[string]string `json:"checks"`
+}
+
+func serveHealthz(wrt http.ResponseWriter, req *http.Request) {
+	if req.Method != http.MethodGet && req.Method != http.MethodHead {
+		wrt.Header().Set("Allow", http.MethodGet+", "+http.MethodHead)
+		http.Error(wrt, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	wrt.Header().Set("Content-Type", "application/json; charset=utf-8")
+
+	checks := map[string]string{}
+	status := http.StatusOK
+	ctx, cancel := context.WithTimeout(req.Context(), 2*time.Second)
+	defer cancel()
+	if err := store.Store.HealthCheck(ctx); err != nil {
+		checks["mysql"] = err.Error()
+		status = http.StatusServiceUnavailable
+	} else {
+		checks["mysql"] = "ok"
+	}
+
+	body := healthzResponse{Status: "ok", Checks: checks}
+	if status != http.StatusOK {
+		body.Status = "error"
+	}
+	wrt.WriteHeader(status)
+	if req.Method != http.MethodHead {
+		_ = json.NewEncoder(wrt).Encode(body)
+	}
+}
+
+func serveMetrics(wrt http.ResponseWriter, req *http.Request) {
+	if req.Method != http.MethodGet && req.Method != http.MethodHead {
+		wrt.Header().Set("Allow", http.MethodGet+", "+http.MethodHead)
+		http.Error(wrt, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	metricsHandler.ServeHTTP(wrt, req)
+}
+
+func monitorHTTP(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(wrt http.ResponseWriter, req *http.Request) {
+		if req.URL.Path == "/metrics" {
+			next.ServeHTTP(wrt, req)
+			return
+		}
+		started := time.Now()
+		rec := &monitoringResponseWriter{ResponseWriter: wrt, status: http.StatusOK}
+		next.ServeHTTP(rec, req)
+		status := strconv.Itoa(rec.status)
+		path := normalizeMetricPath(req.URL.Path)
+		immetrics.HTTPRequestsTotal.WithLabelValues(req.Method, path, status).Inc()
+		immetrics.HTTPRequestDuration.WithLabelValues(req.Method, path, status).Observe(time.Since(started).Seconds())
+	})
+}
+
+type monitoringResponseWriter struct {
+	http.ResponseWriter
+	status int
+}
+
+func (w *monitoringResponseWriter) WriteHeader(status int) {
+	w.status = status
+	w.ResponseWriter.WriteHeader(status)
+}
+
+func (w *monitoringResponseWriter) Flush() {
+	if f, ok := w.ResponseWriter.(http.Flusher); ok {
+		f.Flush()
+	}
+}
+
+func (w *monitoringResponseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
+	h, ok := w.ResponseWriter.(http.Hijacker)
+	if !ok {
+		return nil, nil, http.ErrNotSupported
+	}
+	return h.Hijack()
+}
+
+func (w *monitoringResponseWriter) Push(target string, opts *http.PushOptions) error {
+	p, ok := w.ResponseWriter.(http.Pusher)
+	if !ok {
+		return http.ErrNotSupported
+	}
+	return p.Push(target, opts)
+}
+
+func (w *monitoringResponseWriter) ReadFrom(r io.Reader) (int64, error) {
+	if rf, ok := w.ResponseWriter.(io.ReaderFrom); ok {
+		return rf.ReadFrom(r)
+	}
+	return io.Copy(w.ResponseWriter, r)
+}
+
+func normalizeMetricPath(path string) string {
+	path = strings.TrimSpace(path)
+	if path == "" {
+		return "/"
+	}
+	parts := strings.Split(path, "/")
+	for i, part := range parts {
+		if part == "" {
+			continue
+		}
+		if isMetricDynamicSegment(part) {
+			parts[i] = "{id}"
+		}
+	}
+	return strings.Join(parts, "/")
+}
+
+func isMetricDynamicSegment(segment string) bool {
+	if strings.HasPrefix(segment, "usr") || strings.HasPrefix(segment, "grp") || strings.HasPrefix(segment, "p2p") || strings.HasPrefix(segment, "chn") {
+		return true
+	}
+	if _, err := strconv.ParseInt(segment, 10, 64); err == nil {
+		return true
+	}
+	if uuidLikeMetricSegment.MatchString(segment) {
+		return true
+	}
+	return false
+}

server/monitoring_http_test.go

@@ -0,0 +1,96 @@
+package main
+
+import (
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/golang/mock/gomock"
+	immetrics "github.com/tinode/chat/server/metrics"
+	"github.com/tinode/chat/server/store"
+	"github.com/tinode/chat/server/store/mock_store"
+)
+
+func TestServeHealthzOK(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	mockStore := mock_store.NewMockPersistentStorageInterface(ctrl)
+	oldStore := store.Store
+	store.Store = mockStore
+	t.Cleanup(func() { store.Store = oldStore })
+	mockStore.EXPECT().HealthCheck(gomock.Any()).Return(nil)
+
+	wrt := httptest.NewRecorder()
+	serveHealthz(wrt, httptest.NewRequest(http.MethodGet, "/healthz", nil))
+
+	if wrt.Code != http.StatusOK {
+		t.Fatalf("unexpected status %d body=%s", wrt.Code, wrt.Body.String())
+	}
+	if body := wrt.Body.String(); !strings.Contains(body, `"status":"ok"`) || !strings.Contains(body, `"mysql":"ok"`) {
+		t.Fatalf("unexpected body %s", body)
+	}
+}
+
+func TestServeHealthzError(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	mockStore := mock_store.NewMockPersistentStorageInterface(ctrl)
+	oldStore := store.Store
+	store.Store = mockStore
+	t.Cleanup(func() { store.Store = oldStore })
+	mockStore.EXPECT().HealthCheck(gomock.Any()).Return(errors.New("dial timeout"))
+
+	wrt := httptest.NewRecorder()
+	serveHealthz(wrt, httptest.NewRequest(http.MethodGet, "/healthz", nil))
+
+	if wrt.Code != http.StatusServiceUnavailable {
+		t.Fatalf("unexpected status %d body=%s", wrt.Code, wrt.Body.String())
+	}
+	if body := wrt.Body.String(); !strings.Contains(body, `"status":"error"`) || !strings.Contains(body, `"mysql":"dial timeout"`) {
+		t.Fatalf("unexpected body %s", body)
+	}
+}
+
+func TestServeMetricsIncludesWSConnections(t *testing.T) {
+	wrt := httptest.NewRecorder()
+	serveMetrics(wrt, httptest.NewRequest(http.MethodGet, "/metrics", nil))
+
+	if wrt.Code != http.StatusOK {
+		t.Fatalf("unexpected status %d body=%s", wrt.Code, wrt.Body.String())
+	}
+	body := wrt.Body.String()
+	if !strings.Contains(body, "# TYPE ws_connections_active gauge") || !strings.Contains(body, "ws_connections_active ") {
+		t.Fatalf("unexpected body %s", body)
+	}
+}
+
+func TestMonitorHTTPRecordsRequestMetrics(t *testing.T) {
+	handler := monitorHTTP(http.HandlerFunc(func(wrt http.ResponseWriter, req *http.Request) {
+		wrt.WriteHeader(http.StatusCreated)
+	}))
+
+	handler.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodPost, "/v1/im/users/1234567890123456789/profile", nil))
+	wrt := httptest.NewRecorder()
+	serveMetrics(wrt, httptest.NewRequest(http.MethodGet, "/metrics", nil))
+
+	body := wrt.Body.String()
+	if !strings.Contains(body, `http_requests_total{method="POST",path="/v1/im/users/{id}/profile",status="201"} 1`) {
+		t.Fatalf("request counter was not exported: %s", body)
+	}
+	if !strings.Contains(body, `http_request_duration_seconds_bucket{method="POST",path="/v1/im/users/{id}/profile",status="201"`) {
+		t.Fatalf("request duration was not exported: %s", body)
+	}
+}
+
+func TestServeMetricsIncludesDBQueryDuration(t *testing.T) {
+	immetrics.ObserveDB("mysql", "query", time.Now().Add(-time.Millisecond), nil)
+
+	wrt := httptest.NewRecorder()
+	serveMetrics(wrt, httptest.NewRequest(http.MethodGet, "/metrics", nil))
+
+	body := wrt.Body.String()
+	if !strings.Contains(body, `db_query_duration_seconds_bucket{driver="mysql",operation="query",status="ok"`) {
+		t.Fatalf("db query duration was not exported: %s", body)
+	}
+}

server/stats.go

@@ -77,18 +77,27 @@ func statsInit(mux *http.ServeMux, path string) {
 
 func statsRegisterDbStats() {
 	if f := store.Store.DbStats(); f != nil {
+		if expvar.Get("DbStats") != nil {
+			return
+		}
 		expvar.Publish("DbStats", expvar.Func(f))
 	}
 }
 
 // Register integer variable. Don't check for initialization.
 func statsRegisterInt(name string) {
+	if expvar.Get(name) != nil {
+		return
+	}
 	expvar.Publish(name, new(expvar.Int))
 }
 
 // Register histogram variable. `bounds` specifies histogram buckets/bins
 // (see comment next to the `histogram` struct definition).
 func statsRegisterHistogram(name string, bounds []float64) {
+	if expvar.Get(name) != nil {
+		return
+	}
 	numBuckets := len(bounds) + 1
 	expvar.Publish(name, &histogram{
 		CountPerBucket: make([]int64, numBuckets),

server/store/mock_store/mock_store.go

@@ -5,6 +5,7 @@
 package mock_store
 
 import (
+	context "context"
 	json "encoding/json"
 	reflect "reflect"
 	time "time"
@@ -138,6 +139,20 @@ func (mr *MockPersistentStorageInterfaceMockRecorder) GetAuthNames() *gomock.Cal
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAuthNames", reflect.TypeOf((*MockPersistentStorageInterface)(nil).GetAuthNames))
 }
 
+// HealthCheck mocks base method.
+func (m *MockPersistentStorageInterface) HealthCheck(ctx context.Context) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "HealthCheck", ctx)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// HealthCheck indicates an expected call of HealthCheck.
+func (mr *MockPersistentStorageInterfaceMockRecorder) HealthCheck(ctx interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HealthCheck", reflect.TypeOf((*MockPersistentStorageInterface)(nil).HealthCheck), ctx)
+}
+
 // GetDbVersion mocks base method.
 func (m *MockPersistentStorageInterface) GetDbVersion() int {
 	m.ctrl.T.Helper()

server/store/store.go

@@ -2,6 +2,7 @@
 package store
 
 import (
+	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -120,6 +121,7 @@ type PersistentStorageInterface interface {
 	GetUid() types.Uid
 	GetUidString() string
 	DbStats() func() any
+	HealthCheck(context.Context) error
 	GetAuthNames() []string
 	GetAuthHandler(name string) auth.AuthHandler
 	GetLogicalAuthHandler(name string) auth.AuthHandler
@@ -473,6 +475,19 @@ func (storeObj) GetAdapter() adapter.Adapter {
 	return adp
 }
 
+// HealthCheck verifies that the persistent storage adapter can execute a real operation.
+func (storeObj) HealthCheck(ctx context.Context) error {
+	if adp == nil || !adp.IsOpen() {
+		return errors.New("store is not open")
+	}
+	if checker, ok := adp.(interface {
+		HealthCheck(context.Context) error
+	}); ok {
+		return checker.HealthCheck(ctx)
+	}
+	return errors.New("health check is not supported by " + adp.GetName())
+}
+
 // GetAdapterName returns the name of the current adater.
 func (storeObj) GetAdapterName() string {
 	if adp != nil {