fix: reduce persistence lag and retry audit deadlocks

This commit is contained in:
richjohnrambo
2026-09-27 16:33:05 +08:00
parent 704c65a7a4
commit d96f661f30
8 changed files with 185 additions and 18 deletions
+2 -2
View File
@@ -4671,7 +4671,7 @@ func (a *adapter) FileUserHasAccess(fid string, uid t.Uid) (bool, error) {
return a.db.GetContext(ctx, &allowed, query, args...)
}
err := run()
for attempt := 0; isRetryableMySQLReadErr(err) && attempt < 2; attempt++ {
for attempt := 0; isRetryableMySQLError(err) && attempt < 2; attempt++ {
time.Sleep(time.Duration(attempt+1) * 25 * time.Millisecond)
err = run()
}
@@ -4813,7 +4813,7 @@ func isIDOutOfRange(err error) bool {
return ok && myerr.Number == 1264 && strings.Contains(strings.ToLower(myerr.Message), "id")
}
func isRetryableMySQLReadErr(err error) bool {
func isRetryableMySQLError(err error) bool {
if err == nil {
return false
}
@@ -17,6 +17,7 @@ func ensureIMAdminGovernanceTables(exec sqlExecutor) error {
request_id VARCHAR(64) NOT NULL DEFAULT '', login_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
logout_at DATETIME(3), metadata JSON, created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
PRIMARY KEY(id), INDEX idx_im_user_login_tenant_time(tenant_id, login_at DESC, id DESC),
INDEX idx_im_user_login_session_close(tenant_id, session_id, logout_at),
INDEX idx_im_user_login_user_time(user_id, login_at DESC, id DESC),
INDEX idx_im_user_login_ip_time(ip, login_at DESC, id DESC),
INDEX idx_im_user_login_device_time(device_id(64), login_at DESC),
@@ -53,5 +54,9 @@ func ensureIMAdminGovernanceTables(exec sqlExecutor) error {
return err
}
}
if err := execIgnoreMySQLError(exec,
`ALTER TABLE im_user_login_logs ADD INDEX idx_im_user_login_session_close(tenant_id,session_id,logout_at)`, 1061); err != nil {
return err
}
return nil
}
@@ -13,8 +13,8 @@ func TestEnsureIMAdminGovernanceTablesDefinesLoginAuditTenantScope(t *testing.T)
if err := ensureIMAdminGovernanceTables(exec); err != nil {
t.Fatal(err)
}
if len(exec.statements) != 3 {
t.Fatalf("created %d admin governance tables, want 3", len(exec.statements))
if len(exec.statements) != 4 {
t.Fatalf("executed %d admin governance schema statements, want 4", len(exec.statements))
}
joined := strings.ToLower(strings.Join(exec.statements, "\n"))
for _, table := range []string{"im_user_login_logs", "im_message_admin_actions", "im_user_ip_whitelists"} {
@@ -38,4 +38,8 @@ func TestEnsureIMAdminGovernanceTablesDefinesLoginAuditTenantScope(t *testing.T)
if end < 0 || !strings.Contains(joined[start:start+end], "tenant_id bigint not null") {
t.Fatal("im_user_login_logs must declare tenant_id")
}
if !strings.Contains(joined, "idx_im_user_login_session_close(tenant_id, session_id, logout_at)") &&
!strings.Contains(joined, "idx_im_user_login_session_close(tenant_id,session_id,logout_at)") {
t.Fatal("im_user_login_logs must index tenant-scoped session close updates")
}
}
+12 -1
View File
@@ -93,7 +93,18 @@ func (a *adapter) TenantLoginAuditClose(id t.TenantID, sessionID string, at time
if tenantAdp != a {
return tenantAdp.TenantLoginAuditClose(id, sessionID, at)
}
_, err = a.db.Exec("UPDATE im_user_login_logs SET logout_at=? WHERE tenant_id=? AND session_id=? AND logout_at IS NULL", at, int64(id), sessionID)
return retryMySQLDeadlock(func() error {
_, execErr := a.db.Exec("UPDATE im_user_login_logs SET logout_at=? WHERE tenant_id=? AND session_id=? AND logout_at IS NULL", at, int64(id), sessionID)
return execErr
}, time.Sleep)
}
func retryMySQLDeadlock(operation func() error, sleep func(time.Duration)) error {
err := operation()
for attempt := 1; isRetryableMySQLError(err) && attempt < 4; attempt++ {
sleep(time.Duration(attempt) * 25 * time.Millisecond)
err = operation()
}
return err
}
@@ -3,9 +3,13 @@
package mysql
import (
"errors"
"os"
"strings"
"testing"
"time"
mysqldriver "github.com/go-sql-driver/mysql"
)
func TestLoginAuditQueriesAreTenantScoped(t *testing.T) {
@@ -24,3 +28,48 @@ func TestLoginAuditQueriesAreTenantScoped(t *testing.T) {
}
}
}
func TestRetryMySQLDeadlockEventuallySucceeds(t *testing.T) {
calls := 0
var delays []time.Duration
err := retryMySQLDeadlock(func() error {
calls++
if calls < 3 {
return &mysqldriver.MySQLError{Number: 1213, SQLState: [5]byte{'4', '0', '0', '0', '1'}}
}
return nil
}, func(delay time.Duration) { delays = append(delays, delay) })
if err != nil {
t.Fatal(err)
}
if calls != 3 {
t.Fatalf("operation calls=%d want=3", calls)
}
want := []time.Duration{25 * time.Millisecond, 50 * time.Millisecond}
if len(delays) != len(want) || delays[0] != want[0] || delays[1] != want[1] {
t.Fatalf("retry delays=%v want=%v", delays, want)
}
}
func TestRetryMySQLDeadlockStopsOnNonRetryableError(t *testing.T) {
want := errors.New("invalid update")
calls := 0
err := retryMySQLDeadlock(func() error {
calls++
return want
}, func(time.Duration) { t.Fatal("non-retryable error must not sleep") })
if !errors.Is(err, want) || calls != 1 {
t.Fatalf("err=%v calls=%d want=%v/1", err, calls, want)
}
}
func TestRetryMySQLDeadlockStopsAfterFourAttempts(t *testing.T) {
calls := 0
err := retryMySQLDeadlock(func() error {
calls++
return &mysqldriver.MySQLError{Number: 1205}
}, func(time.Duration) {})
if err == nil || calls != 4 {
t.Fatalf("err=%v calls=%d want non-nil/4", err, calls)
}
}