fix: reduce persistence lag and retry audit deadlocks
This commit is contained in:
@@ -4671,7 +4671,7 @@ func (a *adapter) FileUserHasAccess(fid string, uid t.Uid) (bool, error) {
|
||||
return a.db.GetContext(ctx, &allowed, query, args...)
|
||||
}
|
||||
err := run()
|
||||
for attempt := 0; isRetryableMySQLReadErr(err) && attempt < 2; attempt++ {
|
||||
for attempt := 0; isRetryableMySQLError(err) && attempt < 2; attempt++ {
|
||||
time.Sleep(time.Duration(attempt+1) * 25 * time.Millisecond)
|
||||
err = run()
|
||||
}
|
||||
@@ -4813,7 +4813,7 @@ func isIDOutOfRange(err error) bool {
|
||||
return ok && myerr.Number == 1264 && strings.Contains(strings.ToLower(myerr.Message), "id")
|
||||
}
|
||||
|
||||
func isRetryableMySQLReadErr(err error) bool {
|
||||
func isRetryableMySQLError(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -17,6 +17,7 @@ func ensureIMAdminGovernanceTables(exec sqlExecutor) error {
|
||||
request_id VARCHAR(64) NOT NULL DEFAULT '', login_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
|
||||
logout_at DATETIME(3), metadata JSON, created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
|
||||
PRIMARY KEY(id), INDEX idx_im_user_login_tenant_time(tenant_id, login_at DESC, id DESC),
|
||||
INDEX idx_im_user_login_session_close(tenant_id, session_id, logout_at),
|
||||
INDEX idx_im_user_login_user_time(user_id, login_at DESC, id DESC),
|
||||
INDEX idx_im_user_login_ip_time(ip, login_at DESC, id DESC),
|
||||
INDEX idx_im_user_login_device_time(device_id(64), login_at DESC),
|
||||
@@ -53,5 +54,9 @@ func ensureIMAdminGovernanceTables(exec sqlExecutor) error {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := execIgnoreMySQLError(exec,
|
||||
`ALTER TABLE im_user_login_logs ADD INDEX idx_im_user_login_session_close(tenant_id,session_id,logout_at)`, 1061); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -13,8 +13,8 @@ func TestEnsureIMAdminGovernanceTablesDefinesLoginAuditTenantScope(t *testing.T)
|
||||
if err := ensureIMAdminGovernanceTables(exec); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(exec.statements) != 3 {
|
||||
t.Fatalf("created %d admin governance tables, want 3", len(exec.statements))
|
||||
if len(exec.statements) != 4 {
|
||||
t.Fatalf("executed %d admin governance schema statements, want 4", len(exec.statements))
|
||||
}
|
||||
joined := strings.ToLower(strings.Join(exec.statements, "\n"))
|
||||
for _, table := range []string{"im_user_login_logs", "im_message_admin_actions", "im_user_ip_whitelists"} {
|
||||
@@ -38,4 +38,8 @@ func TestEnsureIMAdminGovernanceTablesDefinesLoginAuditTenantScope(t *testing.T)
|
||||
if end < 0 || !strings.Contains(joined[start:start+end], "tenant_id bigint not null") {
|
||||
t.Fatal("im_user_login_logs must declare tenant_id")
|
||||
}
|
||||
if !strings.Contains(joined, "idx_im_user_login_session_close(tenant_id, session_id, logout_at)") &&
|
||||
!strings.Contains(joined, "idx_im_user_login_session_close(tenant_id,session_id,logout_at)") {
|
||||
t.Fatal("im_user_login_logs must index tenant-scoped session close updates")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -93,7 +93,18 @@ func (a *adapter) TenantLoginAuditClose(id t.TenantID, sessionID string, at time
|
||||
if tenantAdp != a {
|
||||
return tenantAdp.TenantLoginAuditClose(id, sessionID, at)
|
||||
}
|
||||
_, err = a.db.Exec("UPDATE im_user_login_logs SET logout_at=? WHERE tenant_id=? AND session_id=? AND logout_at IS NULL", at, int64(id), sessionID)
|
||||
return retryMySQLDeadlock(func() error {
|
||||
_, execErr := a.db.Exec("UPDATE im_user_login_logs SET logout_at=? WHERE tenant_id=? AND session_id=? AND logout_at IS NULL", at, int64(id), sessionID)
|
||||
return execErr
|
||||
}, time.Sleep)
|
||||
}
|
||||
|
||||
func retryMySQLDeadlock(operation func() error, sleep func(time.Duration)) error {
|
||||
err := operation()
|
||||
for attempt := 1; isRetryableMySQLError(err) && attempt < 4; attempt++ {
|
||||
sleep(time.Duration(attempt) * 25 * time.Millisecond)
|
||||
err = operation()
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
|
||||
@@ -3,9 +3,13 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
mysqldriver "github.com/go-sql-driver/mysql"
|
||||
)
|
||||
|
||||
func TestLoginAuditQueriesAreTenantScoped(t *testing.T) {
|
||||
@@ -24,3 +28,48 @@ func TestLoginAuditQueriesAreTenantScoped(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetryMySQLDeadlockEventuallySucceeds(t *testing.T) {
|
||||
calls := 0
|
||||
var delays []time.Duration
|
||||
err := retryMySQLDeadlock(func() error {
|
||||
calls++
|
||||
if calls < 3 {
|
||||
return &mysqldriver.MySQLError{Number: 1213, SQLState: [5]byte{'4', '0', '0', '0', '1'}}
|
||||
}
|
||||
return nil
|
||||
}, func(delay time.Duration) { delays = append(delays, delay) })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if calls != 3 {
|
||||
t.Fatalf("operation calls=%d want=3", calls)
|
||||
}
|
||||
want := []time.Duration{25 * time.Millisecond, 50 * time.Millisecond}
|
||||
if len(delays) != len(want) || delays[0] != want[0] || delays[1] != want[1] {
|
||||
t.Fatalf("retry delays=%v want=%v", delays, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetryMySQLDeadlockStopsOnNonRetryableError(t *testing.T) {
|
||||
want := errors.New("invalid update")
|
||||
calls := 0
|
||||
err := retryMySQLDeadlock(func() error {
|
||||
calls++
|
||||
return want
|
||||
}, func(time.Duration) { t.Fatal("non-retryable error must not sleep") })
|
||||
if !errors.Is(err, want) || calls != 1 {
|
||||
t.Fatalf("err=%v calls=%d want=%v/1", err, calls, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetryMySQLDeadlockStopsAfterFourAttempts(t *testing.T) {
|
||||
calls := 0
|
||||
err := retryMySQLDeadlock(func() error {
|
||||
calls++
|
||||
return &mysqldriver.MySQLError{Number: 1205}
|
||||
}, func(time.Duration) {})
|
||||
if err == nil || calls != 4 {
|
||||
t.Fatalf("err=%v calls=%d want non-nil/4", err, calls)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user